From 7642e80723386d113ee5c95647164f72e3d68d6d Mon Sep 17 00:00:00 2001 From: Brent Traut Date: Fri, 20 Nov 2026 01:33:52 +0000 Subject: [PATCH] Release persisted MCP tool results from active-turn memory (#62641) ## Why Active turns in paginated threads retain completed MCP tool results in memory even when history can supply the same payloads. History can also store a truncated preview, so dropping a result requires checking that the full payload is recoverable. ## What changed For persisted paginated threads, check successful MCP calls in batches and release an active-turn result only when the stored result matches it exactly. When resuming with full history, fetch released results by item ID so a stale history page cannot supply an older result. ## Testing Add a regression test that restores the latest result despite a stale page and checks that truncated, metadata-mismatched, missing, and failed results remain in memory. GitOrigin-RevId: 49f6f1414201e4327433043050d65a1f4adc9720 --- .../src/protocol/thread_history.rs | 50 +++++-- codex-rs/app-server/src/message_processor.rs | 1 + .../request_processors/thread_lifecycle.rs | 112 +++++++++++++++- .../thread_listener_shutdown_tests.rs | 1 + .../request_processors/thread_processor.rs | 12 +- .../thread_processor_tests.rs | 122 ++++++++++++++++++ .../turn_interrupt_tests.rs | 1 + .../src/request_processors/turn_processor.rs | 4 + codex-rs/app-server/src/thread_state.rs | 5 + 9 files changed, 281 insertions(+), 27 deletions(-) diff --git a/codex-rs/app-server-protocol/src/protocol/thread_history.rs b/codex-rs/app-server-protocol/src/protocol/thread_history.rs index a0819c1d25..c7341ba7c5 100644 --- a/codex-rs/app-server-protocol/src/protocol/thread_history.rs +++ b/codex-rs/app-server-protocol/src/protocol/thread_history.rs @@ -18,6 +18,7 @@ use crate::protocol::v2::McpToolCallError; use crate::protocol::v2::McpToolCallResult; use crate::protocol::v2::McpToolCallStatus; use crate::protocol::v2::ThreadItem; +use crate::protocol::v2::ThreadItemEntry; use crate::protocol::v2::ThreadItemPayload; use crate::protocol::v2::Turn; use crate::protocol::v2::TurnError as V2TurnError; @@ -430,6 +431,32 @@ impl ThreadHistoryBuilder { } } + /// Drops only completed MCP results that the active turn can recover in full by ID. + pub fn release_saved_mcp_results(&mut self, entries: &[ThreadItemEntry]) { + let Some(turn) = self.current_turn.as_mut() else { + return; + }; + for entry in entries { + if entry.turn_id == turn.id + && let ThreadItemPayload::McpToolCall { + id, + status: McpToolCallStatus::Completed, + result: Some(saved), + .. + } = &entry.item.payload + && let Some(index) = turn.item_index.position(&turn.items, id) + && let ThreadItemPayload::McpToolCall { + status: McpToolCallStatus::Completed, + result, + .. + } = &mut turn.items[index] + && result.as_ref() == Some(saved) + { + *result = None; + } + } + } + /// Shared reducer for persisted rollout replay and in-memory current-turn /// tracking used by running thread resume/rejoin. /// @@ -1893,11 +1920,7 @@ impl TurnItemIndex { items.push(item); } - fn upsert<'a>( - &mut self, - items: &'a mut Vec, - item: ThreadItemPayload, - ) -> &'a ThreadItemPayload { + fn position(&mut self, items: &[ThreadItemPayload], id: &str) -> Option { if self.positions.is_none() && items.len() >= TURN_ITEM_INDEX_THRESHOLD { let mut positions = HashMap::with_capacity(items.len()); for (index, existing) in items.iter().enumerate() { @@ -1906,11 +1929,18 @@ impl TurnItemIndex { self.positions = Some(positions); } - let existing_index = match &self.positions { - Some(positions) => positions.get(item.id()).copied(), - None => items.iter().position(|existing| existing.id() == item.id()), - }; - if let Some(index) = existing_index { + match &self.positions { + Some(positions) => positions.get(id).copied(), + None => items.iter().position(|existing| existing.id() == id), + } + } + + fn upsert<'a>( + &mut self, + items: &'a mut Vec, + item: ThreadItemPayload, + ) -> &'a ThreadItemPayload { + if let Some(index) = self.position(items, item.id()) { items[index] = item; &items[index] } else { diff --git a/codex-rs/app-server/src/message_processor.rs b/codex-rs/app-server/src/message_processor.rs index 4774dd4ad4..503c41365f 100644 --- a/codex-rs/app-server/src/message_processor.rs +++ b/codex-rs/app-server/src/message_processor.rs @@ -580,6 +580,7 @@ impl MessageProcessor { ); let turn_processor = TurnRequestProcessor::new( Arc::clone(&thread_manager), + Arc::clone(&thread_store), outgoing.clone(), analytics_events_client.clone(), Arc::clone(&config), diff --git a/codex-rs/app-server/src/request_processors/thread_lifecycle.rs b/codex-rs/app-server/src/request_processors/thread_lifecycle.rs index 3654234966..d3777a7ef3 100644 --- a/codex-rs/app-server/src/request_processors/thread_lifecycle.rs +++ b/codex-rs/app-server/src/request_processors/thread_lifecycle.rs @@ -5,6 +5,7 @@ use crate::bespoke_event_handling::complete_command_execution_item; use crate::extensions::send_thread_warning; use crate::thread_state::ListenerStateCandidate; use codex_app_server_protocol::CommandExecutionStatus; +use codex_app_server_protocol::McpToolCallStatus; use codex_app_server_protocol::ThreadItem; use codex_app_server_protocol::ThreadQueueChangedNotification; use codex_core::ThreadEvictionOutcome; @@ -27,6 +28,7 @@ mod replay_tests; #[derive(Clone)] pub(super) struct ListenerTaskContext { pub(super) thread_manager: Arc, + pub(super) thread_store: Arc, pub(super) thread_state_manager: ThreadStateManager, pub(super) outgoing: Arc, pub(super) pending_thread_unloads: PendingThreadUnloads, @@ -503,6 +505,7 @@ async fn start_listener( let ListenerTaskContext { outgoing, thread_manager, + thread_store, thread_state_manager, pending_thread_unloads, thread_watch_manager, @@ -511,8 +514,13 @@ async fn start_listener( .. } = listener_task_context; let outgoing_for_task = Arc::clone(&outgoing); + let verify_mcp_results = config_snapshot.history_mode + == codex_protocol::protocol::ThreadHistoryMode::Paginated + && conversation.rollout_path().is_some(); tokio::spawn(async move { let mut replays = JoinSet::new(); + let mut mcp_turn_id = String::new(); + let mut mcp_result_ids = Vec::new(); let shutdown_complete = loop { tokio::select! { biased; @@ -600,6 +608,23 @@ async fn start_listener( ).await; } let is_shutdown_complete = matches!(event.msg, EventMsg::ShutdownComplete); + let batch = if verify_mcp_results + && let EventMsg::McpToolCallEnd(call) = &event.msg + && call.is_success() && !call.turn_id.is_empty() + { + if mcp_turn_id != call.turn_id { + mcp_turn_id.clone_from(&call.turn_id); + mcp_result_ids.clear(); + } + mcp_result_ids.push(call.call_id.clone()); + (mcp_result_ids.len() == THREAD_ITEMS_MAX_LIMIT).then(|| ThreadItemsReadParams { + thread_id: conversation_id.to_string(), + turn_id: mcp_turn_id.clone(), + item_ids: std::mem::take(&mut mcp_result_ids), + }) + } else { + None + }; apply_bespoke_event_handling( event, conversation_id, @@ -613,6 +638,15 @@ async fn start_listener( if is_shutdown_complete { break true; } + // Storage may hold only a preview. A failed lookup leaves the live results intact. + if let Some(batch) = batch + && let Ok(saved) = ThreadRequestProcessor::read_thread_items(thread_store.as_ref(), batch).await + { + let mut state = thread_state.lock().await; + if state.listener_generation == listener_generation { + state.release_saved_mcp_results(&saved.data); + } + } } idle_guard = async { if unloading_state.wait_for_unloading_trigger().await { @@ -1134,7 +1168,7 @@ async fn handle_pending_thread_resume_request( .iter_mut() .flat_map(|page| &mut page.data), ); - if let Err(error) = restore_evicted_agent_text(store, conversation_id, turn, stored).await { + if let Err(error) = restore_evicted_payloads(store, conversation_id, turn, stored).await { outgoing.send_error(pending.request_id, error).await; return; } @@ -1442,6 +1476,16 @@ pub(super) async fn resolve_pending_server_request( #[derive(Clone, Eq, Hash, PartialEq)] struct HistoryItemId<'a>(Cow<'a, str>); +pub(super) async fn restore_evicted_payloads<'a>( + store: &dyn ThreadStore, + thread_id: ThreadId, + turn: &mut Turn, + stored: impl Iterator, +) -> Result<(), JSONRPCErrorError> { + restore_evicted_mcp_results(store, thread_id, turn).await?; + restore_evicted_agent_text(store, thread_id, turn, stored).await +} + async fn restore_evicted_agent_text<'a>( store: &dyn ThreadStore, thread_id: ThreadId, @@ -1471,12 +1515,7 @@ async fn restore_evicted_agent_text<'a>( // The prepared page can predate some of the listener's items. Fetch only those gaps. let ids: Vec<_> = missing.keys().cloned().collect(); for chunk in ids.chunks(THREAD_ITEMS_MAX_LIMIT) { - let params = ThreadItemsReadParams { - thread_id: thread_id.to_string(), - turn_id: turn.id.clone(), - item_ids: chunk.iter().map(|id| id.0.to_string()).collect(), - }; - let saved = ThreadRequestProcessor::read_thread_items(store, params).await?; + let saved = read_evicted_items(store, thread_id, &turn.id, chunk).await?; fill_agent_text(&mut missing, saved.data.into_iter().map(|entry| entry.item)); } if missing.is_empty() { @@ -1503,6 +1542,65 @@ fn fill_agent_text( } } +async fn restore_evicted_mcp_results( + store: &dyn ThreadStore, + thread_id: ThreadId, + turn: &mut Turn, +) -> Result<(), JSONRPCErrorError> { + let mut missing: HashMap<_, _> = turn + .items + .iter_mut() + .filter_map(|item| match &mut item.payload { + ThreadItemPayload::McpToolCall { + id, + status: McpToolCallStatus::Completed, + result, + .. + } if result.is_none() => Some((HistoryItemId(Cow::Borrowed(id)), result)), + _ => None, + }) + .collect(); + // A prepared page may contain a stale result for the same call ID. + let ids: Vec<_> = missing.keys().cloned().collect(); + for chunk in ids.chunks(THREAD_ITEMS_MAX_LIMIT) { + let saved = read_evicted_items(store, thread_id, &turn.id, chunk).await?; + for entry in saved.data { + if let ThreadItemPayload::McpToolCall { + id, + status: McpToolCallStatus::Completed, + result: Some(result), + .. + } = entry.item.payload + && let Some(slot) = missing.remove(&HistoryItemId(Cow::Owned(id))) + { + *slot = Some(result); + } + } + } + if missing.is_empty() { + Ok(()) + } else { + Err(internal_error(format!( + "saved history is missing completed MCP results for turn {}", + turn.id + ))) + } +} + +async fn read_evicted_items( + store: &dyn ThreadStore, + thread_id: ThreadId, + turn_id: &str, + ids: &[HistoryItemId<'_>], +) -> Result { + let params = ThreadItemsReadParams { + thread_id: thread_id.to_string(), + turn_id: turn_id.to_owned(), + item_ids: ids.iter().map(|id| id.0.to_string()).collect(), + }; + ThreadRequestProcessor::read_thread_items(store, params).await +} + pub(super) fn merge_turn_history_with_active_turn(turns: &mut Vec, active_turn: Turn) { turns.retain(|turn| turn.id != active_turn.id); turns.push(active_turn); diff --git a/codex-rs/app-server/src/request_processors/thread_listener_shutdown_tests.rs b/codex-rs/app-server/src/request_processors/thread_listener_shutdown_tests.rs index dca218d5ef..62930aa1db 100644 --- a/codex-rs/app-server/src/request_processors/thread_listener_shutdown_tests.rs +++ b/codex-rs/app-server/src/request_processors/thread_listener_shutdown_tests.rs @@ -690,6 +690,7 @@ async fn revert_settles_interrupts_before_replacing_a_runtime() -> Result<()> { .await; let turn_processor = TurnRequestProcessor::new( test.thread_manager.clone(), + test.thread_store.clone(), processor.outgoing.clone(), AnalyticsEventsClient::disabled(), processor.config.clone(), diff --git a/codex-rs/app-server/src/request_processors/thread_processor.rs b/codex-rs/app-server/src/request_processors/thread_processor.rs index 80ae8b7e41..556cc2cf05 100644 --- a/codex-rs/app-server/src/request_processors/thread_processor.rs +++ b/codex-rs/app-server/src/request_processors/thread_processor.rs @@ -1270,6 +1270,7 @@ impl ThreadRequestProcessor { fn listener_task_context(&self) -> ListenerTaskContext { ListenerTaskContext { thread_manager: Arc::clone(&self.thread_manager), + thread_store: Arc::clone(&self.thread_store), thread_state_manager: self.thread_state_manager.clone(), outgoing: Arc::clone(&self.outgoing), pending_thread_unloads: Arc::clone(&self.pending_thread_unloads), @@ -1400,16 +1401,7 @@ impl ThreadRequestProcessor { personality, ); typesafe_overrides.ephemeral = ephemeral; - let listener_task_context = ListenerTaskContext { - thread_manager: Arc::clone(&self.thread_manager), - thread_state_manager: self.thread_state_manager.clone(), - outgoing: Arc::clone(&self.outgoing), - pending_thread_unloads: Arc::clone(&self.pending_thread_unloads), - thread_watch_manager: self.thread_watch_manager.clone(), - codex_home: self.config.codex_home.to_path_buf(), - skills_watcher: Arc::clone(&self.skills_watcher), - turn_cost_worker: self.turn_cost_worker.clone(), - }; + let listener_task_context = self.listener_task_context(); let request_trace = request_context.request_trace(); let config_manager = self.config_manager.clone(); let thread_store = Arc::clone(&self.thread_store); diff --git a/codex-rs/app-server/src/request_processors/thread_processor_tests.rs b/codex-rs/app-server/src/request_processors/thread_processor_tests.rs index 4d91562585..ee85727abb 100644 --- a/codex-rs/app-server/src/request_processors/thread_processor_tests.rs +++ b/codex-rs/app-server/src/request_processors/thread_processor_tests.rs @@ -223,6 +223,127 @@ mod thread_processor_behavior_tests { use std::sync::Arc; use tempfile::TempDir; + #[tokio::test] + async fn full_resume_restores_only_recoverable_mcp_results() -> Result<()> { + use codex_protocol::protocol::HasLegacyEvent; + use core_test_support::responses; + + let server = responses::start_mock_server().await; + responses::mount_sse_once( + &server, + responses::sse(vec![ + responses::ev_response_created("seed"), + responses::ev_completed("seed"), + ]), + ) + .await; + let test = core_test_support::test_codex::test_codex() + .with_history_mode(ThreadHistoryMode::Paginated) + .build_with_auto_env(&server) + .await?; + test.submit_turn("seed history").await?; + let store = test.thread_store.as_ref(); + let thread_id = test.session_configured.thread_id; + let started: EventMsg = + serde_json::from_value(json!({"type": "turn_started", "turn_id": "turn"}))?; + let mut live = ThreadHistoryBuilder::new(); + live.handle_live_paginated_event(&started); + let mut saved = vec![RolloutItem::EventMsg(started)]; + let completion = |id: &str, text: &str, view: &str| -> serde_json::Result { + serde_json::from_value(json!({ + "type": "item_completed", "thread_id": thread_id, "turn_id": "turn", + "started_at_ms": 2000, "completed_at_ms": 2007, + "item": { + "type": "McpToolCall", "id": id, "server": "docs", "tool": "read", + "arguments": {}, "status": if id == "failed" { "failed" } else { "completed" }, + "duration": {"secs": 0, "nanos": 7000000}, + "result": {"content": [{"type": "text", "text": text}], "isError": id == "failed", + "structuredContent": {"id": id}, "_meta": {"view": view}} + } + })) + }; + let deliver = |live: &mut ThreadHistoryBuilder, event: &EventMsg| { + live.handle_live_paginated_event(event); + for legacy in event.as_legacy_events(/*show_raw_agent_reasoning*/ false) { + live.handle_live_paginated_event(&legacy); + } + }; + + // The prepared full page predates a correction to this same call ID. + let old = completion("exact", "old result", "detail")?; + deliver(&mut live, &old); + saved.push(RolloutItem::EventMsg(old)); + let mut page = live.active_turn_snapshot().expect("prepared turn"); + let ids = ["exact", "preview", "metadata", "missing", "failed"]; + let oversized = "x".repeat(200_000); + for id in ids { + let text = if id == "preview" { + &oversized + } else { + "full result" + }; + let event = completion(id, text, "detail")?; + deliver(&mut live, &event); + match id { + "missing" => {} + "metadata" => saved.push(RolloutItem::EventMsg(completion(id, text, "summary")?)), + _ => saved.push(RolloutItem::EventMsg(event)), + } + } + store + .append_items(codex_thread_store::AppendThreadItemsParams { + thread_id, + items: saved, + }) + .await?; + let rows = ThreadRequestProcessor::read_thread_items( + store, + ThreadItemsReadParams { + thread_id: thread_id.to_string(), + turn_id: "turn".to_string(), + item_ids: ids.map(str::to_string).into(), + }, + ) + .await + .expect("read saved results"); + assert_eq!( + rows.data + .iter() + .map(|entry| entry.item.id()) + .collect::>(), + ["exact", "preview", "metadata", "failed"] + ); + + let expected = live.active_turn_snapshot().expect("full turn"); + assert_eq!( + expected + .items + .iter() + .map(ThreadItem::id) + .collect::>(), + ids + ); + live.release_saved_mcp_results(&rows.data); + let mut resumed = live.active_turn_snapshot().expect("evicted turn"); + let mut evicted = expected.clone(); + let ThreadItemPayload::McpToolCall { result, .. } = &mut evicted.items[0].payload else { + panic!("expected MCP call") + }; + assert!(result.take().is_some()); + assert_eq!(resumed, evicted); + + crate::request_processors::thread_lifecycle::restore_evicted_payloads( + store, + thread_id, + &mut resumed, + std::iter::once(&mut page), + ) + .await + .expect("full reconnect"); + assert_eq!(resumed, expected); + Ok(()) + } + fn dynamic_tool( namespace: Option<&str>, name: impl Into, @@ -1326,6 +1447,7 @@ mod thread_processor_behavior_tests { ensure_conversation_listener( ListenerTaskContext { thread_manager: test.thread_manager.clone(), + thread_store: Arc::clone(&test.thread_store), thread_state_manager, outgoing: outgoing.clone(), pending_thread_unloads: Default::default(), diff --git a/codex-rs/app-server/src/request_processors/turn_interrupt_tests.rs b/codex-rs/app-server/src/request_processors/turn_interrupt_tests.rs index 47649b189a..0b31d5a666 100644 --- a/codex-rs/app-server/src/request_processors/turn_interrupt_tests.rs +++ b/codex-rs/app-server/src/request_processors/turn_interrupt_tests.rs @@ -99,6 +99,7 @@ async fn interrupt_reply_waits_for_the_requested_turn(queued: bool) -> anyhow::R ); let processor = TurnRequestProcessor::new( test.thread_manager.clone(), + test.thread_store.clone(), outgoing.clone(), analytics, Arc::new(test.config.clone()), diff --git a/codex-rs/app-server/src/request_processors/turn_processor.rs b/codex-rs/app-server/src/request_processors/turn_processor.rs index a16604de81..3b1fa45b12 100644 --- a/codex-rs/app-server/src/request_processors/turn_processor.rs +++ b/codex-rs/app-server/src/request_processors/turn_processor.rs @@ -90,6 +90,7 @@ fn validate_response_item_image_urls(items: &[ResponseItem]) -> Result<(), JSONR #[derive(Clone)] pub(crate) struct TurnRequestProcessor { thread_manager: Arc, + thread_store: Arc, outgoing: Arc, analytics_events_client: AnalyticsEventsClient, config: Arc, @@ -147,6 +148,7 @@ impl TurnRequestProcessor { #[allow(clippy::too_many_arguments)] pub(crate) fn new( thread_manager: Arc, + thread_store: Arc, outgoing: Arc, analytics_events_client: AnalyticsEventsClient, config: Arc, @@ -159,6 +161,7 @@ impl TurnRequestProcessor { ) -> Self { Self { thread_manager, + thread_store, outgoing, analytics_events_client, config, @@ -1666,6 +1669,7 @@ impl TurnRequestProcessor { fn listener_task_context(&self) -> ListenerTaskContext { ListenerTaskContext { thread_manager: Arc::clone(&self.thread_manager), + thread_store: Arc::clone(&self.thread_store), thread_state_manager: self.thread_state_manager.clone(), outgoing: Arc::clone(&self.outgoing), pending_thread_unloads: Arc::clone(&self.pending_thread_unloads), diff --git a/codex-rs/app-server/src/thread_state.rs b/codex-rs/app-server/src/thread_state.rs index 68cb7db958..cd4b9ca489 100644 --- a/codex-rs/app-server/src/thread_state.rs +++ b/codex-rs/app-server/src/thread_state.rs @@ -6,6 +6,7 @@ use codex_app_server_protocol::ThreadGoal; use codex_app_server_protocol::ThreadHistoryBuilder; use codex_app_server_protocol::ThreadHistoryTurnMetadata; use codex_app_server_protocol::ThreadItem; +use codex_app_server_protocol::ThreadItemEntry; use codex_app_server_protocol::ThreadSettings; use codex_app_server_protocol::Turn; use codex_app_server_protocol::TurnError; @@ -225,6 +226,10 @@ impl ThreadState { self.current_turn_history.active_turn_snapshot() } + pub(crate) fn release_saved_mcp_results(&mut self, entries: &[ThreadItemEntry]) { + self.current_turn_history.release_saved_mcp_results(entries); + } + /// Returns the same turn ID as `active_turn_snapshot` without cloning its items. pub(crate) fn active_turn_id(&self) -> Option<&str> { self.current_turn_history.active_turn_id()