Release persisted MCP tool results from active-turn memory (#62641)

## Why

Active turns in paginated threads retain completed MCP tool results in
memory even when history can supply the same payloads. History can also
store a truncated preview, so dropping a result requires checking that
the full payload is recoverable.

## What changed

For persisted paginated threads, check successful MCP calls in batches
and release an active-turn result only when the stored result matches
it exactly. When resuming with full history, fetch released results by
item ID so a stale history page cannot supply an older result.

## Testing

Add a regression test that restores the latest result despite a stale
page and checks that truncated, metadata-mismatched, missing, and failed
results remain in memory.

GitOrigin-RevId: 49f6f1414201e4327433043050d65a1f4adc9720
This commit is contained in:
Brent Traut
2026-11-20 01:49:15 +00:00
committed by copyberry
parent e720e6c80b
commit 7642e80723
9 changed files with 281 additions and 27 deletions
@@ -18,6 +18,7 @@ use crate::protocol::v2::McpToolCallError;
use crate::protocol::v2::McpToolCallResult;
use crate::protocol::v2::McpToolCallStatus;
use crate::protocol::v2::ThreadItem;
use crate::protocol::v2::ThreadItemEntry;
use crate::protocol::v2::ThreadItemPayload;
use crate::protocol::v2::Turn;
use crate::protocol::v2::TurnError as V2TurnError;
@@ -430,6 +431,32 @@ impl ThreadHistoryBuilder {
}
}
/// Drops only completed MCP results that the active turn can recover in full by ID.
pub fn release_saved_mcp_results(&mut self, entries: &[ThreadItemEntry]) {
let Some(turn) = self.current_turn.as_mut() else {
return;
};
for entry in entries {
if entry.turn_id == turn.id
&& let ThreadItemPayload::McpToolCall {
id,
status: McpToolCallStatus::Completed,
result: Some(saved),
..
} = &entry.item.payload
&& let Some(index) = turn.item_index.position(&turn.items, id)
&& let ThreadItemPayload::McpToolCall {
status: McpToolCallStatus::Completed,
result,
..
} = &mut turn.items[index]
&& result.as_ref() == Some(saved)
{
*result = None;
}
}
}
/// Shared reducer for persisted rollout replay and in-memory current-turn
/// tracking used by running thread resume/rejoin.
///
@@ -1893,11 +1920,7 @@ impl TurnItemIndex {
items.push(item);
}
fn upsert<'a>(
&mut self,
items: &'a mut Vec<ThreadItemPayload>,
item: ThreadItemPayload,
) -> &'a ThreadItemPayload {
fn position(&mut self, items: &[ThreadItemPayload], id: &str) -> Option<usize> {
if self.positions.is_none() && items.len() >= TURN_ITEM_INDEX_THRESHOLD {
let mut positions = HashMap::with_capacity(items.len());
for (index, existing) in items.iter().enumerate() {
@@ -1906,11 +1929,18 @@ impl TurnItemIndex {
self.positions = Some(positions);
}
let existing_index = match &self.positions {
Some(positions) => positions.get(item.id()).copied(),
None => items.iter().position(|existing| existing.id() == item.id()),
};
if let Some(index) = existing_index {
match &self.positions {
Some(positions) => positions.get(id).copied(),
None => items.iter().position(|existing| existing.id() == id),
}
}
fn upsert<'a>(
&mut self,
items: &'a mut Vec<ThreadItemPayload>,
item: ThreadItemPayload,
) -> &'a ThreadItemPayload {
if let Some(index) = self.position(items, item.id()) {
items[index] = item;
&items[index]
} else {
@@ -580,6 +580,7 @@ impl MessageProcessor {
);
let turn_processor = TurnRequestProcessor::new(
Arc::clone(&thread_manager),
Arc::clone(&thread_store),
outgoing.clone(),
analytics_events_client.clone(),
Arc::clone(&config),
@@ -5,6 +5,7 @@ use crate::bespoke_event_handling::complete_command_execution_item;
use crate::extensions::send_thread_warning;
use crate::thread_state::ListenerStateCandidate;
use codex_app_server_protocol::CommandExecutionStatus;
use codex_app_server_protocol::McpToolCallStatus;
use codex_app_server_protocol::ThreadItem;
use codex_app_server_protocol::ThreadQueueChangedNotification;
use codex_core::ThreadEvictionOutcome;
@@ -27,6 +28,7 @@ mod replay_tests;
#[derive(Clone)]
pub(super) struct ListenerTaskContext {
pub(super) thread_manager: Arc<ThreadManager>,
pub(super) thread_store: Arc<dyn ThreadStore>,
pub(super) thread_state_manager: ThreadStateManager,
pub(super) outgoing: Arc<OutgoingMessageSender>,
pub(super) pending_thread_unloads: PendingThreadUnloads,
@@ -503,6 +505,7 @@ async fn start_listener(
let ListenerTaskContext {
outgoing,
thread_manager,
thread_store,
thread_state_manager,
pending_thread_unloads,
thread_watch_manager,
@@ -511,8 +514,13 @@ async fn start_listener(
..
} = listener_task_context;
let outgoing_for_task = Arc::clone(&outgoing);
let verify_mcp_results = config_snapshot.history_mode
== codex_protocol::protocol::ThreadHistoryMode::Paginated
&& conversation.rollout_path().is_some();
tokio::spawn(async move {
let mut replays = JoinSet::new();
let mut mcp_turn_id = String::new();
let mut mcp_result_ids = Vec::new();
let shutdown_complete = loop {
tokio::select! {
biased;
@@ -600,6 +608,23 @@ async fn start_listener(
).await;
}
let is_shutdown_complete = matches!(event.msg, EventMsg::ShutdownComplete);
let batch = if verify_mcp_results
&& let EventMsg::McpToolCallEnd(call) = &event.msg
&& call.is_success() && !call.turn_id.is_empty()
{
if mcp_turn_id != call.turn_id {
mcp_turn_id.clone_from(&call.turn_id);
mcp_result_ids.clear();
}
mcp_result_ids.push(call.call_id.clone());
(mcp_result_ids.len() == THREAD_ITEMS_MAX_LIMIT).then(|| ThreadItemsReadParams {
thread_id: conversation_id.to_string(),
turn_id: mcp_turn_id.clone(),
item_ids: std::mem::take(&mut mcp_result_ids),
})
} else {
None
};
apply_bespoke_event_handling(
event,
conversation_id,
@@ -613,6 +638,15 @@ async fn start_listener(
if is_shutdown_complete {
break true;
}
// Storage may hold only a preview. A failed lookup leaves the live results intact.
if let Some(batch) = batch
&& let Ok(saved) = ThreadRequestProcessor::read_thread_items(thread_store.as_ref(), batch).await
{
let mut state = thread_state.lock().await;
if state.listener_generation == listener_generation {
state.release_saved_mcp_results(&saved.data);
}
}
}
idle_guard = async {
if unloading_state.wait_for_unloading_trigger().await {
@@ -1134,7 +1168,7 @@ async fn handle_pending_thread_resume_request(
.iter_mut()
.flat_map(|page| &mut page.data),
);
if let Err(error) = restore_evicted_agent_text(store, conversation_id, turn, stored).await {
if let Err(error) = restore_evicted_payloads(store, conversation_id, turn, stored).await {
outgoing.send_error(pending.request_id, error).await;
return;
}
@@ -1442,6 +1476,16 @@ pub(super) async fn resolve_pending_server_request(
#[derive(Clone, Eq, Hash, PartialEq)]
struct HistoryItemId<'a>(Cow<'a, str>);
pub(super) async fn restore_evicted_payloads<'a>(
store: &dyn ThreadStore,
thread_id: ThreadId,
turn: &mut Turn,
stored: impl Iterator<Item = &'a mut Turn>,
) -> Result<(), JSONRPCErrorError> {
restore_evicted_mcp_results(store, thread_id, turn).await?;
restore_evicted_agent_text(store, thread_id, turn, stored).await
}
async fn restore_evicted_agent_text<'a>(
store: &dyn ThreadStore,
thread_id: ThreadId,
@@ -1471,12 +1515,7 @@ async fn restore_evicted_agent_text<'a>(
// The prepared page can predate some of the listener's items. Fetch only those gaps.
let ids: Vec<_> = missing.keys().cloned().collect();
for chunk in ids.chunks(THREAD_ITEMS_MAX_LIMIT) {
let params = ThreadItemsReadParams {
thread_id: thread_id.to_string(),
turn_id: turn.id.clone(),
item_ids: chunk.iter().map(|id| id.0.to_string()).collect(),
};
let saved = ThreadRequestProcessor::read_thread_items(store, params).await?;
let saved = read_evicted_items(store, thread_id, &turn.id, chunk).await?;
fill_agent_text(&mut missing, saved.data.into_iter().map(|entry| entry.item));
}
if missing.is_empty() {
@@ -1503,6 +1542,65 @@ fn fill_agent_text(
}
}
async fn restore_evicted_mcp_results(
store: &dyn ThreadStore,
thread_id: ThreadId,
turn: &mut Turn,
) -> Result<(), JSONRPCErrorError> {
let mut missing: HashMap<_, _> = turn
.items
.iter_mut()
.filter_map(|item| match &mut item.payload {
ThreadItemPayload::McpToolCall {
id,
status: McpToolCallStatus::Completed,
result,
..
} if result.is_none() => Some((HistoryItemId(Cow::Borrowed(id)), result)),
_ => None,
})
.collect();
// A prepared page may contain a stale result for the same call ID.
let ids: Vec<_> = missing.keys().cloned().collect();
for chunk in ids.chunks(THREAD_ITEMS_MAX_LIMIT) {
let saved = read_evicted_items(store, thread_id, &turn.id, chunk).await?;
for entry in saved.data {
if let ThreadItemPayload::McpToolCall {
id,
status: McpToolCallStatus::Completed,
result: Some(result),
..
} = entry.item.payload
&& let Some(slot) = missing.remove(&HistoryItemId(Cow::Owned(id)))
{
*slot = Some(result);
}
}
}
if missing.is_empty() {
Ok(())
} else {
Err(internal_error(format!(
"saved history is missing completed MCP results for turn {}",
turn.id
)))
}
}
async fn read_evicted_items(
store: &dyn ThreadStore,
thread_id: ThreadId,
turn_id: &str,
ids: &[HistoryItemId<'_>],
) -> Result<codex_app_server_protocol::ThreadItemsReadResponse, JSONRPCErrorError> {
let params = ThreadItemsReadParams {
thread_id: thread_id.to_string(),
turn_id: turn_id.to_owned(),
item_ids: ids.iter().map(|id| id.0.to_string()).collect(),
};
ThreadRequestProcessor::read_thread_items(store, params).await
}
pub(super) fn merge_turn_history_with_active_turn(turns: &mut Vec<Turn>, active_turn: Turn) {
turns.retain(|turn| turn.id != active_turn.id);
turns.push(active_turn);
@@ -690,6 +690,7 @@ async fn revert_settles_interrupts_before_replacing_a_runtime() -> Result<()> {
.await;
let turn_processor = TurnRequestProcessor::new(
test.thread_manager.clone(),
test.thread_store.clone(),
processor.outgoing.clone(),
AnalyticsEventsClient::disabled(),
processor.config.clone(),
@@ -1270,6 +1270,7 @@ impl ThreadRequestProcessor {
fn listener_task_context(&self) -> ListenerTaskContext {
ListenerTaskContext {
thread_manager: Arc::clone(&self.thread_manager),
thread_store: Arc::clone(&self.thread_store),
thread_state_manager: self.thread_state_manager.clone(),
outgoing: Arc::clone(&self.outgoing),
pending_thread_unloads: Arc::clone(&self.pending_thread_unloads),
@@ -1400,16 +1401,7 @@ impl ThreadRequestProcessor {
personality,
);
typesafe_overrides.ephemeral = ephemeral;
let listener_task_context = ListenerTaskContext {
thread_manager: Arc::clone(&self.thread_manager),
thread_state_manager: self.thread_state_manager.clone(),
outgoing: Arc::clone(&self.outgoing),
pending_thread_unloads: Arc::clone(&self.pending_thread_unloads),
thread_watch_manager: self.thread_watch_manager.clone(),
codex_home: self.config.codex_home.to_path_buf(),
skills_watcher: Arc::clone(&self.skills_watcher),
turn_cost_worker: self.turn_cost_worker.clone(),
};
let listener_task_context = self.listener_task_context();
let request_trace = request_context.request_trace();
let config_manager = self.config_manager.clone();
let thread_store = Arc::clone(&self.thread_store);
@@ -223,6 +223,127 @@ mod thread_processor_behavior_tests {
use std::sync::Arc;
use tempfile::TempDir;
#[tokio::test]
async fn full_resume_restores_only_recoverable_mcp_results() -> Result<()> {
use codex_protocol::protocol::HasLegacyEvent;
use core_test_support::responses;
let server = responses::start_mock_server().await;
responses::mount_sse_once(
&server,
responses::sse(vec![
responses::ev_response_created("seed"),
responses::ev_completed("seed"),
]),
)
.await;
let test = core_test_support::test_codex::test_codex()
.with_history_mode(ThreadHistoryMode::Paginated)
.build_with_auto_env(&server)
.await?;
test.submit_turn("seed history").await?;
let store = test.thread_store.as_ref();
let thread_id = test.session_configured.thread_id;
let started: EventMsg =
serde_json::from_value(json!({"type": "turn_started", "turn_id": "turn"}))?;
let mut live = ThreadHistoryBuilder::new();
live.handle_live_paginated_event(&started);
let mut saved = vec![RolloutItem::EventMsg(started)];
let completion = |id: &str, text: &str, view: &str| -> serde_json::Result<EventMsg> {
serde_json::from_value(json!({
"type": "item_completed", "thread_id": thread_id, "turn_id": "turn",
"started_at_ms": 2000, "completed_at_ms": 2007,
"item": {
"type": "McpToolCall", "id": id, "server": "docs", "tool": "read",
"arguments": {}, "status": if id == "failed" { "failed" } else { "completed" },
"duration": {"secs": 0, "nanos": 7000000},
"result": {"content": [{"type": "text", "text": text}], "isError": id == "failed",
"structuredContent": {"id": id}, "_meta": {"view": view}}
}
}))
};
let deliver = |live: &mut ThreadHistoryBuilder, event: &EventMsg| {
live.handle_live_paginated_event(event);
for legacy in event.as_legacy_events(/*show_raw_agent_reasoning*/ false) {
live.handle_live_paginated_event(&legacy);
}
};
// The prepared full page predates a correction to this same call ID.
let old = completion("exact", "old result", "detail")?;
deliver(&mut live, &old);
saved.push(RolloutItem::EventMsg(old));
let mut page = live.active_turn_snapshot().expect("prepared turn");
let ids = ["exact", "preview", "metadata", "missing", "failed"];
let oversized = "x".repeat(200_000);
for id in ids {
let text = if id == "preview" {
&oversized
} else {
"full result"
};
let event = completion(id, text, "detail")?;
deliver(&mut live, &event);
match id {
"missing" => {}
"metadata" => saved.push(RolloutItem::EventMsg(completion(id, text, "summary")?)),
_ => saved.push(RolloutItem::EventMsg(event)),
}
}
store
.append_items(codex_thread_store::AppendThreadItemsParams {
thread_id,
items: saved,
})
.await?;
let rows = ThreadRequestProcessor::read_thread_items(
store,
ThreadItemsReadParams {
thread_id: thread_id.to_string(),
turn_id: "turn".to_string(),
item_ids: ids.map(str::to_string).into(),
},
)
.await
.expect("read saved results");
assert_eq!(
rows.data
.iter()
.map(|entry| entry.item.id())
.collect::<Vec<_>>(),
["exact", "preview", "metadata", "failed"]
);
let expected = live.active_turn_snapshot().expect("full turn");
assert_eq!(
expected
.items
.iter()
.map(ThreadItem::id)
.collect::<Vec<_>>(),
ids
);
live.release_saved_mcp_results(&rows.data);
let mut resumed = live.active_turn_snapshot().expect("evicted turn");
let mut evicted = expected.clone();
let ThreadItemPayload::McpToolCall { result, .. } = &mut evicted.items[0].payload else {
panic!("expected MCP call")
};
assert!(result.take().is_some());
assert_eq!(resumed, evicted);
crate::request_processors::thread_lifecycle::restore_evicted_payloads(
store,
thread_id,
&mut resumed,
std::iter::once(&mut page),
)
.await
.expect("full reconnect");
assert_eq!(resumed, expected);
Ok(())
}
fn dynamic_tool(
namespace: Option<&str>,
name: impl Into<String>,
@@ -1326,6 +1447,7 @@ mod thread_processor_behavior_tests {
ensure_conversation_listener(
ListenerTaskContext {
thread_manager: test.thread_manager.clone(),
thread_store: Arc::clone(&test.thread_store),
thread_state_manager,
outgoing: outgoing.clone(),
pending_thread_unloads: Default::default(),
@@ -99,6 +99,7 @@ async fn interrupt_reply_waits_for_the_requested_turn(queued: bool) -> anyhow::R
);
let processor = TurnRequestProcessor::new(
test.thread_manager.clone(),
test.thread_store.clone(),
outgoing.clone(),
analytics,
Arc::new(test.config.clone()),
@@ -90,6 +90,7 @@ fn validate_response_item_image_urls(items: &[ResponseItem]) -> Result<(), JSONR
#[derive(Clone)]
pub(crate) struct TurnRequestProcessor {
thread_manager: Arc<ThreadManager>,
thread_store: Arc<dyn ThreadStore>,
outgoing: Arc<OutgoingMessageSender>,
analytics_events_client: AnalyticsEventsClient,
config: Arc<Config>,
@@ -147,6 +148,7 @@ impl TurnRequestProcessor {
#[allow(clippy::too_many_arguments)]
pub(crate) fn new(
thread_manager: Arc<ThreadManager>,
thread_store: Arc<dyn ThreadStore>,
outgoing: Arc<OutgoingMessageSender>,
analytics_events_client: AnalyticsEventsClient,
config: Arc<Config>,
@@ -159,6 +161,7 @@ impl TurnRequestProcessor {
) -> Self {
Self {
thread_manager,
thread_store,
outgoing,
analytics_events_client,
config,
@@ -1666,6 +1669,7 @@ impl TurnRequestProcessor {
fn listener_task_context(&self) -> ListenerTaskContext {
ListenerTaskContext {
thread_manager: Arc::clone(&self.thread_manager),
thread_store: Arc::clone(&self.thread_store),
thread_state_manager: self.thread_state_manager.clone(),
outgoing: Arc::clone(&self.outgoing),
pending_thread_unloads: Arc::clone(&self.pending_thread_unloads),
+5
View File
@@ -6,6 +6,7 @@ use codex_app_server_protocol::ThreadGoal;
use codex_app_server_protocol::ThreadHistoryBuilder;
use codex_app_server_protocol::ThreadHistoryTurnMetadata;
use codex_app_server_protocol::ThreadItem;
use codex_app_server_protocol::ThreadItemEntry;
use codex_app_server_protocol::ThreadSettings;
use codex_app_server_protocol::Turn;
use codex_app_server_protocol::TurnError;
@@ -225,6 +226,10 @@ impl ThreadState {
self.current_turn_history.active_turn_snapshot()
}
pub(crate) fn release_saved_mcp_results(&mut self, entries: &[ThreadItemEntry]) {
self.current_turn_history.release_saved_mcp_results(entries);
}
/// Returns the same turn ID as `active_turn_snapshot` without cloning its items.
pub(crate) fn active_turn_id(&self) -> Option<&str> {
self.current_turn_history.active_turn_id()