diff --git a/codex-rs/core/src/agent/api.rs b/codex-rs/core/src/agent/api.rs index abae992f99..800ceed3c1 100644 --- a/codex-rs/core/src/agent/api.rs +++ b/codex-rs/core/src/agent/api.rs @@ -210,6 +210,8 @@ pub enum AgentInput { Message { message: AgentMessage, mode: MessageDeliveryMode, + /// Native tool invocation to link when the recipient commits this message to history. + source_tool_call_id: Option, }, } diff --git a/codex-rs/core/src/agent/control.rs b/codex-rs/core/src/agent/control.rs index 6d6ae79c8d..4de156fe01 100644 --- a/codex-rs/core/src/agent/control.rs +++ b/codex-rs/core/src/agent/control.rs @@ -271,6 +271,7 @@ impl LocalAgentControl { let publish = || { MessageDelivery::publish( context.sender_thread_id, + context.source_tool_call_id.clone(), state.extensions.message_lifecycle_contributors(), terminal_result.then_some(&self.runtime.unconfirmed_results), ) diff --git a/codex-rs/core/src/agent/control/api.rs b/codex-rs/core/src/agent/control/api.rs index a0b1c75593..c02b6f3088 100644 --- a/codex-rs/core/src/agent/control/api.rs +++ b/codex-rs/core/src/agent/control/api.rs @@ -74,7 +74,11 @@ impl AgentControl for LocalAgentControl { } = request; let input = match input { AgentInput::UserInput(input) => SpawnInitialInput::UserInput(input), - AgentInput::Message { message, mode } => { + AgentInput::Message { + message, + mode, + source_tool_call_id, + } => { if mode != MessageDeliveryMode::TriggerTurn { return Err(CodexErr::InvalidRequest( "spawn input must start the child turn".to_string(), @@ -94,7 +98,11 @@ impl AgentControl for LocalAgentControl { })?; SpawnInitialInput::InterAgentCommunication( message.into_communication(author, recipient, mode), - AgentCommunicationContext::new(AgentCommunicationKind::Spawn, caller), + AgentCommunicationContext { + kind: AgentCommunicationKind::Spawn, + sender_thread_id: caller, + source_tool_call_id, + }, ) } }; @@ -124,7 +132,11 @@ impl AgentControl for LocalAgentControl { let submission_id = self.send_input(target, input, start_options).await?; (receiver.unwrap_or_default(), submission_id) } - AgentInput::Message { message, mode } => { + AgentInput::Message { + message, + mode, + source_tool_call_id, + } => { let receiver = self.runtime.ensure_agent_known(target)?; let author = self .runtime @@ -168,7 +180,11 @@ impl AgentControl for LocalAgentControl { .send_inter_agent_communication( target, communication, - AgentCommunicationContext::new(kind, caller), + AgentCommunicationContext { + kind, + sender_thread_id: caller, + source_tool_call_id, + }, start_options, ) .await?; diff --git a/codex-rs/core/src/agent/control_tests.rs b/codex-rs/core/src/agent/control_tests.rs index d97faac0e7..8b37647f3c 100644 --- a/codex-rs/core/src/agent/control_tests.rs +++ b/codex-rs/core/src/agent/control_tests.rs @@ -1127,6 +1127,7 @@ async fn cancelled_v2_send_finishes_reload_registration() { input: AgentInput::Message { message: AgentMessage::Plaintext(message.to_string()), mode: MessageDeliveryMode::TriggerTurn, + source_tool_call_id: None, }, start_options: Default::default(), }; @@ -1979,6 +1980,7 @@ async fn v2_reload_preserves_shared_instructions_after_root_unloads( input: crate::AgentInput::Message { message: AgentMessage::Plaintext("wake the unloaded target".to_string()), mode: MessageDeliveryMode::QueueOnly, + source_tool_call_id: None, }, start_options: TurnStartOptions { root_turn_id: sender_turn.turn_metadata_state.root_turn_id(), @@ -2151,6 +2153,7 @@ async fn v2_spawn_resolves_reported_effort_without_changing_child_selection( input: AgentInput::Message { message: AgentMessage::Plaintext("child task".to_string()), mode: MessageDeliveryMode::TriggerTurn, + source_tool_call_id: None, }, source, options: SpawnAgentOptions { @@ -4498,7 +4501,15 @@ async fn multi_agent_v2_completion_ignores_dead_direct_parent() { #[test] fn terminal_result_writes_require_confirmation_after_the_parent_claims_mail() { let count = Arc::new(std::sync::atomic::AtomicUsize::default()); - let publish = || MessageDelivery::publish(ThreadId::new(), &[], Some(&count)).unwrap(); + let publish = || { + MessageDelivery::publish( + ThreadId::new(), + /*source_tool_call_id*/ None, + &[], + Some(&count), + ) + .unwrap() + }; let parent = ThreadId::new(); drop(publish().consumed(parent)); let saved = publish(); diff --git a/codex-rs/core/src/agent_communication.rs b/codex-rs/core/src/agent_communication.rs index ebcd23cfe5..1a3252596d 100644 --- a/codex-rs/core/src/agent_communication.rs +++ b/codex-rs/core/src/agent_communication.rs @@ -46,6 +46,7 @@ impl AgentCommunicationKind { pub(crate) struct AgentCommunicationContext { pub(crate) kind: AgentCommunicationKind, pub(crate) sender_thread_id: ThreadId, + pub(crate) source_tool_call_id: Option, } impl AgentCommunicationContext { @@ -53,6 +54,7 @@ impl AgentCommunicationContext { Self { kind, sender_thread_id, + source_tool_call_id: None, } } } @@ -174,16 +176,23 @@ struct MessagePublication { /// Clones follow one logical delivery through queues and share its consumption state. #[derive(Clone)] -pub(crate) struct MessageDelivery(Arc>>); +pub(crate) struct MessageDelivery { + pub(crate) source_tool_call_id: Option, + publication: Option>>>, +} impl MessageDelivery { pub(crate) fn publish( sender: ThreadId, + source_tool_call_id: Option, observers: &[Arc], unconfirmed_results: Option<&Arc>, ) -> Option { if observers.is_empty() && unconfirmed_results.is_none() { - return None; + return source_tool_call_id.map(|id| Self { + source_tool_call_id: Some(id), + publication: None, + }); } let input = MessageLifecycleInput { sender, @@ -201,21 +210,27 @@ impl MessageDelivery { observers: observers.into(), result_write, }; - Some(Self(Arc::new(Mutex::new(Some(publication))))) + Some(Self { + source_tool_call_id, + publication: Some(Arc::new(Mutex::new(Some(publication)))), + }) } pub(crate) fn awaits_result_write(&self) -> bool { - self.0 - .lock() - .unwrap_or_else(PoisonError::into_inner) - .as_ref() - .is_some_and(|publication| publication.result_write.is_some()) + self.publication.as_ref().is_some_and(|publication| { + publication + .lock() + .unwrap_or_else(PoisonError::into_inner) + .as_ref() + .is_some_and(|publication| publication.result_write.is_some()) + }) } /// The first consumer owns both the publication hooks and the write receipt. pub(crate) fn consumed(&self, recipient: ThreadId) -> Option { let publication = self - .0 + .publication + .as_ref()? .lock() .unwrap_or_else(PoisonError::into_inner) .take()?; diff --git a/codex-rs/core/src/session/mod.rs b/codex-rs/core/src/session/mod.rs index 7d4e9f4b4b..c08a92fda9 100644 --- a/codex-rs/core/src/session/mod.rs +++ b/codex-rs/core/src/session/mod.rs @@ -4866,6 +4866,17 @@ impl Session { std::slice::from_mut(&mut response_item), model_info.truncation_policy.into(), ); + // Record the item and its sender at the history write, before shutdown can cancel persistence. + self.services + .rollout_thread_trace + .record_history_item_accepted( + &turn_context.sub_id, + &response_item.item, + communication + .delivery + .as_ref() + .and_then(|d| d.source_tool_call_id.as_deref()), + ); } if let Some(delivery) = delivery.take() { result_write = delivery.consumed(self.thread_id); @@ -4885,7 +4896,12 @@ impl Session { write.confirm(); } drop(boundary); - self.send_raw_response_items(turn_context, &items).await; + let [item] = items; + self.send_event( + turn_context, + EventMsg::RawResponseItem(RawResponseItemEvent { item }), + ) + .await; } async fn maybe_warn_on_server_model_mismatch( @@ -5193,6 +5209,13 @@ impl Session { #[tracing::instrument(level = "trace", skip_all, fields(item_count = items.len()))] async fn send_raw_response_items(&self, turn_context: &TurnContext, items: &[ResponseItem]) { for item in items { + self.services + .rollout_thread_trace + .record_history_item_accepted( + &turn_context.sub_id, + item, + /*source_tool_call_id*/ None, + ); self.send_event( turn_context, EventMsg::RawResponseItem(RawResponseItemEvent { item: item.clone() }), diff --git a/codex-rs/core/src/thread_spawn.rs b/codex-rs/core/src/thread_spawn.rs index f7f864a33e..65b86ec007 100644 --- a/codex-rs/core/src/thread_spawn.rs +++ b/codex-rs/core/src/thread_spawn.rs @@ -110,6 +110,11 @@ pub(crate) async fn spawn( input: AgentInput::Message { message, mode: MessageDeliveryMode::TriggerTurn, + source_tool_call_id: session + .services + .rollout_thread_trace + .is_enabled() + .then(|| call_id.to_owned()), }, source: settings.source, options: settings.options, diff --git a/codex-rs/core/src/tools/handlers/multi_agents_v2/message_tool.rs b/codex-rs/core/src/tools/handlers/multi_agents_v2/message_tool.rs index 8392960121..260a603f85 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents_v2/message_tool.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents_v2/message_tool.rs @@ -68,6 +68,11 @@ pub(super) async fn handle_message_string_tool( input: AgentInput::Message { message: agent_message_from_tool(message, &source), mode, + source_tool_call_id: session + .services + .rollout_thread_trace + .is_enabled() + .then(|| call_id.clone()), }, start_options: TurnStartOptions { parent_turn_id: (mode == MessageDeliveryMode::TriggerTurn) diff --git a/codex-rs/rollout-trace/src/capture_tests.rs b/codex-rs/rollout-trace/src/capture_tests.rs index 2eb309b101..1aa0425e92 100644 --- a/codex-rs/rollout-trace/src/capture_tests.rs +++ b/codex-rs/rollout-trace/src/capture_tests.rs @@ -128,6 +128,7 @@ fn compressed_budget_admits_small_later_events_after_an_omission() -> Result<()> writer.append_with_context( RawTraceEventContext::turn(ROOT_THREAD_ID, "turn"), RawTraceEventPayload::HistoryItemAccepted { + source_tool_call_id: None, item_id: "missing-receipt".into(), agent_message_payload: Some(omitted.clone()), tool_output_payload: None, diff --git a/codex-rs/rollout-trace/src/raw_event.rs b/codex-rs/rollout-trace/src/raw_event.rs index 14675db4d3..89d565e3c6 100644 --- a/codex-rs/rollout-trace/src/raw_event.rs +++ b/codex-rs/rollout-trace/src/raw_event.rs @@ -112,6 +112,8 @@ pub enum RawTraceEventPayload { /// evidence of its rendered content, which may be truncated before sampling. HistoryItemAccepted { item_id: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + source_tool_call_id: Option, /// A delivery may never appear in a subsequent inference or compaction snapshot. #[serde(default, skip_serializing_if = "Option::is_none")] agent_message_payload: Option, diff --git a/codex-rs/rollout-trace/src/reducer/acceptance_tests.rs b/codex-rs/rollout-trace/src/reducer/acceptance_tests.rs index 51c61bc744..cafdf47115 100644 --- a/codex-rs/rollout-trace/src/reducer/acceptance_tests.rs +++ b/codex-rs/rollout-trace/src/reducer/acceptance_tests.rs @@ -41,6 +41,7 @@ fn identical_text_from_a_discarded_attempt_does_not_gain_acceptance() -> anyhow: writer.append_with_context( trace_context("turn-1"), RawTraceEventPayload::HistoryItemAccepted { + source_tool_call_id: None, item_id: "accepted".into(), agent_message_payload: None, tool_output_payload: None, @@ -225,6 +226,7 @@ fn inherited_output_is_child_prefill_without_copying_parent_sampling_provenance( parent.request_item_ids ); let accepted = RawTraceEventPayload::HistoryItemAccepted { + source_tool_call_id: None, item_id: "msg-parent".into(), agent_message_payload: None, tool_output_payload: None, @@ -280,6 +282,7 @@ fn accept_output(writer: &crate::TraceWriter, item: &serde_json::Value) -> anyho writer.append_with_context( trace_context("turn-1"), RawTraceEventPayload::HistoryItemAccepted { + source_tool_call_id: None, item_id: item["id"].as_str().unwrap().into(), agent_message_payload: None, tool_output_payload: Some(payload), diff --git a/codex-rs/rollout-trace/src/reducer/conversation.rs b/codex-rs/rollout-trace/src/reducer/conversation.rs index ba6e22e4ea..3d996fc1e3 100644 --- a/codex-rs/rollout-trace/src/reducer/conversation.rs +++ b/codex-rs/rollout-trace/src/reducer/conversation.rs @@ -62,7 +62,13 @@ impl TraceReducer { if is_delivery { self.resolve_pending_agent_edges_for_item(&id)?; } - self.receivable_ids.insert(key, Some(id)); + self.receivable_ids.insert( + key, + super::ReceivableItem { + item_id: id, + awaiting_snapshot: true, + }, + ); Ok(()) } @@ -539,8 +545,10 @@ impl TraceReducer { && let Some(wire_id) = &item.responses_item_id { let key = (thread_id.to_owned(), wire_id.clone()); - if let Some(Some(id)) = self.receivable_ids.get(&key) { - return id.clone(); + if let Some(receipt) = self.receivable_ids.get(&key) + && receipt.awaiting_snapshot + { + return receipt.item_id.clone(); } } let item_id = self.next_conversation_item_id(); @@ -592,9 +600,15 @@ impl TraceReducer { && let Some(wire_id) = &normalized.responses_item_id { let key = (item.thread_id.clone(), wire_id.clone()); - let accepted_id = self.receivable_ids.entry(key).or_default(); - if accepted_id.as_deref() == Some(item_id) { - *accepted_id = None; + let receipt = self + .receivable_ids + .entry(key) + .or_insert_with(|| super::ReceivableItem { + item_id: item_id.to_owned(), + awaiting_snapshot: false, + }); + if receipt.awaiting_snapshot && receipt.item_id == item_id { + receipt.awaiting_snapshot = false; item.body.clone_from(&normalized.body); if let Some(hash) = normalized.structured_output_hash { self.structured_output_hashes diff --git a/codex-rs/rollout-trace/src/reducer/conversation_tests.rs b/codex-rs/rollout-trace/src/reducer/conversation_tests.rs index 42c62c59ad..5e8652aa58 100644 --- a/codex-rs/rollout-trace/src/reducer/conversation_tests.rs +++ b/codex-rs/rollout-trace/src/reducer/conversation_tests.rs @@ -224,6 +224,7 @@ fn response_outputs_require_harness_acceptance() -> anyhow::Result<()> { writer.append_with_context( trace_context("turn-1"), RawTraceEventPayload::HistoryItemAccepted { + source_tool_call_id: None, item_id: "msg-accepted".into(), agent_message_payload: None, tool_output_payload: None, diff --git a/codex-rs/rollout-trace/src/reducer/inference_tests.rs b/codex-rs/rollout-trace/src/reducer/inference_tests.rs index 642a826359..4f9750a360 100644 --- a/codex-rs/rollout-trace/src/reducer/inference_tests.rs +++ b/codex-rs/rollout-trace/src/reducer/inference_tests.rs @@ -30,6 +30,7 @@ fn cancelled_inference_retains_accepted_partial_output() -> anyhow::Result<()> { writer.append_with_context( trace_context("turn-1"), RawTraceEventPayload::HistoryItemAccepted { + source_tool_call_id: None, item_id: "msg-partial".into(), agent_message_payload: None, tool_output_payload: None, diff --git a/codex-rs/rollout-trace/src/reducer/mod.rs b/codex-rs/rollout-trace/src/reducer/mod.rs index c1d1f1fa95..0f4caba6c6 100644 --- a/codex-rs/rollout-trace/src/reducer/mod.rs +++ b/codex-rs/rollout-trace/src/reducer/mod.rs @@ -80,6 +80,7 @@ pub fn replay_bundle(bundle_dir: impl AsRef) -> Result { pending_code_cell_starts: BTreeMap::new(), pending_code_cell_lifecycle_events: BTreeMap::new(), pending_agent_interaction_edges: Vec::new(), + native_tool_deliveries: BTreeMap::new(), first_acceptances: BTreeMap::new(), receivable_ids: BTreeMap::new(), snapshot_output_anchors: BTreeSet::new(), @@ -139,6 +140,7 @@ pub fn replay_bundle(bundle_dir: impl AsRef) -> Result { // Spawn edges prefer the child task message as their target, but a child can // fail before that message is ever reduced. Only after replaying the whole // bundle do we know which spawn deliveries need the child-thread fallback. + reducer.finish_native_tool_deliveries()?; reducer.resolve_pending_spawn_edge_fallbacks()?; // Cell starts can be queued until their model output arrives. Link only // materialized cells, after both sides of the dispatch have been replayed. @@ -203,14 +205,18 @@ pub fn replay_bundle(bundle_dir: impl AsRef) -> Result { Ok(reducer.rollout) } +struct ReceivableItem { + item_id: String, + awaiting_snapshot: bool, +} + struct TraceReducer { rollout: RolloutTrace, payload_reader: PayloadReader, /// First (wall time, sequence) by receiving thread and Responses item ID. first_acceptances: BTreeMap>, - /// A native delivery or tool output has one graph ID until its first snapshot. - /// None means a snapshot already supplied its body (and a later acceptance is a duplicate). - receivable_ids: BTreeMap<(String, String), Option>, + /// First graph item for each receiving thread and Responses wire ID. + receivable_ids: BTreeMap<(String, String), ReceivableItem>, /// Accepted tool outputs first encountered in the batch about to be appended. snapshot_output_anchors: BTreeSet, next_conversation_item_ordinal: u64, @@ -254,13 +260,10 @@ struct TraceReducer { /// queued for ownership validation; these lifecycle events wait with it and /// are replayed in raw sequence order once the cell materializes. pending_code_cell_lifecycle_events: BTreeMap>, - /// Multi-agent deliveries whose recipient-side transcript item has not been observed yet. - /// - /// V2 agent tools enqueue mailbox messages in the target thread. The trace event for the - /// sending tool arrives before the recipient inference request materializes that mailbox item - /// as a `ConversationItem`, so the reducer keeps the delivery edge pending until it can point - /// at the exact model-visible item instead of a coarse thread. + /// Legacy collaboration events and automatic agent results lack a source-to-item ID. pending_agent_interaction_edges: Vec, + /// V2 tool activity and the recipient's history item can arrive in either order. + native_tool_deliveries: BTreeMap, } impl TraceReducer { @@ -353,6 +356,7 @@ impl TraceReducer { } RawTraceEventPayload::HistoryItemAccepted { item_id, + source_tool_call_id, agent_message_payload, tool_output_payload, } => { @@ -360,6 +364,9 @@ impl TraceReducer { .thread_id .context("history acceptance has no thread")?; self.thread_mut(&thread_id)?; + if let Some(source) = source_tool_call_id { + self.record_agent_message_target(source, thread_id.clone(), item_id.clone())?; + } // Keep the acceptance edge even when a capture limit hid the receipt body. if let Some(payload) = agent_message_payload.or(tool_output_payload) && payload.capture.is_complete() diff --git a/codex-rs/rollout-trace/src/reducer/tool.rs b/codex-rs/rollout-trace/src/reducer/tool.rs index 22758a1e53..b028e827bd 100644 --- a/codex-rs/rollout-trace/src/reducer/tool.rs +++ b/codex-rs/rollout-trace/src/reducer/tool.rs @@ -25,6 +25,7 @@ mod terminal; #[path = "tool/mcp_tests.rs"] mod mcp_tests; +pub(super) use agents::NativeToolDelivery; pub(super) use agents::ObservedAgentResultEdge; pub(super) use agents::PendingAgentInteractionEdge; pub(super) use agents::spawn_edge_id; diff --git a/codex-rs/rollout-trace/src/reducer/tool/agents.rs b/codex-rs/rollout-trace/src/reducer/tool/agents.rs index 7152c60d32..fb90dcfc94 100644 --- a/codex-rs/rollout-trace/src/reducer/tool/agents.rs +++ b/codex-rs/rollout-trace/src/reducer/tool/agents.rs @@ -8,8 +8,9 @@ use codex_protocol::protocol::CollabCloseBeginEvent; use codex_protocol::protocol::CollabCloseEndEvent; use codex_protocol::protocol::SubAgentActivityEvent; use codex_protocol::protocol::SubAgentActivityKind; -use serde::Deserialize; -use serde_json::Value; +use std::collections::BTreeMap; +use std::collections::HashSet; +use std::collections::btree_map::Entry; use super::super::TraceReducer; use crate::model::ConversationItemKind; @@ -17,15 +18,12 @@ use crate::model::ConversationPart; use crate::model::ConversationRole; use crate::model::InteractionEdge; use crate::model::InteractionEdgeKind; +use crate::model::ProducerRef; use crate::model::ToolCallKind; use crate::model::TraceAnchor; use crate::payload::RawPayloadRef; -/// Agent delivery edge waiting for the recipient-side conversation item. -/// -/// Multi-agent v2 records the sender tool before the target thread necessarily -/// includes the delivered mailbox message in a model-visible request. The edge -/// stays pending so it can target that exact conversation item when possible. +/// A legacy collaboration event or an automatic child result awaiting its recipient item. pub(in crate::reducer) struct PendingAgentInteractionEdge { pub(in crate::reducer) edge_id: String, pub(in crate::reducer) kind: InteractionEdgeKind, @@ -55,9 +53,11 @@ pub(in crate::reducer) struct ObservedAgentResultEdge { pub(in crate::reducer) carried_payload: Option, } -#[derive(Deserialize)] -struct AgentMessageInvocationArgs { - message: String, +/// Sender completion and recipient history can arrive in either order. +pub(in crate::reducer) struct NativeToolDelivery { + receiver_thread_id: String, + activity_at_unix_ms: Option, + wire_item_id: Option, } /// Builds the stable edge id for the spawn relationship between two threads. @@ -197,43 +197,15 @@ impl TraceReducer { ) -> Result<()> { let target_thread_id = payload.agent_thread_id.to_string(); match (tool_kind, &payload.kind) { - (ToolCallKind::SpawnAgent, SubAgentActivityKind::Started) => { - let parent_thread_id = self - .rollout - .tool_calls - .get(tool_call_id) - .with_context(|| { - format!("agent activity referenced unknown tool call {tool_call_id}") - })? - .thread_id - .clone(); - self.queue_sub_agent_activity_message_edge( - wall_time_unix_ms, - tool_call_id, - spawn_edge_id(&parent_thread_id, &target_thread_id), - InteractionEdgeKind::SpawnAgent, - target_thread_id.clone(), - Some(target_thread_id), - ) + (ToolCallKind::SpawnAgent, SubAgentActivityKind::Started) + | ( + ToolCallKind::AssignAgentTask | ToolCallKind::SendMessage, + SubAgentActivityKind::Interacted, + ) => { + self.native_tool_delivery(tool_call_id.to_owned(), target_thread_id)? + .activity_at_unix_ms = Some(wall_time_unix_ms); + Ok(()) } - (ToolCallKind::AssignAgentTask, SubAgentActivityKind::Interacted) => self - .queue_sub_agent_activity_message_edge( - wall_time_unix_ms, - tool_call_id, - tool_edge_id(tool_call_id), - InteractionEdgeKind::AssignAgentTask, - target_thread_id, - /*unresolved_spawn_thread_id*/ None, - ), - (ToolCallKind::SendMessage, SubAgentActivityKind::Interacted) => self - .queue_sub_agent_activity_message_edge( - wall_time_unix_ms, - tool_call_id, - tool_edge_id(tool_call_id), - InteractionEdgeKind::SendMessage, - target_thread_id, - /*unresolved_spawn_thread_id*/ None, - ), (ToolCallKind::CloseAgent, SubAgentActivityKind::Interrupted) => self .upsert_close_agent_interaction( tool_call_id, @@ -248,77 +220,129 @@ impl TraceReducer { } } - fn queue_sub_agent_activity_message_edge( + fn native_tool_delivery( &mut self, - wall_time_unix_ms: i64, - tool_call_id: &str, - edge_id: String, - edge_kind: InteractionEdgeKind, - target_thread_id: String, - unresolved_spawn_thread_id: Option, - ) -> Result<()> { - let tool_call = self.rollout.tool_calls.get(tool_call_id).with_context(|| { - format!("agent activity referenced unknown tool call {tool_call_id}") - })?; - let started_at_unix_ms = tool_call.execution.started_at_unix_ms; - let message_author = self.agent_path_for_thread(&tool_call.thread_id)?; - let Some(message_content) = self.agent_message_content_from_invocation(tool_call_id)? - else { - return Ok(()); - }; - let carried_raw_payload_ids = self.agent_tool_payload_ids(tool_call_id)?; - self.queue_or_resolve_agent_interaction_edge(PendingAgentInteractionEdge { - edge_id, - kind: edge_kind, - source: TraceAnchor::ToolCall { - tool_call_id: tool_call_id.to_string(), - }, - target_thread_id, - message_author, - message_content, - unresolved_spawn_thread_id, - started_at_unix_ms, - ended_at_unix_ms: Some(wall_time_unix_ms), - carried_raw_payload_ids, - }) + call_id: String, + receiver: String, + ) -> Result<&mut NativeToolDelivery> { + match self.native_tool_deliveries.entry(call_id) { + Entry::Occupied(entry) if entry.get().receiver_thread_id != receiver => { + bail!("conflicting recipients for tool {}", entry.key()) + } + Entry::Occupied(entry) => Ok(entry.into_mut()), + Entry::Vacant(entry) => Ok(entry.insert(NativeToolDelivery { + receiver_thread_id: receiver, + activity_at_unix_ms: None, + wire_item_id: None, + })), + } } - fn agent_message_content_from_invocation( + pub(in crate::reducer) fn record_agent_message_target( &mut self, - tool_call_id: &str, - ) -> Result> { - let tool_call = self.rollout.tool_calls.get(tool_call_id).with_context(|| { - format!("agent activity referenced unknown tool call {tool_call_id}") - })?; - let invocation_payload_id = tool_call - .raw_invocation_payload_id - .as_deref() - .with_context(|| { - format!("agent activity tool call {tool_call_id} missing invocation payload") - })?; - let invocation_payload = self - .rollout - .raw_payloads - .get(invocation_payload_id) - .with_context(|| { - format!( - "agent activity tool call {tool_call_id} referenced missing invocation payload {invocation_payload_id}" - ) - })?; - if !invocation_payload.capture.is_complete() { - return Ok(None); + call_id: String, + receiver: String, + item_id: String, + ) -> Result<()> { + let delivery = self.native_tool_delivery(call_id, receiver)?; + if delivery.wire_item_id.get_or_insert(item_id.clone()) != &item_id { + bail!("one agent tool delivered different history items"); } - let invocation = self.payload_reader.read_json(invocation_payload)?; - let arguments = invocation - .get("payload") - .and_then(|payload| payload.get("arguments")) - .and_then(Value::as_str) - .with_context(|| { - format!("agent activity tool call {tool_call_id} missing function arguments") + Ok(()) + } + + /// Collect the distinct graph items actually sent to each receiver's model. A reset + /// can give the same Responses item a new graph ID. Repeated prompt prefixes reuse it. + fn model_visible_deliveries(&self) -> BTreeMap<(String, String), Vec> { + let mut visible = BTreeMap::<_, Vec<_>>::new(); + let mut visited = HashSet::new(); + let mut requests: Vec<_> = self.rollout.inference_calls.values().collect(); + requests.sort_unstable_by_key(|call| call.execution.started_seq); + for call in requests { + for id in &call.request_item_ids { + if !visited.insert((&call.thread_id, id)) { + continue; + } + let item = &self.rollout.conversation_items[id]; + if item.thread_id == call.thread_id + && item.agent_message.is_some() + && let Some(wire_id) = &item.responses_item_id + { + visible + .entry((call.thread_id.clone(), wire_id.clone())) + .or_default() + .push(id.clone()); + } + } + } + visible + } + + pub(in crate::reducer) fn finish_native_tool_deliveries(&mut self) -> Result<()> { + if self.native_tool_deliveries.is_empty() { + return Ok(()); + } + let visible = self.model_visible_deliveries(); + for (call_id, delivery) in std::mem::take(&mut self.native_tool_deliveries) { + let Some(tool) = self.rollout.tool_calls.get(&call_id) else { + // Mail can survive shutdown and reach a recipient in a later recording. + continue; + }; + let receiver = delivery.receiver_thread_id; + let (kind, edge_id) = match tool.kind { + ToolCallKind::SpawnAgent => ( + InteractionEdgeKind::SpawnAgent, + spawn_edge_id(&tool.thread_id, &receiver), + ), + ToolCallKind::AssignAgentTask => { + (InteractionEdgeKind::AssignAgentTask, tool_edge_id(&call_id)) + } + ToolCallKind::SendMessage => { + (InteractionEdgeKind::SendMessage, tool_edge_id(&call_id)) + } + _ => bail!("non-agent tool {call_id} supplied an agent delivery"), + }; + // The successful tool identifies the recipient even when it never samples again. + // Keep acceptance on the history item. Only actual receiver inputs enrich this edge. + let mut carried_item_ids = Vec::new(); + if let Some(wire_id) = delivery.wire_item_id { + let key = (receiver.clone(), wire_id); + if let Some(receipt) = self.receivable_ids.get(&key) { + let item = self + .rollout + .conversation_items + .get_mut(&receipt.item_id) + .context("accepted agent message missing from conversation items")?; + item.produced_by.push(ProducerRef::InteractionEdge { + edge_id: edge_id.clone(), + }); + } + carried_item_ids = visible.get(&key).cloned().unwrap_or_default(); + } + let target = match carried_item_ids.first() { + Some(item_id) => TraceAnchor::ConversationItem { + item_id: item_id.clone(), + }, + None => TraceAnchor::Thread { + thread_id: receiver, + }, + }; + self.upsert_interaction_edge(InteractionEdge { + edge_id, + kind, + source: TraceAnchor::ToolCall { + tool_call_id: call_id.clone(), + }, + target, + started_at_unix_ms: tool.execution.started_at_unix_ms, + ended_at_unix_ms: delivery + .activity_at_unix_ms + .or(tool.execution.ended_at_unix_ms), + carried_item_ids, + carried_raw_payload_ids: self.agent_tool_payload_ids(&call_id)?, })?; - let args: AgentMessageInvocationArgs = serde_json::from_str(arguments) - .with_context(|| format!("parse agent activity tool call {tool_call_id} arguments"))?; - Ok(Some(args.message)) + } + Ok(()) } /// Adds the canonical tool result payload to an already reduced multi-agent edge. diff --git a/codex-rs/rollout-trace/src/reducer/tool/agents_tests.rs b/codex-rs/rollout-trace/src/reducer/tool/agents_tests.rs index 2be2f69eea..28da3f8107 100644 --- a/codex-rs/rollout-trace/src/reducer/tool/agents_tests.rs +++ b/codex-rs/rollout-trace/src/reducer/tool/agents_tests.rs @@ -255,6 +255,7 @@ fn sub_agent_started_activity_creates_spawn_edge() -> anyhow::Result<()> { ] }); let receipt = RawTraceEventPayload::HistoryItemAccepted { + source_tool_call_id: Some("call-spawn-v2".into()), item_id: "amsg-delivery".into(), tool_output_payload: None, agent_message_payload: Some( @@ -264,10 +265,25 @@ fn sub_agent_started_activity_creates_spawn_edge() -> anyhow::Result<()> { let context = trace_context_for_thread(child_thread_id, "turn-child-1"); writer.append_with_context(context.clone(), receipt.clone())?; - // Interrupt or reset here: the receipt already has its delivery edge without any child request. + // Interrupt before the first request. The send and history acceptance are both known. let received = replay_bundle(temp.path())?; let edge_id = format!("edge:spawn:019d0000-0000-7000-8000-000000000001:{child_thread_id}"); - let receipt_id = target_conversation_item_id(&received.interaction_edges[&edge_id].target); + let edge = &received.interaction_edges[&edge_id]; + assert_eq!( + edge.target, + TraceAnchor::Thread { + thread_id: child_thread_id.to_owned() + } + ); + assert!(edge.carried_item_ids.is_empty()); + let receipt_id = &received.threads[child_thread_id].conversation_item_ids[0]; + assert!( + received.conversation_items[receipt_id] + .produced_by + .contains(&crate::model::ProducerRef::InteractionEdge { + edge_id: edge_id.clone() + }) + ); let delivered_item_ids = std::slice::from_ref(receipt_id); assert_eq!( received.threads[child_thread_id].conversation_item_ids, @@ -287,9 +303,9 @@ fn sub_agent_started_activity_creates_spawn_edge() -> anyhow::Result<()> { child_thread_id, "turn-child-1", "inference-child-1", - vec![delivery], + vec![delivery.clone()], )?; - writer.append_with_context(context, receipt)?; + writer.append_with_context(context.clone(), receipt)?; let replayed = replay_bundle(temp.path())?; assert_eq!( @@ -319,6 +335,38 @@ fn sub_agent_started_activity_creates_spawn_edge() -> anyhow::Result<()> { activity_payload.raw_payload_id, ] ); + // A checkpoint can copy the same wire item to a new graph node. Only a later + // inference using that copy adds a second observation to the send edge. + let checkpoint_payload = writer.write_json_payload( + RawPayloadKind::CompactionCheckpoint, + &json!({"input_history": [&delivery], "replacement_history": [&delivery]}), + )?; + writer.append_with_context( + context, + RawTraceEventPayload::CompactionInstalled { + compaction_id: "reset".to_owned(), + checkpoint_payload, + }, + )?; + let reset = replay_bundle(temp.path())?; + assert_eq!( + reset.interaction_edges[&edge_id].carried_item_ids, + delivered_item_ids + ); + let replacement = &reset.compactions["reset"].replacement_item_ids[0]; + assert_ne!(replacement, receipt_id); + append_inference_request( + &writer, + child_thread_id, + "turn-child-1", + "after-reset", + vec![delivery], + )?; + let after = replay_bundle(temp.path())?; + assert_eq!( + after.interaction_edges[&edge_id].carried_item_ids, + [receipt_id.clone(), replacement.clone()] + ); Ok(()) } @@ -470,6 +518,39 @@ fn send_message_activity_targets_delivered_child_message() -> anyhow::Result<()> "kind": "interacted" }), )?; + start_thread(&writer, child_thread_id, "/root/child")?; + start_turn_for_thread(&writer, child_thread_id, "turn-child-1")?; + // The recipient may have seen a task with the same payload before this ordinary send. + let task = inter_agent_message( + "/root", + "/root/child", + "Message Type: NEW_TASK\nTask name: /root/child\nSender: /root\nPayload:\nhello again", + ); + let mut delivered = inter_agent_message( + "/root", + "/root/child", + "Message Type: MESSAGE\nTask name: /root/child\nSender: /root\nPayload:\nhello again", + ); + delivered["id"] = json!("amsg-send"); + append_inference_request( + &writer, + child_thread_id, + "turn-child-1", + "inference-child-1", + vec![task, delivered.clone()], + )?; + writer.append_with_context( + trace_context_for_thread(child_thread_id, "turn-child-1"), + RawTraceEventPayload::HistoryItemAccepted { + source_tool_call_id: Some("call-send-v2".into()), + agent_message_payload: Some( + writer.write_json_payload(RawPayloadKind::ProtocolEvent, &delivered)?, + ), + tool_output_payload: None, + item_id: "amsg-send".into(), + }, + )?; + writer.append_with_context( trace_context_for_agent("turn-1"), RawTraceEventPayload::ToolCallRuntimeEnded { @@ -478,16 +559,6 @@ fn send_message_activity_targets_delivered_child_message() -> anyhow::Result<()> runtime_payload: activity_payload.clone(), }, )?; - start_thread(&writer, child_thread_id, "/root/child")?; - start_turn_for_thread(&writer, child_thread_id, "turn-child-1")?; - let delivered = inter_agent_message("/root", "/root/child", "hello again"); - append_inference_request( - &writer, - child_thread_id, - "turn-child-1", - "inference-child-1", - vec![delivered], - )?; let replayed = replay_bundle(temp.path())?; let edge = &replayed.interaction_edges["edge:tool:call-send-v2"]; @@ -495,8 +566,10 @@ fn send_message_activity_targets_delivered_child_message() -> anyhow::Result<()> let target_item_id = target_conversation_item_id(&edge.target); assert_eq!(edge.carried_item_ids, vec![target_item_id.clone()]); assert_eq!( - replayed.conversation_items[target_item_id].thread_id, - child_thread_id + replayed.conversation_items[target_item_id] + .responses_item_id + .as_deref(), + Some("amsg-send") ); assert_eq!( edge.carried_raw_payload_ids, @@ -515,6 +588,22 @@ fn followup_activity_targets_delivered_child_message() -> anyhow::Result<()> { let writer = create_started_agent_writer(&temp)?; start_agent_turn(&writer, "turn-1")?; let child_thread_id = "019d0000-0000-7000-8000-000000000002"; + start_thread(&writer, child_thread_id, "/root/child")?; + start_turn_for_thread(&writer, child_thread_id, "turn-child-1")?; + let mut previous = inter_agent_message( + "/root", + "/root/child", + "Message Type: NEW_TASK\nTask name: /root/child\nSender: /root\nPayload:\ncontinue", + ); + previous["id"] = json!("amsg-host"); + append_inference_request( + &writer, + child_thread_id, + "turn-child-1", + "inference-child-1", + vec![previous.clone()], + )?; + let invocation_payload = writer.write_json_payload( RawPayloadKind::ToolInvocation, &json!({ @@ -559,21 +648,46 @@ fn followup_activity_targets_delivered_child_message() -> anyhow::Result<()> { runtime_payload: activity_payload.clone(), }, )?; - start_thread(&writer, child_thread_id, "/root/child")?; - start_turn_for_thread(&writer, child_thread_id, "turn-child-1")?; - let delivered = inter_agent_message("/root", "/root/child", "continue"); + let sent = replay_bundle(temp.path())?; + let edge = &sent.interaction_edges["edge:tool:call-followup-v2"]; + assert_eq!( + edge.target, + TraceAnchor::Thread { + thread_id: child_thread_id.to_owned() + } + ); + assert!(edge.carried_item_ids.is_empty()); + let mut delivered = previous.clone(); + delivered["id"] = json!("amsg-followup"); + writer.append_with_context( + trace_context_for_thread(child_thread_id, "turn-child-1"), + RawTraceEventPayload::HistoryItemAccepted { + source_tool_call_id: Some("call-followup-v2".into()), + tool_output_payload: None, + item_id: "amsg-followup".into(), + agent_message_payload: Some( + writer.write_json_payload(RawPayloadKind::ProtocolEvent, &delivered)?, + ), + }, + )?; append_inference_request( &writer, child_thread_id, "turn-child-1", - "inference-child-1", - vec![delivered], + "inference-child-2", + vec![previous, delivered], )?; let replayed = replay_bundle(temp.path())?; let edge = &replayed.interaction_edges["edge:tool:call-followup-v2"]; assert_eq!(edge.kind, InteractionEdgeKind::AssignAgentTask); let target_item_id = target_conversation_item_id(&edge.target); + assert_eq!( + replayed.conversation_items[target_item_id] + .responses_item_id + .as_deref(), + Some("amsg-followup") + ); assert_eq!(edge.carried_item_ids, vec![target_item_id.clone()]); assert_eq!( replayed.conversation_items[target_item_id].thread_id, diff --git a/codex-rs/rollout-trace/src/thread.rs b/codex-rs/rollout-trace/src/thread.rs index bb2b15fd70..e7169c2199 100644 --- a/codex-rs/rollout-trace/src/thread.rs +++ b/codex-rs/rollout-trace/src/thread.rs @@ -240,49 +240,11 @@ impl ThreadTraceContext { ); } - /// Records turn lifecycle and history acceptance from existing protocol events. + /// Records turn lifecycle from existing protocol events. pub fn record_codex_turn_event(&self, default_turn_id: &str, event: &EventMsg) { let ThreadTraceContextState::Enabled(context) = &self.state else { return; }; - if let EventMsg::RawResponseItem(event) = event { - let Some(item_id) = event.item.id().map(ToString::to_string) else { - return; - }; - let accepted = move |agent_message_payload, tool_output_payload| { - RawTraceEventPayload::HistoryItemAccepted { - item_id, - agent_message_payload, - tool_output_payload, - } - }; - let turn = RawTraceEventContext::turn(&context.thread_id, default_turn_id); - let is_delivery = matches!(event.item, ResponseItem::AgentMessage { .. }); - if is_delivery - || matches!( - event.item, - ResponseItem::FunctionCallOutput { .. } - | ResponseItem::CustomToolCallOutput { .. } - ) - { - // Publish the receipt and its history event together, in worker order. - let _ = context.writer.event_with_payload( - turn, - RawPayloadKind::ProtocolEvent, - &event.item, - move |payload| { - if is_delivery { - accepted(Some(payload), None) - } else { - accepted(None, Some(payload)) - } - }, - ); - } else { - let _ = context.writer.enqueue_event(turn, accepted(None, None)); - } - return; - } let Some(trace_event) = codex_turn_trace_event(context.thread_id.clone(), default_turn_id, event) else { @@ -378,6 +340,54 @@ impl ThreadTraceContext { ); } + /// Records the accepted wire item and, for agent-tool deliveries, its originating call. + pub fn record_history_item_accepted( + &self, + default_turn_id: &str, + item: &ResponseItem, + source_tool_call_id: Option<&str>, + ) { + let ThreadTraceContextState::Enabled(context) = &self.state else { + return; + }; + let Some(item_id) = item.id().map(ToString::to_string) else { + return; + }; + let source_tool_call_id = source_tool_call_id.map(str::to_owned); + let accepted = move |agent_message_payload, tool_output_payload| { + RawTraceEventPayload::HistoryItemAccepted { + item_id, + source_tool_call_id, + agent_message_payload, + tool_output_payload, + } + }; + let turn = RawTraceEventContext::turn(&context.thread_id, default_turn_id); + let is_delivery = matches!(item, ResponseItem::AgentMessage { .. }); + if is_delivery + || matches!( + item, + ResponseItem::FunctionCallOutput { .. } | ResponseItem::CustomToolCallOutput { .. } + ) + { + // Publish the receipt and its history event together, in worker order. + let _ = context.writer.event_with_payload( + turn, + RawPayloadKind::ProtocolEvent, + item, + move |payload| { + if is_delivery { + accepted(Some(payload), None) + } else { + accepted(None, Some(payload)) + } + }, + ); + } else { + let _ = context.writer.enqueue_event(turn, accepted(None, None)); + } + } + /// Records the exec actually selected by stdin, since its process handle can be reused. pub fn record_stdin_target( &self,