Link agent tool calls to delivered messages in rollout traces (#62709)

## Why

Matching messages by text can link a tool call to an earlier message
with the same content. A recipient can also accept a message without
ever including it in a model request.

## What changed

Record the originating tool call ID when the recipient adds a spawn,
task assignment, or message to its history. On replay, use that ID to
link the accepted message to its sender even if the recipient records
it before the sender tool finishes.

Keep the interaction edge pointed at the recipient thread until the
message appears in a model request. Track distinct conversation items
across history resets and count repeated prompt prefixes only once.

## Testing

Replay tests cover identical message text, receipt before tool
completion, and messages observed again after compaction.

GitOrigin-RevId: ccab7c33993ba5fce7fceb733dc90c9d428ec78f
This commit is contained in:
cassirer-openai
2026-11-20 08:49:07 +00:00
committed by copyberry
parent 0e8c7683c4
commit 076ac7856b
19 changed files with 457 additions and 201 deletions
+2
View File
@@ -210,6 +210,8 @@ pub enum AgentInput {
Message {
message: AgentMessage,
mode: MessageDeliveryMode,
/// Native tool invocation to link when the recipient commits this message to history.
source_tool_call_id: Option<String>,
},
}
+1
View File
@@ -271,6 +271,7 @@ impl LocalAgentControl {
let publish = || {
MessageDelivery::publish(
context.sender_thread_id,
context.source_tool_call_id.clone(),
state.extensions.message_lifecycle_contributors(),
terminal_result.then_some(&self.runtime.unconfirmed_results),
)
+20 -4
View File
@@ -74,7 +74,11 @@ impl AgentControl for LocalAgentControl {
} = request;
let input = match input {
AgentInput::UserInput(input) => SpawnInitialInput::UserInput(input),
AgentInput::Message { message, mode } => {
AgentInput::Message {
message,
mode,
source_tool_call_id,
} => {
if mode != MessageDeliveryMode::TriggerTurn {
return Err(CodexErr::InvalidRequest(
"spawn input must start the child turn".to_string(),
@@ -94,7 +98,11 @@ impl AgentControl for LocalAgentControl {
})?;
SpawnInitialInput::InterAgentCommunication(
message.into_communication(author, recipient, mode),
AgentCommunicationContext::new(AgentCommunicationKind::Spawn, caller),
AgentCommunicationContext {
kind: AgentCommunicationKind::Spawn,
sender_thread_id: caller,
source_tool_call_id,
},
)
}
};
@@ -124,7 +132,11 @@ impl AgentControl for LocalAgentControl {
let submission_id = self.send_input(target, input, start_options).await?;
(receiver.unwrap_or_default(), submission_id)
}
AgentInput::Message { message, mode } => {
AgentInput::Message {
message,
mode,
source_tool_call_id,
} => {
let receiver = self.runtime.ensure_agent_known(target)?;
let author = self
.runtime
@@ -168,7 +180,11 @@ impl AgentControl for LocalAgentControl {
.send_inter_agent_communication(
target,
communication,
AgentCommunicationContext::new(kind, caller),
AgentCommunicationContext {
kind,
sender_thread_id: caller,
source_tool_call_id,
},
start_options,
)
.await?;
+12 -1
View File
@@ -1127,6 +1127,7 @@ async fn cancelled_v2_send_finishes_reload_registration() {
input: AgentInput::Message {
message: AgentMessage::Plaintext(message.to_string()),
mode: MessageDeliveryMode::TriggerTurn,
source_tool_call_id: None,
},
start_options: Default::default(),
};
@@ -1979,6 +1980,7 @@ async fn v2_reload_preserves_shared_instructions_after_root_unloads(
input: crate::AgentInput::Message {
message: AgentMessage::Plaintext("wake the unloaded target".to_string()),
mode: MessageDeliveryMode::QueueOnly,
source_tool_call_id: None,
},
start_options: TurnStartOptions {
root_turn_id: sender_turn.turn_metadata_state.root_turn_id(),
@@ -2151,6 +2153,7 @@ async fn v2_spawn_resolves_reported_effort_without_changing_child_selection(
input: AgentInput::Message {
message: AgentMessage::Plaintext("child task".to_string()),
mode: MessageDeliveryMode::TriggerTurn,
source_tool_call_id: None,
},
source,
options: SpawnAgentOptions {
@@ -4498,7 +4501,15 @@ async fn multi_agent_v2_completion_ignores_dead_direct_parent() {
#[test]
fn terminal_result_writes_require_confirmation_after_the_parent_claims_mail() {
let count = Arc::new(std::sync::atomic::AtomicUsize::default());
let publish = || MessageDelivery::publish(ThreadId::new(), &[], Some(&count)).unwrap();
let publish = || {
MessageDelivery::publish(
ThreadId::new(),
/*source_tool_call_id*/ None,
&[],
Some(&count),
)
.unwrap()
};
let parent = ThreadId::new();
drop(publish().consumed(parent));
let saved = publish();
+24 -9
View File
@@ -46,6 +46,7 @@ impl AgentCommunicationKind {
pub(crate) struct AgentCommunicationContext {
pub(crate) kind: AgentCommunicationKind,
pub(crate) sender_thread_id: ThreadId,
pub(crate) source_tool_call_id: Option<String>,
}
impl AgentCommunicationContext {
@@ -53,6 +54,7 @@ impl AgentCommunicationContext {
Self {
kind,
sender_thread_id,
source_tool_call_id: None,
}
}
}
@@ -174,16 +176,23 @@ struct MessagePublication {
/// Clones follow one logical delivery through queues and share its consumption state.
#[derive(Clone)]
pub(crate) struct MessageDelivery(Arc<Mutex<Option<MessagePublication>>>);
pub(crate) struct MessageDelivery {
pub(crate) source_tool_call_id: Option<String>,
publication: Option<Arc<Mutex<Option<MessagePublication>>>>,
}
impl MessageDelivery {
pub(crate) fn publish(
sender: ThreadId,
source_tool_call_id: Option<String>,
observers: &[Arc<dyn MessageLifecycleContributor>],
unconfirmed_results: Option<&Arc<AtomicUsize>>,
) -> Option<Self> {
if observers.is_empty() && unconfirmed_results.is_none() {
return None;
return source_tool_call_id.map(|id| Self {
source_tool_call_id: Some(id),
publication: None,
});
}
let input = MessageLifecycleInput {
sender,
@@ -201,21 +210,27 @@ impl MessageDelivery {
observers: observers.into(),
result_write,
};
Some(Self(Arc::new(Mutex::new(Some(publication)))))
Some(Self {
source_tool_call_id,
publication: Some(Arc::new(Mutex::new(Some(publication)))),
})
}
pub(crate) fn awaits_result_write(&self) -> bool {
self.0
.lock()
.unwrap_or_else(PoisonError::into_inner)
.as_ref()
.is_some_and(|publication| publication.result_write.is_some())
self.publication.as_ref().is_some_and(|publication| {
publication
.lock()
.unwrap_or_else(PoisonError::into_inner)
.as_ref()
.is_some_and(|publication| publication.result_write.is_some())
})
}
/// The first consumer owns both the publication hooks and the write receipt.
pub(crate) fn consumed(&self, recipient: ThreadId) -> Option<ResultWrite> {
let publication = self
.0
.publication
.as_ref()?
.lock()
.unwrap_or_else(PoisonError::into_inner)
.take()?;
+24 -1
View File
@@ -4866,6 +4866,17 @@ impl Session {
std::slice::from_mut(&mut response_item),
model_info.truncation_policy.into(),
);
// Record the item and its sender at the history write, before shutdown can cancel persistence.
self.services
.rollout_thread_trace
.record_history_item_accepted(
&turn_context.sub_id,
&response_item.item,
communication
.delivery
.as_ref()
.and_then(|d| d.source_tool_call_id.as_deref()),
);
}
if let Some(delivery) = delivery.take() {
result_write = delivery.consumed(self.thread_id);
@@ -4885,7 +4896,12 @@ impl Session {
write.confirm();
}
drop(boundary);
self.send_raw_response_items(turn_context, &items).await;
let [item] = items;
self.send_event(
turn_context,
EventMsg::RawResponseItem(RawResponseItemEvent { item }),
)
.await;
}
async fn maybe_warn_on_server_model_mismatch(
@@ -5193,6 +5209,13 @@ impl Session {
#[tracing::instrument(level = "trace", skip_all, fields(item_count = items.len()))]
async fn send_raw_response_items(&self, turn_context: &TurnContext, items: &[ResponseItem]) {
for item in items {
self.services
.rollout_thread_trace
.record_history_item_accepted(
&turn_context.sub_id,
item,
/*source_tool_call_id*/ None,
);
self.send_event(
turn_context,
EventMsg::RawResponseItem(RawResponseItemEvent { item: item.clone() }),
+5
View File
@@ -110,6 +110,11 @@ pub(crate) async fn spawn(
input: AgentInput::Message {
message,
mode: MessageDeliveryMode::TriggerTurn,
source_tool_call_id: session
.services
.rollout_thread_trace
.is_enabled()
.then(|| call_id.to_owned()),
},
source: settings.source,
options: settings.options,
@@ -68,6 +68,11 @@ pub(super) async fn handle_message_string_tool(
input: AgentInput::Message {
message: agent_message_from_tool(message, &source),
mode,
source_tool_call_id: session
.services
.rollout_thread_trace
.is_enabled()
.then(|| call_id.clone()),
},
start_options: TurnStartOptions {
parent_turn_id: (mode == MessageDeliveryMode::TriggerTurn)
@@ -128,6 +128,7 @@ fn compressed_budget_admits_small_later_events_after_an_omission() -> Result<()>
writer.append_with_context(
RawTraceEventContext::turn(ROOT_THREAD_ID, "turn"),
RawTraceEventPayload::HistoryItemAccepted {
source_tool_call_id: None,
item_id: "missing-receipt".into(),
agent_message_payload: Some(omitted.clone()),
tool_output_payload: None,
+2
View File
@@ -112,6 +112,8 @@ pub enum RawTraceEventPayload {
/// evidence of its rendered content, which may be truncated before sampling.
HistoryItemAccepted {
item_id: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
source_tool_call_id: Option<ToolCallId>,
/// A delivery may never appear in a subsequent inference or compaction snapshot.
#[serde(default, skip_serializing_if = "Option::is_none")]
agent_message_payload: Option<RawPayloadRef>,
@@ -41,6 +41,7 @@ fn identical_text_from_a_discarded_attempt_does_not_gain_acceptance() -> anyhow:
writer.append_with_context(
trace_context("turn-1"),
RawTraceEventPayload::HistoryItemAccepted {
source_tool_call_id: None,
item_id: "accepted".into(),
agent_message_payload: None,
tool_output_payload: None,
@@ -225,6 +226,7 @@ fn inherited_output_is_child_prefill_without_copying_parent_sampling_provenance(
parent.request_item_ids
);
let accepted = RawTraceEventPayload::HistoryItemAccepted {
source_tool_call_id: None,
item_id: "msg-parent".into(),
agent_message_payload: None,
tool_output_payload: None,
@@ -280,6 +282,7 @@ fn accept_output(writer: &crate::TraceWriter, item: &serde_json::Value) -> anyho
writer.append_with_context(
trace_context("turn-1"),
RawTraceEventPayload::HistoryItemAccepted {
source_tool_call_id: None,
item_id: item["id"].as_str().unwrap().into(),
agent_message_payload: None,
tool_output_payload: Some(payload),
@@ -62,7 +62,13 @@ impl TraceReducer {
if is_delivery {
self.resolve_pending_agent_edges_for_item(&id)?;
}
self.receivable_ids.insert(key, Some(id));
self.receivable_ids.insert(
key,
super::ReceivableItem {
item_id: id,
awaiting_snapshot: true,
},
);
Ok(())
}
@@ -539,8 +545,10 @@ impl TraceReducer {
&& let Some(wire_id) = &item.responses_item_id
{
let key = (thread_id.to_owned(), wire_id.clone());
if let Some(Some(id)) = self.receivable_ids.get(&key) {
return id.clone();
if let Some(receipt) = self.receivable_ids.get(&key)
&& receipt.awaiting_snapshot
{
return receipt.item_id.clone();
}
}
let item_id = self.next_conversation_item_id();
@@ -592,9 +600,15 @@ impl TraceReducer {
&& let Some(wire_id) = &normalized.responses_item_id
{
let key = (item.thread_id.clone(), wire_id.clone());
let accepted_id = self.receivable_ids.entry(key).or_default();
if accepted_id.as_deref() == Some(item_id) {
*accepted_id = None;
let receipt = self
.receivable_ids
.entry(key)
.or_insert_with(|| super::ReceivableItem {
item_id: item_id.to_owned(),
awaiting_snapshot: false,
});
if receipt.awaiting_snapshot && receipt.item_id == item_id {
receipt.awaiting_snapshot = false;
item.body.clone_from(&normalized.body);
if let Some(hash) = normalized.structured_output_hash {
self.structured_output_hashes
@@ -224,6 +224,7 @@ fn response_outputs_require_harness_acceptance() -> anyhow::Result<()> {
writer.append_with_context(
trace_context("turn-1"),
RawTraceEventPayload::HistoryItemAccepted {
source_tool_call_id: None,
item_id: "msg-accepted".into(),
agent_message_payload: None,
tool_output_payload: None,
@@ -30,6 +30,7 @@ fn cancelled_inference_retains_accepted_partial_output() -> anyhow::Result<()> {
writer.append_with_context(
trace_context("turn-1"),
RawTraceEventPayload::HistoryItemAccepted {
source_tool_call_id: None,
item_id: "msg-partial".into(),
agent_message_payload: None,
tool_output_payload: None,
+16 -9
View File
@@ -80,6 +80,7 @@ pub fn replay_bundle(bundle_dir: impl AsRef<Path>) -> Result<RolloutTrace> {
pending_code_cell_starts: BTreeMap::new(),
pending_code_cell_lifecycle_events: BTreeMap::new(),
pending_agent_interaction_edges: Vec::new(),
native_tool_deliveries: BTreeMap::new(),
first_acceptances: BTreeMap::new(),
receivable_ids: BTreeMap::new(),
snapshot_output_anchors: BTreeSet::new(),
@@ -139,6 +140,7 @@ pub fn replay_bundle(bundle_dir: impl AsRef<Path>) -> Result<RolloutTrace> {
// Spawn edges prefer the child task message as their target, but a child can
// fail before that message is ever reduced. Only after replaying the whole
// bundle do we know which spawn deliveries need the child-thread fallback.
reducer.finish_native_tool_deliveries()?;
reducer.resolve_pending_spawn_edge_fallbacks()?;
// Cell starts can be queued until their model output arrives. Link only
// materialized cells, after both sides of the dispatch have been replayed.
@@ -203,14 +205,18 @@ pub fn replay_bundle(bundle_dir: impl AsRef<Path>) -> Result<RolloutTrace> {
Ok(reducer.rollout)
}
struct ReceivableItem {
item_id: String,
awaiting_snapshot: bool,
}
struct TraceReducer {
rollout: RolloutTrace,
payload_reader: PayloadReader,
/// First (wall time, sequence) by receiving thread and Responses item ID.
first_acceptances: BTreeMap<String, BTreeMap<String, (i64, crate::RawEventSeq)>>,
/// A native delivery or tool output has one graph ID until its first snapshot.
/// None means a snapshot already supplied its body (and a later acceptance is a duplicate).
receivable_ids: BTreeMap<(String, String), Option<String>>,
/// First graph item for each receiving thread and Responses wire ID.
receivable_ids: BTreeMap<(String, String), ReceivableItem>,
/// Accepted tool outputs first encountered in the batch about to be appended.
snapshot_output_anchors: BTreeSet<String>,
next_conversation_item_ordinal: u64,
@@ -254,13 +260,10 @@ struct TraceReducer {
/// queued for ownership validation; these lifecycle events wait with it and
/// are replayed in raw sequence order once the cell materializes.
pending_code_cell_lifecycle_events: BTreeMap<String, Vec<PendingCodeCellLifecycleEvent>>,
/// Multi-agent deliveries whose recipient-side transcript item has not been observed yet.
///
/// V2 agent tools enqueue mailbox messages in the target thread. The trace event for the
/// sending tool arrives before the recipient inference request materializes that mailbox item
/// as a `ConversationItem`, so the reducer keeps the delivery edge pending until it can point
/// at the exact model-visible item instead of a coarse thread.
/// Legacy collaboration events and automatic agent results lack a source-to-item ID.
pending_agent_interaction_edges: Vec<PendingAgentInteractionEdge>,
/// V2 tool activity and the recipient's history item can arrive in either order.
native_tool_deliveries: BTreeMap<String, tool::NativeToolDelivery>,
}
impl TraceReducer {
@@ -353,6 +356,7 @@ impl TraceReducer {
}
RawTraceEventPayload::HistoryItemAccepted {
item_id,
source_tool_call_id,
agent_message_payload,
tool_output_payload,
} => {
@@ -360,6 +364,9 @@ impl TraceReducer {
.thread_id
.context("history acceptance has no thread")?;
self.thread_mut(&thread_id)?;
if let Some(source) = source_tool_call_id {
self.record_agent_message_target(source, thread_id.clone(), item_id.clone())?;
}
// Keep the acceptance edge even when a capture limit hid the receipt body.
if let Some(payload) = agent_message_payload.or(tool_output_payload)
&& payload.capture.is_complete()
@@ -25,6 +25,7 @@ mod terminal;
#[path = "tool/mcp_tests.rs"]
mod mcp_tests;
pub(super) use agents::NativeToolDelivery;
pub(super) use agents::ObservedAgentResultEdge;
pub(super) use agents::PendingAgentInteractionEdge;
pub(super) use agents::spawn_edge_id;
+135 -111
View File
@@ -8,8 +8,9 @@ use codex_protocol::protocol::CollabCloseBeginEvent;
use codex_protocol::protocol::CollabCloseEndEvent;
use codex_protocol::protocol::SubAgentActivityEvent;
use codex_protocol::protocol::SubAgentActivityKind;
use serde::Deserialize;
use serde_json::Value;
use std::collections::BTreeMap;
use std::collections::HashSet;
use std::collections::btree_map::Entry;
use super::super::TraceReducer;
use crate::model::ConversationItemKind;
@@ -17,15 +18,12 @@ use crate::model::ConversationPart;
use crate::model::ConversationRole;
use crate::model::InteractionEdge;
use crate::model::InteractionEdgeKind;
use crate::model::ProducerRef;
use crate::model::ToolCallKind;
use crate::model::TraceAnchor;
use crate::payload::RawPayloadRef;
/// Agent delivery edge waiting for the recipient-side conversation item.
///
/// Multi-agent v2 records the sender tool before the target thread necessarily
/// includes the delivered mailbox message in a model-visible request. The edge
/// stays pending so it can target that exact conversation item when possible.
/// A legacy collaboration event or an automatic child result awaiting its recipient item.
pub(in crate::reducer) struct PendingAgentInteractionEdge {
pub(in crate::reducer) edge_id: String,
pub(in crate::reducer) kind: InteractionEdgeKind,
@@ -55,9 +53,11 @@ pub(in crate::reducer) struct ObservedAgentResultEdge {
pub(in crate::reducer) carried_payload: Option<RawPayloadRef>,
}
#[derive(Deserialize)]
struct AgentMessageInvocationArgs {
message: String,
/// Sender completion and recipient history can arrive in either order.
pub(in crate::reducer) struct NativeToolDelivery {
receiver_thread_id: String,
activity_at_unix_ms: Option<i64>,
wire_item_id: Option<String>,
}
/// Builds the stable edge id for the spawn relationship between two threads.
@@ -197,43 +197,15 @@ impl TraceReducer {
) -> Result<()> {
let target_thread_id = payload.agent_thread_id.to_string();
match (tool_kind, &payload.kind) {
(ToolCallKind::SpawnAgent, SubAgentActivityKind::Started) => {
let parent_thread_id = self
.rollout
.tool_calls
.get(tool_call_id)
.with_context(|| {
format!("agent activity referenced unknown tool call {tool_call_id}")
})?
.thread_id
.clone();
self.queue_sub_agent_activity_message_edge(
wall_time_unix_ms,
tool_call_id,
spawn_edge_id(&parent_thread_id, &target_thread_id),
InteractionEdgeKind::SpawnAgent,
target_thread_id.clone(),
Some(target_thread_id),
)
(ToolCallKind::SpawnAgent, SubAgentActivityKind::Started)
| (
ToolCallKind::AssignAgentTask | ToolCallKind::SendMessage,
SubAgentActivityKind::Interacted,
) => {
self.native_tool_delivery(tool_call_id.to_owned(), target_thread_id)?
.activity_at_unix_ms = Some(wall_time_unix_ms);
Ok(())
}
(ToolCallKind::AssignAgentTask, SubAgentActivityKind::Interacted) => self
.queue_sub_agent_activity_message_edge(
wall_time_unix_ms,
tool_call_id,
tool_edge_id(tool_call_id),
InteractionEdgeKind::AssignAgentTask,
target_thread_id,
/*unresolved_spawn_thread_id*/ None,
),
(ToolCallKind::SendMessage, SubAgentActivityKind::Interacted) => self
.queue_sub_agent_activity_message_edge(
wall_time_unix_ms,
tool_call_id,
tool_edge_id(tool_call_id),
InteractionEdgeKind::SendMessage,
target_thread_id,
/*unresolved_spawn_thread_id*/ None,
),
(ToolCallKind::CloseAgent, SubAgentActivityKind::Interrupted) => self
.upsert_close_agent_interaction(
tool_call_id,
@@ -248,77 +220,129 @@ impl TraceReducer {
}
}
fn queue_sub_agent_activity_message_edge(
fn native_tool_delivery(
&mut self,
wall_time_unix_ms: i64,
tool_call_id: &str,
edge_id: String,
edge_kind: InteractionEdgeKind,
target_thread_id: String,
unresolved_spawn_thread_id: Option<String>,
) -> Result<()> {
let tool_call = self.rollout.tool_calls.get(tool_call_id).with_context(|| {
format!("agent activity referenced unknown tool call {tool_call_id}")
})?;
let started_at_unix_ms = tool_call.execution.started_at_unix_ms;
let message_author = self.agent_path_for_thread(&tool_call.thread_id)?;
let Some(message_content) = self.agent_message_content_from_invocation(tool_call_id)?
else {
return Ok(());
};
let carried_raw_payload_ids = self.agent_tool_payload_ids(tool_call_id)?;
self.queue_or_resolve_agent_interaction_edge(PendingAgentInteractionEdge {
edge_id,
kind: edge_kind,
source: TraceAnchor::ToolCall {
tool_call_id: tool_call_id.to_string(),
},
target_thread_id,
message_author,
message_content,
unresolved_spawn_thread_id,
started_at_unix_ms,
ended_at_unix_ms: Some(wall_time_unix_ms),
carried_raw_payload_ids,
})
call_id: String,
receiver: String,
) -> Result<&mut NativeToolDelivery> {
match self.native_tool_deliveries.entry(call_id) {
Entry::Occupied(entry) if entry.get().receiver_thread_id != receiver => {
bail!("conflicting recipients for tool {}", entry.key())
}
Entry::Occupied(entry) => Ok(entry.into_mut()),
Entry::Vacant(entry) => Ok(entry.insert(NativeToolDelivery {
receiver_thread_id: receiver,
activity_at_unix_ms: None,
wire_item_id: None,
})),
}
}
fn agent_message_content_from_invocation(
pub(in crate::reducer) fn record_agent_message_target(
&mut self,
tool_call_id: &str,
) -> Result<Option<String>> {
let tool_call = self.rollout.tool_calls.get(tool_call_id).with_context(|| {
format!("agent activity referenced unknown tool call {tool_call_id}")
})?;
let invocation_payload_id = tool_call
.raw_invocation_payload_id
.as_deref()
.with_context(|| {
format!("agent activity tool call {tool_call_id} missing invocation payload")
})?;
let invocation_payload = self
.rollout
.raw_payloads
.get(invocation_payload_id)
.with_context(|| {
format!(
"agent activity tool call {tool_call_id} referenced missing invocation payload {invocation_payload_id}"
)
})?;
if !invocation_payload.capture.is_complete() {
return Ok(None);
call_id: String,
receiver: String,
item_id: String,
) -> Result<()> {
let delivery = self.native_tool_delivery(call_id, receiver)?;
if delivery.wire_item_id.get_or_insert(item_id.clone()) != &item_id {
bail!("one agent tool delivered different history items");
}
let invocation = self.payload_reader.read_json(invocation_payload)?;
let arguments = invocation
.get("payload")
.and_then(|payload| payload.get("arguments"))
.and_then(Value::as_str)
.with_context(|| {
format!("agent activity tool call {tool_call_id} missing function arguments")
Ok(())
}
/// Collect the distinct graph items actually sent to each receiver's model. A reset
/// can give the same Responses item a new graph ID. Repeated prompt prefixes reuse it.
fn model_visible_deliveries(&self) -> BTreeMap<(String, String), Vec<String>> {
let mut visible = BTreeMap::<_, Vec<_>>::new();
let mut visited = HashSet::new();
let mut requests: Vec<_> = self.rollout.inference_calls.values().collect();
requests.sort_unstable_by_key(|call| call.execution.started_seq);
for call in requests {
for id in &call.request_item_ids {
if !visited.insert((&call.thread_id, id)) {
continue;
}
let item = &self.rollout.conversation_items[id];
if item.thread_id == call.thread_id
&& item.agent_message.is_some()
&& let Some(wire_id) = &item.responses_item_id
{
visible
.entry((call.thread_id.clone(), wire_id.clone()))
.or_default()
.push(id.clone());
}
}
}
visible
}
pub(in crate::reducer) fn finish_native_tool_deliveries(&mut self) -> Result<()> {
if self.native_tool_deliveries.is_empty() {
return Ok(());
}
let visible = self.model_visible_deliveries();
for (call_id, delivery) in std::mem::take(&mut self.native_tool_deliveries) {
let Some(tool) = self.rollout.tool_calls.get(&call_id) else {
// Mail can survive shutdown and reach a recipient in a later recording.
continue;
};
let receiver = delivery.receiver_thread_id;
let (kind, edge_id) = match tool.kind {
ToolCallKind::SpawnAgent => (
InteractionEdgeKind::SpawnAgent,
spawn_edge_id(&tool.thread_id, &receiver),
),
ToolCallKind::AssignAgentTask => {
(InteractionEdgeKind::AssignAgentTask, tool_edge_id(&call_id))
}
ToolCallKind::SendMessage => {
(InteractionEdgeKind::SendMessage, tool_edge_id(&call_id))
}
_ => bail!("non-agent tool {call_id} supplied an agent delivery"),
};
// The successful tool identifies the recipient even when it never samples again.
// Keep acceptance on the history item. Only actual receiver inputs enrich this edge.
let mut carried_item_ids = Vec::new();
if let Some(wire_id) = delivery.wire_item_id {
let key = (receiver.clone(), wire_id);
if let Some(receipt) = self.receivable_ids.get(&key) {
let item = self
.rollout
.conversation_items
.get_mut(&receipt.item_id)
.context("accepted agent message missing from conversation items")?;
item.produced_by.push(ProducerRef::InteractionEdge {
edge_id: edge_id.clone(),
});
}
carried_item_ids = visible.get(&key).cloned().unwrap_or_default();
}
let target = match carried_item_ids.first() {
Some(item_id) => TraceAnchor::ConversationItem {
item_id: item_id.clone(),
},
None => TraceAnchor::Thread {
thread_id: receiver,
},
};
self.upsert_interaction_edge(InteractionEdge {
edge_id,
kind,
source: TraceAnchor::ToolCall {
tool_call_id: call_id.clone(),
},
target,
started_at_unix_ms: tool.execution.started_at_unix_ms,
ended_at_unix_ms: delivery
.activity_at_unix_ms
.or(tool.execution.ended_at_unix_ms),
carried_item_ids,
carried_raw_payload_ids: self.agent_tool_payload_ids(&call_id)?,
})?;
let args: AgentMessageInvocationArgs = serde_json::from_str(arguments)
.with_context(|| format!("parse agent activity tool call {tool_call_id} arguments"))?;
Ok(Some(args.message))
}
Ok(())
}
/// Adds the canonical tool result payload to an already reduced multi-agent edge.
@@ -255,6 +255,7 @@ fn sub_agent_started_activity_creates_spawn_edge() -> anyhow::Result<()> {
]
});
let receipt = RawTraceEventPayload::HistoryItemAccepted {
source_tool_call_id: Some("call-spawn-v2".into()),
item_id: "amsg-delivery".into(),
tool_output_payload: None,
agent_message_payload: Some(
@@ -264,10 +265,25 @@ fn sub_agent_started_activity_creates_spawn_edge() -> anyhow::Result<()> {
let context = trace_context_for_thread(child_thread_id, "turn-child-1");
writer.append_with_context(context.clone(), receipt.clone())?;
// Interrupt or reset here: the receipt already has its delivery edge without any child request.
// Interrupt before the first request. The send and history acceptance are both known.
let received = replay_bundle(temp.path())?;
let edge_id = format!("edge:spawn:019d0000-0000-7000-8000-000000000001:{child_thread_id}");
let receipt_id = target_conversation_item_id(&received.interaction_edges[&edge_id].target);
let edge = &received.interaction_edges[&edge_id];
assert_eq!(
edge.target,
TraceAnchor::Thread {
thread_id: child_thread_id.to_owned()
}
);
assert!(edge.carried_item_ids.is_empty());
let receipt_id = &received.threads[child_thread_id].conversation_item_ids[0];
assert!(
received.conversation_items[receipt_id]
.produced_by
.contains(&crate::model::ProducerRef::InteractionEdge {
edge_id: edge_id.clone()
})
);
let delivered_item_ids = std::slice::from_ref(receipt_id);
assert_eq!(
received.threads[child_thread_id].conversation_item_ids,
@@ -287,9 +303,9 @@ fn sub_agent_started_activity_creates_spawn_edge() -> anyhow::Result<()> {
child_thread_id,
"turn-child-1",
"inference-child-1",
vec![delivery],
vec![delivery.clone()],
)?;
writer.append_with_context(context, receipt)?;
writer.append_with_context(context.clone(), receipt)?;
let replayed = replay_bundle(temp.path())?;
assert_eq!(
@@ -319,6 +335,38 @@ fn sub_agent_started_activity_creates_spawn_edge() -> anyhow::Result<()> {
activity_payload.raw_payload_id,
]
);
// A checkpoint can copy the same wire item to a new graph node. Only a later
// inference using that copy adds a second observation to the send edge.
let checkpoint_payload = writer.write_json_payload(
RawPayloadKind::CompactionCheckpoint,
&json!({"input_history": [&delivery], "replacement_history": [&delivery]}),
)?;
writer.append_with_context(
context,
RawTraceEventPayload::CompactionInstalled {
compaction_id: "reset".to_owned(),
checkpoint_payload,
},
)?;
let reset = replay_bundle(temp.path())?;
assert_eq!(
reset.interaction_edges[&edge_id].carried_item_ids,
delivered_item_ids
);
let replacement = &reset.compactions["reset"].replacement_item_ids[0];
assert_ne!(replacement, receipt_id);
append_inference_request(
&writer,
child_thread_id,
"turn-child-1",
"after-reset",
vec![delivery],
)?;
let after = replay_bundle(temp.path())?;
assert_eq!(
after.interaction_edges[&edge_id].carried_item_ids,
[receipt_id.clone(), replacement.clone()]
);
Ok(())
}
@@ -470,6 +518,39 @@ fn send_message_activity_targets_delivered_child_message() -> anyhow::Result<()>
"kind": "interacted"
}),
)?;
start_thread(&writer, child_thread_id, "/root/child")?;
start_turn_for_thread(&writer, child_thread_id, "turn-child-1")?;
// The recipient may have seen a task with the same payload before this ordinary send.
let task = inter_agent_message(
"/root",
"/root/child",
"Message Type: NEW_TASK\nTask name: /root/child\nSender: /root\nPayload:\nhello again",
);
let mut delivered = inter_agent_message(
"/root",
"/root/child",
"Message Type: MESSAGE\nTask name: /root/child\nSender: /root\nPayload:\nhello again",
);
delivered["id"] = json!("amsg-send");
append_inference_request(
&writer,
child_thread_id,
"turn-child-1",
"inference-child-1",
vec![task, delivered.clone()],
)?;
writer.append_with_context(
trace_context_for_thread(child_thread_id, "turn-child-1"),
RawTraceEventPayload::HistoryItemAccepted {
source_tool_call_id: Some("call-send-v2".into()),
agent_message_payload: Some(
writer.write_json_payload(RawPayloadKind::ProtocolEvent, &delivered)?,
),
tool_output_payload: None,
item_id: "amsg-send".into(),
},
)?;
writer.append_with_context(
trace_context_for_agent("turn-1"),
RawTraceEventPayload::ToolCallRuntimeEnded {
@@ -478,16 +559,6 @@ fn send_message_activity_targets_delivered_child_message() -> anyhow::Result<()>
runtime_payload: activity_payload.clone(),
},
)?;
start_thread(&writer, child_thread_id, "/root/child")?;
start_turn_for_thread(&writer, child_thread_id, "turn-child-1")?;
let delivered = inter_agent_message("/root", "/root/child", "hello again");
append_inference_request(
&writer,
child_thread_id,
"turn-child-1",
"inference-child-1",
vec![delivered],
)?;
let replayed = replay_bundle(temp.path())?;
let edge = &replayed.interaction_edges["edge:tool:call-send-v2"];
@@ -495,8 +566,10 @@ fn send_message_activity_targets_delivered_child_message() -> anyhow::Result<()>
let target_item_id = target_conversation_item_id(&edge.target);
assert_eq!(edge.carried_item_ids, vec![target_item_id.clone()]);
assert_eq!(
replayed.conversation_items[target_item_id].thread_id,
child_thread_id
replayed.conversation_items[target_item_id]
.responses_item_id
.as_deref(),
Some("amsg-send")
);
assert_eq!(
edge.carried_raw_payload_ids,
@@ -515,6 +588,22 @@ fn followup_activity_targets_delivered_child_message() -> anyhow::Result<()> {
let writer = create_started_agent_writer(&temp)?;
start_agent_turn(&writer, "turn-1")?;
let child_thread_id = "019d0000-0000-7000-8000-000000000002";
start_thread(&writer, child_thread_id, "/root/child")?;
start_turn_for_thread(&writer, child_thread_id, "turn-child-1")?;
let mut previous = inter_agent_message(
"/root",
"/root/child",
"Message Type: NEW_TASK\nTask name: /root/child\nSender: /root\nPayload:\ncontinue",
);
previous["id"] = json!("amsg-host");
append_inference_request(
&writer,
child_thread_id,
"turn-child-1",
"inference-child-1",
vec![previous.clone()],
)?;
let invocation_payload = writer.write_json_payload(
RawPayloadKind::ToolInvocation,
&json!({
@@ -559,21 +648,46 @@ fn followup_activity_targets_delivered_child_message() -> anyhow::Result<()> {
runtime_payload: activity_payload.clone(),
},
)?;
start_thread(&writer, child_thread_id, "/root/child")?;
start_turn_for_thread(&writer, child_thread_id, "turn-child-1")?;
let delivered = inter_agent_message("/root", "/root/child", "continue");
let sent = replay_bundle(temp.path())?;
let edge = &sent.interaction_edges["edge:tool:call-followup-v2"];
assert_eq!(
edge.target,
TraceAnchor::Thread {
thread_id: child_thread_id.to_owned()
}
);
assert!(edge.carried_item_ids.is_empty());
let mut delivered = previous.clone();
delivered["id"] = json!("amsg-followup");
writer.append_with_context(
trace_context_for_thread(child_thread_id, "turn-child-1"),
RawTraceEventPayload::HistoryItemAccepted {
source_tool_call_id: Some("call-followup-v2".into()),
tool_output_payload: None,
item_id: "amsg-followup".into(),
agent_message_payload: Some(
writer.write_json_payload(RawPayloadKind::ProtocolEvent, &delivered)?,
),
},
)?;
append_inference_request(
&writer,
child_thread_id,
"turn-child-1",
"inference-child-1",
vec![delivered],
"inference-child-2",
vec![previous, delivered],
)?;
let replayed = replay_bundle(temp.path())?;
let edge = &replayed.interaction_edges["edge:tool:call-followup-v2"];
assert_eq!(edge.kind, InteractionEdgeKind::AssignAgentTask);
let target_item_id = target_conversation_item_id(&edge.target);
assert_eq!(
replayed.conversation_items[target_item_id]
.responses_item_id
.as_deref(),
Some("amsg-followup")
);
assert_eq!(edge.carried_item_ids, vec![target_item_id.clone()]);
assert_eq!(
replayed.conversation_items[target_item_id].thread_id,
+49 -39
View File
@@ -240,49 +240,11 @@ impl ThreadTraceContext {
);
}
/// Records turn lifecycle and history acceptance from existing protocol events.
/// Records turn lifecycle from existing protocol events.
pub fn record_codex_turn_event(&self, default_turn_id: &str, event: &EventMsg) {
let ThreadTraceContextState::Enabled(context) = &self.state else {
return;
};
if let EventMsg::RawResponseItem(event) = event {
let Some(item_id) = event.item.id().map(ToString::to_string) else {
return;
};
let accepted = move |agent_message_payload, tool_output_payload| {
RawTraceEventPayload::HistoryItemAccepted {
item_id,
agent_message_payload,
tool_output_payload,
}
};
let turn = RawTraceEventContext::turn(&context.thread_id, default_turn_id);
let is_delivery = matches!(event.item, ResponseItem::AgentMessage { .. });
if is_delivery
|| matches!(
event.item,
ResponseItem::FunctionCallOutput { .. }
| ResponseItem::CustomToolCallOutput { .. }
)
{
// Publish the receipt and its history event together, in worker order.
let _ = context.writer.event_with_payload(
turn,
RawPayloadKind::ProtocolEvent,
&event.item,
move |payload| {
if is_delivery {
accepted(Some(payload), None)
} else {
accepted(None, Some(payload))
}
},
);
} else {
let _ = context.writer.enqueue_event(turn, accepted(None, None));
}
return;
}
let Some(trace_event) =
codex_turn_trace_event(context.thread_id.clone(), default_turn_id, event)
else {
@@ -378,6 +340,54 @@ impl ThreadTraceContext {
);
}
/// Records the accepted wire item and, for agent-tool deliveries, its originating call.
pub fn record_history_item_accepted(
&self,
default_turn_id: &str,
item: &ResponseItem,
source_tool_call_id: Option<&str>,
) {
let ThreadTraceContextState::Enabled(context) = &self.state else {
return;
};
let Some(item_id) = item.id().map(ToString::to_string) else {
return;
};
let source_tool_call_id = source_tool_call_id.map(str::to_owned);
let accepted = move |agent_message_payload, tool_output_payload| {
RawTraceEventPayload::HistoryItemAccepted {
item_id,
source_tool_call_id,
agent_message_payload,
tool_output_payload,
}
};
let turn = RawTraceEventContext::turn(&context.thread_id, default_turn_id);
let is_delivery = matches!(item, ResponseItem::AgentMessage { .. });
if is_delivery
|| matches!(
item,
ResponseItem::FunctionCallOutput { .. } | ResponseItem::CustomToolCallOutput { .. }
)
{
// Publish the receipt and its history event together, in worker order.
let _ = context.writer.event_with_payload(
turn,
RawPayloadKind::ProtocolEvent,
item,
move |payload| {
if is_delivery {
accepted(Some(payload), None)
} else {
accepted(None, Some(payload))
}
},
);
} else {
let _ = context.writer.enqueue_event(turn, accepted(None, None));
}
}
/// Records the exec actually selected by stdin, since its process handle can be reused.
pub fn record_stdin_target(
&self,