Add OpenTelemetry for subagent lifecycles (#62567)

## Why

Correlating a subagent's creation, turn attempts, and stop requests
helps explain what happened when those actions span workers.

## What changed

- Emit structured events and counters for successful spawns and accepted
  `close_agent` and `interrupt_agent` calls. Count accepted stops even
  when the target is already idle or unloaded.
- Track initial, follow-up, and resumed turn attempts for spawned
  subagents. Record each start and at most one outcome per attempt:
  completed, failed, interrupted, or suspended.
- Use thread, turn, and call IDs to correlate structured events while
  keeping those IDs out of metric labels.

## Testing

Add coverage for lifecycle counts, duplicate outcomes, stop handling,
and event correlation across workers. Improve the Windows elevated
sandbox test for an approved `git pull` by provisioning the sandbox
before the turn and reporting setup or launch failures with diagnostics.

GitOrigin-RevId: bc1190302e1dffbdd79c7ac984b6debe1dac4a65
This commit is contained in:
Bryan Ashley
2026-11-19 18:39:32 +00:00
committed by copyberry
parent 617ad20df0
commit 65454c7b99
18 changed files with 779 additions and 35 deletions
+1
View File
@@ -8,6 +8,7 @@ pub(crate) mod host;
mod registry;
pub(crate) mod role;
pub(crate) mod status;
pub(crate) mod turn_metrics;
pub(crate) mod types;
pub(crate) use codex_protocol::protocol::AgentStatus;
+95
View File
@@ -0,0 +1,95 @@
//! Tracks attempts on native spawned children, including across suspend and resume.
//! Each regular attempt records at most one end, even if suspension is followed by shutdown.
use std::sync::atomic::AtomicBool;
use std::sync::atomic::Ordering;
use codex_extension_api::ExtensionData;
use codex_history::InitialHistory;
use codex_otel::SubagentTurnStatus;
use codex_protocol::protocol::EventMsg;
use codex_protocol::protocol::SessionSource;
use codex_protocol::protocol::SubAgentSource;
use crate::session::turn_context::TurnContext;
enum SpawnedThread {
New { first_turn_pending: AtomicBool },
Resumed,
}
struct ActiveSubagentTurn(&'static str);
pub(crate) fn register_thread(
thread_store: &ExtensionData,
source: &SessionSource,
history: &InitialHistory,
) {
if !matches!(
source,
SessionSource::SubAgent(SubAgentSource::ThreadSpawn { .. })
) {
return;
}
let thread = match history {
InitialHistory::Resumed(_) => SpawnedThread::Resumed,
// A full fork can already contain many turns from its parent.
InitialHistory::New | InitialHistory::Cleared | InitialHistory::Forked(_) => {
SpawnedThread::New {
first_turn_pending: AtomicBool::new(true),
}
}
};
thread_store.insert(thread);
}
pub(crate) fn record_start(thread_store: &ExtensionData, turn: &TurnContext) {
let Some(thread) = thread_store.get::<SpawnedThread>() else {
return;
};
let kind = match thread.as_ref() {
SpawnedThread::New { first_turn_pending }
if first_turn_pending.swap(false, Ordering::Relaxed) =>
{
"initial"
}
SpawnedThread::New { .. } => "follow_up",
// On reload the attempt can be a recovery of an existing turn ID.
SpawnedThread::Resumed => "resumed_thread",
};
turn.extension_data.insert(ActiveSubagentTurn(kind));
turn.session_telemetry.record_multi_agent_turn_lifecycle(
&turn.sub_id,
kind,
SubagentTurnStatus::Started,
);
}
pub(crate) fn record_terminal_event(turn: &TurnContext, event: &EventMsg) {
let status = match event {
EventMsg::TurnComplete(event) => match &event.error {
Some(error) => SubagentTurnStatus::Failed(error.codex_error_info.as_ref()),
None => SubagentTurnStatus::Completed,
},
EventMsg::TurnAborted(event) => SubagentTurnStatus::Interrupted {
reason: &event.reason,
error: event
.error
.as_ref()
.and_then(|error| error.codex_error_info.as_ref()),
},
_ => return,
};
record_end(turn, status);
}
pub(crate) fn record_suspension(turn: &TurnContext) {
record_end(turn, SubagentTurnStatus::Suspended);
}
fn record_end(turn: &TurnContext, status: SubagentTurnStatus<'_>) {
if let Some(kind) = turn.extension_data.remove::<ActiveSubagentTurn>() {
turn.session_telemetry
.record_multi_agent_turn_lifecycle(&turn.sub_id, kind.0, status);
}
}
+1
View File
@@ -2742,6 +2742,7 @@ impl Session {
/// Persist the event to rollout and send it to clients.
pub(crate) async fn send_event(&self, turn_context: &TurnContext, mut msg: EventMsg) {
crate::agent::turn_metrics::record_terminal_event(turn_context, &msg);
if let Some((item_id, started_at_ms)) = msg.command_approval_start() {
// The client can see the command before execution. Keep its first start
// for the canonical lifecycle events that will be saved in history.
+5
View File
@@ -1083,6 +1083,11 @@ impl Session {
thread_id.to_string(),
thread_extension_init,
);
crate::agent::turn_metrics::register_thread(
&thread_extension_data,
&session_source,
&initial_history,
);
if let Some(required) = &required_cyber_access_programs {
thread_extension_data.insert(
crate::cyber_access_program::RequiredCyberAccessProgramsSnapshot(Arc::clone(
@@ -112,6 +112,7 @@ pub(super) async fn suspend_turn_and_shutdown(
CodexErr::Fatal(format!("close suspended root turn writer failed: {error}"))
})?;
recorded?;
crate::agent::turn_metrics::record_suspension(&task.turn_context);
// Announce completion only after extension cleanup and writer closure so a
// replacement worker cannot write the same thread concurrently.
session
+6
View File
@@ -526,6 +526,12 @@ impl Session {
done_clone.notify_waiters();
}
.instrument(task_span);
if task_kind == TaskKind::Regular {
crate::agent::turn_metrics::record_start(
&self.services.thread_extension_data,
&turn_context,
);
}
let handle = tokio::spawn(storage_originator.scope(task_future));
let timer = turn_context
.session_telemetry
+147
View File
@@ -43,6 +43,153 @@ use tokio_util::sync::CancellationToken;
struct PendingTask;
#[tokio::test]
async fn subagent_lifecycle_counts_each_regular_attempt_with_bounded_labels() {
use crate::agent::turn_metrics;
use codex_history::InitialHistory;
use codex_history::ResumedHistory;
use codex_history::RolloutItem;
use codex_otel::MULTI_AGENT_TURN_LIFECYCLE_METRIC;
use codex_protocol::models::ResponseInputItem;
use codex_protocol::protocol::SubAgentSource;
use codex_protocol::protocol::TurnAbortReason;
let metrics = MetricsClient::new(
MetricsConfig::in_memory(
"test",
"codex-core",
env!("CARGO_PKG_VERSION"),
InMemoryMetricExporter::default(),
)
.with_runtime_reader(),
)
.expect("metrics");
let (mut session, _, _rx) = make_session_and_context_with_rx().await;
let services = &mut Arc::get_mut(&mut session)
.expect("unique test session")
.services;
services.session_telemetry = services
.session_telemetry
.clone()
.with_product_sku(Some("codex"))
.with_metrics(metrics.clone());
let child = SessionSource::SubAgent(SubAgentSource::ThreadSpawn {
parent_thread_id: ThreadId::new(),
depth: 1,
agent_path: None,
agent_nickname: None,
agent_role: None,
});
let parent_input = ResponseInputItem::Message {
role: "user".into(),
content: vec![ContentItem::InputText {
text: "a turn from the parent".into(),
}],
phase: None,
};
let inherited = vec![RolloutItem::ResponseItem(
ResponseItem::from(parent_input).into(),
)];
let fork = InitialHistory::Forked(inherited.clone());
assert!(crate::thread_rollout_truncation::initial_history_has_prior_user_turns(&fork));
for source in [
SessionSource::Cli,
SessionSource::SubAgent(SubAgentSource::Review),
] {
turn_metrics::register_thread(
&session.services.thread_extension_data,
&source,
&InitialHistory::New,
);
let turn = session.new_default_turn().await;
session
.spawn_task(Arc::clone(&turn), Vec::new(), PendingTask)
.await;
session.on_task_finished(turn, Ok(None)).await;
}
turn_metrics::register_thread(&session.services.thread_extension_data, &child, &fork);
let initial = session.new_default_turn().await;
session
.spawn_task(Arc::clone(&initial), Vec::new(), PendingTask)
.await;
session
.on_task_finished(Arc::clone(&initial), Ok(None))
.await;
// A second observer or shutdown action must not report another outcome.
turn_metrics::record_suspension(&initial);
let follow_up = session.new_default_turn().await;
session
.spawn_task(Arc::clone(&follow_up), Vec::new(), PendingTask)
.await;
session
.on_task_finished(follow_up, Err(CodexErr::ContextWindowExceeded))
.await;
turn_metrics::register_thread(
&session.services.thread_extension_data,
&child,
&InitialHistory::Resumed(ResumedHistory {
conversation_id: session.thread_id,
history: Arc::new(inherited),
history_revision: None,
rollout_path: None,
}),
);
let resumed = session.new_default_turn().await;
session
.spawn_task(Arc::clone(&resumed), Vec::new(), PendingTask)
.await;
session.abort_all_tasks(TurnAbortReason::Interrupted).await;
let suspended = session.new_default_turn().await;
session
.spawn_task(Arc::clone(&suspended), Vec::new(), PendingTask)
.await;
turn_metrics::record_suspension(&suspended);
session.abort_all_tasks(TurnAbortReason::Interrupted).await;
let snapshot = metrics.snapshot().expect("snapshot metrics");
let metric = find_metric(&snapshot, MULTI_AGENT_TURN_LIFECYCLE_METRIC);
let AggregatedMetrics::U64(MetricData::Sum(sum)) = metric.data() else {
panic!("expected a counter");
};
let actual: BTreeMap<_, _> = sum
.data_points()
.map(|point| {
let mut tags = attributes_to_map(point.attributes());
assert_eq!(tags.remove("product_sku").as_deref(), Some("codex"));
assert_eq!(tags.remove("originator").as_deref(), Some("other"));
let kind = tags.remove("turn_kind").expect("turn kind");
let status = tags.remove("status").expect("turn status");
match status.as_str() {
"failed" => assert_eq!(
tags.remove("error_kind").as_deref(),
Some("context_window_exceeded")
),
"interrupted" => {
assert_eq!(tags.remove("abort_reason").as_deref(), Some("interrupted"))
}
_ => {}
}
assert!(tags.is_empty(), "unexpected metric labels: {tags:?}");
((kind, status), point.value())
})
.collect();
let expected = [
("initial", "started", 1),
("initial", "completed", 1),
("follow_up", "started", 1),
("follow_up", "failed", 1),
("resumed_thread", "started", 2),
("resumed_thread", "interrupted", 1),
("resumed_thread", "suspended", 1),
]
.into_iter()
.map(|(kind, status, count)| ((kind.to_owned(), status.to_owned()), count))
.collect();
assert_eq!(actual, expected);
}
impl SessionTask for PendingTask {
fn kind(&self) -> TaskKind {
TaskKind::Regular
+8
View File
@@ -115,6 +115,14 @@ pub(crate) async fn spawn(
options: settings.options,
};
let (agent, config) = session.services.agent_control.spawn(request).await?;
step_context
.turn
.session_telemetry
.record_multi_agent_spawned(
session.session_id(),
&step_context.turn.sub_id,
agent.thread_id,
);
Ok(ThreadSpawnResult {
agent,
config,
@@ -1,5 +1,6 @@
use super::*;
use crate::tools::handlers::multi_agents_spec::create_close_agent_tool_v1;
use codex_otel::SubagentStopAction;
use codex_tools::ToolSpec;
pub(crate) struct Handler;
@@ -68,10 +69,24 @@ async fn handle_close_agent(
let previous_status = local_agent_control.get_status(agent_id).await;
let result = local_agent_control.close_agent(agent_id).await;
let (status, receiver_agent) = match &result {
Ok(snapshot) => (
snapshot.status().cloned().unwrap_or(AgentStatus::NotFound),
snapshot.metadata().clone(),
),
Ok(snapshot) => {
// Legacy close can also address unrelated loaded threads.
if snapshot.metadata().agent_id.is_some()
&& agent_id != ThreadId::from(session.session_id())
{
turn.session_telemetry.record_multi_agent_stop_accepted(
session.session_id(),
agent_id,
&call_id,
SubagentStopAction::Close,
snapshot.status(),
);
}
(
snapshot.status().cloned().unwrap_or(AgentStatus::NotFound),
snapshot.metadata().clone(),
)
}
Err(_) => (previous_status, receiver_agent),
};
session
@@ -161,7 +161,14 @@ async fn handle_spawn_agent(
collab_spawn_error(err)
});
let (new_thread_id, status) = match &result {
Ok((spawned_agent, _)) => (Some(spawned_agent.thread_id), spawned_agent.status.clone()),
Ok((spawned_agent, _)) => {
turn.session_telemetry.record_multi_agent_spawned(
session.session_id(),
&turn.sub_id,
spawned_agent.thread_id,
);
(Some(spawned_agent.thread_id), spawned_agent.status.clone())
}
Err(_) => (None, AgentStatus::NotFound),
};
let agent_snapshot = result.as_ref().ok().map(|(_, config)| config);
@@ -3841,23 +3841,29 @@ async fn multi_agent_v2_wait_agent_does_not_return_completed_content() {
#[tokio::test]
async fn multi_agent_v2_interrupt_agent_accepts_task_name_target() {
let (mut session, mut turn) = make_session_and_context().await;
let (_, turn) = make_session_and_context().await;
let mut config = (*turn.config).clone();
config.features.enable(Feature::MultiAgentV2).unwrap();
config.apps_mcp_product_sku = Some("codex".into());
let manager = thread_manager();
let root = manager
.start_thread(StartThreadOptions::new((*turn.config).clone()))
.start_thread(StartThreadOptions::new(config))
.await
.expect("root thread should start");
set_agent_control(&mut session, manager.agent_control());
session.thread_id = root.thread_id;
let mut config = (*turn.config).clone();
config
.features
.enable(Feature::MultiAgentV2)
.expect("test config should allow feature update");
set_turn_config(&mut turn, config);
let session = Arc::new(session);
let turn = Arc::new(turn);
let session = root.thread.session.clone();
let metrics = MetricsClient::new(
MetricsConfig::in_memory("test", "core", "1", Default::default()).with_runtime_reader(),
)
.unwrap();
let attach_metrics = |mut turn: Arc<TurnContext>| {
let context = Arc::get_mut(&mut turn).unwrap();
context.session_telemetry = context
.session_telemetry
.clone()
.with_metrics(metrics.clone());
turn
};
let turn = attach_metrics(session.new_default_turn().await);
SpawnAgentHandlerV2::default()
.handle(invocation(
session.clone(),
@@ -3882,10 +3888,11 @@ async fn multi_agent_v2_interrupt_agent_accepts_task_name_target() {
.await
.expect("worker thread should be resident");
let worker_session = worker_thread.session.clone();
let worker_turn = attach_metrics(worker_session.new_default_turn().await);
SpawnAgentHandlerV2::default()
.handle(invocation(
worker_session.clone(),
worker_session.new_default_turn().await,
worker_turn.clone(),
"spawn_agent",
function_payload(json!({
"message": "inspect a child task",
@@ -3944,6 +3951,82 @@ async fn multi_agent_v2_interrupt_agent_accepts_task_name_target() {
!ops.iter()
.any(|(thread_id, op)| *thread_id == child_id && matches!(op, Op::InterruptAgent))
);
InterruptAgentHandler
.handle(invocation(
worker_session,
worker_turn,
"interrupt_agent",
function_payload(json!({"target": child_id.to_string()})),
))
.await
.unwrap();
assert!(
InterruptAgentHandler
.handle(invocation(
session.clone(),
turn.clone(),
"interrupt_agent",
function_payload(json!({"target": "/root"}))
))
.await
.is_err()
);
CloseAgentHandler
.handle(invocation(
session,
turn,
"close_agent",
function_payload(json!({"target": child_id.to_string()})),
))
.await
.unwrap();
let snapshot = metrics.snapshot().unwrap();
let observations = snapshot
.scope_metrics()
.flat_map(opentelemetry_sdk::metrics::data::ScopeMetrics::metrics);
let mut counts = BTreeMap::new();
for metric in observations.filter(|m| {
matches!(
m.name(),
codex_otel::MULTI_AGENT_SPAWNED_METRIC | codex_otel::MULTI_AGENT_STOP_ACCEPTED_METRIC
)
}) {
let AggregatedMetrics::U64(MetricData::Sum(sum)) = metric.data() else {
panic!("subagent lifecycle metric should be a counter");
};
for point in sum.data_points() {
let label = |key| {
point
.attributes()
.find(|a| a.key.as_str() == key)
.map(|a| a.value.as_str().into_owned())
.unwrap_or_default()
};
assert_eq!(label("product_sku"), "codex");
*counts
.entry((metric.name().to_string(), label("caller"), label("action")))
.or_insert(0) += point.value();
}
}
let stop = codex_otel::MULTI_AGENT_STOP_ACCEPTED_METRIC;
assert_eq!(
counts,
BTreeMap::from([
(
(
codex_otel::MULTI_AGENT_SPAWNED_METRIC.into(),
String::new(),
String::new()
),
2
),
((stop.into(), "root".into(), "interrupt".into()), 1),
((stop.into(), "root".into(), "close".into()), 1),
((stop.into(), "subagent".into(), "interrupt".into()), 1),
])
);
}
#[tokio::test]
@@ -2,6 +2,7 @@ use super::analytics::ToolCallAnalytics;
use super::*;
use crate::agent::api::AgentTarget;
use crate::tools::handlers::multi_agents_spec::create_interrupt_agent_tool_v2;
use codex_otel::SubagentStopAction;
use codex_protocol::protocol::MultiAgentVersion;
use codex_tools::ToolSpec;
@@ -58,6 +59,13 @@ async fn handle_interrupt_agent(
let agent_path = snapshot.metadata().agent_path.clone().ok_or_else(|| {
FunctionCallError::RespondToModel("target agent is missing an agent_path".to_string())
})?;
turn.session_telemetry.record_multi_agent_stop_accepted(
session.session_id(),
agent_id,
&call_id,
SubagentStopAction::Interrupt,
snapshot.status(),
);
let previous_status = snapshot.status().cloned().unwrap_or(AgentStatus::NotFound);
emit_sub_agent_activity(
&session,