Add OpenTelemetry for subagent lifecycles (#62567)

## Why

Correlating a subagent's creation, turn attempts, and stop requests
helps explain what happened when those actions span workers.

## What changed

- Emit structured events and counters for successful spawns and accepted
  `close_agent` and `interrupt_agent` calls. Count accepted stops even
  when the target is already idle or unloaded.
- Track initial, follow-up, and resumed turn attempts for spawned
  subagents. Record each start and at most one outcome per attempt:
  completed, failed, interrupted, or suspended.
- Use thread, turn, and call IDs to correlate structured events while
  keeping those IDs out of metric labels.

## Testing

Add coverage for lifecycle counts, duplicate outcomes, stop handling,
and event correlation across workers. Improve the Windows elevated
sandbox test for an approved `git pull` by provisioning the sandbox
before the turn and reporting setup or launch failures with diagnostics.

GitOrigin-RevId: bc1190302e1dffbdd79c7ac984b6debe1dac4a65
This commit is contained in:
Bryan Ashley
2026-11-19 18:39:32 +00:00
committed by copyberry
parent 617ad20df0
commit 65454c7b99
18 changed files with 779 additions and 35 deletions
+1
View File
@@ -8,6 +8,7 @@ pub(crate) mod host;
mod registry;
pub(crate) mod role;
pub(crate) mod status;
pub(crate) mod turn_metrics;
pub(crate) mod types;
pub(crate) use codex_protocol::protocol::AgentStatus;
+95
View File
@@ -0,0 +1,95 @@
//! Tracks attempts on native spawned children, including across suspend and resume.
//! Each regular attempt records at most one end, even if suspension is followed by shutdown.
use std::sync::atomic::AtomicBool;
use std::sync::atomic::Ordering;
use codex_extension_api::ExtensionData;
use codex_history::InitialHistory;
use codex_otel::SubagentTurnStatus;
use codex_protocol::protocol::EventMsg;
use codex_protocol::protocol::SessionSource;
use codex_protocol::protocol::SubAgentSource;
use crate::session::turn_context::TurnContext;
enum SpawnedThread {
New { first_turn_pending: AtomicBool },
Resumed,
}
struct ActiveSubagentTurn(&'static str);
pub(crate) fn register_thread(
thread_store: &ExtensionData,
source: &SessionSource,
history: &InitialHistory,
) {
if !matches!(
source,
SessionSource::SubAgent(SubAgentSource::ThreadSpawn { .. })
) {
return;
}
let thread = match history {
InitialHistory::Resumed(_) => SpawnedThread::Resumed,
// A full fork can already contain many turns from its parent.
InitialHistory::New | InitialHistory::Cleared | InitialHistory::Forked(_) => {
SpawnedThread::New {
first_turn_pending: AtomicBool::new(true),
}
}
};
thread_store.insert(thread);
}
pub(crate) fn record_start(thread_store: &ExtensionData, turn: &TurnContext) {
let Some(thread) = thread_store.get::<SpawnedThread>() else {
return;
};
let kind = match thread.as_ref() {
SpawnedThread::New { first_turn_pending }
if first_turn_pending.swap(false, Ordering::Relaxed) =>
{
"initial"
}
SpawnedThread::New { .. } => "follow_up",
// On reload the attempt can be a recovery of an existing turn ID.
SpawnedThread::Resumed => "resumed_thread",
};
turn.extension_data.insert(ActiveSubagentTurn(kind));
turn.session_telemetry.record_multi_agent_turn_lifecycle(
&turn.sub_id,
kind,
SubagentTurnStatus::Started,
);
}
pub(crate) fn record_terminal_event(turn: &TurnContext, event: &EventMsg) {
let status = match event {
EventMsg::TurnComplete(event) => match &event.error {
Some(error) => SubagentTurnStatus::Failed(error.codex_error_info.as_ref()),
None => SubagentTurnStatus::Completed,
},
EventMsg::TurnAborted(event) => SubagentTurnStatus::Interrupted {
reason: &event.reason,
error: event
.error
.as_ref()
.and_then(|error| error.codex_error_info.as_ref()),
},
_ => return,
};
record_end(turn, status);
}
pub(crate) fn record_suspension(turn: &TurnContext) {
record_end(turn, SubagentTurnStatus::Suspended);
}
fn record_end(turn: &TurnContext, status: SubagentTurnStatus<'_>) {
if let Some(kind) = turn.extension_data.remove::<ActiveSubagentTurn>() {
turn.session_telemetry
.record_multi_agent_turn_lifecycle(&turn.sub_id, kind.0, status);
}
}
+1
View File
@@ -2742,6 +2742,7 @@ impl Session {
/// Persist the event to rollout and send it to clients.
pub(crate) async fn send_event(&self, turn_context: &TurnContext, mut msg: EventMsg) {
crate::agent::turn_metrics::record_terminal_event(turn_context, &msg);
if let Some((item_id, started_at_ms)) = msg.command_approval_start() {
// The client can see the command before execution. Keep its first start
// for the canonical lifecycle events that will be saved in history.
+5
View File
@@ -1083,6 +1083,11 @@ impl Session {
thread_id.to_string(),
thread_extension_init,
);
crate::agent::turn_metrics::register_thread(
&thread_extension_data,
&session_source,
&initial_history,
);
if let Some(required) = &required_cyber_access_programs {
thread_extension_data.insert(
crate::cyber_access_program::RequiredCyberAccessProgramsSnapshot(Arc::clone(
@@ -112,6 +112,7 @@ pub(super) async fn suspend_turn_and_shutdown(
CodexErr::Fatal(format!("close suspended root turn writer failed: {error}"))
})?;
recorded?;
crate::agent::turn_metrics::record_suspension(&task.turn_context);
// Announce completion only after extension cleanup and writer closure so a
// replacement worker cannot write the same thread concurrently.
session
+6
View File
@@ -526,6 +526,12 @@ impl Session {
done_clone.notify_waiters();
}
.instrument(task_span);
if task_kind == TaskKind::Regular {
crate::agent::turn_metrics::record_start(
&self.services.thread_extension_data,
&turn_context,
);
}
let handle = tokio::spawn(storage_originator.scope(task_future));
let timer = turn_context
.session_telemetry
+147
View File
@@ -43,6 +43,153 @@ use tokio_util::sync::CancellationToken;
struct PendingTask;
#[tokio::test]
async fn subagent_lifecycle_counts_each_regular_attempt_with_bounded_labels() {
use crate::agent::turn_metrics;
use codex_history::InitialHistory;
use codex_history::ResumedHistory;
use codex_history::RolloutItem;
use codex_otel::MULTI_AGENT_TURN_LIFECYCLE_METRIC;
use codex_protocol::models::ResponseInputItem;
use codex_protocol::protocol::SubAgentSource;
use codex_protocol::protocol::TurnAbortReason;
let metrics = MetricsClient::new(
MetricsConfig::in_memory(
"test",
"codex-core",
env!("CARGO_PKG_VERSION"),
InMemoryMetricExporter::default(),
)
.with_runtime_reader(),
)
.expect("metrics");
let (mut session, _, _rx) = make_session_and_context_with_rx().await;
let services = &mut Arc::get_mut(&mut session)
.expect("unique test session")
.services;
services.session_telemetry = services
.session_telemetry
.clone()
.with_product_sku(Some("codex"))
.with_metrics(metrics.clone());
let child = SessionSource::SubAgent(SubAgentSource::ThreadSpawn {
parent_thread_id: ThreadId::new(),
depth: 1,
agent_path: None,
agent_nickname: None,
agent_role: None,
});
let parent_input = ResponseInputItem::Message {
role: "user".into(),
content: vec![ContentItem::InputText {
text: "a turn from the parent".into(),
}],
phase: None,
};
let inherited = vec![RolloutItem::ResponseItem(
ResponseItem::from(parent_input).into(),
)];
let fork = InitialHistory::Forked(inherited.clone());
assert!(crate::thread_rollout_truncation::initial_history_has_prior_user_turns(&fork));
for source in [
SessionSource::Cli,
SessionSource::SubAgent(SubAgentSource::Review),
] {
turn_metrics::register_thread(
&session.services.thread_extension_data,
&source,
&InitialHistory::New,
);
let turn = session.new_default_turn().await;
session
.spawn_task(Arc::clone(&turn), Vec::new(), PendingTask)
.await;
session.on_task_finished(turn, Ok(None)).await;
}
turn_metrics::register_thread(&session.services.thread_extension_data, &child, &fork);
let initial = session.new_default_turn().await;
session
.spawn_task(Arc::clone(&initial), Vec::new(), PendingTask)
.await;
session
.on_task_finished(Arc::clone(&initial), Ok(None))
.await;
// A second observer or shutdown action must not report another outcome.
turn_metrics::record_suspension(&initial);
let follow_up = session.new_default_turn().await;
session
.spawn_task(Arc::clone(&follow_up), Vec::new(), PendingTask)
.await;
session
.on_task_finished(follow_up, Err(CodexErr::ContextWindowExceeded))
.await;
turn_metrics::register_thread(
&session.services.thread_extension_data,
&child,
&InitialHistory::Resumed(ResumedHistory {
conversation_id: session.thread_id,
history: Arc::new(inherited),
history_revision: None,
rollout_path: None,
}),
);
let resumed = session.new_default_turn().await;
session
.spawn_task(Arc::clone(&resumed), Vec::new(), PendingTask)
.await;
session.abort_all_tasks(TurnAbortReason::Interrupted).await;
let suspended = session.new_default_turn().await;
session
.spawn_task(Arc::clone(&suspended), Vec::new(), PendingTask)
.await;
turn_metrics::record_suspension(&suspended);
session.abort_all_tasks(TurnAbortReason::Interrupted).await;
let snapshot = metrics.snapshot().expect("snapshot metrics");
let metric = find_metric(&snapshot, MULTI_AGENT_TURN_LIFECYCLE_METRIC);
let AggregatedMetrics::U64(MetricData::Sum(sum)) = metric.data() else {
panic!("expected a counter");
};
let actual: BTreeMap<_, _> = sum
.data_points()
.map(|point| {
let mut tags = attributes_to_map(point.attributes());
assert_eq!(tags.remove("product_sku").as_deref(), Some("codex"));
assert_eq!(tags.remove("originator").as_deref(), Some("other"));
let kind = tags.remove("turn_kind").expect("turn kind");
let status = tags.remove("status").expect("turn status");
match status.as_str() {
"failed" => assert_eq!(
tags.remove("error_kind").as_deref(),
Some("context_window_exceeded")
),
"interrupted" => {
assert_eq!(tags.remove("abort_reason").as_deref(), Some("interrupted"))
}
_ => {}
}
assert!(tags.is_empty(), "unexpected metric labels: {tags:?}");
((kind, status), point.value())
})
.collect();
let expected = [
("initial", "started", 1),
("initial", "completed", 1),
("follow_up", "started", 1),
("follow_up", "failed", 1),
("resumed_thread", "started", 2),
("resumed_thread", "interrupted", 1),
("resumed_thread", "suspended", 1),
]
.into_iter()
.map(|(kind, status, count)| ((kind.to_owned(), status.to_owned()), count))
.collect();
assert_eq!(actual, expected);
}
impl SessionTask for PendingTask {
fn kind(&self) -> TaskKind {
TaskKind::Regular
+8
View File
@@ -115,6 +115,14 @@ pub(crate) async fn spawn(
options: settings.options,
};
let (agent, config) = session.services.agent_control.spawn(request).await?;
step_context
.turn
.session_telemetry
.record_multi_agent_spawned(
session.session_id(),
&step_context.turn.sub_id,
agent.thread_id,
);
Ok(ThreadSpawnResult {
agent,
config,
@@ -1,5 +1,6 @@
use super::*;
use crate::tools::handlers::multi_agents_spec::create_close_agent_tool_v1;
use codex_otel::SubagentStopAction;
use codex_tools::ToolSpec;
pub(crate) struct Handler;
@@ -68,10 +69,24 @@ async fn handle_close_agent(
let previous_status = local_agent_control.get_status(agent_id).await;
let result = local_agent_control.close_agent(agent_id).await;
let (status, receiver_agent) = match &result {
Ok(snapshot) => (
snapshot.status().cloned().unwrap_or(AgentStatus::NotFound),
snapshot.metadata().clone(),
),
Ok(snapshot) => {
// Legacy close can also address unrelated loaded threads.
if snapshot.metadata().agent_id.is_some()
&& agent_id != ThreadId::from(session.session_id())
{
turn.session_telemetry.record_multi_agent_stop_accepted(
session.session_id(),
agent_id,
&call_id,
SubagentStopAction::Close,
snapshot.status(),
);
}
(
snapshot.status().cloned().unwrap_or(AgentStatus::NotFound),
snapshot.metadata().clone(),
)
}
Err(_) => (previous_status, receiver_agent),
};
session
@@ -161,7 +161,14 @@ async fn handle_spawn_agent(
collab_spawn_error(err)
});
let (new_thread_id, status) = match &result {
Ok((spawned_agent, _)) => (Some(spawned_agent.thread_id), spawned_agent.status.clone()),
Ok((spawned_agent, _)) => {
turn.session_telemetry.record_multi_agent_spawned(
session.session_id(),
&turn.sub_id,
spawned_agent.thread_id,
);
(Some(spawned_agent.thread_id), spawned_agent.status.clone())
}
Err(_) => (None, AgentStatus::NotFound),
};
let agent_snapshot = result.as_ref().ok().map(|(_, config)| config);
@@ -3841,23 +3841,29 @@ async fn multi_agent_v2_wait_agent_does_not_return_completed_content() {
#[tokio::test]
async fn multi_agent_v2_interrupt_agent_accepts_task_name_target() {
let (mut session, mut turn) = make_session_and_context().await;
let (_, turn) = make_session_and_context().await;
let mut config = (*turn.config).clone();
config.features.enable(Feature::MultiAgentV2).unwrap();
config.apps_mcp_product_sku = Some("codex".into());
let manager = thread_manager();
let root = manager
.start_thread(StartThreadOptions::new((*turn.config).clone()))
.start_thread(StartThreadOptions::new(config))
.await
.expect("root thread should start");
set_agent_control(&mut session, manager.agent_control());
session.thread_id = root.thread_id;
let mut config = (*turn.config).clone();
config
.features
.enable(Feature::MultiAgentV2)
.expect("test config should allow feature update");
set_turn_config(&mut turn, config);
let session = Arc::new(session);
let turn = Arc::new(turn);
let session = root.thread.session.clone();
let metrics = MetricsClient::new(
MetricsConfig::in_memory("test", "core", "1", Default::default()).with_runtime_reader(),
)
.unwrap();
let attach_metrics = |mut turn: Arc<TurnContext>| {
let context = Arc::get_mut(&mut turn).unwrap();
context.session_telemetry = context
.session_telemetry
.clone()
.with_metrics(metrics.clone());
turn
};
let turn = attach_metrics(session.new_default_turn().await);
SpawnAgentHandlerV2::default()
.handle(invocation(
session.clone(),
@@ -3882,10 +3888,11 @@ async fn multi_agent_v2_interrupt_agent_accepts_task_name_target() {
.await
.expect("worker thread should be resident");
let worker_session = worker_thread.session.clone();
let worker_turn = attach_metrics(worker_session.new_default_turn().await);
SpawnAgentHandlerV2::default()
.handle(invocation(
worker_session.clone(),
worker_session.new_default_turn().await,
worker_turn.clone(),
"spawn_agent",
function_payload(json!({
"message": "inspect a child task",
@@ -3944,6 +3951,82 @@ async fn multi_agent_v2_interrupt_agent_accepts_task_name_target() {
!ops.iter()
.any(|(thread_id, op)| *thread_id == child_id && matches!(op, Op::InterruptAgent))
);
InterruptAgentHandler
.handle(invocation(
worker_session,
worker_turn,
"interrupt_agent",
function_payload(json!({"target": child_id.to_string()})),
))
.await
.unwrap();
assert!(
InterruptAgentHandler
.handle(invocation(
session.clone(),
turn.clone(),
"interrupt_agent",
function_payload(json!({"target": "/root"}))
))
.await
.is_err()
);
CloseAgentHandler
.handle(invocation(
session,
turn,
"close_agent",
function_payload(json!({"target": child_id.to_string()})),
))
.await
.unwrap();
let snapshot = metrics.snapshot().unwrap();
let observations = snapshot
.scope_metrics()
.flat_map(opentelemetry_sdk::metrics::data::ScopeMetrics::metrics);
let mut counts = BTreeMap::new();
for metric in observations.filter(|m| {
matches!(
m.name(),
codex_otel::MULTI_AGENT_SPAWNED_METRIC | codex_otel::MULTI_AGENT_STOP_ACCEPTED_METRIC
)
}) {
let AggregatedMetrics::U64(MetricData::Sum(sum)) = metric.data() else {
panic!("subagent lifecycle metric should be a counter");
};
for point in sum.data_points() {
let label = |key| {
point
.attributes()
.find(|a| a.key.as_str() == key)
.map(|a| a.value.as_str().into_owned())
.unwrap_or_default()
};
assert_eq!(label("product_sku"), "codex");
*counts
.entry((metric.name().to_string(), label("caller"), label("action")))
.or_insert(0) += point.value();
}
}
let stop = codex_otel::MULTI_AGENT_STOP_ACCEPTED_METRIC;
assert_eq!(
counts,
BTreeMap::from([
(
(
codex_otel::MULTI_AGENT_SPAWNED_METRIC.into(),
String::new(),
String::new()
),
2
),
((stop.into(), "root".into(), "interrupt".into()), 1),
((stop.into(), "root".into(), "close".into()), 1),
((stop.into(), "subagent".into(), "interrupt".into()), 1),
])
);
}
#[tokio::test]
@@ -2,6 +2,7 @@ use super::analytics::ToolCallAnalytics;
use super::*;
use crate::agent::api::AgentTarget;
use crate::tools::handlers::multi_agents_spec::create_interrupt_agent_tool_v2;
use codex_otel::SubagentStopAction;
use codex_protocol::protocol::MultiAgentVersion;
use codex_tools::ToolSpec;
@@ -58,6 +59,13 @@ async fn handle_interrupt_agent(
let agent_path = snapshot.metadata().agent_path.clone().ok_or_else(|| {
FunctionCallError::RespondToModel("target agent is missing an agent_path".to_string())
})?;
turn.session_telemetry.record_multi_agent_stop_accepted(
session.session_id(),
agent_id,
&call_id,
SubagentStopAction::Interrupt,
snapshot.status(),
);
let previous_status = snapshot.status().cloned().unwrap_or(AgentStatus::NotFound);
emit_sub_agent_activity(
&session,
+81 -13
View File
@@ -54,8 +54,6 @@ use core_test_support::responses::mount_sse_sequence;
use core_test_support::responses::sse;
use core_test_support::test_codex::TestCodexHarness;
use core_test_support::test_codex::test_codex;
use core_test_support::wait_for_event;
use core_test_support::wait_for_event_with_timeout;
use pretty_assertions::assert_eq;
use serde_json::json;
use serial_test::serial;
@@ -1269,6 +1267,7 @@ async fn windows_elevated_unified_exec_enforces_large_recursive_deny_reads() ->
#[serial(codex_home)]
async fn windows_elevated_approved_git_pull_preserves_deny_read() -> anyhow::Result<()> {
use EventMsg::ExecApprovalRequest;
use EventMsg::ExecCommandBegin;
use EventMsg::ExecCommandEnd;
use EventMsg::TurnComplete;
@@ -1351,6 +1350,19 @@ async fn windows_elevated_approved_git_pull_preserves_deny_read() -> anyhow::Res
let bundle_path = bundle.to_str().context("UTF-8 bundle path")?;
git(upstream, &["bundle", "create", bundle_path, "--all"])?;
// Provision the sandbox before starting the turn. Otherwise a setup/launch failure
// is delivered to the mock model as a tool result without any process to wait for.
let config = &harness.test().config;
eprintln!("approved Git pull: provisioning the elevated sandbox");
codex_core::windows_sandbox::prepare_elevated_sandbox(
&config.permissions.effective_permission_profile(),
&config.workspace_roots,
cwd,
&std::env::vars().collect(),
codex_home.path(),
)
.context("provision elevated sandbox for the approved Git pull")?;
let args = json!({"cmd": concat!(
"git pull --quiet --ff-only .\\upstream.bundle main || exit /b 10 & ",
"type tracked.txt >NUL || exit /b 11 & ",
@@ -1362,7 +1374,7 @@ async fn windows_elevated_approved_git_pull_preserves_deny_read() -> anyhow::Res
"yield_time_ms": 30_000, "tty": false, "login": false,
});
let call_id = "git-pull";
core_test_support::responses::mount_function_call_agent_response(
let mocks = core_test_support::responses::mount_function_call_agent_response(
harness.server(),
call_id,
&args.to_string(),
@@ -1376,13 +1388,33 @@ async fn windows_elevated_approved_git_pull_preserves_deny_read() -> anyhow::Res
text_elements: Vec::new(),
}]))
.await?;
let approval = wait_for_event(codex, |event| {
matches!(event, ExecApprovalRequest(_) | TurnComplete(_))
let tool_output = || mocks.completion.function_call_output_text(call_id);
// Account-lock contention before the turn counts against nextest's timeout. Report
// phase changes as they happen so the last one is visible even on a runner kill.
eprintln!("approved Git pull: waiting for the approval request");
let event_budget = Duration::from_secs(120);
let approval = tokio::time::timeout(event_budget, async {
loop {
match codex.next_event().await?.msg {
ExecApprovalRequest(request) => return anyhow::Ok(request),
event @ (TurnComplete(_) | EventMsg::TurnAborted(_) | EventMsg::Error(_)) => {
anyhow::bail!(
"no Git approval before {event:?}; tool output: {:?}",
tool_output()
);
}
_ => {}
}
}
})
.await;
let ExecApprovalRequest(approval) = approval else {
anyhow::bail!("expected approval before git ran: {approval:?}");
};
.await
.with_context(|| {
format!(
"waiting for Git approval; model requests: {}; tool output: {:?}",
mocks.function_call.requests().len(),
tool_output(),
)
})??;
assert_eq!(approval.call_id, call_id);
assert!(!fetch_head.exists());
assert!(!cwd.join("git-and-denials-ok.txt").exists());
@@ -1390,16 +1422,52 @@ async fn windows_elevated_approved_git_pull_preserves_deny_read() -> anyhow::Res
codex
.submit(Op::ExecApproval {
id: approval.effective_approval_id(),
turn_id: None,
turn_id: Some(approval.turn_id),
decision: ReviewDecision::Approved,
})
.await?;
eprintln!("approved Git pull: approval submitted, waiting for the process");
let mut process_end = None;
let mut turn_completed = false;
let mut process_started = false;
// The approved launch can refresh Windows ACLs before it emits ExecCommandBegin.
let startup_budget = Duration::from_secs(300);
let mut deadline = tokio::time::Instant::now() + startup_budget;
while process_end.is_none() || !turn_completed {
match wait_for_event_with_timeout(codex, |_| true, Duration::from_secs(120)).await {
ExecCommandEnd(event) if event.call_id == call_id => process_end = Some(event),
TurnComplete(_) => turn_completed = true,
let event = tokio::time::timeout_at(deadline, codex.next_event())
.await
.with_context(|| format!(
"waiting after Git approval: started={process_started}, exited={}, turn completed={turn_completed}; tool output: {:?}",
process_end.is_some(), tool_output(),
))??;
match event.msg {
ExecCommandBegin(event) if event.call_id == call_id && !process_started => {
eprintln!("approved Git pull: process started");
process_started = true;
deadline = tokio::time::Instant::now() + event_budget;
}
ExecCommandEnd(event) if event.call_id == call_id => {
eprintln!("approved Git pull: process exited with {}", event.exit_code);
process_end = Some(event);
}
TurnComplete(turn) => {
anyhow::ensure!(turn.error.is_none(), "Git turn failed: {:?}", turn.error);
// A yielded process may finish after the turn. A launch failure cannot:
// no command started, so its ExecCommandEnd would never arrive.
anyhow::ensure!(
process_started || process_end.is_some(),
"Git did not launch after approval; tool output: {:?}",
tool_output(),
);
eprintln!("approved Git pull: turn completed");
turn_completed = true;
}
event @ (EventMsg::TurnAborted(_) | EventMsg::Error(_)) => {
anyhow::bail!(
"approved Git turn failed: {event:?}; tool output: {:?}",
tool_output()
)
}
ExecApprovalRequest(request) => {
anyhow::bail!("unexpected additional Git approval: {request:?}")
}
+3
View File
@@ -11,6 +11,7 @@ mod headers_helper;
mod network_policy;
mod otlp;
mod skill_invocation;
mod subagent;
mod targets;
mod tool_result;
@@ -39,6 +40,8 @@ pub use crate::metrics::*;
pub use crate::provider::OtelProvider;
pub use crate::skill_invocation::SkillInvocationEvent;
pub use crate::skill_invocation::SkillInvocationType;
pub use crate::subagent::SubagentStopAction;
pub use crate::subagent::SubagentTurnStatus;
pub use crate::trace_context::context_from_w3c_trace_context;
pub use crate::trace_context::current_span_trace_id;
pub use crate::trace_context::current_span_w3c_trace_context;
+3
View File
@@ -11,6 +11,9 @@ pub const TOOL_CALL_UNIFIED_EXEC_METRIC: &str = "codex.tool.unified_exec";
pub const MULTI_AGENT_SPAWN_FAILURE_METRIC: &str = "codex.multi_agent.spawn.failure";
pub const MULTI_AGENT_SPAWN_PHASE_DURATION_METRIC: &str =
"codex.multi_agent.spawn.phase.duration_ms";
pub const MULTI_AGENT_TURN_LIFECYCLE_METRIC: &str = "codex.multi_agent.turn.lifecycle";
pub const MULTI_AGENT_SPAWNED_METRIC: &str = "codex.multi_agent.spawned";
pub const MULTI_AGENT_STOP_ACCEPTED_METRIC: &str = "codex.multi_agent.stop_accepted";
pub const ARTIFACT_OPERATION_STARTED_METRIC: &str = "codex.artifact.operation.started";
pub const ARTIFACT_OPERATION_EXPECTED_OUTPUT_COUNT_METRIC: &str =
"codex.artifact.operation.expected_output_count";
+158
View File
@@ -0,0 +1,158 @@
//! Records accepted spawns, child turn attempts, and accepted stop commands.
//! Stable IDs in events link the same child across workers. Counter labels are bounded.
use crate::MULTI_AGENT_SPAWNED_METRIC;
use crate::MULTI_AGENT_STOP_ACCEPTED_METRIC;
use crate::MULTI_AGENT_TURN_LIFECYCLE_METRIC;
use crate::SessionTelemetry;
use crate::bounded_originator_tag_value;
use crate::events::shared::trace_event;
use codex_protocol::SessionId;
use codex_protocol::ThreadId;
use codex_protocol::protocol::AgentStatus;
use codex_protocol::protocol::CodexErrorInfo;
use codex_protocol::protocol::TurnAbortReason;
use strum_macros::IntoStaticStr;
/// A child turn attempt can be suspended and later started again on another worker.
#[derive(Clone, Copy, IntoStaticStr)]
#[strum(serialize_all = "snake_case")]
pub enum SubagentTurnStatus<'a> {
Started,
Completed,
Failed(Option<&'a CodexErrorInfo>),
Interrupted {
reason: &'a TurnAbortReason,
error: Option<&'a CodexErrorInfo>,
},
Suspended,
}
/// The operation accepted by the target's agent backend.
#[derive(Clone, Copy, IntoStaticStr)]
#[strum(serialize_all = "snake_case")]
pub enum SubagentStopAction {
Close,
Interrupt,
}
impl SessionTelemetry {
/// Links a successful spawn to the child that will report turn attempts.
pub fn record_multi_agent_spawned(
&self,
root_thread_id: SessionId,
parent_turn_id: &str,
child_thread_id: ThreadId,
) {
trace_event!(
self,
event.name = MULTI_AGENT_SPAWNED_METRIC,
root_thread_id = %root_thread_id,
child_thread_id = %child_thread_id,
parent_turn_id,
product_sku = self.metadata.product_sku.unwrap_or("unknown"),
);
self.subagent_counter(MULTI_AGENT_SPAWNED_METRIC, &[]);
}
/// Records a turn boundary using only the error class, never its message or payload.
pub fn record_multi_agent_turn_lifecycle(
&self,
turn_id: &str,
turn_kind: &'static str,
status: SubagentTurnStatus<'_>,
) {
let (error, abort_reason) = match status {
SubagentTurnStatus::Failed(error) => (error, None),
SubagentTurnStatus::Interrupted { reason, error } => {
(error, Some(<&str>::from(reason)))
}
SubagentTurnStatus::Started
| SubagentTurnStatus::Completed
| SubagentTurnStatus::Suspended => (None, None),
};
let error_kind = error.map(<&str>::from);
let status: &str = status.into();
trace_event!(
self,
event.name = MULTI_AGENT_TURN_LIFECYCLE_METRIC,
child_thread_id = %self.metadata.conversation_id,
turn_id,
turn_kind,
status,
abort_reason,
error_kind,
product_sku = self.metadata.product_sku.unwrap_or("unknown"),
);
self.subagent_counter(
MULTI_AGENT_TURN_LIFECYCLE_METRIC,
&[
("turn_kind", Some(turn_kind)),
("status", Some(status)),
("error_kind", error_kind),
("abort_reason", abort_reason),
],
);
}
/// Records an accepted stop even when the target was already idle or unloaded.
pub fn record_multi_agent_stop_accepted(
&self,
root_thread_id: SessionId,
child_thread_id: ThreadId,
call_id: &str,
action: SubagentStopAction,
previous_status: Option<&AgentStatus>,
) {
let caller_thread_id = self.metadata.conversation_id;
let caller = if ThreadId::from(root_thread_id) == caller_thread_id {
"root"
} else {
"subagent"
};
let action: &str = action.into();
let previous_status = previous_status.map_or("unknown", <&str>::from);
trace_event!(
self,
event.name = MULTI_AGENT_STOP_ACCEPTED_METRIC,
root_thread_id = %root_thread_id,
child_thread_id = %child_thread_id,
caller_thread_id = %caller_thread_id,
call_id,
caller,
action,
previous_status,
product_sku = self.metadata.product_sku.unwrap_or("unknown"),
);
self.subagent_counter(
MULTI_AGENT_STOP_ACCEPTED_METRIC,
&[
("caller", Some(caller)),
("action", Some(action)),
("previous_status", Some(previous_status)),
],
);
}
fn subagent_counter(&self, name: &str, labels: &[(&str, Option<&str>)]) {
let Some(metrics) = &self.metrics else {
return;
};
// Default session tags can include model and the parent's thread ID.
let mut tags = Vec::with_capacity(labels.len() + 2);
tags.extend(
labels
.iter()
.filter_map(|(key, value)| Some((*key, (*value)?))),
);
tags.push((
"product_sku",
self.metadata.product_sku.unwrap_or("unknown"),
));
tags.push((
"originator",
bounded_originator_tag_value(&self.metadata.originator),
));
let _ = metrics.counter(name, /*inc*/ 1, &tags);
}
}
@@ -4,10 +4,15 @@ use crate::harness::find_metric;
use crate::harness::latest_metrics;
use codex_api::AgentIdentityTelemetry;
use codex_otel::AuthEnvTelemetryMetadata;
use codex_otel::MULTI_AGENT_SPAWNED_METRIC;
use codex_otel::MULTI_AGENT_STOP_ACCEPTED_METRIC;
use codex_otel::MULTI_AGENT_TURN_LIFECYCLE_METRIC;
use codex_otel::OtelProvider;
use codex_otel::SSE_EVENT_COUNT_METRIC;
use codex_otel::SSE_EVENT_DURATION_METRIC;
use codex_otel::SessionTelemetry;
use codex_otel::SubagentStopAction;
use codex_otel::SubagentTurnStatus;
use codex_otel::TelemetryAuthMode;
use opentelemetry::KeyValue;
use opentelemetry::logs::AnyValue;
@@ -17,6 +22,7 @@ use opentelemetry_sdk::logs::SdkLogRecord;
use opentelemetry_sdk::logs::SdkLoggerProvider;
use opentelemetry_sdk::metrics::data::AggregatedMetrics;
use opentelemetry_sdk::metrics::data::MetricData;
use opentelemetry_sdk::metrics::data::SumDataPoint;
use opentelemetry_sdk::trace::InMemorySpanExporter;
use opentelemetry_sdk::trace::SdkTracerProvider;
use pretty_assertions::assert_eq;
@@ -41,6 +47,7 @@ use codex_protocol::permissions::FileSystemSandboxEntry;
use codex_protocol::permissions::FileSystemSandboxPolicy;
use codex_protocol::permissions::FileSystemSpecialPath;
use codex_protocol::permissions::NetworkSandboxPolicy;
use codex_protocol::protocol::AgentStatus;
use codex_protocol::protocol::AskForApproval;
use codex_protocol::protocol::GuardianAssessmentEvent;
use codex_protocol::protocol::GuardianAssessmentOutcome;
@@ -107,6 +114,128 @@ fn find_span_event_by_name_attr<'a>(
.expect("span event should exist")
}
#[test]
fn subagent_lifecycle_links_every_spawn_and_accepted_stop_across_workers() {
let exporter = InMemoryLogExporter::default();
let provider = SdkLoggerProvider::builder()
.with_simple_exporter(exporter.clone())
.build();
// A host's structured log subscriber can observe trace-safe events without an active span.
let subscriber = tracing_subscriber::registry().with(
opentelemetry_appender_tracing::layer::OpenTelemetryTracingBridge::new(&provider)
.with_filter(filter_fn(|metadata| {
metadata.target() == "codex_otel.trace_safe"
})),
);
let root = ThreadId::from_u128(/*value*/ 1);
let parent = ThreadId::from_u128(/*value*/ 100);
let child = ThreadId::from_u128(/*value*/ 101);
let (metrics, metric_exporter) = build_metrics_with_defaults(&[]).expect("metrics");
let (product_sku, originator) = ("codex", "codex_cli_rs");
let telemetry = |thread_id| {
SessionTelemetry::new(
thread_id,
"model",
"model",
/*account_id*/ None,
/*account_email*/ None,
/*auth_mode*/ None,
originator.to_string(),
/*log_user_prompts*/ false,
"tty".into(),
SessionSource::Cli,
)
.with_metrics_without_metadata_tags(metrics.clone())
.with_product_sku(Some(product_sku))
};
tracing::subscriber::with_default(subscriber, || {
tracing::callsite::rebuild_interest_cache();
let root_id = root.into();
telemetry(parent).record_multi_agent_spawned(root_id, "parent-turn", child);
telemetry(child).record_multi_agent_turn_lifecycle(
"child-turn",
"initial",
SubagentTurnStatus::Started,
);
telemetry(child).record_multi_agent_turn_lifecycle(
"child-turn",
"resumed_thread",
SubagentTurnStatus::Completed,
);
let status = AgentStatus::Completed(Some("private child output".into()));
telemetry(parent).record_multi_agent_stop_accepted(
root_id,
child,
"parent-call",
SubagentStopAction::Interrupt,
Some(&status),
);
telemetry(root).record_multi_agent_stop_accepted(
root_id,
child,
"root-call",
SubagentStopAction::Close,
/*previous_status*/ None,
);
});
let logs = exporter.get_emitted_logs().expect("export logs");
let events = logs
.iter()
.map(|log| log_attributes(&log.record))
.collect::<Vec<_>>();
assert_eq!(events.len(), 5);
assert!(
events
.iter()
.all(|fields| fields["child_thread_id"] == child.to_string()
&& fields["product_sku"] == product_sku
&& fields["originator"] == originator)
);
assert_eq!(events[0]["event.name"], MULTI_AGENT_SPAWNED_METRIC);
assert_eq!(events[0]["parent_turn_id"], "parent-turn");
assert_eq!(events[0]["root_thread_id"], root.to_string());
assert_eq!(events[1]["turn_id"], events[2]["turn_id"]);
assert_eq!(events[2]["status"], "completed");
assert_eq!(events[3]["caller"], "subagent");
assert_eq!(events[3]["previous_status"], "completed");
assert_eq!(events[4]["event.name"], MULTI_AGENT_STOP_ACCEPTED_METRIC);
assert_eq!(events[4]["caller"], "root");
assert_eq!(events[4]["action"], "close");
assert_eq!(events[4]["call_id"], "root-call");
assert_eq!(events[4]["caller_thread_id"], events[0]["root_thread_id"]);
metrics.shutdown().expect("flush counters");
let snapshot = latest_metrics(&metric_exporter);
for (name, count) in [
(MULTI_AGENT_SPAWNED_METRIC, 1),
(MULTI_AGENT_TURN_LIFECYCLE_METRIC, 2),
(MULTI_AGENT_STOP_ACCEPTED_METRIC, 2),
] {
let counter = find_metric(&snapshot, name).expect("subagent counter");
let AggregatedMetrics::U64(MetricData::Sum(sum)) = counter.data() else {
panic!("expected subagent counter");
};
assert_eq!(
sum.data_points().map(SumDataPoint::value).sum::<u64>(),
count
);
for point in sum.data_points() {
let tags = attributes_to_map(point.attributes());
assert_eq!(tags["product_sku"], product_sku);
assert_eq!(tags["originator"], originator);
assert!(tags.keys().all(|key| matches!(
key.as_str(),
"product_sku"
| "originator"
| "turn_kind"
| "status"
| "action"
| "caller"
| "previous_status"
)));
}
}
}
fn auth_env_metadata() -> AuthEnvTelemetryMetadata {
AuthEnvTelemetryMetadata {
openai_api_key_env_present: true,
+9 -3
View File
@@ -81,6 +81,7 @@ use serde_json::Map;
use serde_json::Value;
use serde_with::serde_as;
use strum_macros::Display;
use strum_macros::IntoStaticStr;
use tokio::sync::oneshot;
use tokio::sync::watch;
use tracing::error;
@@ -2082,8 +2083,11 @@ impl From<SubAgentActivityEvent> for EventMsg {
}
/// Agent lifecycle status, derived from emitted events.
#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq, JsonSchema, TS, Default)]
#[derive(
Debug, Clone, Deserialize, Serialize, PartialEq, Eq, JsonSchema, TS, Default, IntoStaticStr,
)]
#[serde(rename_all = "snake_case")]
#[strum(serialize_all = "snake_case")]
#[ts(rename_all = "snake_case")]
pub enum AgentStatus {
/// Agent is waiting for initialization.
@@ -2113,8 +2117,9 @@ pub enum NonSteerableTurnKind {
}
/// Codex errors that we expose to clients.
#[derive(Clone, Debug, PartialEq, Eq, JsonSchema, TS)]
#[derive(Clone, Debug, PartialEq, Eq, JsonSchema, TS, IntoStaticStr)]
#[schemars(rename_all = "snake_case")]
#[strum(serialize_all = "snake_case")]
#[ts(rename_all = "snake_case")]
pub enum CodexErrorInfo {
ContextWindowExceeded,
@@ -4733,8 +4738,9 @@ pub struct TurnAbortedEvent {
pub duration_ms: Option<i64>,
}
#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, JsonSchema, TS)]
#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, JsonSchema, TS, IntoStaticStr)]
#[serde(rename_all = "snake_case")]
#[strum(serialize_all = "snake_case")]
pub enum TurnAbortReason {
Interrupted,
Replaced,