mirror of
https://github.com/openai/codex.git
synced 2026-11-19 22:30:42 +00:00
Add OpenTelemetry for subagent lifecycles (#62567)
## Why Correlating a subagent's creation, turn attempts, and stop requests helps explain what happened when those actions span workers. ## What changed - Emit structured events and counters for successful spawns and accepted `close_agent` and `interrupt_agent` calls. Count accepted stops even when the target is already idle or unloaded. - Track initial, follow-up, and resumed turn attempts for spawned subagents. Record each start and at most one outcome per attempt: completed, failed, interrupted, or suspended. - Use thread, turn, and call IDs to correlate structured events while keeping those IDs out of metric labels. ## Testing Add coverage for lifecycle counts, duplicate outcomes, stop handling, and event correlation across workers. Improve the Windows elevated sandbox test for an approved `git pull` by provisioning the sandbox before the turn and reporting setup or launch failures with diagnostics. GitOrigin-RevId: bc1190302e1dffbdd79c7ac984b6debe1dac4a65
This commit is contained in:
@@ -8,6 +8,7 @@ pub(crate) mod host;
|
||||
mod registry;
|
||||
pub(crate) mod role;
|
||||
pub(crate) mod status;
|
||||
pub(crate) mod turn_metrics;
|
||||
pub(crate) mod types;
|
||||
|
||||
pub(crate) use codex_protocol::protocol::AgentStatus;
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
//! Tracks attempts on native spawned children, including across suspend and resume.
|
||||
//! Each regular attempt records at most one end, even if suspension is followed by shutdown.
|
||||
|
||||
use std::sync::atomic::AtomicBool;
|
||||
use std::sync::atomic::Ordering;
|
||||
|
||||
use codex_extension_api::ExtensionData;
|
||||
use codex_history::InitialHistory;
|
||||
use codex_otel::SubagentTurnStatus;
|
||||
use codex_protocol::protocol::EventMsg;
|
||||
use codex_protocol::protocol::SessionSource;
|
||||
use codex_protocol::protocol::SubAgentSource;
|
||||
|
||||
use crate::session::turn_context::TurnContext;
|
||||
|
||||
enum SpawnedThread {
|
||||
New { first_turn_pending: AtomicBool },
|
||||
Resumed,
|
||||
}
|
||||
|
||||
struct ActiveSubagentTurn(&'static str);
|
||||
|
||||
pub(crate) fn register_thread(
|
||||
thread_store: &ExtensionData,
|
||||
source: &SessionSource,
|
||||
history: &InitialHistory,
|
||||
) {
|
||||
if !matches!(
|
||||
source,
|
||||
SessionSource::SubAgent(SubAgentSource::ThreadSpawn { .. })
|
||||
) {
|
||||
return;
|
||||
}
|
||||
let thread = match history {
|
||||
InitialHistory::Resumed(_) => SpawnedThread::Resumed,
|
||||
// A full fork can already contain many turns from its parent.
|
||||
InitialHistory::New | InitialHistory::Cleared | InitialHistory::Forked(_) => {
|
||||
SpawnedThread::New {
|
||||
first_turn_pending: AtomicBool::new(true),
|
||||
}
|
||||
}
|
||||
};
|
||||
thread_store.insert(thread);
|
||||
}
|
||||
|
||||
pub(crate) fn record_start(thread_store: &ExtensionData, turn: &TurnContext) {
|
||||
let Some(thread) = thread_store.get::<SpawnedThread>() else {
|
||||
return;
|
||||
};
|
||||
let kind = match thread.as_ref() {
|
||||
SpawnedThread::New { first_turn_pending }
|
||||
if first_turn_pending.swap(false, Ordering::Relaxed) =>
|
||||
{
|
||||
"initial"
|
||||
}
|
||||
SpawnedThread::New { .. } => "follow_up",
|
||||
// On reload the attempt can be a recovery of an existing turn ID.
|
||||
SpawnedThread::Resumed => "resumed_thread",
|
||||
};
|
||||
turn.extension_data.insert(ActiveSubagentTurn(kind));
|
||||
turn.session_telemetry.record_multi_agent_turn_lifecycle(
|
||||
&turn.sub_id,
|
||||
kind,
|
||||
SubagentTurnStatus::Started,
|
||||
);
|
||||
}
|
||||
|
||||
pub(crate) fn record_terminal_event(turn: &TurnContext, event: &EventMsg) {
|
||||
let status = match event {
|
||||
EventMsg::TurnComplete(event) => match &event.error {
|
||||
Some(error) => SubagentTurnStatus::Failed(error.codex_error_info.as_ref()),
|
||||
None => SubagentTurnStatus::Completed,
|
||||
},
|
||||
EventMsg::TurnAborted(event) => SubagentTurnStatus::Interrupted {
|
||||
reason: &event.reason,
|
||||
error: event
|
||||
.error
|
||||
.as_ref()
|
||||
.and_then(|error| error.codex_error_info.as_ref()),
|
||||
},
|
||||
_ => return,
|
||||
};
|
||||
record_end(turn, status);
|
||||
}
|
||||
|
||||
pub(crate) fn record_suspension(turn: &TurnContext) {
|
||||
record_end(turn, SubagentTurnStatus::Suspended);
|
||||
}
|
||||
|
||||
fn record_end(turn: &TurnContext, status: SubagentTurnStatus<'_>) {
|
||||
if let Some(kind) = turn.extension_data.remove::<ActiveSubagentTurn>() {
|
||||
turn.session_telemetry
|
||||
.record_multi_agent_turn_lifecycle(&turn.sub_id, kind.0, status);
|
||||
}
|
||||
}
|
||||
@@ -2742,6 +2742,7 @@ impl Session {
|
||||
|
||||
/// Persist the event to rollout and send it to clients.
|
||||
pub(crate) async fn send_event(&self, turn_context: &TurnContext, mut msg: EventMsg) {
|
||||
crate::agent::turn_metrics::record_terminal_event(turn_context, &msg);
|
||||
if let Some((item_id, started_at_ms)) = msg.command_approval_start() {
|
||||
// The client can see the command before execution. Keep its first start
|
||||
// for the canonical lifecycle events that will be saved in history.
|
||||
|
||||
@@ -1083,6 +1083,11 @@ impl Session {
|
||||
thread_id.to_string(),
|
||||
thread_extension_init,
|
||||
);
|
||||
crate::agent::turn_metrics::register_thread(
|
||||
&thread_extension_data,
|
||||
&session_source,
|
||||
&initial_history,
|
||||
);
|
||||
if let Some(required) = &required_cyber_access_programs {
|
||||
thread_extension_data.insert(
|
||||
crate::cyber_access_program::RequiredCyberAccessProgramsSnapshot(Arc::clone(
|
||||
|
||||
@@ -112,6 +112,7 @@ pub(super) async fn suspend_turn_and_shutdown(
|
||||
CodexErr::Fatal(format!("close suspended root turn writer failed: {error}"))
|
||||
})?;
|
||||
recorded?;
|
||||
crate::agent::turn_metrics::record_suspension(&task.turn_context);
|
||||
// Announce completion only after extension cleanup and writer closure so a
|
||||
// replacement worker cannot write the same thread concurrently.
|
||||
session
|
||||
|
||||
@@ -526,6 +526,12 @@ impl Session {
|
||||
done_clone.notify_waiters();
|
||||
}
|
||||
.instrument(task_span);
|
||||
if task_kind == TaskKind::Regular {
|
||||
crate::agent::turn_metrics::record_start(
|
||||
&self.services.thread_extension_data,
|
||||
&turn_context,
|
||||
);
|
||||
}
|
||||
let handle = tokio::spawn(storage_originator.scope(task_future));
|
||||
let timer = turn_context
|
||||
.session_telemetry
|
||||
|
||||
@@ -43,6 +43,153 @@ use tokio_util::sync::CancellationToken;
|
||||
|
||||
struct PendingTask;
|
||||
|
||||
#[tokio::test]
|
||||
async fn subagent_lifecycle_counts_each_regular_attempt_with_bounded_labels() {
|
||||
use crate::agent::turn_metrics;
|
||||
use codex_history::InitialHistory;
|
||||
use codex_history::ResumedHistory;
|
||||
use codex_history::RolloutItem;
|
||||
use codex_otel::MULTI_AGENT_TURN_LIFECYCLE_METRIC;
|
||||
use codex_protocol::models::ResponseInputItem;
|
||||
use codex_protocol::protocol::SubAgentSource;
|
||||
use codex_protocol::protocol::TurnAbortReason;
|
||||
|
||||
let metrics = MetricsClient::new(
|
||||
MetricsConfig::in_memory(
|
||||
"test",
|
||||
"codex-core",
|
||||
env!("CARGO_PKG_VERSION"),
|
||||
InMemoryMetricExporter::default(),
|
||||
)
|
||||
.with_runtime_reader(),
|
||||
)
|
||||
.expect("metrics");
|
||||
let (mut session, _, _rx) = make_session_and_context_with_rx().await;
|
||||
let services = &mut Arc::get_mut(&mut session)
|
||||
.expect("unique test session")
|
||||
.services;
|
||||
services.session_telemetry = services
|
||||
.session_telemetry
|
||||
.clone()
|
||||
.with_product_sku(Some("codex"))
|
||||
.with_metrics(metrics.clone());
|
||||
let child = SessionSource::SubAgent(SubAgentSource::ThreadSpawn {
|
||||
parent_thread_id: ThreadId::new(),
|
||||
depth: 1,
|
||||
agent_path: None,
|
||||
agent_nickname: None,
|
||||
agent_role: None,
|
||||
});
|
||||
let parent_input = ResponseInputItem::Message {
|
||||
role: "user".into(),
|
||||
content: vec![ContentItem::InputText {
|
||||
text: "a turn from the parent".into(),
|
||||
}],
|
||||
phase: None,
|
||||
};
|
||||
let inherited = vec![RolloutItem::ResponseItem(
|
||||
ResponseItem::from(parent_input).into(),
|
||||
)];
|
||||
let fork = InitialHistory::Forked(inherited.clone());
|
||||
assert!(crate::thread_rollout_truncation::initial_history_has_prior_user_turns(&fork));
|
||||
for source in [
|
||||
SessionSource::Cli,
|
||||
SessionSource::SubAgent(SubAgentSource::Review),
|
||||
] {
|
||||
turn_metrics::register_thread(
|
||||
&session.services.thread_extension_data,
|
||||
&source,
|
||||
&InitialHistory::New,
|
||||
);
|
||||
let turn = session.new_default_turn().await;
|
||||
session
|
||||
.spawn_task(Arc::clone(&turn), Vec::new(), PendingTask)
|
||||
.await;
|
||||
session.on_task_finished(turn, Ok(None)).await;
|
||||
}
|
||||
turn_metrics::register_thread(&session.services.thread_extension_data, &child, &fork);
|
||||
let initial = session.new_default_turn().await;
|
||||
session
|
||||
.spawn_task(Arc::clone(&initial), Vec::new(), PendingTask)
|
||||
.await;
|
||||
session
|
||||
.on_task_finished(Arc::clone(&initial), Ok(None))
|
||||
.await;
|
||||
// A second observer or shutdown action must not report another outcome.
|
||||
turn_metrics::record_suspension(&initial);
|
||||
|
||||
let follow_up = session.new_default_turn().await;
|
||||
session
|
||||
.spawn_task(Arc::clone(&follow_up), Vec::new(), PendingTask)
|
||||
.await;
|
||||
session
|
||||
.on_task_finished(follow_up, Err(CodexErr::ContextWindowExceeded))
|
||||
.await;
|
||||
|
||||
turn_metrics::register_thread(
|
||||
&session.services.thread_extension_data,
|
||||
&child,
|
||||
&InitialHistory::Resumed(ResumedHistory {
|
||||
conversation_id: session.thread_id,
|
||||
history: Arc::new(inherited),
|
||||
history_revision: None,
|
||||
rollout_path: None,
|
||||
}),
|
||||
);
|
||||
let resumed = session.new_default_turn().await;
|
||||
session
|
||||
.spawn_task(Arc::clone(&resumed), Vec::new(), PendingTask)
|
||||
.await;
|
||||
session.abort_all_tasks(TurnAbortReason::Interrupted).await;
|
||||
let suspended = session.new_default_turn().await;
|
||||
session
|
||||
.spawn_task(Arc::clone(&suspended), Vec::new(), PendingTask)
|
||||
.await;
|
||||
turn_metrics::record_suspension(&suspended);
|
||||
session.abort_all_tasks(TurnAbortReason::Interrupted).await;
|
||||
|
||||
let snapshot = metrics.snapshot().expect("snapshot metrics");
|
||||
let metric = find_metric(&snapshot, MULTI_AGENT_TURN_LIFECYCLE_METRIC);
|
||||
let AggregatedMetrics::U64(MetricData::Sum(sum)) = metric.data() else {
|
||||
panic!("expected a counter");
|
||||
};
|
||||
let actual: BTreeMap<_, _> = sum
|
||||
.data_points()
|
||||
.map(|point| {
|
||||
let mut tags = attributes_to_map(point.attributes());
|
||||
assert_eq!(tags.remove("product_sku").as_deref(), Some("codex"));
|
||||
assert_eq!(tags.remove("originator").as_deref(), Some("other"));
|
||||
let kind = tags.remove("turn_kind").expect("turn kind");
|
||||
let status = tags.remove("status").expect("turn status");
|
||||
match status.as_str() {
|
||||
"failed" => assert_eq!(
|
||||
tags.remove("error_kind").as_deref(),
|
||||
Some("context_window_exceeded")
|
||||
),
|
||||
"interrupted" => {
|
||||
assert_eq!(tags.remove("abort_reason").as_deref(), Some("interrupted"))
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
assert!(tags.is_empty(), "unexpected metric labels: {tags:?}");
|
||||
((kind, status), point.value())
|
||||
})
|
||||
.collect();
|
||||
let expected = [
|
||||
("initial", "started", 1),
|
||||
("initial", "completed", 1),
|
||||
("follow_up", "started", 1),
|
||||
("follow_up", "failed", 1),
|
||||
("resumed_thread", "started", 2),
|
||||
("resumed_thread", "interrupted", 1),
|
||||
("resumed_thread", "suspended", 1),
|
||||
]
|
||||
.into_iter()
|
||||
.map(|(kind, status, count)| ((kind.to_owned(), status.to_owned()), count))
|
||||
.collect();
|
||||
assert_eq!(actual, expected);
|
||||
}
|
||||
|
||||
impl SessionTask for PendingTask {
|
||||
fn kind(&self) -> TaskKind {
|
||||
TaskKind::Regular
|
||||
|
||||
@@ -115,6 +115,14 @@ pub(crate) async fn spawn(
|
||||
options: settings.options,
|
||||
};
|
||||
let (agent, config) = session.services.agent_control.spawn(request).await?;
|
||||
step_context
|
||||
.turn
|
||||
.session_telemetry
|
||||
.record_multi_agent_spawned(
|
||||
session.session_id(),
|
||||
&step_context.turn.sub_id,
|
||||
agent.thread_id,
|
||||
);
|
||||
Ok(ThreadSpawnResult {
|
||||
agent,
|
||||
config,
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
use super::*;
|
||||
use crate::tools::handlers::multi_agents_spec::create_close_agent_tool_v1;
|
||||
use codex_otel::SubagentStopAction;
|
||||
use codex_tools::ToolSpec;
|
||||
|
||||
pub(crate) struct Handler;
|
||||
@@ -68,10 +69,24 @@ async fn handle_close_agent(
|
||||
let previous_status = local_agent_control.get_status(agent_id).await;
|
||||
let result = local_agent_control.close_agent(agent_id).await;
|
||||
let (status, receiver_agent) = match &result {
|
||||
Ok(snapshot) => (
|
||||
snapshot.status().cloned().unwrap_or(AgentStatus::NotFound),
|
||||
snapshot.metadata().clone(),
|
||||
),
|
||||
Ok(snapshot) => {
|
||||
// Legacy close can also address unrelated loaded threads.
|
||||
if snapshot.metadata().agent_id.is_some()
|
||||
&& agent_id != ThreadId::from(session.session_id())
|
||||
{
|
||||
turn.session_telemetry.record_multi_agent_stop_accepted(
|
||||
session.session_id(),
|
||||
agent_id,
|
||||
&call_id,
|
||||
SubagentStopAction::Close,
|
||||
snapshot.status(),
|
||||
);
|
||||
}
|
||||
(
|
||||
snapshot.status().cloned().unwrap_or(AgentStatus::NotFound),
|
||||
snapshot.metadata().clone(),
|
||||
)
|
||||
}
|
||||
Err(_) => (previous_status, receiver_agent),
|
||||
};
|
||||
session
|
||||
|
||||
@@ -161,7 +161,14 @@ async fn handle_spawn_agent(
|
||||
collab_spawn_error(err)
|
||||
});
|
||||
let (new_thread_id, status) = match &result {
|
||||
Ok((spawned_agent, _)) => (Some(spawned_agent.thread_id), spawned_agent.status.clone()),
|
||||
Ok((spawned_agent, _)) => {
|
||||
turn.session_telemetry.record_multi_agent_spawned(
|
||||
session.session_id(),
|
||||
&turn.sub_id,
|
||||
spawned_agent.thread_id,
|
||||
);
|
||||
(Some(spawned_agent.thread_id), spawned_agent.status.clone())
|
||||
}
|
||||
Err(_) => (None, AgentStatus::NotFound),
|
||||
};
|
||||
let agent_snapshot = result.as_ref().ok().map(|(_, config)| config);
|
||||
|
||||
@@ -3841,23 +3841,29 @@ async fn multi_agent_v2_wait_agent_does_not_return_completed_content() {
|
||||
|
||||
#[tokio::test]
|
||||
async fn multi_agent_v2_interrupt_agent_accepts_task_name_target() {
|
||||
let (mut session, mut turn) = make_session_and_context().await;
|
||||
let (_, turn) = make_session_and_context().await;
|
||||
let mut config = (*turn.config).clone();
|
||||
config.features.enable(Feature::MultiAgentV2).unwrap();
|
||||
config.apps_mcp_product_sku = Some("codex".into());
|
||||
let manager = thread_manager();
|
||||
let root = manager
|
||||
.start_thread(StartThreadOptions::new((*turn.config).clone()))
|
||||
.start_thread(StartThreadOptions::new(config))
|
||||
.await
|
||||
.expect("root thread should start");
|
||||
set_agent_control(&mut session, manager.agent_control());
|
||||
session.thread_id = root.thread_id;
|
||||
let mut config = (*turn.config).clone();
|
||||
config
|
||||
.features
|
||||
.enable(Feature::MultiAgentV2)
|
||||
.expect("test config should allow feature update");
|
||||
set_turn_config(&mut turn, config);
|
||||
|
||||
let session = Arc::new(session);
|
||||
let turn = Arc::new(turn);
|
||||
let session = root.thread.session.clone();
|
||||
let metrics = MetricsClient::new(
|
||||
MetricsConfig::in_memory("test", "core", "1", Default::default()).with_runtime_reader(),
|
||||
)
|
||||
.unwrap();
|
||||
let attach_metrics = |mut turn: Arc<TurnContext>| {
|
||||
let context = Arc::get_mut(&mut turn).unwrap();
|
||||
context.session_telemetry = context
|
||||
.session_telemetry
|
||||
.clone()
|
||||
.with_metrics(metrics.clone());
|
||||
turn
|
||||
};
|
||||
let turn = attach_metrics(session.new_default_turn().await);
|
||||
SpawnAgentHandlerV2::default()
|
||||
.handle(invocation(
|
||||
session.clone(),
|
||||
@@ -3882,10 +3888,11 @@ async fn multi_agent_v2_interrupt_agent_accepts_task_name_target() {
|
||||
.await
|
||||
.expect("worker thread should be resident");
|
||||
let worker_session = worker_thread.session.clone();
|
||||
let worker_turn = attach_metrics(worker_session.new_default_turn().await);
|
||||
SpawnAgentHandlerV2::default()
|
||||
.handle(invocation(
|
||||
worker_session.clone(),
|
||||
worker_session.new_default_turn().await,
|
||||
worker_turn.clone(),
|
||||
"spawn_agent",
|
||||
function_payload(json!({
|
||||
"message": "inspect a child task",
|
||||
@@ -3944,6 +3951,82 @@ async fn multi_agent_v2_interrupt_agent_accepts_task_name_target() {
|
||||
!ops.iter()
|
||||
.any(|(thread_id, op)| *thread_id == child_id && matches!(op, Op::InterruptAgent))
|
||||
);
|
||||
|
||||
InterruptAgentHandler
|
||||
.handle(invocation(
|
||||
worker_session,
|
||||
worker_turn,
|
||||
"interrupt_agent",
|
||||
function_payload(json!({"target": child_id.to_string()})),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(
|
||||
InterruptAgentHandler
|
||||
.handle(invocation(
|
||||
session.clone(),
|
||||
turn.clone(),
|
||||
"interrupt_agent",
|
||||
function_payload(json!({"target": "/root"}))
|
||||
))
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
CloseAgentHandler
|
||||
.handle(invocation(
|
||||
session,
|
||||
turn,
|
||||
"close_agent",
|
||||
function_payload(json!({"target": child_id.to_string()})),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let snapshot = metrics.snapshot().unwrap();
|
||||
let observations = snapshot
|
||||
.scope_metrics()
|
||||
.flat_map(opentelemetry_sdk::metrics::data::ScopeMetrics::metrics);
|
||||
let mut counts = BTreeMap::new();
|
||||
for metric in observations.filter(|m| {
|
||||
matches!(
|
||||
m.name(),
|
||||
codex_otel::MULTI_AGENT_SPAWNED_METRIC | codex_otel::MULTI_AGENT_STOP_ACCEPTED_METRIC
|
||||
)
|
||||
}) {
|
||||
let AggregatedMetrics::U64(MetricData::Sum(sum)) = metric.data() else {
|
||||
panic!("subagent lifecycle metric should be a counter");
|
||||
};
|
||||
for point in sum.data_points() {
|
||||
let label = |key| {
|
||||
point
|
||||
.attributes()
|
||||
.find(|a| a.key.as_str() == key)
|
||||
.map(|a| a.value.as_str().into_owned())
|
||||
.unwrap_or_default()
|
||||
};
|
||||
assert_eq!(label("product_sku"), "codex");
|
||||
*counts
|
||||
.entry((metric.name().to_string(), label("caller"), label("action")))
|
||||
.or_insert(0) += point.value();
|
||||
}
|
||||
}
|
||||
let stop = codex_otel::MULTI_AGENT_STOP_ACCEPTED_METRIC;
|
||||
assert_eq!(
|
||||
counts,
|
||||
BTreeMap::from([
|
||||
(
|
||||
(
|
||||
codex_otel::MULTI_AGENT_SPAWNED_METRIC.into(),
|
||||
String::new(),
|
||||
String::new()
|
||||
),
|
||||
2
|
||||
),
|
||||
((stop.into(), "root".into(), "interrupt".into()), 1),
|
||||
((stop.into(), "root".into(), "close".into()), 1),
|
||||
((stop.into(), "subagent".into(), "interrupt".into()), 1),
|
||||
])
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -2,6 +2,7 @@ use super::analytics::ToolCallAnalytics;
|
||||
use super::*;
|
||||
use crate::agent::api::AgentTarget;
|
||||
use crate::tools::handlers::multi_agents_spec::create_interrupt_agent_tool_v2;
|
||||
use codex_otel::SubagentStopAction;
|
||||
use codex_protocol::protocol::MultiAgentVersion;
|
||||
use codex_tools::ToolSpec;
|
||||
|
||||
@@ -58,6 +59,13 @@ async fn handle_interrupt_agent(
|
||||
let agent_path = snapshot.metadata().agent_path.clone().ok_or_else(|| {
|
||||
FunctionCallError::RespondToModel("target agent is missing an agent_path".to_string())
|
||||
})?;
|
||||
turn.session_telemetry.record_multi_agent_stop_accepted(
|
||||
session.session_id(),
|
||||
agent_id,
|
||||
&call_id,
|
||||
SubagentStopAction::Interrupt,
|
||||
snapshot.status(),
|
||||
);
|
||||
let previous_status = snapshot.status().cloned().unwrap_or(AgentStatus::NotFound);
|
||||
emit_sub_agent_activity(
|
||||
&session,
|
||||
|
||||
Reference in New Issue
Block a user