mirror of
https://github.com/openai/codex.git
synced 2026-11-18 22:20:48 +00:00
Report why turn critical path capture is incomplete (#62357)
## Why The `incomplete` status alone gives no indication of which timing dependency was missing or which operation prevented capture. ## What changed Emit deduplicated `critical_path_incomplete_reasons` in turn analytics for incomplete captures. Reasons cover unsupported Code Mode and agent operations, unattributed reviews and user waits, missing turn starts, and forced teardown, with an `unclassified` fallback. External tools that share a built-in name can still invalidate capture without being classified as the built-in operation. During interruption, use the task's join result to distinguish forced teardown from normal completion so a task that already finished can still report a complete path. ## Testing Add coverage for analytics serialization, concurrent reason deduplication, tool name collisions, and interrupted tasks that finish normally or require forced teardown. GitOrigin-RevId: fab9aa1ff881997a34dcf75adc5638133cedc7fe
This commit is contained in:
@@ -1,6 +1,7 @@
|
||||
//! Calls the decision extension for each approval and enforces host constraints.
|
||||
//! The synchronous service captures one action; no outcome is stored by tool-call ID.
|
||||
|
||||
use codex_analytics::IncompleteReason;
|
||||
use codex_async_utils::THREAD_STACK_SIZE_BYTES;
|
||||
use codex_protocol::config_types::ApprovalsReviewer;
|
||||
use codex_protocol::openai_models::GuardianScope;
|
||||
@@ -101,7 +102,10 @@ pub(crate) fn decide_approval(
|
||||
.start_review_for_turn(call_id, &context.turn().sub_id)
|
||||
});
|
||||
if timing.is_none() {
|
||||
context.turn().turn_timing_state.invalidate_critical_path();
|
||||
context
|
||||
.turn()
|
||||
.turn_timing_state
|
||||
.invalidate_critical_path(&[IncompleteReason::UnattributedReview]);
|
||||
}
|
||||
timing
|
||||
} else {
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
#![cfg(test)]
|
||||
//! Verifies review timing binds to its invocation before scheduling and respects the capture gate.
|
||||
|
||||
use codex_analytics::IncompleteReason;
|
||||
use codex_direct_tool_timing::ToolBatchTiming;
|
||||
use codex_utils_critical_path::CriticalPath;
|
||||
use codex_utils_critical_path::PathDurations;
|
||||
@@ -97,6 +98,10 @@ async fn decision_binds_before_scheduling_and_does_not_follow_reused_ids(
|
||||
.is_none(),
|
||||
network && critical_path_timing
|
||||
);
|
||||
assert_eq!(
|
||||
turn.turn_timing_state.incomplete_reasons().first(),
|
||||
(network && critical_path_timing).then_some(&IncompleteReason::UnattributedReview)
|
||||
);
|
||||
if await_decision {
|
||||
assert_eq!(
|
||||
future.await,
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
use codex_analytics::IncompleteReason;
|
||||
use codex_direct_tool_timing::ToolBatchTiming;
|
||||
use codex_direct_tool_timing::ToolTimingGuard;
|
||||
use codex_features::Feature;
|
||||
@@ -17,6 +18,7 @@ use codex_protocol::request_permissions::RequestPermissionProfile;
|
||||
use codex_protocol::request_permissions::RequestPermissionsResponse;
|
||||
use codex_protocol::request_user_input::RequestUserInputArgs;
|
||||
use codex_protocol::request_user_input::RequestUserInputResponse;
|
||||
use pretty_assertions::assert_eq;
|
||||
use tokio::sync::watch;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
@@ -299,4 +301,8 @@ async fn unowned_human_wait_does_not_claim_a_complete_path(capture: bool) {
|
||||
.is_none(),
|
||||
capture
|
||||
);
|
||||
assert_eq!(
|
||||
turn_context.turn_timing_state.incomplete_reasons().first(),
|
||||
capture.then_some(&IncompleteReason::UnattributedUserWait)
|
||||
);
|
||||
}
|
||||
|
||||
@@ -849,7 +849,9 @@ impl Session {
|
||||
.features
|
||||
.enabled(Feature::CriticalPathTiming)
|
||||
{
|
||||
turn_context.turn_timing_state.invalidate_critical_path();
|
||||
turn_context
|
||||
.turn_timing_state
|
||||
.invalidate_critical_path(&[IncompleteReason::UnattributedUserWait]);
|
||||
}
|
||||
let _elicitation = self.services.elicitations.register();
|
||||
let (tx_response, rx_response) = oneshot::channel();
|
||||
|
||||
@@ -68,6 +68,7 @@ use chrono::Utc;
|
||||
use codex_analytics::AnalyticsEventsClient;
|
||||
use codex_analytics::ImagePreparationFact;
|
||||
use codex_analytics::ImagePreparationMetadata;
|
||||
use codex_analytics::IncompleteReason;
|
||||
use codex_analytics::SubAgentThreadStartedInput;
|
||||
use codex_analytics::ThreadInitializationMode;
|
||||
use codex_analytics::TurnCodexErrorFact;
|
||||
@@ -3312,7 +3313,9 @@ impl Session {
|
||||
.features
|
||||
.enabled(Feature::CriticalPathTiming)
|
||||
{
|
||||
turn_context.turn_timing_state.invalidate_critical_path();
|
||||
turn_context
|
||||
.turn_timing_state
|
||||
.invalidate_critical_path(&[IncompleteReason::UnattributedUserWait]);
|
||||
}
|
||||
timing
|
||||
}
|
||||
|
||||
@@ -15,7 +15,6 @@ use codex_diagnostics::Gauge;
|
||||
use codex_exec_server::ExecutorCapabilityDiscoveryV2Cache;
|
||||
use codex_extension_api::ThreadIdleCause;
|
||||
use futures::future::BoxFuture;
|
||||
use tokio::select;
|
||||
use tokio::sync::Mutex;
|
||||
use tokio::sync::MutexGuard;
|
||||
use tokio::sync::Notify;
|
||||
@@ -47,6 +46,7 @@ use crate::state::RunningTask;
|
||||
use crate::state::TaskKind;
|
||||
use crate::state::TurnState;
|
||||
use crate::turn_diff_tracker::TurnDiffTracker;
|
||||
use codex_analytics::IncompleteReason;
|
||||
use codex_analytics::TurnProfileFact;
|
||||
use codex_analytics::TurnTokenUsageFact;
|
||||
use codex_context_fragments::RenderedFragment;
|
||||
@@ -1229,22 +1229,28 @@ impl Session {
|
||||
.cancel_git_enrichment_task();
|
||||
let session_task = task.task;
|
||||
|
||||
select! {
|
||||
_ = task.done.notified() => {
|
||||
},
|
||||
_ = tokio::time::sleep(Duration::from_millis(GRACEFULL_INTERRUPTION_TIMEOUT_MS)) => {
|
||||
warn!("task {sub_id} didn't complete gracefully after {}ms", GRACEFULL_INTERRUPTION_TIMEOUT_MS);
|
||||
// Forced teardown can abort independently spawned tool dispatches
|
||||
// whose guards are destroyed later. Do not publish complete totals
|
||||
// for a dependency graph that has not settled.
|
||||
task.turn_context.turn_timing_state.invalidate_critical_path();
|
||||
let mut handle = task.handle;
|
||||
let grace = Duration::from_millis(GRACEFULL_INTERRUPTION_TIMEOUT_MS);
|
||||
let joined = match tokio::time::timeout(grace, &mut handle).await {
|
||||
Ok(joined) => joined,
|
||||
Err(_) => {
|
||||
handle.abort();
|
||||
handle.await
|
||||
}
|
||||
};
|
||||
// A task may finish normally between the deadline and the abort.
|
||||
if let Err(error) = joined {
|
||||
warn!(%error, "task {sub_id} did not finish gracefully");
|
||||
let reason = if error.is_cancelled() {
|
||||
IncompleteReason::ForcedTeardown
|
||||
} else {
|
||||
IncompleteReason::Unclassified
|
||||
};
|
||||
task.turn_context
|
||||
.turn_timing_state
|
||||
.invalidate_critical_path(&[reason]);
|
||||
}
|
||||
|
||||
task.handle.abort();
|
||||
// Aborting schedules destruction; join before freezing the turn's timing.
|
||||
let _ = task.handle.await;
|
||||
|
||||
if let Err(error) = self
|
||||
.record_pending_initial_input(
|
||||
&task.turn_context,
|
||||
|
||||
@@ -600,6 +600,7 @@ fn emit_compact_metric_records_auto_local() {
|
||||
|
||||
struct UncooperativeTimedTask {
|
||||
entered: Arc<tokio::sync::Notify>,
|
||||
finish: Arc<tokio::sync::Notify>,
|
||||
}
|
||||
|
||||
impl SessionTask for UncooperativeTimedTask {
|
||||
@@ -631,45 +632,73 @@ impl SessionTask for UncooperativeTimedTask {
|
||||
_blocking = timing.begin_tool_blocking(Some(batch));
|
||||
let _review = _work.start_review();
|
||||
self.entered.notify_one();
|
||||
std::future::pending().await
|
||||
self.finish.notified().await;
|
||||
Ok(None)
|
||||
}
|
||||
}
|
||||
|
||||
#[test_case::test_case(false; "forced teardown")]
|
||||
#[test_case::test_case(true; "finished before wait")]
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn hard_abort_drops_timing_guards_before_profile_completion() {
|
||||
async fn abort_drops_timing_guards_before_profile_completion(finished_before_wait: bool) {
|
||||
use codex_analytics::IncompleteReason::ForcedTeardown;
|
||||
use codex_analytics::TurnCriticalPath;
|
||||
use codex_protocol::protocol::TurnAbortReason;
|
||||
|
||||
let (session, context, _receiver) = make_session_and_context_with_rx().await;
|
||||
let entered = Arc::new(tokio::sync::Notify::new());
|
||||
let finish = Arc::new(tokio::sync::Notify::new());
|
||||
session
|
||||
.spawn_task(
|
||||
Arc::clone(&context),
|
||||
Vec::new(),
|
||||
UncooperativeTimedTask {
|
||||
entered: Arc::clone(&entered),
|
||||
finish: Arc::clone(&finish),
|
||||
},
|
||||
)
|
||||
.await;
|
||||
entered.notified().await;
|
||||
let turn = session
|
||||
.active_turn
|
||||
.lock()
|
||||
.await
|
||||
.take()
|
||||
.expect("active turn");
|
||||
if finished_before_wait {
|
||||
// The task exits after the abort claims the turn but before it waits.
|
||||
let done = turn.task.as_ref().expect("running task").done.notified();
|
||||
finish.notify_one();
|
||||
done.await;
|
||||
}
|
||||
session
|
||||
.abort_all_tasks(codex_protocol::protocol::TurnAbortReason::Interrupted)
|
||||
.abort_taken_tasks(Some(turn), TurnAbortReason::Interrupted)
|
||||
.await;
|
||||
// The task's review and tool guards are gone before abort completion.
|
||||
|
||||
let timing = &context.turn_timing_state;
|
||||
assert!(
|
||||
context
|
||||
.turn_timing_state
|
||||
timing
|
||||
.direct_tool_timings
|
||||
.owner_for_turn("uncooperative", &context.sub_id)
|
||||
.is_none()
|
||||
);
|
||||
assert_eq!(
|
||||
context
|
||||
.turn_timing_state
|
||||
.snapshot_path(std::time::Instant::now()),
|
||||
None
|
||||
);
|
||||
let (_, _, profile) = timing.complete_profile_and_duration_ms().await;
|
||||
if finished_before_wait {
|
||||
assert!(matches!(
|
||||
profile.critical_path,
|
||||
TurnCriticalPath::Complete(_)
|
||||
));
|
||||
} else {
|
||||
assert_eq!(
|
||||
profile.critical_path,
|
||||
TurnCriticalPath::Incomplete(vec![ForcedTeardown])
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn turn_critical_path_metrics_count_each_capture_once() {
|
||||
use codex_analytics::IncompleteReason::Unclassified;
|
||||
use codex_analytics::TurnCriticalPath;
|
||||
use codex_analytics::TurnCriticalPathDurations;
|
||||
|
||||
@@ -690,11 +719,12 @@ async fn turn_critical_path_metrics_count_each_capture_once() {
|
||||
});
|
||||
let _interest_cache_guard =
|
||||
tracing::Dispatch::new(tracing::subscriber::NoSubscriber::default());
|
||||
let incomplete = TurnCriticalPath::Incomplete(vec![Unclassified]);
|
||||
for (sku, path) in [
|
||||
(Some("codex"), complete),
|
||||
(Some("codex"), TurnCriticalPath::Disabled),
|
||||
(None, TurnCriticalPath::Incomplete),
|
||||
(Some("unrecognized-sku"), TurnCriticalPath::Incomplete),
|
||||
(None, incomplete.clone()),
|
||||
(Some("unrecognized-sku"), incomplete),
|
||||
] {
|
||||
Arc::make_mut(&mut context.config).apps_mcp_product_sku = sku.map(str::to_owned);
|
||||
context.turn_timing_state = Arc::default();
|
||||
@@ -725,7 +755,7 @@ async fn turn_critical_path_metrics_count_each_capture_once() {
|
||||
assert!(trace.contains("critical_path_inference_ms=31"));
|
||||
assert!(trace.contains("critical_path_user_wait_ms=0"));
|
||||
}
|
||||
TurnCriticalPath::Disabled | TurnCriticalPath::Incomplete => {
|
||||
TurnCriticalPath::Disabled | TurnCriticalPath::Incomplete(_) => {
|
||||
assert!(!trace.contains("critical_path_duration_ms="));
|
||||
let status = if matches!(profile.critical_path, TurnCriticalPath::Disabled) {
|
||||
"disabled"
|
||||
|
||||
@@ -16,15 +16,22 @@ use crate::function_tool::OrCancelToolExt;
|
||||
use crate::session::session::Session;
|
||||
use crate::session::step_context::StepContext;
|
||||
use crate::tools::call_trace;
|
||||
use crate::tools::code_mode::PUBLIC_TOOL_NAME;
|
||||
use crate::tools::code_mode::WAIT_TOOL_NAME;
|
||||
use crate::tools::context::AbortedToolOutput;
|
||||
use crate::tools::context::SharedTurnDiffTracker;
|
||||
use crate::tools::context::ToolCallState;
|
||||
use crate::tools::context::ToolPayload;
|
||||
use crate::tools::lifecycle::notify_tool_aborted;
|
||||
use crate::tools::registry::AnyToolResult;
|
||||
use crate::tools::registry::RegisteredTool;
|
||||
use crate::tools::registry::ToolArgumentDiffConsumer;
|
||||
use crate::tools::router::ToolCall;
|
||||
use crate::tools::router::ToolCallSource;
|
||||
use codex_analytics::IncompleteReason;
|
||||
use codex_analytics::IncompleteReason::CodeModeUnsupported;
|
||||
use codex_analytics::IncompleteReason::Unclassified;
|
||||
use codex_analytics::IncompleteReason::UnsupportedOperation;
|
||||
use codex_direct_tool_timing::ToolAdmission;
|
||||
use codex_direct_tool_timing::ToolBatchTiming;
|
||||
use codex_direct_tool_timing::ToolGateTiming;
|
||||
@@ -79,15 +86,32 @@ impl From<FunctionCallError> for ToolDispatchError {
|
||||
|
||||
// Direct spawn and wait calls use this turn's tool time; child work stays on its own turn.
|
||||
// CodeMode and follow-up agent operations still lack runtime capture.
|
||||
fn direct_path_supported(tool_name: &codex_tools::ToolName, source: &ToolCallSource) -> bool {
|
||||
!(matches!(source, ToolCallSource::CodeMode { .. })
|
||||
|| (tool_name.is_default_namespace()
|
||||
&& matches!(tool_name.name.as_str(),
|
||||
crate::tools::code_mode::PUBLIC_TOOL_NAME | crate::tools::code_mode::WAIT_TOOL_NAME))
|
||||
// Multi-agent tool namespaces can be overridden by the tool catalog.
|
||||
|| matches!(tool_name.name.as_str(), "send_input"
|
||||
| "close_agent" | "resume_agent" | "list_agents"
|
||||
| "followup_task" | "interrupt_agent" | "send_message"))
|
||||
fn direct_path_incomplete_reasons(
|
||||
tool_name: &codex_tools::ToolName,
|
||||
registered_tool: Option<&RegisteredTool>,
|
||||
source: &ToolCallSource,
|
||||
) -> &'static [IncompleteReason] {
|
||||
// The registry resolves the call by this name, normalizing only the default namespace.
|
||||
// Keep legacy name-based invalidation even when the name resolves to an external tool.
|
||||
let reasons: &[_] = match tool_name.name.as_str() {
|
||||
PUBLIC_TOOL_NAME | WAIT_TOOL_NAME if tool_name.is_default_namespace() => {
|
||||
&[CodeModeUnsupported]
|
||||
}
|
||||
"send_input" | "close_agent" | "resume_agent" | "list_agents" | "followup_task"
|
||||
| "interrupt_agent" | "send_message" => &[UnsupportedOperation],
|
||||
_ => &[],
|
||||
};
|
||||
let trusted = registered_tool.is_some_and(|tool| tool.is_trusted);
|
||||
match (source, reasons, trusted) {
|
||||
(ToolCallSource::CodeMode { .. }, [UnsupportedOperation], true) => {
|
||||
&[CodeModeUnsupported, UnsupportedOperation]
|
||||
}
|
||||
(ToolCallSource::CodeMode { .. }, [_], false) => &[CodeModeUnsupported, Unclassified],
|
||||
(ToolCallSource::CodeMode { .. }, _, _) => &[CodeModeUnsupported],
|
||||
(_, [], _) => &[],
|
||||
(_, reasons, true) => reasons,
|
||||
(_, _, false) => &[Unclassified],
|
||||
}
|
||||
}
|
||||
|
||||
impl ToolCallRuntime {
|
||||
@@ -203,9 +227,8 @@ impl ToolCallRuntime {
|
||||
.record_tool_call(&call, &source, &step_context);
|
||||
let router = &step_context.tool_router;
|
||||
let supports_parallel = router.tool_supports_parallel(&call);
|
||||
let tool_runtime = router
|
||||
.registered_tool(&call.tool_name)
|
||||
.map(|tool| Arc::clone(&tool.runtime));
|
||||
let registered_tool = router.registered_tool(&call.tool_name);
|
||||
let tool_runtime = registered_tool.map(|tool| Arc::clone(&tool.runtime));
|
||||
let finishes_on_cancellation = tool_runtime
|
||||
.as_ref()
|
||||
.is_some_and(|runtime| runtime.finishes_on_cancellation());
|
||||
@@ -220,14 +243,11 @@ impl ToolCallRuntime {
|
||||
.config
|
||||
.features
|
||||
.enabled(codex_features::Feature::CriticalPathTiming)
|
||||
&& !direct_path_supported(
|
||||
&tool_runtime
|
||||
.as_ref()
|
||||
.map_or_else(|| call.tool_name.clone(), |runtime| runtime.tool_name()),
|
||||
&source,
|
||||
)
|
||||
{
|
||||
turn.turn_timing_state.invalidate_critical_path();
|
||||
let reasons = direct_path_incomplete_reasons(&call.tool_name, registered_tool, &source);
|
||||
if !reasons.is_empty() {
|
||||
turn.turn_timing_state.invalidate_critical_path(reasons);
|
||||
}
|
||||
}
|
||||
let work_timing = self.timing.and_then(|timing| match &source {
|
||||
ToolCallSource::Direct | ToolCallSource::DirectPlaintextMessage => {
|
||||
@@ -658,44 +678,53 @@ mod tests {
|
||||
use codex_protocol::models::FunctionCallOutputBody;
|
||||
use codex_protocol::models::FunctionCallOutputPayload;
|
||||
use codex_protocol::openai_models::ToolMode;
|
||||
use codex_tools::ToolName;
|
||||
use pretty_assertions::assert_eq;
|
||||
use tokio::sync::Notify;
|
||||
use tracing_test::internal::MockWriter;
|
||||
|
||||
#[test]
|
||||
fn direct_tool_support_follows_namespace_resolution() {
|
||||
let handler = crate::tools::multi_agent_tool::multi_agent_v2_handler(
|
||||
crate::tools::handlers::multi_agents_v2::WaitAgentHandler::default(),
|
||||
Some("custom_collaboration"),
|
||||
/*description_override*/ None,
|
||||
/*parameters_override*/ None,
|
||||
);
|
||||
let registry = ToolRegistry::from_tools([handler]);
|
||||
let registered = registry
|
||||
.registered_tool(&codex_tools::ToolName::namespaced(
|
||||
"custom_collaboration",
|
||||
"wait_agent",
|
||||
))
|
||||
.unwrap();
|
||||
assert!(direct_path_supported(
|
||||
®istered.runtime.tool_name(),
|
||||
&ToolCallSource::Direct
|
||||
));
|
||||
for name in [
|
||||
crate::tools::code_mode::PUBLIC_TOOL_NAME,
|
||||
crate::tools::code_mode::WAIT_TOOL_NAME,
|
||||
"followup_task",
|
||||
"send_message",
|
||||
fn direct_path_reasons_follow_namespace_resolution() {
|
||||
let nested = ToolCallSource::CodeMode {
|
||||
cell_id: "cell".into(),
|
||||
runtime_tool_call_id: "call".into(),
|
||||
};
|
||||
let code: &[IncompleteReason] = &[CodeModeUnsupported];
|
||||
let mixed: &[IncompleteReason] = &[CodeModeUnsupported, UnsupportedOperation];
|
||||
let mixed_unknown: &[IncompleteReason] = &[CodeModeUnsupported, Unclassified];
|
||||
let namespaced_agent = ToolName::namespaced("custom_collaboration", "send_message");
|
||||
let wait_agent = ToolName::namespaced("custom_collaboration", "wait_agent");
|
||||
let external_wait = ToolName::plain(WAIT_TOOL_NAME).with_default_namespace();
|
||||
let mcp_message = ToolName::namespaced("mcp__chat", "send_message");
|
||||
let unknown = &[Unclassified][..];
|
||||
for (name, builtin, direct, indirect) in [
|
||||
(ToolName::plain(PUBLIC_TOOL_NAME), true, code, code),
|
||||
(ToolName::plain(WAIT_TOOL_NAME), true, code, code),
|
||||
(external_wait, false, unknown, mixed_unknown),
|
||||
(wait_agent, true, &[], code),
|
||||
(namespaced_agent, true, &[UnsupportedOperation], mixed),
|
||||
(mcp_message, false, unknown, mixed_unknown),
|
||||
(ToolName::namespaced("mcp", "wait"), false, &[], code),
|
||||
] {
|
||||
assert!(!direct_path_supported(
|
||||
&codex_tools::ToolName::plain(name).with_default_namespace(),
|
||||
&ToolCallSource::Direct
|
||||
));
|
||||
let mut registry = ToolRegistry::default();
|
||||
let handler = ImmediateHandler {
|
||||
tool_name: name.clone(),
|
||||
supports_parallel: false,
|
||||
calls: AtomicUsize::default(),
|
||||
};
|
||||
if builtin {
|
||||
registry.add(handler);
|
||||
} else {
|
||||
assert!(registry.register_external(Arc::new(handler)));
|
||||
}
|
||||
let registered = registry.registered_tool(&name).unwrap();
|
||||
let reasons = |source| direct_path_incomplete_reasons(&name, Some(registered), source);
|
||||
assert_eq!(
|
||||
(reasons(&ToolCallSource::Direct), reasons(&nested)),
|
||||
(direct, indirect),
|
||||
"{name}"
|
||||
);
|
||||
}
|
||||
assert!(direct_path_supported(
|
||||
&codex_tools::ToolName::namespaced("mcp", "exec"),
|
||||
&ToolCallSource::Direct
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -441,6 +441,7 @@ pub(crate) struct RegisteredTool {
|
||||
pub(crate) runtime: Arc<dyn CoreToolRuntime>,
|
||||
pub(crate) exposure: ToolExposure,
|
||||
pub(crate) native_review: NativeReview,
|
||||
pub(super) is_trusted: bool,
|
||||
/// Per-step description override; never written back to a shared runtime.
|
||||
pub(crate) model_spec: Option<Arc<ToolSpec>>,
|
||||
}
|
||||
@@ -515,6 +516,7 @@ impl ToolRegistry {
|
||||
runtime,
|
||||
exposure,
|
||||
native_review: NativeReview::Disabled,
|
||||
is_trusted: false,
|
||||
model_spec,
|
||||
}
|
||||
}
|
||||
@@ -576,6 +578,7 @@ impl ToolRegistry {
|
||||
}
|
||||
let mut tool = self.prepare_tool(runtime, exposure);
|
||||
tool.native_review = native_review;
|
||||
tool.is_trusted = true;
|
||||
match self.tools.entry(tool_name) {
|
||||
Entry::Vacant(entry) => {
|
||||
entry.insert(tool);
|
||||
@@ -598,7 +601,8 @@ impl ToolRegistry {
|
||||
}
|
||||
|
||||
let exposure = runtime.exposure();
|
||||
let tool = self.prepare_tool(runtime, exposure);
|
||||
let mut tool = self.prepare_tool(runtime, exposure);
|
||||
tool.is_trusted = true;
|
||||
self.tools.shift_insert(0, tool_name, tool);
|
||||
}
|
||||
|
||||
|
||||
@@ -609,6 +609,7 @@ fn mcp_runtime(tool_info: codex_mcp::ToolInfo) -> RegisteredTool {
|
||||
exposure: runtime.exposure(),
|
||||
native_review: Default::default(),
|
||||
runtime,
|
||||
is_trusted: false,
|
||||
model_spec: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -433,6 +433,7 @@ fn mcp_runtime(
|
||||
runtime: handler,
|
||||
exposure,
|
||||
native_review: Default::default(),
|
||||
is_trusted: false,
|
||||
model_spec: None,
|
||||
}
|
||||
}
|
||||
@@ -1749,6 +1750,7 @@ async fn strict_namespace_ownership_requires_tool_namespace_inventory_opt_in() {
|
||||
runtime: Arc::new(McpHandler::new(tool).expect("MCP tool spec should build")),
|
||||
exposure,
|
||||
native_review: Default::default(),
|
||||
is_trusted: false,
|
||||
model_spec: None,
|
||||
}
|
||||
})
|
||||
@@ -2046,6 +2048,7 @@ async fn strict_tool_collisions_allow_multiple_tools_in_one_namespace() {
|
||||
),
|
||||
exposure: ToolExposure::Direct,
|
||||
native_review: Default::default(),
|
||||
is_trusted: false,
|
||||
model_spec: None,
|
||||
},
|
||||
mcp_runtime("shared", "shared", "lookup", ToolExposure::Direct),
|
||||
@@ -2089,6 +2092,7 @@ async fn relaxed_tool_collisions_preserve_first_nonempty_namespace_description()
|
||||
runtime: Arc::new(McpHandler::new(tool).expect("MCP tool spec should build")),
|
||||
exposure: ToolExposure::Direct,
|
||||
native_review: Default::default(),
|
||||
is_trusted: false,
|
||||
model_spec: None,
|
||||
}
|
||||
};
|
||||
|
||||
@@ -6,6 +6,7 @@ use std::time::Instant;
|
||||
use std::time::SystemTime;
|
||||
use std::time::UNIX_EPOCH;
|
||||
|
||||
use codex_analytics::IncompleteReason;
|
||||
use codex_analytics::TurnProfile;
|
||||
use codex_direct_tool_timing::DirectToolTimings;
|
||||
use codex_otel::TURN_TTFM_DURATION_METRIC;
|
||||
@@ -108,6 +109,7 @@ struct TurnTimingStateInner {
|
||||
struct TurnProfileState {
|
||||
capture_enabled: bool,
|
||||
path: Option<CriticalPath>,
|
||||
incomplete_reasons: Vec<IncompleteReason>,
|
||||
tool_dependency: Option<CriticalPath>,
|
||||
started_at: Option<Instant>,
|
||||
last_transition_at: Option<Instant>,
|
||||
@@ -149,10 +151,15 @@ impl TurnTimingState {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn invalidate_critical_path(&self) {
|
||||
pub(crate) fn invalidate_critical_path(&self, reasons: &[IncompleteReason]) {
|
||||
let mut state = self.profile_state();
|
||||
if state.completed_profile.is_none() {
|
||||
state.path = None;
|
||||
for reason in reasons {
|
||||
if !state.incomplete_reasons.contains(reason) {
|
||||
state.incomplete_reasons.push(*reason);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -342,6 +349,7 @@ impl TurnProfileState {
|
||||
*self = Self {
|
||||
path,
|
||||
capture_enabled: self.capture_enabled,
|
||||
incomplete_reasons: std::mem::take(&mut self.incomplete_reasons),
|
||||
started_at: Some(started_at),
|
||||
last_transition_at: Some(started_at),
|
||||
..Self::default()
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
use super::TurnProfileState;
|
||||
use super::duration_to_u64_ms;
|
||||
use codex_analytics::IncompleteReason;
|
||||
use codex_analytics::TurnCriticalPath;
|
||||
use codex_analytics::TurnCriticalPathDurations;
|
||||
use codex_otel::TURN_CRITICAL_PATH_CAPTURE_METRIC;
|
||||
@@ -14,7 +15,14 @@ pub(super) fn completed_critical_path(state: &TurnProfileState, end: Instant) ->
|
||||
return TurnCriticalPath::Disabled;
|
||||
}
|
||||
let (Some(path), Some(start)) = (&state.path, state.started_at) else {
|
||||
return TurnCriticalPath::Incomplete;
|
||||
let mut reasons = state.incomplete_reasons.clone();
|
||||
if state.started_at.is_none() && !reasons.contains(&IncompleteReason::MissingTurnStart) {
|
||||
reasons.push(IncompleteReason::MissingTurnStart);
|
||||
}
|
||||
if reasons.is_empty() {
|
||||
reasons.push(IncompleteReason::Unclassified);
|
||||
}
|
||||
return TurnCriticalPath::Incomplete(reasons);
|
||||
};
|
||||
let totals = path.durations(start, end);
|
||||
TurnCriticalPath::Complete(TurnCriticalPathDurations {
|
||||
|
||||
@@ -1,4 +1,7 @@
|
||||
use super::ToolBatchTiming;
|
||||
use codex_analytics::IncompleteReason;
|
||||
use codex_analytics::IncompleteReason::*;
|
||||
use codex_analytics::TurnCriticalPath;
|
||||
use codex_analytics::TurnProfile;
|
||||
use codex_protocol::items::AgentMessageItem;
|
||||
use codex_protocol::items::TurnItem;
|
||||
@@ -21,6 +24,12 @@ use super::TurnTimingState;
|
||||
use super::response_item_records_turn_ttft;
|
||||
use crate::ResponseEvent;
|
||||
|
||||
impl TurnTimingState {
|
||||
pub(crate) fn incomplete_reasons(&self) -> Vec<IncompleteReason> {
|
||||
self.profile_state().incomplete_reasons.clone()
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn turn_timing_state_records_ttft_only_once_per_turn() {
|
||||
let state = TurnTimingState::default();
|
||||
@@ -414,17 +423,49 @@ fn capture_preserves_phases_and_freezes_missing_dependencies() {
|
||||
async fn invalidation_and_late_guard_drop_cannot_restore_a_frozen_path() {
|
||||
let timing = Arc::new(TurnTimingState::default());
|
||||
timing.enable_critical_path();
|
||||
timing.invalidate_critical_path(&[CodeModeUnsupported]);
|
||||
timing.mark_turn_started(Instant::now()).await;
|
||||
let guard = timing.begin_tool_blocking(Some(ToolBatchTiming::default()));
|
||||
timing.invalidate_critical_path();
|
||||
let (_, _, profile) = timing.complete_profile_and_duration_ms().await;
|
||||
assert_eq!(
|
||||
profile.critical_path,
|
||||
codex_analytics::TurnCriticalPath::Incomplete
|
||||
);
|
||||
let barrier = std::sync::Barrier::new(2);
|
||||
std::thread::scope(|scope| {
|
||||
for reason in [UnsupportedOperation, UnattributedUserWait] {
|
||||
let (timing, barrier) = (&timing, &barrier);
|
||||
scope.spawn(move || {
|
||||
barrier.wait();
|
||||
timing.invalidate_critical_path(&[CodeModeUnsupported, reason]);
|
||||
});
|
||||
}
|
||||
});
|
||||
let result = timing.complete_profile_and_duration_ms().await;
|
||||
let TurnCriticalPath::Incomplete(reasons) = &result.2.critical_path else {
|
||||
panic!("expected incomplete capture");
|
||||
};
|
||||
assert_eq!(reasons.first(), Some(&CodeModeUnsupported));
|
||||
assert_eq!(reasons.len(), 3);
|
||||
assert!(reasons.contains(&UnsupportedOperation) && reasons.contains(&UnattributedUserWait));
|
||||
timing.invalidate_critical_path(&[ForcedTeardown]);
|
||||
timing.enable_critical_path();
|
||||
drop(guard);
|
||||
assert!(timing.snapshot_path(Instant::now()).is_none());
|
||||
assert_eq!(timing.complete_profile_and_duration_ms().await, result);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn critical_path_fallback_depends_on_whether_the_turn_started() {
|
||||
let now = Instant::now();
|
||||
let mut state = TurnProfileState {
|
||||
capture_enabled: true,
|
||||
..Default::default()
|
||||
};
|
||||
assert_eq!(
|
||||
state.complete(now).critical_path,
|
||||
TurnCriticalPath::Incomplete(vec![MissingTurnStart])
|
||||
);
|
||||
state.start(now);
|
||||
assert_eq!(
|
||||
state.complete(now).critical_path,
|
||||
TurnCriticalPath::Incomplete(vec![Unclassified])
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
Reference in New Issue
Block a user