From 0be58e088aeb70b6796d5548f101b482c33f5367 Mon Sep 17 00:00:00 2001 From: Channing Conger Date: Tue, 7 Jul 2026 03:53:01 +0000 Subject: [PATCH] Benchmark unified exec through code mode --- codex-rs/app-server-test-client/src/lib.rs | 102 +++- .../src/loopback_responses_server.rs | 37 +- .../src/unified_exec_benchmark.rs | 477 ++++++++++++++++++ 3 files changed, 587 insertions(+), 29 deletions(-) create mode 100644 codex-rs/app-server-test-client/src/unified_exec_benchmark.rs diff --git a/codex-rs/app-server-test-client/src/lib.rs b/codex-rs/app-server-test-client/src/lib.rs index a70945c468..5393868e05 100644 --- a/codex-rs/app-server-test-client/src/lib.rs +++ b/codex-rs/app-server-test-client/src/lib.rs @@ -92,6 +92,7 @@ mod plugin_analytics_capture; mod plugin_analytics_mutation_smoke; mod plugin_analytics_smoke; mod request_user_input; +mod unified_exec_benchmark; const NOTIFICATIONS_TO_OPT_OUT: &[&str] = &[ // v2 item deltas. @@ -278,6 +279,22 @@ enum CliCommand { #[arg(long, default_value_t = 15)] hold_seconds: u64, }, + /// Compare direct and code-mode unified exec using a deterministic local model. + #[command(name = "unified-exec-benchmark")] + UnifiedExecBenchmark { + /// Measured samples per invocation path. + #[arg(long, default_value_t = 20)] + samples: usize, + /// Unmeasured warmup samples per invocation path. + #[arg(long, default_value_t = 3)] + warmups: usize, + /// Duration of the command run by every sample. + #[arg(long, default_value_t = 50)] + command_duration_ms: u64, + /// Existing workspace path used as the turn cwd. + #[arg(long, value_name = "path", default_value = ".")] + workspace: PathBuf, + }, /// Exercise remote plugin analytics through production app-server RPC paths. #[command(name = "plugin-analytics-smoke")] PluginAnalyticsSmoke { @@ -462,6 +479,26 @@ pub async fn run() -> Result<()> { hold_seconds, ) } + CliCommand::UnifiedExecBenchmark { + samples, + warmups, + command_duration_ms, + workspace, + } => { + ensure_dynamic_tools_unused(&dynamic_tools, "unified-exec-benchmark")?; + if url.is_some() { + bail!("unified-exec-benchmark requires --codex-bin and does not support --url"); + } + let codex_bin = codex_bin.context("unified-exec-benchmark requires --codex-bin")?; + unified_exec_benchmark::run( + &codex_bin, + &config_overrides, + samples, + warmups, + command_duration_ms, + &workspace, + ) + } CliCommand::PluginAnalyticsSmoke { plugin_id, capture_file, @@ -1503,6 +1540,7 @@ struct CodexClient { unexpected_items_before_helper_done: Vec, last_turn_status: Option, last_turn_error_message: Option, + quiet: bool, } #[derive(Debug, Clone, Copy)] @@ -1592,6 +1630,7 @@ impl CodexClient { unexpected_items_before_helper_done: Vec::new(), last_turn_status: None, last_turn_error_message: None, + quiet: false, }) } @@ -1631,6 +1670,7 @@ impl CodexClient { unexpected_items_before_helper_done: Vec::new(), last_turn_status: None, last_turn_error_message: None, + quiet: false, }) } @@ -1833,27 +1873,33 @@ impl CodexClient { match server_notification { ServerNotification::ThreadStarted(payload) => { - if payload.thread.id == thread_id { + if payload.thread.id == thread_id && !self.quiet { println!("< thread/started notification: {:?}", payload.thread); } } ServerNotification::TurnStarted(payload) => { - if payload.turn.id == turn_id { + if payload.turn.id == turn_id && !self.quiet { println!("< turn/started notification: {:?}", payload.turn.status); } } ServerNotification::AgentMessageDelta(delta) => { - print!("{}", delta.delta); - std::io::stdout().flush().ok(); + if !self.quiet { + print!("{}", delta.delta); + std::io::stdout().flush().ok(); + } } ServerNotification::CommandExecutionOutputDelta(delta) => { self.note_helper_output(&delta.delta); - print!("{}", delta.delta); - std::io::stdout().flush().ok(); + if !self.quiet { + print!("{}", delta.delta); + std::io::stdout().flush().ok(); + } } ServerNotification::TerminalInteraction(delta) => { - println!("[stdin sent: {}]", delta.stdin); - std::io::stdout().flush().ok(); + if !self.quiet { + println!("[stdin sent: {}]", delta.stdin); + std::io::stdout().flush().ok(); + } } ServerNotification::ItemStarted(payload) => { if matches!(payload.item, ThreadItem::CommandExecution { .. }) { @@ -1870,7 +1916,9 @@ impl CodexClient { self.unexpected_items_before_helper_done .push(payload.item.clone()); } - println!("\n< item started: {:?}", payload.item); + if !self.quiet { + println!("\n< item started: {:?}", payload.item); + } } ServerNotification::ItemCompleted(payload) => { if let ThreadItem::CommandExecution { @@ -1885,7 +1933,9 @@ impl CodexClient { self.command_execution_outputs.push(aggregated_output); } } - println!("< item completed: {:?}", payload.item); + if !self.quiet { + println!("< item completed: {:?}", payload.item); + } } ServerNotification::TurnCompleted(payload) => { if payload.turn.id == turn_id { @@ -1898,20 +1948,26 @@ impl CodexClient { .error .as_ref() .map(|error| error.message.clone()); - println!("\n< turn/completed notification: {:?}", payload.turn.status); - if payload.turn.status == TurnStatus::Failed - && let Some(error) = payload.turn.error - { - println!("[turn error] {}", error.message); + if !self.quiet { + println!("\n< turn/completed notification: {:?}", payload.turn.status); + if payload.turn.status == TurnStatus::Failed + && let Some(error) = payload.turn.error + { + println!("[turn error] {}", error.message); + } } break; } } ServerNotification::McpToolCallProgress(payload) => { - println!("< MCP tool progress: {}", payload.message); + if !self.quiet { + println!("< MCP tool progress: {}", payload.message); + } } _ => { - println!("[UNKNOWN SERVER NOTIFICATION] {server_notification:?}"); + if !self.quiet { + println!("[UNKNOWN SERVER NOTIFICATION] {server_notification:?}"); + } } } } @@ -1955,7 +2011,9 @@ impl CodexClient { request.trace = current_span_w3c_trace_context(); let request_json = serde_json::to_string(&request)?; let request_pretty = serde_json::to_string_pretty(&request)?; - print_multiline_with_prefix("> ", &request_pretty); + if !self.quiet { + print_multiline_with_prefix("> ", &request_pretty); + } self.write_payload(&request_json) } @@ -2020,7 +2078,9 @@ impl CodexClient { let parsed: Value = serde_json::from_str(trimmed).context("response was not valid JSON-RPC")?; let pretty = serde_json::to_string_pretty(&parsed)?; - print_multiline_with_prefix("< ", &pretty); + if !self.quiet { + print_multiline_with_prefix("< ", &pretty); + } let message: JSONRPCMessage = serde_json::from_value(parsed) .context("response was not a valid JSON-RPC message")?; return Ok(message); @@ -2180,7 +2240,9 @@ impl CodexClient { fn write_jsonrpc_message(&mut self, message: JSONRPCMessage) -> Result<()> { let payload = serde_json::to_string(&message)?; let pretty = serde_json::to_string_pretty(&message)?; - print_multiline_with_prefix("> ", &pretty); + if !self.quiet { + print_multiline_with_prefix("> ", &pretty); + } self.write_payload(&payload) } diff --git a/codex-rs/app-server-test-client/src/loopback_responses_server.rs b/codex-rs/app-server-test-client/src/loopback_responses_server.rs index 74b7c753e0..259a2228dc 100644 --- a/codex-rs/app-server-test-client/src/loopback_responses_server.rs +++ b/codex-rs/app-server-test-client/src/loopback_responses_server.rs @@ -20,6 +20,20 @@ pub(super) struct LoopbackResponsesServer { impl LoopbackResponsesServer { pub(super) fn start() -> Result { + Self::start_with_responder(|_| { + Ok(concat!( + "event: response.created\n", + "data: {\"type\":\"response.created\",\"response\":{\"id\":\"resp-plugin-analytics\"}}\n\n", + "event: response.completed\n", + "data: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp-plugin-analytics\",\"usage\":{\"input_tokens\":0,\"input_tokens_details\":null,\"output_tokens\":0,\"output_tokens_details\":null,\"total_tokens\":0}}}\n\n" + ) + .to_string()) + }) + } + + pub(super) fn start_with_responder( + mut responder: impl FnMut(&[u8]) -> io::Result + Send + 'static, + ) -> Result { let listener = TcpListener::bind("127.0.0.1:0").context("bind loopback Responses API server")?; listener @@ -32,7 +46,7 @@ impl LoopbackResponsesServer { while !thread_shutdown.load(Ordering::Relaxed) { match listener.accept() { Ok((stream, _)) => { - if let Err(err) = handle_model_connection(stream) { + if let Err(err) = handle_model_connection(stream, &mut responder) { eprintln!("loopback Responses API server error: {err}"); } } @@ -67,7 +81,10 @@ impl Drop for LoopbackResponsesServer { } } -fn handle_model_connection(mut stream: TcpStream) -> io::Result<()> { +fn handle_model_connection( + mut stream: TcpStream, + responder: &mut impl FnMut(&[u8]) -> io::Result, +) -> io::Result<()> { stream.set_nonblocking(false)?; stream.set_read_timeout(Some(Duration::from_secs(2)))?; let request = read_http_request(&mut stream)?; @@ -77,13 +94,15 @@ fn handle_model_connection(mut stream: TcpStream) -> io::Result<()> { .and_then(|line| std::str::from_utf8(line).ok()) .unwrap_or_default(); if request_line.starts_with("POST ") && request_line.contains("/responses ") { - let body = concat!( - "event: response.created\n", - "data: {\"type\":\"response.created\",\"response\":{\"id\":\"resp-plugin-analytics\"}}\n\n", - "event: response.completed\n", - "data: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp-plugin-analytics\",\"usage\":{\"input_tokens\":0,\"input_tokens_details\":null,\"output_tokens\":0,\"output_tokens_details\":null,\"total_tokens\":0}}}\n\n" - ); - write_http_response(&mut stream, "200 OK", "text/event-stream", body) + let body = responder(&request)?; + write_http_response(&mut stream, "200 OK", "text/event-stream", &body) + } else if request_line.starts_with("GET ") && request_line.contains("/models") { + write_http_response( + &mut stream, + "200 OK", + "application/json", + r#"{"models":[]}"#, + ) } else { write_http_response( &mut stream, diff --git a/codex-rs/app-server-test-client/src/unified_exec_benchmark.rs b/codex-rs/app-server-test-client/src/unified_exec_benchmark.rs new file mode 100644 index 0000000000..afefde5e2e --- /dev/null +++ b/codex-rs/app-server-test-client/src/unified_exec_benchmark.rs @@ -0,0 +1,477 @@ +use super::CodexClient; +use super::loopback_responses_server::LoopbackResponsesServer; +use anyhow::Context; +use anyhow::Result; +use anyhow::bail; +use codex_app_server_protocol::AskForApproval; +use codex_app_server_protocol::CommandExecutionStatus; +use codex_app_server_protocol::SandboxPolicy; +use codex_app_server_protocol::ThreadStartParams; +use codex_app_server_protocol::TurnStartParams; +use codex_app_server_protocol::TurnStatus; +use codex_app_server_protocol::UserInput; +use serde_json::Value; +use serde_json::json; +use std::collections::VecDeque; +use std::env; +use std::ffi::OsString; +use std::fmt; +use std::fs; +use std::io; +use std::path::Path; +use std::path::PathBuf; +use std::sync::Arc; +use std::sync::Mutex; +use std::time::Duration; +use std::time::Instant; +use uuid::Uuid; + +const MOCK_MODEL_SLUG: &str = "unified-exec-benchmark"; +const MOCK_PROVIDER_ID: &str = "unified_exec_benchmark"; +const OUTPUT_MARKER: &str = "unified_exec_benchmark_marker"; +const MAX_COMMAND_DURATION_MS: u64 = 50; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum InvocationPath { + Direct, + CodeMode, +} + +impl fmt::Display for InvocationPath { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Direct => f.write_str("direct"), + Self::CodeMode => f.write_str("code_mode"), + } + } +} + +#[derive(Clone, Copy, Debug)] +struct PlannedSample { + path: InvocationPath, + warmup: bool, +} + +#[derive(Clone, Copy, Debug)] +struct Observation { + path: InvocationPath, + elapsed: Duration, +} + +struct BenchmarkResponder { + samples: VecDeque, + active_sample: Option<(PlannedSample, Instant)>, + observations: Arc>>, + next_response_id: usize, + command: String, + yield_time_ms: u64, +} + +struct TemporaryCodexHome { + path: PathBuf, +} + +impl TemporaryCodexHome { + fn create() -> Result { + let path = env::temp_dir().join(format!("codex-unified-exec-benchmark-{}", Uuid::new_v4())); + fs::create_dir(&path) + .with_context(|| format!("create temporary Codex home {}", path.display()))?; + Ok(Self { path }) + } +} + +impl Drop for TemporaryCodexHome { + fn drop(&mut self) { + if let Err(err) = fs::remove_dir_all(&self.path) { + eprintln!( + "failed to remove temporary Codex home {}: {err}", + self.path.display() + ); + } + } +} + +impl BenchmarkResponder { + fn respond(&mut self, _request: &[u8]) -> io::Result { + if let Some((sample, started_at)) = self.active_sample.take() { + if !sample.warmup { + self.observations + .lock() + .map_err(|_| io::Error::other("benchmark observations lock poisoned"))? + .push(Observation { + path: sample.path, + elapsed: started_at.elapsed(), + }); + } + return Ok(self.completed_response()); + } + + let sample = self + .samples + .pop_front() + .ok_or_else(|| io::Error::other("unexpected extra Responses API request"))?; + self.active_sample = Some((sample, Instant::now())); + Ok(self.tool_call_response(sample.path)) + } + + fn tool_call_response(&mut self, path: InvocationPath) -> String { + let response_id = self.response_id(); + let call_id = format!("call-{response_id}"); + let arguments = json!({ + "cmd": &self.command, + "yield_time_ms": self.yield_time_ms, + }) + .to_string(); + let tool_call = match path { + InvocationPath::Direct => json!({ + "type": "response.output_item.done", + "item": { + "type": "function_call", + "call_id": call_id, + "name": "exec_command", + "arguments": arguments, + } + }), + InvocationPath::CodeMode => { + let code = format!( + "const result = await tools.exec_command({arguments});\ntext(result.output);" + ); + json!({ + "type": "response.output_item.done", + "item": { + "type": "custom_tool_call", + "call_id": call_id, + "name": "exec", + "input": code, + } + }) + } + }; + sse([ + json!({ + "type": "response.created", + "response": { "id": &response_id }, + }), + tool_call, + completed_event(&response_id), + ]) + } + + fn completed_response(&mut self) -> String { + let response_id = self.response_id(); + sse([ + json!({ + "type": "response.output_item.done", + "item": { + "type": "message", + "role": "assistant", + "id": format!("message-{response_id}"), + "content": [{ "type": "output_text", "text": "done" }], + } + }), + completed_event(&response_id), + ]) + } + + fn response_id(&mut self) -> String { + let response_id = format!("benchmark-{}", self.next_response_id); + self.next_response_id += 1; + response_id + } +} + +pub(super) fn run( + codex_bin: &Path, + config_overrides: &[String], + samples: usize, + warmups: usize, + command_duration_ms: u64, + workspace: &Path, +) -> Result<()> { + if cfg!(windows) { + bail!("unified-exec-benchmark currently requires the POSIX sleep utility"); + } + if samples == 0 { + bail!("--samples must be greater than zero"); + } + if command_duration_ms == 0 || command_duration_ms > MAX_COMMAND_DURATION_MS { + bail!( + "--command-duration-ms must be between 1 and {MAX_COMMAND_DURATION_MS} for the single-call benchmark" + ); + } + + let workspace = workspace + .canonicalize() + .with_context(|| format!("resolve workspace {}", workspace.display()))?; + let command_duration = Duration::from_millis(command_duration_ms); + let command = format!( + "sleep {:.3}; printf {OUTPUT_MARKER}", + command_duration.as_secs_f64() + ); + let schedule = sample_schedule(samples, warmups); + let observations = Arc::new(Mutex::new(Vec::with_capacity(samples * 2))); + let mut responder = BenchmarkResponder { + samples: schedule.clone().into(), + active_sample: None, + observations: Arc::clone(&observations), + next_response_id: 1, + command: command.clone(), + yield_time_ms: command_duration_ms + 1_000, + }; + let responses_server = + LoopbackResponsesServer::start_with_responder(move |request| responder.respond(request))?; + + let mut overrides = config_overrides.to_vec(); + overrides.extend(benchmark_config_overrides(responses_server.base_url())?); + let codex_home = TemporaryCodexHome::create()?; + let environment = [( + OsString::from("CODEX_HOME"), + codex_home.path.as_os_str().to_os_string(), + )]; + let mut client = CodexClient::spawn_stdio_with_env(codex_bin, &overrides, &environment)?; + client.quiet = true; + client.initialize()?; + let thread = client.thread_start(ThreadStartParams { + model: Some(MOCK_MODEL_SLUG.to_string()), + model_provider: Some(MOCK_PROVIDER_ID.to_string()), + base_instructions: Some(String::new()), + developer_instructions: Some(String::new()), + ephemeral: Some(true), + ..Default::default() + })?; + + for sample in schedule { + reset_turn_observations(&mut client); + run_sample(&mut client, &thread.thread.id, &workspace)?; + validate_sample(&client, sample)?; + } + + let observations = observations + .lock() + .map_err(|_| anyhow::anyhow!("benchmark observations lock poisoned"))? + .clone(); + if observations.len() != samples * 2 { + bail!( + "expected {} measured samples, recorded {}", + samples * 2, + observations.len() + ); + } + + print_results( + codex_bin, + &command, + samples, + warmups, + command_duration, + &observations, + ); + Ok(()) +} + +fn run_sample(client: &mut CodexClient, thread_id: &str, workspace: &Path) -> Result<()> { + let turn = client.turn_start(TurnStartParams { + thread_id: thread_id.to_string(), + client_user_message_id: None, + input: vec![UserInput::Text { + text: "run the benchmark tool call".to_string(), + text_elements: Vec::new(), + }], + approval_policy: Some(AskForApproval::Never), + sandbox_policy: Some(SandboxPolicy::DangerFullAccess), + cwd: Some(workspace.to_path_buf()), + ..Default::default() + })?; + client.stream_turn(thread_id, &turn.turn.id) +} + +fn validate_sample(client: &CodexClient, sample: PlannedSample) -> Result<()> { + if client.last_turn_status != Some(TurnStatus::Completed) { + bail!( + "{} sample failed: turn status {:?}, error {:?}", + sample.path, + client.last_turn_status, + client.last_turn_error_message + ); + } + if client.command_execution_statuses.last() != Some(&CommandExecutionStatus::Completed) { + bail!( + "{} sample failed: command statuses {:?}", + sample.path, + client.command_execution_statuses + ); + } + if !client + .command_execution_outputs + .last() + .is_some_and(|output| output.contains(OUTPUT_MARKER)) + { + bail!("{} sample did not emit the expected marker", sample.path); + } + Ok(()) +} + +fn reset_turn_observations(client: &mut CodexClient) { + client.command_execution_statuses.clear(); + client.command_execution_outputs.clear(); + client.command_output_stream.clear(); + client.command_item_started = false; + client.helper_done_seen = false; + client.turn_completed_before_helper_done = false; + client.unexpected_items_before_helper_done.clear(); + client.last_turn_status = None; + client.last_turn_error_message = None; +} + +fn sample_schedule(samples: usize, warmups: usize) -> Vec { + let mut schedule = Vec::with_capacity((samples + warmups) * 2); + for round in 0..(samples + warmups) { + let paths = if round % 2 == 0 { + [InvocationPath::Direct, InvocationPath::CodeMode] + } else { + [InvocationPath::CodeMode, InvocationPath::Direct] + }; + for path in paths { + schedule.push(PlannedSample { + path, + warmup: round < warmups, + }); + } + } + schedule +} + +fn benchmark_config_overrides(responses_base_url: &str) -> Result> { + let provider_base_url = quoted(&format!("{responses_base_url}/v1"))?; + Ok(vec![ + format!("model={}", quoted(MOCK_MODEL_SLUG)?), + format!("model_provider={}", quoted(MOCK_PROVIDER_ID)?), + format!( + "model_providers.{MOCK_PROVIDER_ID}.name={}", + quoted("Unified exec benchmark mock provider")? + ), + format!("model_providers.{MOCK_PROVIDER_ID}.base_url={provider_base_url}"), + format!( + "model_providers.{MOCK_PROVIDER_ID}.wire_api={}", + quoted("responses")? + ), + format!("model_providers.{MOCK_PROVIDER_ID}.requires_openai_auth=false"), + format!("model_providers.{MOCK_PROVIDER_ID}.request_max_retries=0"), + format!("model_providers.{MOCK_PROVIDER_ID}.stream_max_retries=0"), + "features.unified_exec=true".to_string(), + "features.code_mode=true".to_string(), + "suppress_unstable_features_warning=true".to_string(), + ]) +} + +fn quoted(value: &str) -> Result { + serde_json::to_string(value).context("serialize config string") +} + +fn completed_event(response_id: &str) -> Value { + json!({ + "type": "response.completed", + "response": { + "id": response_id, + "usage": { + "input_tokens": 0, + "input_tokens_details": null, + "output_tokens": 0, + "output_tokens_details": null, + "total_tokens": 0, + } + } + }) +} + +fn sse(events: impl IntoIterator) -> String { + let mut body = String::new(); + for event in events { + let event_type = event + .get("type") + .and_then(Value::as_str) + .unwrap_or_default(); + body.push_str("event: "); + body.push_str(event_type); + body.push('\n'); + body.push_str("data: "); + body.push_str(&event.to_string()); + body.push_str("\n\n"); + } + body +} + +#[derive(Debug)] +struct Summary { + minimum_ms: f64, + p50_ms: f64, + p95_ms: f64, + mean_ms: f64, +} + +fn summarize(path: InvocationPath, observations: &[Observation]) -> Summary { + let mut elapsed: Vec = observations + .iter() + .filter(|observation| observation.path == path) + .map(|observation| observation.elapsed) + .collect(); + elapsed.sort_unstable(); + let mean = elapsed.iter().map(Duration::as_secs_f64).sum::() / elapsed.len() as f64; + Summary { + minimum_ms: elapsed[0].as_secs_f64() * 1_000.0, + p50_ms: percentile(&elapsed, 50).as_secs_f64() * 1_000.0, + p95_ms: percentile(&elapsed, 95).as_secs_f64() * 1_000.0, + mean_ms: mean * 1_000.0, + } +} + +fn percentile(sorted: &[Duration], percentile: usize) -> Duration { + let index = ((sorted.len() - 1) * percentile).div_ceil(100); + sorted[index] +} + +fn print_results( + codex_bin: &Path, + command: &str, + samples: usize, + warmups: usize, + command_duration: Duration, + observations: &[Observation], +) { + let direct = summarize(InvocationPath::Direct, observations); + let code_mode = summarize(InvocationPath::CodeMode, observations); + let configured_ms = command_duration.as_secs_f64() * 1_000.0; + let delta_ms = code_mode.mean_ms - direct.mean_ms; + let delta_percent = delta_ms / direct.mean_ms * 100.0; + + println!("unified exec benchmark"); + println!("codex: {}", codex_bin.display()); + println!("command: {command}"); + println!("warmups: {warmups}/path; samples: {samples}/path"); + println!("timing: initial Responses request to follow-up Responses request"); + println!(); + println!( + "{:<10} {:>10} {:>10} {:>10} {:>10} {:>14}", + "path", "min", "p50", "p95", "mean", "mean-sleep" + ); + println!( + "{:<10} {:>9.3}ms {:>9.3}ms {:>9.3}ms {:>9.3}ms {:>13.3}ms", + InvocationPath::Direct, + direct.minimum_ms, + direct.p50_ms, + direct.p95_ms, + direct.mean_ms, + direct.mean_ms - configured_ms, + ); + println!( + "{:<10} {:>9.3}ms {:>9.3}ms {:>9.3}ms {:>9.3}ms {:>13.3}ms", + InvocationPath::CodeMode, + code_mode.minimum_ms, + code_mode.p50_ms, + code_mode.p95_ms, + code_mode.mean_ms, + code_mode.mean_ms - configured_ms, + ); + println!(); + println!("code_mode - direct mean: {delta_ms:+.3}ms ({delta_percent:+.2}%)"); +}