mirror of
https://github.com/openai/codex.git
synced 2026-11-15 21:50:46 +00:00
Add opt-in overhead timing to code-mode responses (#46288)
## Why Code-mode responses show host duration without exposing time spent outside the host, including app-server waiting. ## What changed - Add `features.code_mode.experimental_show_cell_overhead`, disabled by default, to show handler duration, host duration, and their difference in `exec` and `wait` response headers. - Reuse the completed handler measurement from tool-call logging, excluding dispatch waiting, and capture it even when logging is disabled. - Preserve the existing timing format when the option is disabled or host timing is unavailable, and preserve boolean feature toggles when merging nested code-mode configuration. ## Testing Add coverage for timing headers, zero and missing host measurements, negative rounding differences, content preservation, and configuration merging. Extend app-server tests across gRPC and stdio to verify timing includes post-host elicitation waiting, and add a response snapshot for successful execution and a failed wait. GitOrigin-RevId: 929b84784d9c6077cf2b5927a90bd2d5f3870078
This commit is contained in:
committed by
copyberry
parent
0a5b999169
commit
0fd1cd8d99
@@ -1,5 +1,4 @@
|
||||
use crate::function_tool::FunctionCallError;
|
||||
use crate::tools::context::FunctionToolOutput;
|
||||
use crate::tools::context::ToolInvocation;
|
||||
use crate::tools::context::ToolPayload;
|
||||
use crate::tools::context::boxed_tool_output;
|
||||
@@ -13,6 +12,7 @@ use super::ExecContext;
|
||||
use super::PUBLIC_TOOL_NAME;
|
||||
use super::handle_runtime_response;
|
||||
use super::is_exec_tool_name;
|
||||
use super::output::CodeModeToolOutput;
|
||||
use super::telemetry::CodeModeToolCallGuard;
|
||||
use super::telemetry::trace_id;
|
||||
|
||||
@@ -39,7 +39,7 @@ impl CodeModeExecuteHandler {
|
||||
originating_call: Option<crate::tools::context::ToolCallOrigin>,
|
||||
code: String,
|
||||
telemetry: &mut CodeModeToolCallGuard,
|
||||
) -> Result<FunctionToolOutput, FunctionCallError> {
|
||||
) -> Result<CodeModeToolOutput, FunctionCallError> {
|
||||
let args =
|
||||
codex_code_mode::parse_exec_source(&code).map_err(FunctionCallError::RespondToModel)?;
|
||||
let exec = ExecContext {
|
||||
@@ -146,14 +146,13 @@ impl CodeModeExecuteHandler {
|
||||
let wall_time = response
|
||||
.code_mode_host_duration()
|
||||
.unwrap_or_else(|| started_at.elapsed());
|
||||
handle_runtime_response(
|
||||
Ok(handle_runtime_response(
|
||||
&step_context.settings.model_info,
|
||||
response,
|
||||
args.max_output_tokens,
|
||||
wall_time,
|
||||
)
|
||||
.await
|
||||
.map_err(FunctionCallError::RespondToModel)
|
||||
exec.turn.config.code_mode.experimental_show_cell_overhead,
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
mod delegate;
|
||||
mod execute_handler;
|
||||
pub(crate) mod execute_spec;
|
||||
mod output;
|
||||
mod response_adapter;
|
||||
mod telemetry;
|
||||
mod wait_handler;
|
||||
@@ -51,6 +52,7 @@ use delegate::CodeModeCellDelegate;
|
||||
use delegate::CodeModeDispatchBroker;
|
||||
use delegate::CodeModeDispatchWorker;
|
||||
pub(crate) use execute_handler::CodeModeExecuteHandler;
|
||||
use output::CodeModeToolOutput;
|
||||
use response_adapter::into_function_call_output_content_items;
|
||||
pub(crate) use wait_handler::CodeModeWaitHandler;
|
||||
|
||||
@@ -251,51 +253,43 @@ impl CodeModeService {
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn handle_runtime_response(
|
||||
fn handle_runtime_response(
|
||||
model_info: &codex_protocol::openai_models::ModelInfo,
|
||||
response: RuntimeResponse,
|
||||
max_output_tokens: Option<usize>,
|
||||
wall_time: Duration,
|
||||
) -> Result<FunctionToolOutput, String> {
|
||||
experimental_show_cell_overhead: bool,
|
||||
) -> CodeModeToolOutput {
|
||||
let script_status = format_script_status(&response);
|
||||
let supports_original = can_request_original_image_detail(model_info);
|
||||
let host_duration = response
|
||||
.code_mode_host_duration()
|
||||
.filter(|_| experimental_show_cell_overhead);
|
||||
|
||||
match response {
|
||||
RuntimeResponse::Yielded { content_items, .. } => {
|
||||
let mut content_items = into_function_call_output_content_items(content_items);
|
||||
sanitize_image_detail_items(supports_original, &mut content_items);
|
||||
content_items = truncate_code_mode_result(content_items, max_output_tokens);
|
||||
prepend_script_status(&mut content_items, &script_status, wall_time);
|
||||
Ok(FunctionToolOutput::from_content(content_items, Some(true)))
|
||||
}
|
||||
RuntimeResponse::Terminated { content_items, .. } => {
|
||||
let mut content_items = into_function_call_output_content_items(content_items);
|
||||
sanitize_image_detail_items(supports_original, &mut content_items);
|
||||
content_items = truncate_code_mode_result(content_items, max_output_tokens);
|
||||
prepend_script_status(&mut content_items, &script_status, wall_time);
|
||||
Ok(FunctionToolOutput::from_content(content_items, Some(true)))
|
||||
}
|
||||
let (content_items, error_text) = match response {
|
||||
RuntimeResponse::Yielded { content_items, .. }
|
||||
| RuntimeResponse::Terminated { content_items, .. } => (content_items, None),
|
||||
RuntimeResponse::Result {
|
||||
content_items,
|
||||
error_text,
|
||||
..
|
||||
} => {
|
||||
let mut content_items = into_function_call_output_content_items(content_items);
|
||||
sanitize_image_detail_items(supports_original, &mut content_items);
|
||||
let success = error_text.is_none();
|
||||
if let Some(error_text) = error_text {
|
||||
content_items.push(FunctionCallOutputContentItem::InputText {
|
||||
text: format!("Script error:\n{error_text}"),
|
||||
});
|
||||
}
|
||||
content_items = truncate_code_mode_result(content_items, max_output_tokens);
|
||||
prepend_script_status(&mut content_items, &script_status, wall_time);
|
||||
Ok(FunctionToolOutput::from_content(
|
||||
content_items,
|
||||
Some(success),
|
||||
))
|
||||
}
|
||||
} => (content_items, error_text),
|
||||
};
|
||||
let mut content_items = into_function_call_output_content_items(content_items);
|
||||
sanitize_image_detail_items(supports_original, &mut content_items);
|
||||
let success = error_text.is_none();
|
||||
if let Some(error_text) = error_text {
|
||||
content_items.push(FunctionCallOutputContentItem::InputText {
|
||||
text: format!("Script error:\n{error_text}"),
|
||||
});
|
||||
}
|
||||
content_items = truncate_code_mode_result(content_items, max_output_tokens);
|
||||
CodeModeToolOutput::new(
|
||||
FunctionToolOutput::from_content(content_items, Some(success)),
|
||||
script_status,
|
||||
wall_time,
|
||||
host_duration,
|
||||
)
|
||||
}
|
||||
|
||||
fn format_script_status(response: &RuntimeResponse) -> String {
|
||||
@@ -314,16 +308,6 @@ fn format_script_status(response: &RuntimeResponse) -> String {
|
||||
}
|
||||
}
|
||||
|
||||
fn prepend_script_status(
|
||||
content_items: &mut Vec<FunctionCallOutputContentItem>,
|
||||
status: &str,
|
||||
wall_time: Duration,
|
||||
) {
|
||||
let wall_time_seconds = ((wall_time.as_secs_f32()) * 10.0).round() / 10.0;
|
||||
let header = format!("{status}\nWall time {wall_time_seconds:.1} seconds\nOutput:\n");
|
||||
content_items.insert(0, FunctionCallOutputContentItem::InputText { text: header });
|
||||
}
|
||||
|
||||
fn truncate_code_mode_result(
|
||||
items: Vec<FunctionCallOutputContentItem>,
|
||||
max_output_tokens: Option<usize>,
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
//! Code-mode response headers use the host measurement and completed handler timing.
|
||||
//! Content is already truncated; only the bounded header changes at completion.
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
use codex_protocol::models::FunctionCallOutputContentItem;
|
||||
use codex_protocol::models::ResponseInputItem;
|
||||
|
||||
use crate::tools::context::FunctionToolOutput;
|
||||
use crate::tools::context::ToolOutput;
|
||||
use crate::tools::context::ToolPayload;
|
||||
|
||||
pub(super) struct CodeModeToolOutput {
|
||||
output: FunctionToolOutput,
|
||||
status: String,
|
||||
host_duration: Option<Duration>,
|
||||
}
|
||||
|
||||
impl CodeModeToolOutput {
|
||||
pub(super) fn new(
|
||||
mut output: FunctionToolOutput,
|
||||
status: String,
|
||||
wall_time: Duration,
|
||||
host_duration: Option<Duration>,
|
||||
) -> Self {
|
||||
// Use the host-only header when overhead is hidden or host timing is unavailable.
|
||||
let wall_time_seconds = (wall_time.as_secs_f32() * 10.0).round() / 10.0;
|
||||
output.body.insert(
|
||||
/*index*/ 0,
|
||||
FunctionCallOutputContentItem::InputText {
|
||||
text: format!("{status}\nWall time {wall_time_seconds:.1} seconds\nOutput:\n"),
|
||||
},
|
||||
);
|
||||
Self {
|
||||
output,
|
||||
status,
|
||||
host_duration,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl ToolOutput for CodeModeToolOutput {
|
||||
fn log_output(&self) -> String {
|
||||
self.output.log_output()
|
||||
}
|
||||
|
||||
fn success_for_logging(&self) -> bool {
|
||||
self.output.success_for_logging()
|
||||
}
|
||||
|
||||
fn set_handler_duration_ms(&mut self, handler_duration_ms: u64) {
|
||||
let Some(host_duration) = self.host_duration else {
|
||||
return;
|
||||
};
|
||||
let total_seconds = handler_duration_ms as f64 / 1_000.0;
|
||||
let host_seconds = host_duration.as_secs_f64();
|
||||
// Match the logged measurements before rounding. Millisecond quantization of the
|
||||
// outer measurement can legitimately produce a small negative difference.
|
||||
let overhead_seconds = total_seconds - host_seconds;
|
||||
let status = &self.status;
|
||||
// The constructor owns this slot; script text and media stay untouched.
|
||||
self.output.body[0] = FunctionCallOutputContentItem::InputText {
|
||||
text: format!(
|
||||
"{status}\nWall time {total_seconds:.3} seconds (code-mode {host_seconds:.3} seconds; overhead {overhead_seconds:.3} seconds)\nOutput:\n"
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
fn to_response_item(&self, call_id: &str, payload: &ToolPayload) -> ResponseInputItem {
|
||||
self.output.to_response_item(call_id, payload)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "output_tests.rs"]
|
||||
mod tests;
|
||||
@@ -0,0 +1,65 @@
|
||||
//! Checks code-mode timing headers, including absent and zero host measurements.
|
||||
|
||||
use super::*;
|
||||
use codex_protocol::models::ImageReference;
|
||||
use pretty_assertions::assert_eq;
|
||||
|
||||
#[test]
|
||||
fn completed_timing_preserves_content_and_distinguishes_zero_from_unavailable() {
|
||||
let content = vec![
|
||||
FunctionCallOutputContentItem::InputText {
|
||||
text: "result".to_string(),
|
||||
},
|
||||
FunctionCallOutputContentItem::InputImage {
|
||||
image: ImageReference::Inline {
|
||||
image_url: "data:image/png;base64,image".to_string(),
|
||||
},
|
||||
detail: None,
|
||||
},
|
||||
FunctionCallOutputContentItem::InputAudio {
|
||||
audio_url: "data:audio/wav;base64,audio".to_string(),
|
||||
},
|
||||
];
|
||||
for (host_duration, expected_timing) in [
|
||||
(
|
||||
Some(Duration::from_millis(/*millis*/ 750)),
|
||||
"Wall time 1.250 seconds (code-mode 0.750 seconds; overhead 0.500 seconds)",
|
||||
),
|
||||
(
|
||||
Some(Duration::ZERO),
|
||||
"Wall time 1.250 seconds (code-mode 0.000 seconds; overhead 1.250 seconds)",
|
||||
),
|
||||
(
|
||||
Some(Duration::from_micros(/*micros*/ 1_251_400)),
|
||||
"Wall time 1.250 seconds (code-mode 1.251 seconds; overhead -0.001 seconds)",
|
||||
),
|
||||
(None, "Wall time 0.8 seconds"),
|
||||
] {
|
||||
let mut output = CodeModeToolOutput::new(
|
||||
FunctionToolOutput::from_content(content.clone(), Some(false)),
|
||||
"Script failed".to_string(),
|
||||
Duration::from_millis(/*millis*/ 750),
|
||||
host_duration,
|
||||
);
|
||||
output.set_handler_duration_ms(/*handler_duration_ms*/ 1_250);
|
||||
let mut expected_content = vec![FunctionCallOutputContentItem::InputText {
|
||||
text: format!("Script failed\n{expected_timing}\nOutput:\n"),
|
||||
}];
|
||||
expected_content.extend(content.clone());
|
||||
let expected = FunctionToolOutput::from_content(expected_content, Some(false));
|
||||
for payload in [
|
||||
ToolPayload::Custom {
|
||||
input: "text('result')".to_string(),
|
||||
},
|
||||
ToolPayload::Function {
|
||||
arguments: r#"{"cell_id":"1"}"#.to_string(),
|
||||
},
|
||||
] {
|
||||
let response = output.to_response_item("call", &payload);
|
||||
assert_eq!(response, expected.to_response_item("call", &payload));
|
||||
// Serialization must reuse the completed value, including on retries.
|
||||
assert_eq!(output.to_response_item("call", &payload), response);
|
||||
}
|
||||
assert_eq!(output.success_for_logging(), false);
|
||||
}
|
||||
}
|
||||
@@ -178,15 +178,13 @@ impl CodeModeWaitHandler {
|
||||
let wall_time = wait_response
|
||||
.code_mode_host_duration()
|
||||
.unwrap_or_else(|| started_at.elapsed());
|
||||
handle_runtime_response(
|
||||
Ok(boxed_tool_output(handle_runtime_response(
|
||||
&step_context.settings.model_info,
|
||||
wait_response.into(),
|
||||
args.max_tokens,
|
||||
wall_time,
|
||||
)
|
||||
.await
|
||||
.map_err(FunctionCallError::RespondToModel)
|
||||
.map(boxed_tool_output)
|
||||
exec.turn.config.code_mode.experimental_show_cell_overhead,
|
||||
)))
|
||||
}
|
||||
_ => Err(FunctionCallError::RespondToModel(format!(
|
||||
"{WAIT_TOOL_NAME} expects JSON arguments"
|
||||
|
||||
Reference in New Issue
Block a user