mirror of
https://github.com/openai/codex.git
synced 2026-11-11 21:10:39 +00:00
Sean Huang
edd1ae37ec
## Why Code Mode sessions always used client-side output handling, leaving server truncation settings and observation budgets unused. ## What changed - Honor each session's `output_truncation` setting in the runtime and both host transports, and advertise the bounded-output capability. - Apply `max_output_tokens` independently to execute, wait, and terminate responses in server mode. Keep client mode as the default when the setting is omitted. - Use bounded output collection and rendering in server mode, preserving head and tail output with omission notices. Include script errors in the output budget while retaining failure status and a separate diagnostic capped at 64 KiB. - Estimate audio tokens without the Tokio-dependent cache when rendering bounded output. ## Testing Add runtime and host regression tests for session mode selection, per-request budgets, output overflow, oversized images, zero-token audio output, diagnostic preservation, and termination under output backpressure. GitOrigin-RevId: 3d9661edcdd2dde31526bb92eb83fa0614f0fa88
341 lines
11 KiB
Rust
341 lines
11 KiB
Rust
//! Real-runtime regressions for bounded output, observation and termination.
|
|
|
|
use std::sync::Arc;
|
|
use std::time::Duration;
|
|
|
|
use codex_code_mode_protocol::ExecuteRequest;
|
|
use codex_code_mode_protocol::FunctionCallOutputContentItem;
|
|
use codex_code_mode_protocol::NoopCodeModeSessionDelegate;
|
|
use codex_code_mode_protocol::RuntimeResponse;
|
|
use codex_code_mode_protocol::WaitOutcome;
|
|
use codex_code_mode_protocol::WaitRequest;
|
|
use pretty_assertions::assert_eq;
|
|
|
|
use super::InProcessCodeModeSession;
|
|
use crate::output::MAX_OUTPUT_ITEMS;
|
|
use crate::output::MAX_SERVER_DIAGNOSTIC_BYTES;
|
|
use codex_code_mode_protocol::CodeModeOutputTruncation;
|
|
use codex_code_mode_protocol::CodeModeSessionOptions;
|
|
use codex_code_mode_protocol::host::BOUNDED_OUTPUT_MAX_BYTES as MAX_OUTPUT_BYTES;
|
|
use codex_code_mode_protocol::host::BOUNDED_OUTPUT_MAX_IMAGE_BYTES as MAX_IMAGE_BYTES;
|
|
|
|
fn server_session() -> InProcessCodeModeSession {
|
|
InProcessCodeModeSession::with_options(
|
|
CodeModeSessionOptions {
|
|
output_truncation: CodeModeOutputTruncation::Server,
|
|
..CodeModeSessionOptions::default()
|
|
},
|
|
/*task_failure_handler*/ None,
|
|
)
|
|
}
|
|
|
|
async fn start(session: &InProcessCodeModeSession, source: &str) -> RuntimeResponse {
|
|
session
|
|
.execute(
|
|
ExecuteRequest {
|
|
tool_call_id: "bounded-output".to_owned(),
|
|
enabled_tools: Vec::new(),
|
|
source: source.to_owned(),
|
|
yield_time_ms: Some(60_000),
|
|
// These tests isolate collector overflow from the per-observation
|
|
// formatter: request enough room to display every retained item.
|
|
max_output_tokens: Some(MAX_OUTPUT_BYTES),
|
|
},
|
|
Arc::new(NoopCodeModeSessionDelegate),
|
|
/*preempt*/ None,
|
|
)
|
|
.await
|
|
.unwrap()
|
|
.initial_response()
|
|
.await
|
|
.unwrap()
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn bounded_output_keeps_head_tail_without_stopping_the_cell() {
|
|
let session = server_session();
|
|
let emitted = MAX_OUTPUT_ITEMS + 100;
|
|
let response = start(
|
|
&session,
|
|
&format!("text('begin'); for (let i = 0; i < {emitted}; i++) text(i); text('end');"),
|
|
)
|
|
.await;
|
|
let RuntimeResponse::Result {
|
|
content_items,
|
|
error_text,
|
|
..
|
|
} = response
|
|
else {
|
|
panic!("expected completed cell");
|
|
};
|
|
assert_eq!(error_text, None);
|
|
assert_eq!(content_items.len(), MAX_OUTPUT_ITEMS);
|
|
assert_eq!(
|
|
content_items.first(),
|
|
Some(&FunctionCallOutputContentItem::InputText {
|
|
text: "begin".to_owned()
|
|
})
|
|
);
|
|
assert_eq!(
|
|
content_items.last(),
|
|
Some(&FunctionCallOutputContentItem::InputText {
|
|
text: "end".to_owned()
|
|
})
|
|
);
|
|
assert!(matches!(
|
|
&content_items[MAX_OUTPUT_ITEMS / 2],
|
|
FunctionCallOutputContentItem::InputText { text }
|
|
if text.starts_with("[Code Mode session output limit reached: omitted ")
|
|
));
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn huge_output_is_bounded_and_yield_drain_starts_a_new_budget() {
|
|
let session = server_session();
|
|
// Continue immediately after yielding to exercise the FIFO boundary as
|
|
// well as discarding the already-delivered bounded snapshot.
|
|
let response = start(
|
|
&session,
|
|
r#"text('😀'.repeat(1000000)); yield_control(); text('after yield');"#,
|
|
)
|
|
.await;
|
|
let RuntimeResponse::Yielded {
|
|
cell_id,
|
|
content_items,
|
|
..
|
|
} = response
|
|
else {
|
|
panic!("expected yielded cell");
|
|
};
|
|
assert_eq!(
|
|
content_items,
|
|
vec![
|
|
FunctionCallOutputContentItem::InputText {
|
|
text: "😀".repeat(MAX_OUTPUT_BYTES / 8)
|
|
},
|
|
FunctionCallOutputContentItem::InputText {
|
|
text: format!(
|
|
"[Code Mode session output limit reached: omitted {} text bytes; discarded 0 text segments, 0 images, 0 audio items]",
|
|
4_000_000 - MAX_OUTPUT_BYTES
|
|
)
|
|
},
|
|
FunctionCallOutputContentItem::InputText {
|
|
text: "😀".repeat(MAX_OUTPUT_BYTES / 8)
|
|
},
|
|
]
|
|
);
|
|
let response = session
|
|
.wait(
|
|
WaitRequest {
|
|
cell_id,
|
|
yield_time_ms: 60_000,
|
|
max_output_tokens: None,
|
|
},
|
|
/*preempt*/ None,
|
|
)
|
|
.await
|
|
.unwrap();
|
|
let WaitOutcome::LiveCell(RuntimeResponse::Result {
|
|
content_items,
|
|
error_text,
|
|
..
|
|
}) = response
|
|
else {
|
|
panic!("expected completed cell");
|
|
};
|
|
assert_eq!(
|
|
(content_items, error_text),
|
|
(
|
|
vec![FunctionCallOutputContentItem::InputText {
|
|
text: "after yield".to_owned()
|
|
}],
|
|
None
|
|
)
|
|
);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn a_single_oversized_image_is_omitted_without_losing_surrounding_text() {
|
|
let session = server_session();
|
|
let response = start(
|
|
&session,
|
|
&format!("text('before'); image('data:image/png;base64,' + 'A'.repeat({MAX_IMAGE_BYTES})); text('after');"),
|
|
)
|
|
.await;
|
|
let RuntimeResponse::Result {
|
|
content_items,
|
|
error_text,
|
|
..
|
|
} = response
|
|
else {
|
|
panic!("expected completed cell");
|
|
};
|
|
assert_eq!(
|
|
(content_items, error_text),
|
|
(
|
|
vec![
|
|
FunctionCallOutputContentItem::InputText {
|
|
text: "before".to_owned()
|
|
},
|
|
FunctionCallOutputContentItem::InputText {
|
|
text: "[Code Mode session output limit reached: omitted 0 text bytes; discarded 0 text segments, 1 images, 0 audio items]".to_owned()
|
|
},
|
|
FunctionCallOutputContentItem::InputText {
|
|
text: "after".to_owned()
|
|
}
|
|
],
|
|
None
|
|
)
|
|
);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn termination_releases_a_flooding_bounded_runtime() {
|
|
let session = server_session();
|
|
let emitted = MAX_OUTPUT_ITEMS + 100;
|
|
let response = start(
|
|
&session,
|
|
&format!(
|
|
"for (let i = 0; ; i++) {{
|
|
text('output');
|
|
if (i === {emitted}) yield_control();
|
|
}}"
|
|
),
|
|
)
|
|
.await;
|
|
let RuntimeResponse::Yielded {
|
|
cell_id,
|
|
content_items,
|
|
..
|
|
} = response
|
|
else {
|
|
panic!("expected yielded cell");
|
|
};
|
|
assert_eq!(content_items.len(), MAX_OUTPUT_ITEMS);
|
|
assert!(matches!(
|
|
&content_items[MAX_OUTPUT_ITEMS / 2],
|
|
FunctionCallOutputContentItem::InputText { text }
|
|
if text.starts_with("[Code Mode session output limit reached: omitted ")
|
|
));
|
|
let response = tokio::time::timeout(Duration::from_secs(5), session.terminate(cell_id.into()))
|
|
.await
|
|
.expect("termination must unblock output")
|
|
.unwrap();
|
|
assert!(matches!(
|
|
response,
|
|
WaitOutcome::LiveCell(RuntimeResponse::Terminated { .. })
|
|
));
|
|
session.shutdown().await.unwrap();
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn terminal_error_shares_the_collector_limit_and_remains_a_failure() {
|
|
let session = server_session();
|
|
let response = start(
|
|
&session,
|
|
"text('before'); throw 'error-start:' + 'x'.repeat(2 * 1024 * 1024) + ':error-end';",
|
|
)
|
|
.await;
|
|
let RuntimeResponse::Result {
|
|
content_items,
|
|
error_text,
|
|
..
|
|
} = response
|
|
else {
|
|
panic!("expected failed cell");
|
|
};
|
|
let diagnostic = error_text.expect("overflowed diagnostics still report failure");
|
|
// The model-output budget is generous, but the diagnostic has its own limit.
|
|
assert!(diagnostic.len() <= MAX_SERVER_DIAGNOSTIC_BYTES);
|
|
assert!(diagnostic.starts_with("error-start:"));
|
|
assert!(diagnostic.ends_with(":error-end"));
|
|
let notice = content_items
|
|
.iter()
|
|
.find_map(|item| match item {
|
|
FunctionCallOutputContentItem::InputText { text }
|
|
if text.starts_with("[Code Mode session output limit reached: omitted ") =>
|
|
{
|
|
Some(text)
|
|
}
|
|
_ => None,
|
|
})
|
|
.expect("terminal overflow notice");
|
|
let total_text_bytes: usize = content_items
|
|
.iter()
|
|
.map(|item| match item {
|
|
FunctionCallOutputContentItem::InputText { text } => text.len(),
|
|
_ => 0,
|
|
})
|
|
.sum();
|
|
assert!(total_text_bytes - notice.len() <= MAX_OUTPUT_BYTES);
|
|
assert!(matches!(content_items.last(),
|
|
Some(FunctionCallOutputContentItem::InputText { text }) if text.ends_with(":error-end")
|
|
));
|
|
session.shutdown().await.unwrap();
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn wait_preserves_diagnostics_across_output_budgets() {
|
|
let source = "yield_control(); text('before'); throw 'start:' + '🎷'.repeat(4096) + ':end';";
|
|
let raw_error = format!("start:{}:end", "🎷".repeat(4096));
|
|
for mode in [
|
|
CodeModeOutputTruncation::Client,
|
|
CodeModeOutputTruncation::Server,
|
|
] {
|
|
let session = InProcessCodeModeSession::with_options(
|
|
CodeModeSessionOptions {
|
|
output_truncation: mode,
|
|
..CodeModeSessionOptions::default()
|
|
},
|
|
/*task_failure_handler*/ None,
|
|
);
|
|
// The error fits the diagnostic limit but exceeds both output budgets.
|
|
for max_output_tokens in [0, 16] {
|
|
let response = start(&session, source).await;
|
|
let RuntimeResponse::Yielded { cell_id, .. } = response else {
|
|
panic!("expected yield before error");
|
|
};
|
|
let WaitOutcome::LiveCell(RuntimeResponse::Result {
|
|
content_items,
|
|
error_text,
|
|
..
|
|
}) = session
|
|
.wait(
|
|
WaitRequest {
|
|
cell_id,
|
|
yield_time_ms: 60_000,
|
|
max_output_tokens: Some(max_output_tokens),
|
|
},
|
|
/*preempt*/ None,
|
|
)
|
|
.await
|
|
.unwrap()
|
|
else {
|
|
panic!("expected completed cell");
|
|
};
|
|
let diagnostic = error_text.expect("errors remain failures at all budgets");
|
|
assert_eq!(diagnostic, raw_error);
|
|
match mode {
|
|
CodeModeOutputTruncation::Client => {
|
|
assert_eq!(
|
|
content_items,
|
|
vec![FunctionCallOutputContentItem::InputText {
|
|
text: "before".to_owned()
|
|
}]
|
|
);
|
|
}
|
|
CodeModeOutputTruncation::Server => {
|
|
if max_output_tokens != 0 {
|
|
assert!(matches!(
|
|
content_items.first(),
|
|
Some(FunctionCallOutputContentItem::InputText { text })
|
|
if text.starts_with("Warning: truncated output")
|
|
));
|
|
}
|
|
}
|
|
}
|
|
}
|
|
session.shutdown().await.unwrap();
|
|
}
|
|
}
|