mirror of
https://github.com/openai/codex.git
synced 2026-11-17 22:10:37 +00:00
Prewarm restored file images on cold resume (#61959)
## Why Cold resumes warm only base instructions, leaving historical file images out of the WebSocket warm-up. ## What changed - Add the disabled-by-default `restored_history_prewarm` feature flag. - Start the WebSocket handshake while history loads, then build the warm-up from restored settings and history when it contains file images. - Retain the socket and normalized-image cache but discard the image warm-up response ID, so generation resends the full current history and subsequent turns continue from the real response. - Keep base warm-up for history without file images and defer image preparation to generation when the handshake falls back to HTTP. ## Testing Add request-history tests and snapshots covering text-only history, interrupted and injected images, resuming with a vision model after image omission, and HTTP fallback without duplicate image resolution. GitOrigin-RevId: 9a828f44ef0952f7335b5a8c0f7844a069f90d80
This commit is contained in:
@@ -1325,6 +1325,9 @@
|
||||
"responses_websockets_v2": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"restored_history_prewarm": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"retain_client_developer_messages": {
|
||||
"type": "boolean"
|
||||
},
|
||||
@@ -8604,6 +8607,9 @@
|
||||
"responses_websockets_v2": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"restored_history_prewarm": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"retain_client_developer_messages": {
|
||||
"type": "boolean"
|
||||
},
|
||||
|
||||
@@ -2610,6 +2610,7 @@ impl ModelClientSession {
|
||||
summary: ReasoningSummaryConfig,
|
||||
service_tier: Option<String>,
|
||||
responses_metadata: &CodexResponsesMetadata,
|
||||
reuse_response: bool,
|
||||
) -> Result<()> {
|
||||
if !self.client.responses_websocket_enabled() {
|
||||
return Ok(());
|
||||
@@ -2637,6 +2638,13 @@ impl ModelClientSession {
|
||||
.await
|
||||
{
|
||||
Ok(WebsocketStreamOutcome::Stream(opened)) => {
|
||||
if !reuse_response {
|
||||
// Keep the socket and its normalized images. Drop the continuation
|
||||
// before waiting so cancellation cannot expose the warm-up ID.
|
||||
self.websocket_session.last_response_rx = None;
|
||||
self.websocket_session.continuation_reset_reason =
|
||||
Some("restored_history_warmup");
|
||||
}
|
||||
let mut stream = (*opened).into_raw_stream()?;
|
||||
// Wait for the v2 warmup request to complete before sending the first turn request.
|
||||
while let Some(event) = stream.next().await {
|
||||
|
||||
@@ -2134,8 +2134,15 @@ impl Session {
|
||||
&inherited_environments,
|
||||
&session_configuration.environments,
|
||||
);
|
||||
sess.schedule_startup_prewarm(super::startup_prewarm::PrewarmInput::Base)
|
||||
.await;
|
||||
let (input, history_ready) = if sess.features().enabled(Feature::RestoredHistoryPrewarm)
|
||||
&& matches!(&initial_history, InitialHistory::Resumed(saved) if !saved.history.is_empty())
|
||||
{
|
||||
let (sender, receiver) = tokio::sync::oneshot::channel();
|
||||
(super::startup_prewarm::PrewarmInput::Restoring(receiver), Some(sender))
|
||||
} else {
|
||||
(super::startup_prewarm::PrewarmInput::Base, None)
|
||||
};
|
||||
sess.schedule_startup_prewarm(input).await;
|
||||
let session_start_source = match &initial_history {
|
||||
InitialHistory::Forked(_) if forked_from_id.is_some() => {
|
||||
codex_hooks::SessionStartSource::Fork
|
||||
@@ -2167,6 +2174,9 @@ impl Session {
|
||||
let mut state = sess.state.lock().await;
|
||||
state.queue_pending_session_start_source(session_start_source);
|
||||
}
|
||||
if let Some(sender) = history_ready {
|
||||
let _ = sender.send(sess.clone_history().await);
|
||||
}
|
||||
Ok(sess)
|
||||
})
|
||||
.await;
|
||||
|
||||
@@ -30,18 +30,24 @@ use codex_otel::STARTUP_PREWARM_DURATION_METRIC;
|
||||
use codex_otel::SessionTelemetry;
|
||||
use codex_protocol::error::Result as CodexResult;
|
||||
use codex_protocol::models::BaseInstructions;
|
||||
use codex_protocol::models::ContentItem;
|
||||
use codex_protocol::models::FunctionCallOutputContentItem;
|
||||
use codex_protocol::models::ImageReference;
|
||||
use codex_protocol::models::ResponseItem;
|
||||
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
#[derive(Debug)]
|
||||
pub(crate) enum PrewarmInput {
|
||||
Base,
|
||||
History,
|
||||
Restoring(tokio::sync::oneshot::Receiver<crate::context_manager::ContextManager>),
|
||||
}
|
||||
|
||||
impl PrewarmInput {
|
||||
fn as_str(self) -> &'static str {
|
||||
fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
Self::Base => "base",
|
||||
Self::History => "history",
|
||||
Self::Restoring(_) => "restored_history",
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -255,6 +261,7 @@ impl Session {
|
||||
let session_telemetry = self.services.session_telemetry.clone();
|
||||
let started_at = Instant::now();
|
||||
let startup_prewarm_session = Arc::clone(self);
|
||||
let input_label = input.as_str();
|
||||
let startup_prewarm = tokio::spawn(
|
||||
async move {
|
||||
let result = schedule_startup_prewarm_inner(startup_prewarm_session, input).await;
|
||||
@@ -267,7 +274,7 @@ impl Session {
|
||||
session_telemetry.record_duration(
|
||||
STARTUP_PREWARM_DURATION_METRIC,
|
||||
started_at.elapsed(),
|
||||
&[("status", status), ("input", input.as_str())],
|
||||
&[("status", status), ("input", input_label)],
|
||||
);
|
||||
result
|
||||
}
|
||||
@@ -275,7 +282,7 @@ impl Session {
|
||||
"startup_prewarm",
|
||||
otel.name = "startup_prewarm",
|
||||
thread.id = %self.thread_id(),
|
||||
prewarm.input = input.as_str(),
|
||||
prewarm.input = input_label,
|
||||
)),
|
||||
);
|
||||
state.set_session_startup_prewarm(SessionStartupPrewarmHandle::new(
|
||||
@@ -302,6 +309,35 @@ impl Session {
|
||||
}
|
||||
}
|
||||
|
||||
fn has_file_image(item: &ResponseItem) -> bool {
|
||||
match item {
|
||||
ResponseItem::Message { content, .. } => content.iter().any(|part| {
|
||||
matches!(
|
||||
part,
|
||||
ContentItem::InputImage {
|
||||
image: ImageReference::File { .. },
|
||||
..
|
||||
}
|
||||
)
|
||||
}),
|
||||
ResponseItem::FunctionCallOutput { output, .. }
|
||||
| ResponseItem::CustomToolCallOutput { output, .. } => {
|
||||
output.content_items().is_some_and(|content| {
|
||||
content.iter().any(|part| {
|
||||
matches!(
|
||||
part,
|
||||
FunctionCallOutputContentItem::InputImage {
|
||||
image: ImageReference::File { .. },
|
||||
..
|
||||
}
|
||||
)
|
||||
})
|
||||
})
|
||||
}
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
async fn schedule_startup_prewarm_inner(
|
||||
session: Arc<Session>,
|
||||
input: PrewarmInput,
|
||||
@@ -328,7 +364,14 @@ async fn schedule_startup_prewarm_inner(
|
||||
if websocket_ready {
|
||||
return Ok(client_session);
|
||||
}
|
||||
let base_instructions = session.get_prompt_base_instructions().await.text;
|
||||
// Existing warm-ups snapshot instructions before building the turn context.
|
||||
// Only a cold-restore preconnect must defer this read until history is ready.
|
||||
let instructions = session.get_prompt_base_instructions();
|
||||
let instructions = if matches!(input, PrewarmInput::Restoring(_)) {
|
||||
instructions.left_future()
|
||||
} else {
|
||||
futures::future::ready(instructions.await).right_future()
|
||||
};
|
||||
let startup_turn_context = session
|
||||
.new_startup_prewarm_turn_with_sub_id(INITIAL_SUBMIT_ID.to_owned())
|
||||
.await;
|
||||
@@ -374,21 +417,36 @@ async fn schedule_startup_prewarm_inner(
|
||||
context_window_id,
|
||||
);
|
||||
// Start the handshake with the expected route while capturing tools for generate=false.
|
||||
let (step_context, ()) = tokio::try_join!(
|
||||
let restoring = matches!(input, PrewarmInput::Restoring(_));
|
||||
let loaded_history = matches!(input, PrewarmInput::History);
|
||||
let ((step_context, base_instructions, restored_history), ()) = tokio::try_join!(
|
||||
async {
|
||||
// Restoration can change settings and the context window. Only the handshake
|
||||
// may use their provisional values.
|
||||
let (prompt_context, restored_history) = if let PrewarmInput::Restoring(ready) = input {
|
||||
let snapshot = ready
|
||||
.await
|
||||
.map_err(|_| codex_protocol::error::CodexErr::TurnAborted)?;
|
||||
(
|
||||
session
|
||||
.new_startup_prewarm_turn_with_sub_id(INITIAL_SUBMIT_ID.to_owned())
|
||||
.await,
|
||||
Some(snapshot),
|
||||
)
|
||||
} else {
|
||||
(Arc::clone(&startup_turn_context), None)
|
||||
};
|
||||
let base_instructions = instructions.await.text;
|
||||
let built_tools_started_at = Instant::now();
|
||||
let step_context = session
|
||||
.capture_step_context(
|
||||
Arc::clone(&startup_turn_context),
|
||||
&startup_cancellation_token,
|
||||
)
|
||||
.capture_step_context(prompt_context, &startup_cancellation_token)
|
||||
.await?;
|
||||
startup_turn_context.session_telemetry.record_startup_phase(
|
||||
"startup_prewarm_build_tools",
|
||||
built_tools_started_at.elapsed(),
|
||||
/*status*/ None,
|
||||
);
|
||||
Ok(step_context)
|
||||
Ok((step_context, base_instructions, restored_history))
|
||||
},
|
||||
client_session.preconnect_websocket(
|
||||
&preconnect_model_info,
|
||||
@@ -397,27 +455,40 @@ async fn schedule_startup_prewarm_inner(
|
||||
&handshake_metadata,
|
||||
),
|
||||
)?;
|
||||
// A 426 disables WebSockets for the session. Generation will prepare its own HTTP input.
|
||||
if restoring && !session.services.model_client.responses_websocket_enabled() {
|
||||
return Ok(client_session);
|
||||
}
|
||||
let build_prompt_started_at = Instant::now();
|
||||
let prompt_input = match input {
|
||||
PrewarmInput::Base => Vec::new(),
|
||||
PrewarmInput::History => {
|
||||
// Use the same history projection and tool metadata as a sampling request.
|
||||
// The real turn still checks this prefix before reusing the prepared response.
|
||||
let mut history = session
|
||||
.clone_history()
|
||||
.await
|
||||
.for_prompt(
|
||||
&step_context.settings.model_info.input_modalities,
|
||||
session.services.image_store.as_ref(),
|
||||
)
|
||||
.await;
|
||||
session
|
||||
.services
|
||||
.executed_tool_calls
|
||||
.attach_to_prompt(&mut history, &mut HashMap::new());
|
||||
history
|
||||
}
|
||||
let history = if loaded_history {
|
||||
Some(session.clone_history().await)
|
||||
} else {
|
||||
restored_history
|
||||
};
|
||||
let prompt_input = if let Some(snapshot) = history {
|
||||
// Use the same history projection and tool metadata as a sampling request.
|
||||
let mut history = snapshot
|
||||
.for_prompt(
|
||||
&step_context.settings.model_info.input_modalities,
|
||||
session.services.image_store.as_ref(),
|
||||
)
|
||||
.await;
|
||||
session
|
||||
.services
|
||||
.executed_tool_calls
|
||||
.attach_to_prompt(&mut history, &mut HashMap::new());
|
||||
history
|
||||
} else {
|
||||
Vec::new()
|
||||
};
|
||||
// Only file images can benefit from the socket's normalized-image cache.
|
||||
let warm_restored_images = restoring && prompt_input.iter().any(has_file_image);
|
||||
let prompt_input = if restoring && !warm_restored_images {
|
||||
Vec::new()
|
||||
} else {
|
||||
prompt_input
|
||||
};
|
||||
let reuse_response = !warm_restored_images;
|
||||
let startup_prompt = build_prompt(
|
||||
prompt_input,
|
||||
step_context.as_ref(),
|
||||
@@ -451,6 +522,7 @@ async fn schedule_startup_prewarm_inner(
|
||||
step_context.settings.reasoning_summary,
|
||||
step_context.settings.service_tier.clone(),
|
||||
&responses_metadata,
|
||||
reuse_response,
|
||||
)
|
||||
.await?;
|
||||
startup_turn_context.session_telemetry.record_startup_phase(
|
||||
|
||||
@@ -791,6 +791,7 @@ async fn responses_websocket_request_prewarm_reuses_connection() {
|
||||
harness.summary,
|
||||
/*service_tier*/ None,
|
||||
&responses_metadata,
|
||||
/*reuse_response*/ true,
|
||||
)
|
||||
.await
|
||||
.expect("websocket prewarm failed");
|
||||
@@ -888,6 +889,7 @@ async fn responses_websocket_request_prewarm_uses_caller_supplied_metadata() {
|
||||
harness.summary,
|
||||
/*service_tier*/ None,
|
||||
&responses_metadata,
|
||||
/*reuse_response*/ true,
|
||||
)
|
||||
.await
|
||||
.expect("websocket prewarm failed");
|
||||
@@ -933,6 +935,7 @@ async fn responses_websocket_request_prewarm_traces_logical_request() {
|
||||
harness.summary,
|
||||
/*service_tier*/ None,
|
||||
&prewarm_responses_metadata,
|
||||
/*reuse_response*/ true,
|
||||
)
|
||||
.await
|
||||
.expect("websocket prewarm failed");
|
||||
@@ -1397,6 +1400,7 @@ async fn responses_websocket_prewarm_reuses_advisory_model_and_tier_routing_hint
|
||||
harness.summary,
|
||||
warmup_tier.map(str::to_string),
|
||||
&responses_metadata,
|
||||
/*reuse_response*/ true,
|
||||
)
|
||||
.await
|
||||
.expect("websocket prewarm failed");
|
||||
|
||||
@@ -135,6 +135,9 @@ mod dynamic_tools;
|
||||
#[path = "scenarios_audio_file_tests.rs"]
|
||||
mod audio_file;
|
||||
|
||||
#[path = "scenarios_restored_history_prewarm.rs"]
|
||||
mod restored_history_prewarm;
|
||||
|
||||
#[path = "scenarios_retained_tool_outputs.rs"]
|
||||
mod retained_tool_outputs;
|
||||
|
||||
|
||||
@@ -0,0 +1,315 @@
|
||||
//! Complete request histories for cold resumes with historical images.
|
||||
|
||||
use super::*;
|
||||
use codex_protocol::models::ResponseInputItem;
|
||||
use core_test_support::responses;
|
||||
use core_test_support::responses::WebSocketConnectionConfig;
|
||||
use core_test_support::responses::WebSocketTestServer;
|
||||
use core_test_support::responses::start_websocket_server_with_headers;
|
||||
use pretty_assertions::assert_eq;
|
||||
|
||||
fn image_input(id: &str) -> UserInput {
|
||||
UserInput::Image {
|
||||
image: ImageReference::File { file_id: id.into() },
|
||||
detail: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn socket(warmup: &str, delay: Option<Duration>) -> WebSocketConnectionConfig {
|
||||
WebSocketConnectionConfig {
|
||||
requests: vec![
|
||||
vec![ev_response_created(warmup), ev_completed(warmup)],
|
||||
vec![
|
||||
ev_response_created("generation"),
|
||||
ev_assistant_message("msg_seen", "seen"),
|
||||
ev_completed("generation"),
|
||||
],
|
||||
],
|
||||
response_headers: vec![],
|
||||
accept_delay: delay,
|
||||
close_after_requests: true,
|
||||
}
|
||||
}
|
||||
|
||||
async fn saved_image_thread(
|
||||
server: &WebSocketTestServer,
|
||||
store: Arc<RecordingFileAttachmentStore>,
|
||||
) -> anyhow::Result<TestCodex> {
|
||||
let test = test_codex()
|
||||
.with_image_store(store)
|
||||
.with_config(|config| {
|
||||
config
|
||||
.features
|
||||
.enable(Feature::RestoredHistoryPrewarm)
|
||||
.expect("enable restored history prewarm");
|
||||
})
|
||||
.build_with_websocket_server(server)
|
||||
.await?;
|
||||
test.codex
|
||||
.start_or_steer_turn(TurnInputRequest::user_input(vec![image_input(
|
||||
"file_prior",
|
||||
)]))
|
||||
.await?;
|
||||
wait_for_event(&test.codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await;
|
||||
Ok(test)
|
||||
}
|
||||
|
||||
async fn reopen(test: &mut TestCodex, base_url: Option<String>) -> anyhow::Result<()> {
|
||||
let path = test.codex.rollout_path().expect("saved thread rollout");
|
||||
test.codex.shutdown_and_wait().await?;
|
||||
let mut options = codex_core::StartThreadOptions::new(test.config.clone());
|
||||
if let Some(url) = base_url {
|
||||
options.config.model_provider.base_url = Some(url);
|
||||
}
|
||||
options.initial_history = codex_rollout::RolloutRecorder::get_rollout_history(&path).await?;
|
||||
test.codex = test.thread_manager.start_thread(options).await?.thread;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn text_history_uses_base_warmup_and_sends_history_once() -> anyhow::Result<()> {
|
||||
skip_if_no_network!(Ok(()));
|
||||
let server = start_websocket_server_with_headers(vec![
|
||||
socket("initial", /*delay*/ None),
|
||||
socket("base", /*delay*/ None),
|
||||
])
|
||||
.await;
|
||||
let mut test = test_codex()
|
||||
.with_config(|config| {
|
||||
config
|
||||
.features
|
||||
.enable(Feature::RestoredHistoryPrewarm)
|
||||
.expect("enable restored history prewarm");
|
||||
})
|
||||
.build_with_websocket_server(&server)
|
||||
.await?;
|
||||
test.submit_text_turn("saved text marker").await?;
|
||||
reopen(&mut test, /*base_url*/ None).await?;
|
||||
test.submit_text_turn("continue").await?;
|
||||
|
||||
let connections = server.connections();
|
||||
let [warm, generation] = connections[1].as_slice() else {
|
||||
panic!("expected warm-up and generation");
|
||||
};
|
||||
assert_eq!(warm.body_json()["generate"], false);
|
||||
assert_eq!(
|
||||
warm.body_json()["input"],
|
||||
connections[0][0].body_json()["input"]
|
||||
);
|
||||
let generation = generation.body_json();
|
||||
assert_eq!(generation["previous_response_id"], "base");
|
||||
assert!(
|
||||
generation["input"]
|
||||
.to_string()
|
||||
.contains("saved text marker")
|
||||
);
|
||||
test.codex.shutdown_and_wait().await?;
|
||||
server.shutdown().await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn warmup_includes_interrupted_image_and_generation_adds_injected_image() -> anyhow::Result<()>
|
||||
{
|
||||
skip_if_no_network!(Ok(()));
|
||||
let server = start_websocket_server_with_headers(vec![
|
||||
socket("initial", /*delay*/ None),
|
||||
socket("held", /*delay*/ None),
|
||||
socket("restored", Some(Duration::from_millis(500))),
|
||||
])
|
||||
.await;
|
||||
let mut test = saved_image_thread(&server, Arc::default()).await?;
|
||||
let release = server.gate_response_event(&ev_completed("held"));
|
||||
reopen(&mut test, /*base_url*/ None).await?;
|
||||
tokio::time::timeout(
|
||||
Duration::from_secs(15),
|
||||
server.wait_for_request(/*connection_index*/ 1, /*request_index*/ 0),
|
||||
)
|
||||
.await?;
|
||||
test.codex
|
||||
.start_or_steer_turn(TurnInputRequest::user_input(vec![image_input(
|
||||
"file_interrupted",
|
||||
)]))
|
||||
.await?;
|
||||
wait_for_event(&test.codex, |ev| matches!(ev, EventMsg::TurnStarted(_))).await;
|
||||
test.codex.submit(Op::Interrupt).await?;
|
||||
wait_for_event(&test.codex, |ev| matches!(ev, EventMsg::TurnAborted(_))).await;
|
||||
let _ = release.send(());
|
||||
reopen(&mut test, /*base_url*/ None).await?;
|
||||
test.codex
|
||||
.inject_response_items(vec![ResponseItem::from(ResponseInputItem::from(vec![
|
||||
image_input("file_injected"),
|
||||
]))])
|
||||
.await?;
|
||||
test.submit_text_turn("continue").await?;
|
||||
|
||||
let connections = server.connections();
|
||||
assert_eq!(connections[1].len(), 1, "interrupt must precede generation");
|
||||
let [warm, generation] = connections[2].as_slice() else {
|
||||
panic!("expected warm-up and generation");
|
||||
};
|
||||
assert_eq!(warm.body_json()["generate"], false);
|
||||
// The snapshot formatter summarizes image slots, so check their file IDs here.
|
||||
let warm = warm.body_json()["input"].to_string();
|
||||
assert!(warm.contains("file_prior"));
|
||||
assert!(warm.contains("file_interrupted"));
|
||||
assert!(!warm.contains("file_injected"));
|
||||
let generation = generation.body_json();
|
||||
assert!(generation.get("previous_response_id").is_none());
|
||||
let items = generation["input"].as_array().expect("generation input");
|
||||
let image_position = |id: &str| {
|
||||
items
|
||||
.iter()
|
||||
.position(|item| {
|
||||
item["content"]
|
||||
.as_array()
|
||||
.is_some_and(|parts| parts.iter().any(|part| part["file_id"] == id))
|
||||
})
|
||||
.expect("submitted image")
|
||||
};
|
||||
let submit = items
|
||||
.iter()
|
||||
.position(|item| {
|
||||
item["role"] == "user"
|
||||
&& item["content"]
|
||||
.as_array()
|
||||
.is_some_and(|parts| parts.iter().any(|part| part["text"] == "continue"))
|
||||
})
|
||||
.expect("current user message");
|
||||
assert!(image_position("file_prior") < image_position("file_interrupted"));
|
||||
assert!(image_position("file_interrupted") < image_position("file_injected"));
|
||||
assert!(image_position("file_injected") < submit);
|
||||
|
||||
let bodies: Vec<_> = connections
|
||||
.iter()
|
||||
.flatten()
|
||||
.map(responses::WebSocketRequest::body_json)
|
||||
.collect();
|
||||
let labels = [
|
||||
"initial warm-up",
|
||||
"original image submitted",
|
||||
"warm-up before interrupt",
|
||||
"resumed warm-up",
|
||||
"generation after injection",
|
||||
];
|
||||
assert_eq!(bodies.len(), labels.len());
|
||||
let entries = bodies
|
||||
.iter()
|
||||
.zip(labels)
|
||||
.map(|(body, label)| SnapshotEntry::body(body).labeled(label))
|
||||
.collect::<Vec<_>>();
|
||||
insta::assert_snapshot!(context_snapshot::format_context_snapshot(
|
||||
"Warm-up includes the previously sampled image and the restored interrupted submit. Generation resends both images and adds the image injected after resume.",
|
||||
&entries,
|
||||
&ContextSnapshotOptions::default()
|
||||
.rewrite_known_segments()
|
||||
.include_request_settings(),
|
||||
));
|
||||
test.codex.shutdown_and_wait().await?;
|
||||
server.shutdown().await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn upgrade_required_prepares_saved_image_only_for_http_generation() -> anyhow::Result<()> {
|
||||
skip_if_no_network!(Ok(()));
|
||||
use wiremock::Mock;
|
||||
use wiremock::ResponseTemplate;
|
||||
use wiremock::matchers::method;
|
||||
use wiremock::matchers::path_regex;
|
||||
let server = start_websocket_server_with_headers(vec![socket("initial", /*delay*/ None)]).await;
|
||||
let store = Arc::new(RecordingFileAttachmentStore::default());
|
||||
let mut test = saved_image_thread(&server, Arc::clone(&store)).await?;
|
||||
let http = responses::start_mock_server().await;
|
||||
Mock::given(method("GET"))
|
||||
.and(path_regex(".*/responses$"))
|
||||
.respond_with(ResponseTemplate::new(426))
|
||||
.expect(1)
|
||||
.mount(&http)
|
||||
.await;
|
||||
let post = responses::mount_sse_once(
|
||||
&http,
|
||||
responses::sse(vec![ev_response_created("http"), ev_completed("http")]),
|
||||
)
|
||||
.await;
|
||||
store.resolves.lock().unwrap().clear();
|
||||
reopen(&mut test, Some(format!("{}/v1", http.uri()))).await?;
|
||||
test.submit_text_turn("continue").await?;
|
||||
let body = post.single_request().body_json();
|
||||
assert_ne!(body["generate"], false);
|
||||
assert!(body["input"].to_string().contains("file_prior"));
|
||||
assert_eq!(store.resolves.lock().unwrap().get("file_prior"), Some(&1));
|
||||
test.codex.shutdown_and_wait().await?;
|
||||
server.shutdown().await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn vision_resume_resends_previously_omitted_image_then_continues_from_generation()
|
||||
-> Result<()> {
|
||||
skip_if_no_network!(Ok(()));
|
||||
let mut resumed = socket("vision", /*delay*/ None);
|
||||
resumed.requests.push(vec![
|
||||
ev_response_created("next"),
|
||||
ev_assistant_message("msg_next", "seen again"),
|
||||
ev_completed("next"),
|
||||
]);
|
||||
let server =
|
||||
start_websocket_server_with_headers(vec![socket("text", /*delay*/ None), resumed]).await;
|
||||
let mut test = test_codex()
|
||||
.with_model_info_override("text-only-prewarm-test", |model| {
|
||||
model.input_modalities = vec![InputModality::Text]
|
||||
})
|
||||
.with_model("text-only-prewarm-test")
|
||||
.with_config(|config| {
|
||||
config
|
||||
.features
|
||||
.enable(Feature::RestoredHistoryPrewarm)
|
||||
.expect("enable restored history prewarm");
|
||||
})
|
||||
.build_with_websocket_server(&server)
|
||||
.await?;
|
||||
test.codex
|
||||
.start_or_steer_turn(TurnInputRequest::user_input(vec![image_input(
|
||||
"file_omitted",
|
||||
)]))
|
||||
.await?;
|
||||
wait_for_event(&test.codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await;
|
||||
test.config.model = Some("gpt-5.5".into());
|
||||
reopen(&mut test, /*base_url*/ None).await?;
|
||||
test.submit_text_turn("Describe the image.").await?;
|
||||
test.submit_text_turn("continue").await?;
|
||||
|
||||
let connections = server.connections();
|
||||
let bodies: Vec<_> = connections
|
||||
.iter()
|
||||
.flatten()
|
||||
.map(responses::WebSocketRequest::body_json)
|
||||
.collect();
|
||||
assert_eq!(connections.len(), 2);
|
||||
assert_eq!(connections[1].len(), 3);
|
||||
assert_eq!(bodies.len(), 5);
|
||||
assert!(
|
||||
bodies[1]["input"]
|
||||
.to_string()
|
||||
.contains("image content omitted")
|
||||
);
|
||||
assert!(!bodies[1]["input"].to_string().contains("file_omitted"));
|
||||
assert_eq!(bodies[2]["model"], "gpt-5.5");
|
||||
assert!(bodies[2]["input"].to_string().contains("file_omitted"));
|
||||
assert!(bodies[3]["input"].to_string().contains("file_omitted"));
|
||||
assert!(bodies[3].get("previous_response_id").is_none());
|
||||
assert_eq!(bodies[4]["previous_response_id"], "generation");
|
||||
assert!(!bodies[4]["input"].to_string().contains("file_omitted"));
|
||||
let entries = bodies.iter().map(SnapshotEntry::body).collect::<Vec<_>>();
|
||||
insta::assert_snapshot!(context_snapshot::format_context_snapshot(
|
||||
"A text-only model sees a placeholder. A vision resume warms the image and submits it again on generation. The following turn continues from the real response.",
|
||||
&entries,
|
||||
&ContextSnapshotOptions::default()
|
||||
.rewrite_known_segments()
|
||||
.include_request_settings(),
|
||||
));
|
||||
test.codex.shutdown_and_wait().await?;
|
||||
server.shutdown().await;
|
||||
Ok(())
|
||||
}
|
||||
+97
@@ -0,0 +1,97 @@
|
||||
---
|
||||
source: core/tests/suite/scenarios_restored_history_prewarm.rs
|
||||
expression: "context_snapshot::format_context_snapshot(\"A text-only model sees a placeholder. A vision resume warms the image and submits it again on generation. The following turn continues from the real response.\",\n&entries,\n&ContextSnapshotOptions::default().rewrite_known_segments().include_request_settings(),)"
|
||||
---
|
||||
Scenario: A text-only model sees a placeholder. A vision resume warms the image and submits it again on generation. The following turn continues from the real response.
|
||||
|
||||
## Window 1
|
||||
Settings:
|
||||
generate: false
|
||||
include: ["reasoning.encrypted_content"]
|
||||
model: "text-only-prewarm-test"
|
||||
parallel_tool_calls: true
|
||||
prompt_cache_key: "<PROMPT_CACHE_KEY 1>"
|
||||
reasoning: {"effort":"medium"}
|
||||
store: false
|
||||
stream: true
|
||||
text: {"verbosity":"low"}
|
||||
tool_choice: "auto"
|
||||
tools (7; hash=b1cff986b4ecf851):
|
||||
- function/exec_command: Runs a command in a PTY, returning output or a session ID for ongoing interaction.; args=[cmd, justification, login, max_output_tokens, prefix_rule, sandbox_permissions, shell, tty, workdir, yield_time_ms]; hash=24f4b6739dd5d940
|
||||
- function/write_stdin: Writes characters to an existing unified exec session and returns recent output.; args=[chars, max_output_tokens, session_id, yield_time_ms]; hash=7fb62d8a3e2df4e2
|
||||
- function/request_user_input: Request user input for one to three short questions and wait for the response. This tool is only ...; args=[questions]; hash=64a729b25c424fc6
|
||||
- custom/apply_patch: The `apply_patch` tool can be used to edit files. This is a FREEFORM tool, so do not wrap the pat...; hash=dea9f12b0111a4ba
|
||||
- function/view_image: View a local image file from the filesystem when visual inspection is needed. Use this for images...; args=[path]; hash=1908de9c0be7118a
|
||||
- tool_search: # Tool discovery Searches over deferred tool metadata with BM25 and exposes matching tools for t...; args=[limit, query]; hash=b573ed69c0b0e6c7
|
||||
- web_search; hash=f103e327c6870a46
|
||||
type: "response.create"
|
||||
-- request 1 (request) --
|
||||
00:message/developer:
|
||||
<MODEL_INSTRUCTIONS>
|
||||
|
||||
## Window 2 (after request 1: input diverged at item 00)
|
||||
Settings: relative to window 1
|
||||
generate: <removed>
|
||||
previous_response_id: "text"
|
||||
-- request 2 (request) --
|
||||
00:message/developer[2]:
|
||||
[01] <PERMISSIONS_INSTRUCTIONS>
|
||||
[02] <tools>
|
||||
Deferred tool namespaces:
|
||||
- multi_agent_v1: Tools for spawning and managing sub-agents.
|
||||
</tools>
|
||||
01:message/user:
|
||||
<ENVIRONMENT_CONTEXT>
|
||||
02:message/user:
|
||||
image content omitted because you do not support image input
|
||||
|
||||
## Window 3 (after request 2: input diverged at item 00)
|
||||
Settings: relative to window 2
|
||||
generate: false
|
||||
model: "gpt-5.5"
|
||||
previous_response_id: <removed>
|
||||
-- request 3 (request) --
|
||||
00:message/developer:
|
||||
<MODEL_INSTRUCTIONS>
|
||||
01:message/developer[2]:
|
||||
[01] <PERMISSIONS_INSTRUCTIONS>
|
||||
[02] <tools>
|
||||
Deferred tool namespaces:
|
||||
- multi_agent_v1: Tools for spawning and managing sub-agents.
|
||||
</tools>
|
||||
02:message/user:
|
||||
<ENVIRONMENT_CONTEXT>
|
||||
03:message/user:
|
||||
<input_image:detail,file_id>
|
||||
04:message/assistant:
|
||||
seen
|
||||
|
||||
## Window 4 (after request 3)
|
||||
Settings: relative to window 3
|
||||
generate: <removed>
|
||||
-- request 4 (request) --
|
||||
00:message/developer:
|
||||
<MODEL_INSTRUCTIONS>
|
||||
01:message/developer[2]:
|
||||
[01] <PERMISSIONS_INSTRUCTIONS>
|
||||
[02] <tools>
|
||||
Deferred tool namespaces:
|
||||
- multi_agent_v1: Tools for spawning and managing sub-agents.
|
||||
</tools>
|
||||
02:message/user:
|
||||
<ENVIRONMENT_CONTEXT>
|
||||
03:message/user:
|
||||
<input_image:detail,file_id>
|
||||
04:message/assistant:
|
||||
seen
|
||||
05:message/developer:
|
||||
<MODEL_SWITCH>
|
||||
06:message/user:
|
||||
Describe the image.
|
||||
|
||||
## Window 5 (after request 4: input diverged at item 00)
|
||||
Settings: relative to window 4
|
||||
previous_response_id: "generation"
|
||||
-- request 5 (request) --
|
||||
00:message/user:
|
||||
continue
|
||||
+100
@@ -0,0 +1,100 @@
|
||||
---
|
||||
source: core/tests/suite/scenarios_restored_history_prewarm.rs
|
||||
expression: "context_snapshot::format_context_snapshot(\"Warm-up includes the previously sampled image and the restored interrupted submit. Generation resends both images and adds the image injected after resume.\",\n&entries,\n&ContextSnapshotOptions::default().rewrite_known_segments().include_request_settings(),)"
|
||||
---
|
||||
Scenario: Warm-up includes the previously sampled image and the restored interrupted submit. Generation resends both images and adds the image injected after resume.
|
||||
|
||||
## Window 1
|
||||
Settings:
|
||||
generate: false
|
||||
include: ["reasoning.encrypted_content"]
|
||||
model: "gpt-5.5"
|
||||
parallel_tool_calls: true
|
||||
prompt_cache_key: "<PROMPT_CACHE_KEY 1>"
|
||||
reasoning: {"effort":"medium"}
|
||||
store: false
|
||||
stream: true
|
||||
text: {"verbosity":"low"}
|
||||
tool_choice: "auto"
|
||||
tools (7; hash=b1cff986b4ecf851):
|
||||
- function/exec_command: Runs a command in a PTY, returning output or a session ID for ongoing interaction.; args=[cmd, justification, login, max_output_tokens, prefix_rule, sandbox_permissions, shell, tty, workdir, yield_time_ms]; hash=24f4b6739dd5d940
|
||||
- function/write_stdin: Writes characters to an existing unified exec session and returns recent output.; args=[chars, max_output_tokens, session_id, yield_time_ms]; hash=7fb62d8a3e2df4e2
|
||||
- function/request_user_input: Request user input for one to three short questions and wait for the response. This tool is only ...; args=[questions]; hash=64a729b25c424fc6
|
||||
- custom/apply_patch: The `apply_patch` tool can be used to edit files. This is a FREEFORM tool, so do not wrap the pat...; hash=dea9f12b0111a4ba
|
||||
- function/view_image: View a local image file from the filesystem when visual inspection is needed. Use this for images...; args=[path]; hash=1908de9c0be7118a
|
||||
- tool_search: # Tool discovery Searches over deferred tool metadata with BM25 and exposes matching tools for t...; args=[limit, query]; hash=b573ed69c0b0e6c7
|
||||
- web_search; hash=f103e327c6870a46
|
||||
type: "response.create"
|
||||
-- request 1 (request; initial warm-up) --
|
||||
00:message/developer:
|
||||
<MODEL_INSTRUCTIONS>
|
||||
|
||||
## Window 2 (after request 1: input diverged at item 00)
|
||||
Settings: relative to window 1
|
||||
generate: <removed>
|
||||
previous_response_id: "initial"
|
||||
-- request 2 (request; original image submitted) --
|
||||
00:message/developer[2]:
|
||||
[01] <PERMISSIONS_INSTRUCTIONS>
|
||||
[02] <tools>
|
||||
Deferred tool namespaces:
|
||||
- multi_agent_v1: Tools for spawning and managing sub-agents.
|
||||
</tools>
|
||||
01:message/user:
|
||||
<ENVIRONMENT_CONTEXT>
|
||||
02:message/user:
|
||||
<input_image:detail,file_id>
|
||||
|
||||
## Window 3 (after request 2: input diverged at item 00)
|
||||
Settings: same as window 1
|
||||
-- request 3 (request; warm-up before interrupt) --
|
||||
00:message/developer:
|
||||
<MODEL_INSTRUCTIONS>
|
||||
01:message/developer[2]:
|
||||
[01] <PERMISSIONS_INSTRUCTIONS>
|
||||
[02] <tools>
|
||||
Deferred tool namespaces:
|
||||
- multi_agent_v1: Tools for spawning and managing sub-agents.
|
||||
</tools>
|
||||
02:message/user:
|
||||
<ENVIRONMENT_CONTEXT>
|
||||
03:message/user:
|
||||
<input_image:detail,file_id>
|
||||
04:message/assistant:
|
||||
seen
|
||||
-- request 4 (request; resumed warm-up) --
|
||||
05:message/user:
|
||||
<input_image:detail,file_id>
|
||||
06:message/user:
|
||||
<turn_aborted>
|
||||
The user interrupted the previous turn on purpose. Any running unified exec processes may still be runni...mands were aborted, they may have partially executed. [hash=8a46c957943b77f4]
|
||||
</turn_aborted>
|
||||
|
||||
## Window 4 (after request 4)
|
||||
Settings: relative to window 3
|
||||
generate: <removed>
|
||||
-- request 5 (request; generation after injection) --
|
||||
00:message/developer:
|
||||
<MODEL_INSTRUCTIONS>
|
||||
01:message/developer[2]:
|
||||
[01] <PERMISSIONS_INSTRUCTIONS>
|
||||
[02] <tools>
|
||||
Deferred tool namespaces:
|
||||
- multi_agent_v1: Tools for spawning and managing sub-agents.
|
||||
</tools>
|
||||
02:message/user:
|
||||
<ENVIRONMENT_CONTEXT>
|
||||
03:message/user:
|
||||
<input_image:detail,file_id>
|
||||
04:message/assistant:
|
||||
seen
|
||||
05:message/user:
|
||||
<input_image:detail,file_id>
|
||||
06:message/user:
|
||||
<turn_aborted>
|
||||
The user interrupted the previous turn on purpose. Any running unified exec processes may still be runni...mands were aborted, they may have partially executed. [hash=8a46c957943b77f4]
|
||||
</turn_aborted>
|
||||
07:message/user:
|
||||
<input_image:detail,file_id>
|
||||
08:message/user:
|
||||
continue
|
||||
@@ -161,6 +161,8 @@ pub enum Feature {
|
||||
CodeModeHostOutputTruncation,
|
||||
/// Establish the code-mode host connection during session startup.
|
||||
CodeModePrewarm,
|
||||
/// Warm persisted thread history before the first generation on cold resume.
|
||||
RestoredHistoryPrewarm,
|
||||
/// Terminate active code mode cells when their turn is interrupted.
|
||||
CodeModeInterrupt,
|
||||
/// Removed compatibility flag for Code Mode tool description ordering.
|
||||
@@ -1366,6 +1368,12 @@ pub const FEATURES: &[FeatureSpec] = &[
|
||||
stage: Stage::UnderDevelopment,
|
||||
default_enabled: false,
|
||||
},
|
||||
FeatureSpec {
|
||||
id: Feature::RestoredHistoryPrewarm,
|
||||
key: "restored_history_prewarm",
|
||||
stage: Stage::UnderDevelopment,
|
||||
default_enabled: false,
|
||||
},
|
||||
FeatureSpec {
|
||||
id: Feature::CodeModeInterrupt,
|
||||
key: "code_mode_interrupt",
|
||||
|
||||
Reference in New Issue
Block a user