mirror of
https://github.com/openai/codex.git
synced 2026-11-19 22:30:42 +00:00
Fix timing races in core integration tests (#62564)
## Why Steering can cancel a paused mock HTTP stream, realtime handoff and turn events can arrive in either order, and streaming tools can run before the response reports token usage. Tests that assume a fixed sequence can fail depending on timing. ## What changed - Pause the hooks test at assistant message completion so both queued prompts enter the same batch before the turn resumes. - Track the handoff and turn events in one pass, waiting for the matching turn to complete on the unblocked path. - Call `get_context_remaining` during the compaction fallback and assert zero remaining tokens using the preceding response's usage. GitOrigin-RevId: ec8a108517c819d7087681e8e2c7b5b57f4b341e
This commit is contained in:
@@ -1287,21 +1287,6 @@ fn sse_event(event: Value) -> String {
|
||||
sse(vec![event])
|
||||
}
|
||||
|
||||
fn request_message_input_texts(body: &[u8], role: &str) -> Vec<String> {
|
||||
let body: Value = serde_json::from_slice(body).expect("parse request body");
|
||||
body.get("input")
|
||||
.and_then(Value::as_array)
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.filter(|item| item.get("type").and_then(Value::as_str) == Some("message"))
|
||||
.filter(|item| item.get("role").and_then(Value::as_str) == Some(role))
|
||||
.filter_map(|item| item.get("content").and_then(Value::as_array))
|
||||
.flatten()
|
||||
.filter(|span| span.get("type").and_then(Value::as_str) == Some("input_text"))
|
||||
.filter_map(|span| span.get("text").and_then(Value::as_str).map(str::to_owned))
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn stop_hook_can_block_multiple_times_in_same_turn() -> Result<()> {
|
||||
skip_if_no_network!(Ok(()));
|
||||
@@ -2837,41 +2822,53 @@ async fn blocked_user_prompt_submit_persists_additional_context_for_next_turn()
|
||||
async fn blocked_queued_prompt_does_not_strand_earlier_accepted_prompt() -> Result<()> {
|
||||
skip_if_no_network!(Ok(()));
|
||||
|
||||
let (gate_completed_tx, gate_completed_rx) = oneshot::channel();
|
||||
let first_chunks = vec![
|
||||
StreamingSseChunk {
|
||||
gate: None,
|
||||
body: sse_event(ev_response_created("resp-1")),
|
||||
},
|
||||
StreamingSseChunk {
|
||||
gate: None,
|
||||
body: sse_event(ev_message_item_added("msg-1", "")),
|
||||
},
|
||||
StreamingSseChunk {
|
||||
gate: None,
|
||||
body: sse_event(ev_output_text_delta("first ")),
|
||||
},
|
||||
StreamingSseChunk {
|
||||
gate: Some(gate_completed_rx),
|
||||
body: sse_event(ev_message_item_done("msg-1", "first response")),
|
||||
},
|
||||
StreamingSseChunk {
|
||||
gate: None,
|
||||
body: sse_event(ev_completed("resp-1")),
|
||||
},
|
||||
];
|
||||
let second_chunks = vec![StreamingSseChunk {
|
||||
gate: None,
|
||||
body: sse(vec![
|
||||
ev_response_created("resp-2"),
|
||||
ev_assistant_message("msg-2", "accepted queued prompt handled"),
|
||||
ev_completed("resp-2"),
|
||||
]),
|
||||
}];
|
||||
let (server, _completions) =
|
||||
start_streaming_sse_server(vec![first_chunks, second_chunks]).await;
|
||||
#[derive(Default)]
|
||||
struct FirstMessageGate {
|
||||
entered: tokio::sync::Notify,
|
||||
release: tokio::sync::Notify,
|
||||
}
|
||||
|
||||
impl codex_extension_api::TurnLifecycleContributor for FirstMessageGate {
|
||||
fn on_item_completed<'a>(
|
||||
&'a self,
|
||||
_thread_store: &'a codex_extension_api::ExtensionData,
|
||||
_turn_store: &'a codex_extension_api::ExtensionData,
|
||||
item: &'a codex_protocol::items::TurnItem,
|
||||
) -> codex_extension_api::ExtensionFuture<'a, ()> {
|
||||
Box::pin(async move {
|
||||
if item.id() == "msg-1" {
|
||||
self.entered.notify_one();
|
||||
self.release.notified().await;
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
let gate = Arc::new(FirstMessageGate::default());
|
||||
let mut extensions = codex_extension_api::ExtensionRegistryBuilder::new();
|
||||
extensions.turn_lifecycle_contributor(gate.clone());
|
||||
let server = start_mock_server().await;
|
||||
let responses = mount_sse_sequence(
|
||||
&server,
|
||||
vec![
|
||||
sse(vec![
|
||||
ev_response_created("resp-1"),
|
||||
ev_message_item_added("msg-1", ""),
|
||||
ev_output_text_delta("first "),
|
||||
ev_message_item_done("msg-1", "first response"),
|
||||
ev_completed("resp-1"),
|
||||
]),
|
||||
sse(vec![
|
||||
ev_response_created("resp-2"),
|
||||
ev_assistant_message("msg-2", "accepted queued prompt handled"),
|
||||
ev_completed("resp-2"),
|
||||
]),
|
||||
],
|
||||
)
|
||||
.await;
|
||||
|
||||
let mut builder = test_codex()
|
||||
.with_extensions(Arc::new(extensions.build()))
|
||||
.with_model("gpt-5.4")
|
||||
.with_pre_build_hook(|home| {
|
||||
write_user_prompt_submit_hook(home, "blocked queued prompt", BLOCKED_PROMPT_CONTEXT)
|
||||
@@ -2881,7 +2878,8 @@ async fn blocked_queued_prompt_does_not_strand_earlier_accepted_prompt() -> Resu
|
||||
trust_discovered_hooks(config);
|
||||
config.otel.log_user_prompt = true;
|
||||
});
|
||||
let test = builder.build_with_streaming_server(&server).await?;
|
||||
// The hook fixture executes a host script from this temporary home.
|
||||
let test = builder.build(&server).await?;
|
||||
|
||||
test.codex
|
||||
.start_or_steer_turn(TurnInputRequest::user_input(vec![UserInput::Text {
|
||||
@@ -2890,10 +2888,9 @@ async fn blocked_queued_prompt_does_not_strand_earlier_accepted_prompt() -> Resu
|
||||
}]))
|
||||
.await?;
|
||||
|
||||
wait_for_event(&test.codex, |event| {
|
||||
matches!(event, EventMsg::AgentMessageContentDelta(_))
|
||||
})
|
||||
.await;
|
||||
// A steer can cancel the HTTP stream even if its mock is gated. Stop the turn
|
||||
// inside message completion instead, so both steers are in the same pending batch.
|
||||
timeout(Duration::from_secs(30), gate.entered.notified()).await?;
|
||||
|
||||
for text in ["accepted queued prompt", "blocked queued prompt"] {
|
||||
test.codex
|
||||
@@ -2904,28 +2901,14 @@ async fn blocked_queued_prompt_does_not_strand_earlier_accepted_prompt() -> Resu
|
||||
.await?;
|
||||
}
|
||||
|
||||
sleep(Duration::from_millis(100)).await;
|
||||
let _ = gate_completed_tx.send(());
|
||||
|
||||
let requests = tokio::time::timeout(Duration::from_secs(30), async {
|
||||
loop {
|
||||
let requests = server.requests().await;
|
||||
if requests.len() >= 2 {
|
||||
break requests;
|
||||
}
|
||||
sleep(Duration::from_millis(50)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("second request should arrive")
|
||||
.into_iter()
|
||||
.collect::<Vec<_>>();
|
||||
gate.release.notify_one();
|
||||
|
||||
wait_for_event(&test.codex, |event| {
|
||||
matches!(event, EventMsg::TurnComplete(_))
|
||||
})
|
||||
.await;
|
||||
|
||||
let requests = responses.requests();
|
||||
assert_eq!(requests.len(), 2);
|
||||
|
||||
logs_assert(|lines| {
|
||||
@@ -2948,7 +2931,7 @@ async fn blocked_queued_prompt_does_not_strand_earlier_accepted_prompt() -> Resu
|
||||
assert!(!rollout.contains("blocked queued prompt"));
|
||||
assert!(rollout.contains("accepted queued prompt"));
|
||||
|
||||
let second_user_texts = request_message_input_texts(&requests[1], "user");
|
||||
let second_user_texts = requests[1].message_input_texts("user");
|
||||
assert!(
|
||||
second_user_texts.contains(&"accepted queued prompt".to_string()),
|
||||
"second request should include the accepted queued prompt",
|
||||
@@ -3024,7 +3007,6 @@ async fn blocked_queued_prompt_does_not_strand_earlier_accepted_prompt() -> Resu
|
||||
],
|
||||
);
|
||||
|
||||
server.shutdown().await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
@@ -5227,24 +5227,31 @@ async fn realtime_handoff_history_scenario_inner(
|
||||
.await;
|
||||
assert_eq!(session_updated, "sess_inbound");
|
||||
|
||||
let _ = wait_for_event_match(&test.codex, |msg| match msg {
|
||||
EventMsg::RealtimeConversationRealtime(RealtimeConversationRealtimeEvent {
|
||||
payload: RealtimeEvent::HandoffRequested(handoff),
|
||||
}) if handoff.handoff_id == "handoff_inbound"
|
||||
&& handoff.input_transcript == "text from realtime" =>
|
||||
{
|
||||
Some(())
|
||||
// Routing starts the turn before the realtime fanout publishes HandoffRequested.
|
||||
// Either notification can win; consume one stream without discarding the other.
|
||||
let mut saw_handoff = false;
|
||||
let mut turn_id = None;
|
||||
let mut completed_turn_id = None;
|
||||
wait_for_event(&test.codex, |event| {
|
||||
match event {
|
||||
EventMsg::RealtimeConversationRealtime(RealtimeConversationRealtimeEvent {
|
||||
payload: RealtimeEvent::HandoffRequested(handoff),
|
||||
}) if handoff.handoff_id == "handoff_inbound"
|
||||
&& handoff.input_transcript == "text from realtime" =>
|
||||
{
|
||||
saw_handoff = true;
|
||||
}
|
||||
EventMsg::TurnStarted(started) => turn_id = Some(started.turn_id.clone()),
|
||||
EventMsg::TurnComplete(completed) => {
|
||||
completed_turn_id = Some(completed.turn_id.clone())
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
_ => None,
|
||||
// A blocked prompt must reach the stop-and-interrupt path below before it can finish.
|
||||
saw_handoff && turn_id.is_some() && (block_prompt || turn_id == completed_turn_id)
|
||||
})
|
||||
.await;
|
||||
|
||||
let turn_id = loop {
|
||||
let event = test.codex.next_event().await?;
|
||||
if let EventMsg::TurnStarted(turn_started) = event.msg {
|
||||
break turn_started.turn_id;
|
||||
}
|
||||
};
|
||||
let turn_id = turn_id.expect("observed routed turn");
|
||||
Uuid::parse_str(&turn_id).context("realtime-routed turn ID should be a UUID")?;
|
||||
|
||||
if block_prompt {
|
||||
@@ -5302,10 +5309,6 @@ async fn realtime_handoff_history_scenario_inner(
|
||||
)), "rejected handoff speech remains available for closure carryover");
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
wait_for_event(&test.codex, |event| {
|
||||
matches!(event, EventMsg::TurnComplete(_))
|
||||
})
|
||||
.await;
|
||||
test.thread_store
|
||||
.flush_thread(test.session_configured.thread_id)
|
||||
.await?;
|
||||
|
||||
@@ -1506,6 +1506,7 @@ async fn token_budget_auto_compact_fallback_uses_buffer_until_new_context() -> R
|
||||
|
||||
let server = start_mock_server().await;
|
||||
let trigger_call_id = "trigger-call";
|
||||
let remaining_call_id = "remaining-in-fallback";
|
||||
let fallback_call_id = "fallback-call";
|
||||
let new_context_call_id = "new-context-call";
|
||||
let responses = mount_sse_sequence(
|
||||
@@ -1518,6 +1519,7 @@ async fn token_budget_auto_compact_fallback_uses_buffer_until_new_context() -> R
|
||||
]),
|
||||
sse(vec![
|
||||
ev_response_created("fallback-tool-resp"),
|
||||
ev_function_call(remaining_call_id, "get_context_remaining", "{}"),
|
||||
ev_exec_command_call(fallback_call_id, "echo fallback-note > fallback-note.txt"),
|
||||
ev_completed_with_tokens("fallback-tool-resp", /*total_tokens*/ 10_000),
|
||||
]),
|
||||
@@ -1570,8 +1572,10 @@ async fn token_budget_auto_compact_fallback_uses_buffer_until_new_context() -> R
|
||||
.any(|name| name == "get_context_remaining"),
|
||||
"fallback should preserve the normal tool surface"
|
||||
);
|
||||
// Streaming tools can start before their own response.completed usage arrives.
|
||||
// The fallback step always observes the preceding response's 9,500 tokens.
|
||||
assert_eq!(
|
||||
fallback_request.function_call_output_text(trigger_call_id),
|
||||
requests[2].function_call_output_text(remaining_call_id),
|
||||
Some("You have 0 tokens left in this context window.".to_string())
|
||||
);
|
||||
let initial_context = token_budget_contexts(&requests[0]);
|
||||
|
||||
Reference in New Issue
Block a user