Fix timing races in core integration tests (#62564)

## Why

Steering can cancel a paused mock HTTP stream, realtime handoff and turn
events can arrive in either order, and streaming tools can run before the
response reports token usage. Tests that assume a fixed sequence can fail
depending on timing.

## What changed

- Pause the hooks test at assistant message completion so both queued prompts
  enter the same batch before the turn resumes.
- Track the handoff and turn events in one pass, waiting for the matching
  turn to complete on the unblocked path.
- Call `get_context_remaining` during the compaction fallback and assert
  zero remaining tokens using the preceding response's usage.

GitOrigin-RevId: ec8a108517c819d7087681e8e2c7b5b57f4b341e
This commit is contained in:
Dylan Hurd
2026-11-19 18:22:41 +00:00
committed by copyberry
parent 308ab1b895
commit 692292695b
3 changed files with 80 additions and 91 deletions
+53 -71
View File
@@ -1287,21 +1287,6 @@ fn sse_event(event: Value) -> String {
sse(vec![event])
}
fn request_message_input_texts(body: &[u8], role: &str) -> Vec<String> {
let body: Value = serde_json::from_slice(body).expect("parse request body");
body.get("input")
.and_then(Value::as_array)
.into_iter()
.flatten()
.filter(|item| item.get("type").and_then(Value::as_str) == Some("message"))
.filter(|item| item.get("role").and_then(Value::as_str) == Some(role))
.filter_map(|item| item.get("content").and_then(Value::as_array))
.flatten()
.filter(|span| span.get("type").and_then(Value::as_str) == Some("input_text"))
.filter_map(|span| span.get("text").and_then(Value::as_str).map(str::to_owned))
.collect()
}
#[tokio::test]
async fn stop_hook_can_block_multiple_times_in_same_turn() -> Result<()> {
skip_if_no_network!(Ok(()));
@@ -2837,41 +2822,53 @@ async fn blocked_user_prompt_submit_persists_additional_context_for_next_turn()
async fn blocked_queued_prompt_does_not_strand_earlier_accepted_prompt() -> Result<()> {
skip_if_no_network!(Ok(()));
let (gate_completed_tx, gate_completed_rx) = oneshot::channel();
let first_chunks = vec![
StreamingSseChunk {
gate: None,
body: sse_event(ev_response_created("resp-1")),
},
StreamingSseChunk {
gate: None,
body: sse_event(ev_message_item_added("msg-1", "")),
},
StreamingSseChunk {
gate: None,
body: sse_event(ev_output_text_delta("first ")),
},
StreamingSseChunk {
gate: Some(gate_completed_rx),
body: sse_event(ev_message_item_done("msg-1", "first response")),
},
StreamingSseChunk {
gate: None,
body: sse_event(ev_completed("resp-1")),
},
];
let second_chunks = vec![StreamingSseChunk {
gate: None,
body: sse(vec![
ev_response_created("resp-2"),
ev_assistant_message("msg-2", "accepted queued prompt handled"),
ev_completed("resp-2"),
]),
}];
let (server, _completions) =
start_streaming_sse_server(vec![first_chunks, second_chunks]).await;
#[derive(Default)]
struct FirstMessageGate {
entered: tokio::sync::Notify,
release: tokio::sync::Notify,
}
impl codex_extension_api::TurnLifecycleContributor for FirstMessageGate {
fn on_item_completed<'a>(
&'a self,
_thread_store: &'a codex_extension_api::ExtensionData,
_turn_store: &'a codex_extension_api::ExtensionData,
item: &'a codex_protocol::items::TurnItem,
) -> codex_extension_api::ExtensionFuture<'a, ()> {
Box::pin(async move {
if item.id() == "msg-1" {
self.entered.notify_one();
self.release.notified().await;
}
})
}
}
let gate = Arc::new(FirstMessageGate::default());
let mut extensions = codex_extension_api::ExtensionRegistryBuilder::new();
extensions.turn_lifecycle_contributor(gate.clone());
let server = start_mock_server().await;
let responses = mount_sse_sequence(
&server,
vec![
sse(vec![
ev_response_created("resp-1"),
ev_message_item_added("msg-1", ""),
ev_output_text_delta("first "),
ev_message_item_done("msg-1", "first response"),
ev_completed("resp-1"),
]),
sse(vec![
ev_response_created("resp-2"),
ev_assistant_message("msg-2", "accepted queued prompt handled"),
ev_completed("resp-2"),
]),
],
)
.await;
let mut builder = test_codex()
.with_extensions(Arc::new(extensions.build()))
.with_model("gpt-5.4")
.with_pre_build_hook(|home| {
write_user_prompt_submit_hook(home, "blocked queued prompt", BLOCKED_PROMPT_CONTEXT)
@@ -2881,7 +2878,8 @@ async fn blocked_queued_prompt_does_not_strand_earlier_accepted_prompt() -> Resu
trust_discovered_hooks(config);
config.otel.log_user_prompt = true;
});
let test = builder.build_with_streaming_server(&server).await?;
// The hook fixture executes a host script from this temporary home.
let test = builder.build(&server).await?;
test.codex
.start_or_steer_turn(TurnInputRequest::user_input(vec![UserInput::Text {
@@ -2890,10 +2888,9 @@ async fn blocked_queued_prompt_does_not_strand_earlier_accepted_prompt() -> Resu
}]))
.await?;
wait_for_event(&test.codex, |event| {
matches!(event, EventMsg::AgentMessageContentDelta(_))
})
.await;
// A steer can cancel the HTTP stream even if its mock is gated. Stop the turn
// inside message completion instead, so both steers are in the same pending batch.
timeout(Duration::from_secs(30), gate.entered.notified()).await?;
for text in ["accepted queued prompt", "blocked queued prompt"] {
test.codex
@@ -2904,28 +2901,14 @@ async fn blocked_queued_prompt_does_not_strand_earlier_accepted_prompt() -> Resu
.await?;
}
sleep(Duration::from_millis(100)).await;
let _ = gate_completed_tx.send(());
let requests = tokio::time::timeout(Duration::from_secs(30), async {
loop {
let requests = server.requests().await;
if requests.len() >= 2 {
break requests;
}
sleep(Duration::from_millis(50)).await;
}
})
.await
.expect("second request should arrive")
.into_iter()
.collect::<Vec<_>>();
gate.release.notify_one();
wait_for_event(&test.codex, |event| {
matches!(event, EventMsg::TurnComplete(_))
})
.await;
let requests = responses.requests();
assert_eq!(requests.len(), 2);
logs_assert(|lines| {
@@ -2948,7 +2931,7 @@ async fn blocked_queued_prompt_does_not_strand_earlier_accepted_prompt() -> Resu
assert!(!rollout.contains("blocked queued prompt"));
assert!(rollout.contains("accepted queued prompt"));
let second_user_texts = request_message_input_texts(&requests[1], "user");
let second_user_texts = requests[1].message_input_texts("user");
assert!(
second_user_texts.contains(&"accepted queued prompt".to_string()),
"second request should include the accepted queued prompt",
@@ -3024,7 +3007,6 @@ async fn blocked_queued_prompt_does_not_strand_earlier_accepted_prompt() -> Resu
],
);
server.shutdown().await;
Ok(())
}
@@ -5227,24 +5227,31 @@ async fn realtime_handoff_history_scenario_inner(
.await;
assert_eq!(session_updated, "sess_inbound");
let _ = wait_for_event_match(&test.codex, |msg| match msg {
EventMsg::RealtimeConversationRealtime(RealtimeConversationRealtimeEvent {
payload: RealtimeEvent::HandoffRequested(handoff),
}) if handoff.handoff_id == "handoff_inbound"
&& handoff.input_transcript == "text from realtime" =>
{
Some(())
// Routing starts the turn before the realtime fanout publishes HandoffRequested.
// Either notification can win; consume one stream without discarding the other.
let mut saw_handoff = false;
let mut turn_id = None;
let mut completed_turn_id = None;
wait_for_event(&test.codex, |event| {
match event {
EventMsg::RealtimeConversationRealtime(RealtimeConversationRealtimeEvent {
payload: RealtimeEvent::HandoffRequested(handoff),
}) if handoff.handoff_id == "handoff_inbound"
&& handoff.input_transcript == "text from realtime" =>
{
saw_handoff = true;
}
EventMsg::TurnStarted(started) => turn_id = Some(started.turn_id.clone()),
EventMsg::TurnComplete(completed) => {
completed_turn_id = Some(completed.turn_id.clone())
}
_ => {}
}
_ => None,
// A blocked prompt must reach the stop-and-interrupt path below before it can finish.
saw_handoff && turn_id.is_some() && (block_prompt || turn_id == completed_turn_id)
})
.await;
let turn_id = loop {
let event = test.codex.next_event().await?;
if let EventMsg::TurnStarted(turn_started) = event.msg {
break turn_started.turn_id;
}
};
let turn_id = turn_id.expect("observed routed turn");
Uuid::parse_str(&turn_id).context("realtime-routed turn ID should be a UUID")?;
if block_prompt {
@@ -5302,10 +5309,6 @@ async fn realtime_handoff_history_scenario_inner(
)), "rejected handoff speech remains available for closure carryover");
return Ok(Vec::new());
}
wait_for_event(&test.codex, |event| {
matches!(event, EventMsg::TurnComplete(_))
})
.await;
test.thread_store
.flush_thread(test.session_configured.thread_id)
.await?;
+5 -1
View File
@@ -1506,6 +1506,7 @@ async fn token_budget_auto_compact_fallback_uses_buffer_until_new_context() -> R
let server = start_mock_server().await;
let trigger_call_id = "trigger-call";
let remaining_call_id = "remaining-in-fallback";
let fallback_call_id = "fallback-call";
let new_context_call_id = "new-context-call";
let responses = mount_sse_sequence(
@@ -1518,6 +1519,7 @@ async fn token_budget_auto_compact_fallback_uses_buffer_until_new_context() -> R
]),
sse(vec![
ev_response_created("fallback-tool-resp"),
ev_function_call(remaining_call_id, "get_context_remaining", "{}"),
ev_exec_command_call(fallback_call_id, "echo fallback-note > fallback-note.txt"),
ev_completed_with_tokens("fallback-tool-resp", /*total_tokens*/ 10_000),
]),
@@ -1570,8 +1572,10 @@ async fn token_budget_auto_compact_fallback_uses_buffer_until_new_context() -> R
.any(|name| name == "get_context_remaining"),
"fallback should preserve the normal tool surface"
);
// Streaming tools can start before their own response.completed usage arrives.
// The fallback step always observes the preceding response's 9,500 tokens.
assert_eq!(
fallback_request.function_call_output_text(trigger_call_id),
requests[2].function_call_output_text(remaining_call_id),
Some("You have 0 tokens left in this context window.".to_string())
);
let initial_context = token_budget_contexts(&requests[0]);