diff --git a/.codex/flaky-test-triage.md b/.codex/flaky-test-triage.md index e09a82c71b..965f6a707c 100644 --- a/.codex/flaky-test-triage.md +++ b/.codex/flaky-test-triage.md @@ -49,10 +49,13 @@ Older failures also appeared on Linux, but the repeated cross-PR signal is stron - Commit `495ef3f76` extended `rust-ci` annotations enough to expose the exact flaky tests on the two remaining Windows jobs. - Windows x64 failure: `codex::tests::run_user_shell_command_does_not_set_reference_context_item` timed out waiting for `TurnComplete`. - Windows arm64 failure: `all::suite::fuzzy_file_search::test_fuzzy_file_search_session_multiple_query_updates_work` timed out during app-server `initialize`, before the fuzzy-search session logic started. +- Commit `d017d0fc3` fixed those two Windows-specific flakes, but the first in-process fuzzy-search harness revision introduced a new ordering regression on Linux. +- Linux failure: `all::suite::fuzzy_file_search::test_fuzzy_file_search_session_update_works_without_waiting_for_start_response` received an error instead of a response because `sessionUpdate` could be enqueued ahead of `sessionStart`. - Current patch set: - Pin the standalone shell test to `cmd.exe` on Windows so it validates reference-context isolation without depending on PowerShell startup behavior. - Replace the fuzzy-file-search suite's spawned `codex-app-server` harness with the in-process app-server runtime so the tests still exercise request/notification behavior without the flaky stdio startup path. -- Rationale: both failures are test-harness flakes, not product behaviors. The fixes keep the assertions intact and remove environment-sensitive startup paths instead of stretching timeouts. + - Preserve in-process request order by enqueueing requests synchronously and storing a pending-response handle instead of spawning `sender.request(...)` tasks that can race. +- Rationale: these failures are test-harness flakes, not product behaviors. The fixes keep the assertions intact and remove environment-sensitive startup and ordering hazards instead of stretching timeouts. ## Constraints @@ -81,3 +84,4 @@ Older failures also appeared on Linux, but the repeated cross-PR signal is stron | `b6e18d2e8` | Record validation pass 2 | failed | Run `23083106021` isolated a new failure in `Tests — ubuntu-24.04 - x86_64-unknown-linux-gnu` while both Windows `Tests` jobs and all Bazel/lint checks passed. The GitHub CLI log download hit EOFs on both `results-receiver.actions.githubusercontent.com` and the signed Azure blob URL again, so the follow-up commit adds CI annotations and step-summary output for parsed nextest `FAIL`/`LEAK` lines. | | `761363008` | Annotate nextest failures in CI | failed | Run `23083522224` flipped to a new `Tests — windows-x64 - x86_64-pc-windows-msvc` failure while `Tests — windows-arm64 - aarch64-pc-windows-msvc` and every non-test job passed. The new annotations proved this failure did not emit a parsable nextest `FAIL`/`LEAK` line, so the next follow-up extends the annotations to the last 80 log lines and requests explicit final failure output from nextest. | | `495ef3f76` | Expose nextest failure tail in CI | failed | Run `23083992418` failed in both Windows `Tests` jobs. The new annotations identified `codex::tests::run_user_shell_command_does_not_set_reference_context_item` timing out on Windows x64 and `all::suite::fuzzy_file_search::test_fuzzy_file_search_session_multiple_query_updates_work` timing out during app-server `initialize` on Windows arm64. | +| `d017d0fc3` | Stabilize Windows shell and fuzzy search tests | failed | Run `23084639316` fixed the original Windows-targeted failures, but introduced a new failure in `Tests — ubuntu-24.04 - x86_64-unknown-linux-gnu`: `all::suite::fuzzy_file_search::test_fuzzy_file_search_session_update_works_without_waiting_for_start_response`. The in-process harness used spawned `sender.request(...)` tasks, which made request submission order nondeterministic and let `sessionUpdate` race ahead of `sessionStart`. `Bazel (experimental)` also failed on macOS in the same patch window, and the still-running Windows jobs were superseded by the next follow-up commit. | diff --git a/codex-rs/app-server/src/in_process.rs b/codex-rs/app-server/src/in_process.rs index 3a9286a5f4..7c83c0065c 100644 --- a/codex-rs/app-server/src/in_process.rs +++ b/codex-rs/app-server/src/in_process.rs @@ -94,6 +94,22 @@ pub const DEFAULT_IN_PROCESS_CHANNEL_CAPACITY: usize = CHANNEL_CAPACITY; type PendingClientRequestResponse = std::result::Result; +/// Handle for a request that has been enqueued but whose response has not yet been awaited. +pub struct PendingInProcessRequest { + response_rx: oneshot::Receiver, +} + +impl PendingInProcessRequest { + pub async fn recv(self) -> IoResult { + self.response_rx.await.map_err(|err| { + IoError::new( + ErrorKind::BrokenPipe, + format!("in-process request response channel closed: {err}"), + ) + }) + } +} + fn server_notification_requires_delivery(notification: &ServerNotification) -> bool { matches!(notification, ServerNotification::TurnCompleted(_)) } @@ -200,18 +216,17 @@ pub struct InProcessClientSender { } impl InProcessClientSender { - pub async fn request(&self, request: ClientRequest) -> IoResult { + pub fn start_request(&self, request: ClientRequest) -> IoResult { let (response_tx, response_rx) = oneshot::channel(); self.try_send_client_message(InProcessClientMessage::Request { request: Box::new(request), response_tx, })?; - response_rx.await.map_err(|err| { - IoError::new( - ErrorKind::BrokenPipe, - format!("in-process request response channel closed: {err}"), - ) - }) + Ok(PendingInProcessRequest { response_rx }) + } + + pub async fn request(&self, request: ClientRequest) -> IoResult { + self.start_request(request)?.recv().await } pub fn notify(&self, notification: ClientNotification) -> IoResult<()> { @@ -263,6 +278,11 @@ pub struct InProcessClientHandle { } impl InProcessClientHandle { + /// Enqueues a typed client request and returns a handle for awaiting the response later. + pub fn start_request(&self, request: ClientRequest) -> IoResult { + self.client.start_request(request) + } + /// Sends a typed client request into the in-process runtime. /// /// The returned value is a transport-level `IoResult` containing either a diff --git a/codex-rs/app-server/tests/suite/fuzzy_file_search.rs b/codex-rs/app-server/tests/suite/fuzzy_file_search.rs index 137d308f9d..99ede428b6 100644 --- a/codex-rs/app-server/tests/suite/fuzzy_file_search.rs +++ b/codex-rs/app-server/tests/suite/fuzzy_file_search.rs @@ -3,6 +3,7 @@ use anyhow::anyhow; use codex_app_server::in_process::InProcessClientHandle; use codex_app_server::in_process::InProcessServerEvent; use codex_app_server::in_process::InProcessStartArgs; +use codex_app_server::in_process::PendingInProcessRequest; use codex_app_server::in_process::start as start_in_process; use codex_app_server_protocol::ClientInfo; use codex_app_server_protocol::ClientRequest; @@ -32,11 +33,9 @@ use std::collections::HashMap; use std::path::Path; use std::sync::Arc; use tempfile::TempDir; -use tokio::task::JoinHandle; use tokio::time::timeout; -type RawRequestResult = std::result::Result; -type PendingRequest = JoinHandle>; +type PendingRequest = PendingInProcessRequest; const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5); const SHORT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_millis(500); @@ -244,10 +243,7 @@ impl McpProcess { request_id: RequestId, ) -> Result { let pending = self.take_pending_request(&request_id)?; - match pending - .await - .map_err(|err| anyhow!("request task failed for {request_id:?}: {err}"))?? - { + match pending.recv().await? { Ok(result) => Ok(JSONRPCResponse { id: request_id, result, @@ -263,10 +259,7 @@ impl McpProcess { request_id: RequestId, ) -> Result { let pending = self.take_pending_request(&request_id)?; - match pending - .await - .map_err(|err| anyhow!("request task failed for {request_id:?}: {err}"))?? - { + match pending.recv().await? { Ok(result) => { anyhow::bail!("expected error for {request_id:?}, got response: {result:?}") } @@ -321,12 +314,10 @@ impl McpProcess { .as_ref() .ok_or_else(|| anyhow!("MCP client not initialized"))? .sender(); + let pending_request = sender.start_request(request)?; if self .pending_requests - .insert( - request_id, - tokio::spawn(async move { sender.request(request).await }), - ) + .insert(request_id, pending_request) .is_some() { anyhow::bail!("duplicate pending request id: {request_id}"); @@ -352,9 +343,7 @@ impl McpProcess { impl Drop for McpProcess { fn drop(&mut self) { - for (_, pending_request) in self.pending_requests.drain() { - pending_request.abort(); - } + self.pending_requests.clear(); if let Some(client) = self.client.take() && let Ok(runtime) = tokio::runtime::Handle::try_current() {