diff --git a/codex-rs/app-server/src/request_processors/thread_list_state.rs b/codex-rs/app-server/src/request_processors/thread_list_state.rs index 9d63321add..6fe440334a 100644 --- a/codex-rs/app-server/src/request_processors/thread_list_state.rs +++ b/codex-rs/app-server/src/request_processors/thread_list_state.rs @@ -1,26 +1,125 @@ -//! Coordinates exclusive access to thread-list mutations and resumes. +//! Orders list writers behind earlier resume probes, including probes that need to upgrade. use super::internal_error; use codex_app_server_protocol::JSONRPCErrorError; +use futures::future::BoxFuture; +use futures::future::MaybeDone; +use futures::future::maybe_done; +use tokio::sync::Mutex; +use tokio::sync::MutexGuard; +use tokio::sync::RwLock; +use tokio::sync::RwLockReadGuard; use tokio::sync::Semaphore; use tokio::sync::SemaphorePermit; +const PERMITS: u32 = 1024; +type ProbeOrder<'a> = MaybeDone>>; + pub(crate) struct ThreadListState { + admission: RwLock<()>, + probe_order: Mutex<()>, semaphore: Semaphore, } impl Default for ThreadListState { fn default() -> Self { Self { - semaphore: Semaphore::new(/*permits*/ 1), + admission: RwLock::new(()), + probe_order: Mutex::new(()), + semaphore: Semaphore::new(PERMITS as usize), } } } impl ThreadListState { + #[expect( + clippy::await_holding_invalid_type, + reason = "keep later probes behind the writer until its semaphore permit is acquired" + )] pub(super) async fn acquire_exclusive(&self) -> Result, JSONRPCErrorError> { - self.semaphore.acquire().await.map_err(|err| { + let _admission = self.admission.write().await; + self.acquire(PERMITS).await + } + + #[expect( + clippy::await_holding_invalid_type, + reason = "keep later writers behind the probe until its shared permit is acquired" + )] + pub(super) async fn acquire_resume( + &self, + shared: bool, + ) -> Result, JSONRPCErrorError> { + if shared { + let admission = self.admission.read().await; + let permit = self.acquire(/*count*/ 1).await?; + let mut order: ProbeOrder<'_> = maybe_done(Box::pin(self.probe_order.lock())); + // Poll once to reserve this probe's FIFO position without blocking other probes. + let _ = futures::poll!(&mut order); + Ok(ThreadListResumePermit { + state: self, + permit, + admission: Some(admission), + order: Some(order), + }) + } else { + let permit = self.acquire_exclusive().await?; + Ok(ThreadListResumePermit { + state: self, + permit, + admission: None, + order: None, + }) + } + } + + async fn acquire(&self, count: u32) -> Result, JSONRPCErrorError> { + self.semaphore.acquire_many(count).await.map_err(|err| { internal_error(format!("failed to acquire thread list state permit: {err}")) }) } } + +pub(super) struct ThreadListResumePermit<'a> { + state: &'a ThreadListState, + permit: SemaphorePermit<'a>, + admission: Option>, + order: Option>, +} + +impl<'a> ThreadListResumePermit<'a> { + pub(super) fn is_shared(&self) -> bool { + self.permit.num_permits() == 1 + } + + pub(super) fn finish_probe(&mut self) { + self.order.take(); + self.admission.take(); + } + + pub(super) async fn upgrade(self) -> Result { + let Self { + state, + permit, + admission, + order, + } = self; + debug_assert!(admission.is_some()); + drop(permit); + let mut order = order.ok_or_else(|| internal_error("missing probe reservation"))?; + (&mut order).await; + // The admission read guard keeps later writers out of the semaphore queue. + let permit = state.acquire(PERMITS).await?; + drop(order); + drop(admission); + Ok(Self { + state, + permit, + admission: None, + order: None, + }) + } +} + +#[cfg(test)] +#[path = "thread_list_state_tests.rs"] +mod tests; diff --git a/codex-rs/app-server/src/request_processors/thread_list_state_tests.rs b/codex-rs/app-server/src/request_processors/thread_list_state_tests.rs new file mode 100644 index 0000000000..df5de1393d --- /dev/null +++ b/codex-rs/app-server/src/request_processors/thread_list_state_tests.rs @@ -0,0 +1,32 @@ +#![cfg(test)] + +use super::*; +use std::time::Duration; + +#[tokio::test] +async fn upgrades_follow_probe_order_even_when_the_second_probe_finishes_first() { + let state = ThreadListState::default(); + let first = state + .acquire_resume(/*shared*/ true) + .await + .expect("first probe"); + let second = state + .acquire_resume(/*shared*/ true) + .await + .expect("second probe"); + let writer = state.acquire_exclusive(); + tokio::pin!(writer); + assert!(futures::poll!(&mut writer).is_pending()); + let second_upgrade = second.upgrade(); + tokio::pin!(second_upgrade); + assert!(futures::poll!(&mut second_upgrade).is_pending()); + tokio::time::timeout(Duration::from_secs(1), async { + let first = first.upgrade().await.expect("first upgrade"); + assert!(futures::poll!(&mut second_upgrade).is_pending()); + drop(first); + drop(second_upgrade.await.expect("second upgrade")); + let _writer = writer.await.expect("writer proceeds"); + }) + .await + .expect("upgrades and writer must finish"); +} diff --git a/codex-rs/app-server/src/request_processors/thread_listener_shutdown_tests.rs b/codex-rs/app-server/src/request_processors/thread_listener_shutdown_tests.rs index e8164da0c5..001e82abee 100644 --- a/codex-rs/app-server/src/request_processors/thread_listener_shutdown_tests.rs +++ b/codex-rs/app-server/src/request_processors/thread_listener_shutdown_tests.rs @@ -168,6 +168,120 @@ async fn late_command_cancellation_reaches_the_current_subscriber() -> Result<() Ok(()) } +#[tokio::test] +async fn only_canonical_id_resumes_share_list_state() -> Result<()> { + let server = wiremock::MockServer::start().await; + let test = test_codex().build_with_auto_env(&server).await?; + let thread_id = test.session_configured.thread_id; + test.thread_store + .persist_thread(thread_id, PersistContext::Standard) + .await?; + let (processor, mut outgoing) = test_processor(&test).await?; + let connection_id = ConnectionId(1); + processor + .connection_initialized(connection_id, ConnectionCapabilities::default()) + .await; + let held = processor + .thread_list_state_permit + .acquire_resume(/*shared*/ true) + .await + .map_err(|error| anyhow::anyhow!(error.message))?; + let resume_id = ConnectionRequestId { + connection_id, + request_id: RequestId::Integer(1), + }; + let resume = tokio::time::timeout( + Duration::from_secs(10), + processor.thread_resume( + ThreadResumeTarget::Client(resume_id), + ThreadResumeParams { + thread_id: thread_id.to_string(), + exclude_turns: true, + ..Default::default() + }, + /*app_server_client_name*/ None, + /*app_server_client_version*/ None, + /*client_mcp_extensions*/ Default::default(), + ), + ) + .await + .context("loaded resume blocked by an unrelated held list-state permit")? + .map_err(|error| anyhow::anyhow!("resume failed: {}", error.message))?; + assert!(resume.is_none()); + assert!( + std::iter::from_fn(|| outgoing.try_recv().ok()).any(|envelope| matches!( + envelope, + OutgoingEnvelope::ToConnection { + message: OutgoingMessage::Response(response), + .. + } if response.id == RequestId::Integer(1) + )) + ); + // The alias uses a different request queue, so it must wait for shared resumes to finish. + let alias_resume = processor.thread_resume( + ThreadResumeTarget::Client(ConnectionRequestId { + connection_id, + request_id: RequestId::Integer(3), + }), + ThreadResumeParams { + thread_id: thread_id.to_string().replace('-', ""), + exclude_turns: true, + ..Default::default() + }, + /*app_server_client_name*/ None, + /*app_server_client_version*/ None, + /*client_mcp_extensions*/ Default::default(), + ); + tokio::pin!(alias_resume); + assert!(futures::poll!(&mut alias_resume).is_pending()); + drop(held); + let alias_response = tokio::time::timeout(Duration::from_secs(10), alias_resume) + .await + .context("aliased resume did not finish after shared resumes")? + .map_err(|error| anyhow::anyhow!("aliased resume failed: {}", error.message))?; + assert!(alias_response.is_none()); + assert!( + std::iter::from_fn(|| outgoing.try_recv().ok()).any(|envelope| matches!( + envelope, + OutgoingEnvelope::ToConnection { message: OutgoingMessage::Response(response), .. } + if response.id == RequestId::Integer(3) + )) + ); + + // A path may not exist until an earlier exclusive restore completes. Do not + // inspect it or return a path error ahead of that operation. + let writer = processor + .acquire_thread_list_state_permit() + .await + .expect("writer permit"); + let path_resume = processor.thread_resume( + ThreadResumeTarget::Client(ConnectionRequestId { + connection_id, + request_id: RequestId::Integer(2), + }), + ThreadResumeParams { + thread_id: thread_id.to_string(), + path: Some( + test.config + .codex_home + .join("missing-rollout.jsonl") + .to_path_buf(), + ), + exclude_turns: true, + ..Default::default() + }, + /*app_server_client_name*/ None, + /*app_server_client_version*/ None, + /*client_mcp_extensions*/ Default::default(), + ); + tokio::pin!(path_resume); + assert!(futures::poll!(&mut path_resume).is_pending()); + drop(writer); + let error = path_resume.await.expect_err("missing running-thread path"); + assert!(error.message.contains("stale path")); + Ok(()) +} + #[test_case(true; "listener attached before shutdown")] #[test_case(false; "listener attached after shutdown")] #[tokio::test] diff --git a/codex-rs/app-server/src/request_processors/thread_processor.rs b/codex-rs/app-server/src/request_processors/thread_processor.rs index 75d9ce5a9c..eccce8072c 100644 --- a/codex-rs/app-server/src/request_processors/thread_processor.rs +++ b/codex-rs/app-server/src/request_processors/thread_processor.rs @@ -7,7 +7,9 @@ mod daemon_snapshot; #[path = "realtime_history.rs"] mod realtime_history; +mod resume_permit; mod workspace_switching; +use resume_permit::ResumePermit; #[path = "thread_resume_account.rs"] mod thread_resume_account; @@ -122,6 +124,21 @@ struct ResumeConfigState { persisted_settings: Option, } +#[derive(Default)] +struct ResumeRetryState<'a> { + prepared_config: Option, + force_exclusive: bool, + held_permit: Option>, +} + +impl<'a> ResumeRetryState<'a> { + async fn upgrade(&mut self, permit: ResumePermit<'a>) -> Result<(), JSONRPCErrorError> { + self.force_exclusive = true; + self.held_permit = Some(permit.upgrade().await?); + Ok(()) + } +} + struct PreparedResumeConfig { state: ResumeConfigState, config: Config, @@ -540,6 +557,8 @@ enum RunningThreadResumeResult { /// The optional stored thread contains the history-bearing probe that cold /// resume can reuse instead of reading the rollout again. NotRunning(Option>), + /// The source changed ownership and must use exclusive list ordering. + NeedsExclusive, } impl ThreadRequestProcessor { @@ -626,7 +645,7 @@ impl ThreadRequestProcessor { app_server_client_version: Option, client_mcp_extensions: ClientMcpExtensions, ) -> Result, JSONRPCErrorError> { - let mut prepared_config = None; + let mut retry = ResumeRetryState::default(); // Keep the resume future off the request handler's stack. while Box::pin(self.thread_resume_inner( &target, @@ -634,7 +653,7 @@ impl ThreadRequestProcessor { app_server_client_name.clone(), app_server_client_version.clone(), client_mcp_extensions.clone(), - &mut prepared_config, + &mut retry, )) .await? .is_continue() @@ -3903,14 +3922,14 @@ impl ThreadRequestProcessor { .await; } - async fn thread_resume_inner( - &self, + async fn thread_resume_inner<'a>( + &'a self, target: &ThreadResumeTarget, params: &ThreadResumeParams, app_server_client_name: Option, app_server_client_version: Option, client_mcp_extensions: ClientMcpExtensions, - prepared_config: &mut Option, + retry: &mut ResumeRetryState<'a>, ) -> Result, JSONRPCErrorError> { if let Ok(thread_id) = ThreadId::from_string(¶ms.thread_id) && self @@ -3930,12 +3949,18 @@ impl ThreadRequestProcessor { let redact_resume_payloads = should_redact_thread_resume_payloads(app_server_client_name.as_deref()); - let _thread_list_state_permit = self.acquire_thread_list_state_permit().await?; + let exclusive = + retry.force_exclusive || matches!(target, ThreadResumeTarget::DaemonRecovery(_)); + let mut permit = match retry.held_permit.take() { + Some(permit) => permit, + None => ResumePermit::acquire(self, params, exclusive).await?, + }; let stored_thread_from_running_probe = match target { ThreadResumeTarget::Client(request_id) => match self .resume_running_thread( request_id, params, + &mut permit, app_server_client_name.clone(), app_server_client_version.clone(), /*cold_resume_history*/ None, @@ -3944,7 +3969,7 @@ impl ThreadRequestProcessor { { RunningThreadResumeResult::Handled(completion) => { // The listener may need this permit to finish the response. - drop(_thread_list_state_permit); + drop(permit); // A stopped listener can discard an accepted command before responding. completion .await @@ -3952,6 +3977,10 @@ impl ThreadRequestProcessor { return Ok(ControlFlow::Break(())); } RunningThreadResumeResult::NotRunning(stored_thread) => stored_thread, + RunningThreadResumeResult::NeedsExclusive => { + retry.upgrade(permit).await?; + return Ok(ControlFlow::Continue(())); + } }, ThreadResumeTarget::DaemonRecovery(saved) => { let thread_id = ThreadId::from_string(¶ms.thread_id) @@ -3990,7 +4019,8 @@ impl ThreadRequestProcessor { } = params.clone(); let include_turns = !exclude_turns; - let prepared_model_context = prepared_config + let prepared_model_context = retry + .prepared_config .as_mut() .and_then(|prepared| prepared.model_context.take()); let (thread_history, resume_source_thread) = if let Some(history) = history { @@ -4010,6 +4040,11 @@ impl ThreadRequestProcessor { .await? } }; + permit.check_id(stored_thread.thread_id)?; + if permit.is_shared() && !resume_permit::source_can_share(&stored_thread) { + retry.upgrade(permit).await?; + return Ok(ControlFlow::Continue(())); + } thread_resume_account::check( &self.config.features, app_server_client_name.as_deref(), @@ -4022,8 +4057,26 @@ impl ThreadRequestProcessor { prepared_model_context, ) .await?; + permit.check_id(stored_thread.thread_id)?; + if permit.is_shared() && !resume_permit::source_can_share(&stored_thread) { + retry.upgrade(permit).await?; + return Ok(ControlFlow::Continue(())); + } (thread_history, Some(stored_thread)) }; + if let InitialHistory::Resumed(resumed) = &thread_history { + permit.check_id(resumed.conversation_id)?; + } + if permit.is_shared() + && let Some((source, _)) = thread_history.get_resumed_session_sources() + && !resume_permit::source_kind_can_share( + &source, + thread_history.get_resumed_parent_thread_id(), + ) + { + retry.upgrade(permit).await?; + return Ok(ControlFlow::Continue(())); + } // Path-based resume can use an empty request thread ID. Coordinate once its real // identity is known; unrelated loaded threads never wait for this cold startup. let _goal_resume_guard = if let InitialHistory::Resumed(resumed) = &thread_history { @@ -4059,7 +4112,7 @@ impl ThreadRequestProcessor { let paginated_resume = paginated_thread_id.is_some(); if paginated_resume && include_turns - && prepared_config.is_none() + && retry.prepared_config.is_none() && let ThreadResumeTarget::Client(request_id) = target { self.send_deprecation_notice( @@ -4104,6 +4157,7 @@ impl ThreadRequestProcessor { .resume_running_thread( request_id, &attach_params, + &mut permit, app_server_client_name, app_server_client_version, cold_resume_history, @@ -4111,7 +4165,7 @@ impl ThreadRequestProcessor { .await? { RunningThreadResumeResult::Handled(completion) => { - drop(_thread_list_state_permit); + drop(permit); completion .await .map_err(|_| thread_closing_for_resume(child_thread_id))?; @@ -4120,9 +4174,14 @@ impl ThreadRequestProcessor { RunningThreadResumeResult::NotRunning(_) => Err(invalid_request( "cannot resume an unloaded multi-agent v2 sub-agent through its parent; resume the parent first, or use thread/read to inspect it", )), + RunningThreadResumeResult::NeedsExclusive => { + retry.upgrade(permit).await?; + Ok(ControlFlow::Continue(())) + } }; } + permit.finish_probe(); // Copied or referenced history can contain another thread's settings. Only snapshots // explicitly owned by this thread can restore its cwd and workspace folders. let history_settings = if let InitialHistory::Resumed(resumed) = &thread_history { @@ -4245,12 +4304,12 @@ impl ThreadRequestProcessor { persisted_metadata, persisted_settings, }; - let mut config = match prepared_config.take() { + let mut config = match retry.prepared_config.take() { Some(prepared) if prepared.state == config_state => prepared.config, _ => { // Config loading can call back into Desktop; release both locks during host work. drop(_goal_resume_guard); - drop(_thread_list_state_permit); + drop(permit); let config = self .config_manager .load_for_resume( @@ -4281,7 +4340,7 @@ impl ThreadRequestProcessor { ) .await .map_err(|err| config_load_error(&err))?; - *prepared_config = Some(PreparedResumeConfig { + retry.prepared_config = Some(PreparedResumeConfig { state: config_state, config, model_context: if paginated_resume @@ -4617,6 +4676,7 @@ impl ThreadRequestProcessor { &self, request_id: &ConnectionRequestId, params: &ThreadResumeParams, + permit: &mut ResumePermit<'_>, app_server_client_name: Option, app_server_client_version: Option, cold_resume_history: Option<&[RolloutItem]>, @@ -4686,8 +4746,18 @@ impl ThreadRequestProcessor { active_path.display() ))); } + permit.check_id(existing_thread_id)?; let (config_snapshot, selected_service_tier) = existing_thread.resume_config_snapshot().await; + if permit.is_shared() + && (!resume_permit::source_can_share(&source_thread) + || !resume_permit::source_kind_can_share( + &config_snapshot.session_source, + /*parent_thread_id*/ None, + )) + { + return Ok(RunningThreadResumeResult::NeedsExclusive); + } let mismatch_details = collect_resume_override_mismatches( params, &config_snapshot, @@ -4744,6 +4814,7 @@ impl ThreadRequestProcessor { mismatch_details.join("; ") ); } + permit.finish_probe(); self.config_manager .check_thread_model_provider(existing_thread.config().await.as_ref()) .await diff --git a/codex-rs/app-server/src/request_processors/thread_processor/resume_permit.rs b/codex-rs/app-server/src/request_processors/thread_processor/resume_permit.rs new file mode 100644 index 0000000000..aa4b27f7fc --- /dev/null +++ b/codex-rs/app-server/src/request_processors/thread_processor/resume_permit.rs @@ -0,0 +1,76 @@ +//! Coordinates independent root resumes while keeping list mutations and uncertain sources exclusive. +//! +//! The request queue serializes canonical thread IDs across listener replacement. UUID aliases, +//! paths, and uncertain sources use exclusive list ordering before resolving their identity. + +use super::*; +use crate::request_processors::thread_list_state::ThreadListResumePermit; +use codex_protocol::protocol::SessionSource; + +pub(super) struct ResumePermit<'a> { + list: ThreadListResumePermit<'a>, + thread_id: Option, +} + +impl<'a> ResumePermit<'a> { + /// Only canonical IDs can share: all requests for that spelling use the same queue. + /// Callers check source ownership under the permit and retry exclusively when needed. + pub(super) async fn acquire( + processor: &'a ThreadRequestProcessor, + params: &ThreadResumeParams, + force_exclusive: bool, + ) -> Result { + let thread_id = if !force_exclusive && params.history.is_none() && params.path.is_none() { + ThreadId::from_string(¶ms.thread_id).ok() + } else { + None + }; + let shared = thread_id.is_some_and(|id| id.to_string() == params.thread_id); + let list = processor + .thread_list_state_permit + .acquire_resume(shared) + .await?; + Ok(Self { list, thread_id }) + } + + pub(super) fn is_shared(&self) -> bool { + self.list.is_shared() + } + + pub(super) fn finish_probe(&mut self) { + self.list.finish_probe(); + } + + pub(super) async fn upgrade(self) -> Result { + let Self { list, thread_id } = self; + Ok(Self { + list: list.upgrade().await?, + thread_id, + }) + } + + /// Never use one thread's shared permit for another. + pub(super) fn check_id(&self, actual: ThreadId) -> Result<(), JSONRPCErrorError> { + if self.thread_id.is_some_and(|requested| requested != actual) { + return Err(invalid_request( + "thread source changed during resume; retry with the current thread ID or path", + )); + } + Ok(()) + } +} + +pub(super) fn source_can_share(source: &StoredThread) -> bool { + source.attached_to_thread_id.is_none() + && source_kind_can_share(&source.source, source.parent_thread_id) +} + +/// Agent-owned, unknown and parent-linked sources retain the original exclusive ordering. +pub(super) fn source_kind_can_share( + source: &SessionSource, + parent_thread_id: Option, +) -> bool { + parent_thread_id.is_none() + && !source.is_non_root_agent() + && !matches!(source, SessionSource::Unknown) +} diff --git a/codex-rs/app-server/tests/suite/v2/thread_resume.rs b/codex-rs/app-server/tests/suite/v2/thread_resume.rs index 976418dad5..cd090af5fd 100644 --- a/codex-rs/app-server/tests/suite/v2/thread_resume.rs +++ b/codex-rs/app-server/tests/suite/v2/thread_resume.rs @@ -170,6 +170,60 @@ const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs #[cfg(not(windows))] const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); +#[tokio::test] +async fn independent_cold_resumes_overlap() -> Result<()> { + let responses = create_mock_responses_server_repeating_assistant("Done").await; + let codex_home = TempDir::new()?; + mock_responses_config(&responses.uri()).write(codex_home.path())?; + let make_rollout = |preview| { + create_fake_rollout( + codex_home.path(), + "2025-01-05T12-00-00", + "2025-01-05T12:00:00Z", + preview, + Some("mock_provider"), + /*git_info*/ None, + ) + }; + let slow = make_rollout("slow")?; + let quick = make_rollout("quick")?; + let mut client = TestAppServer::builder() + .with_codex_home(codex_home.path()) + .build_initialized() + .await?; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; + let url = format!("http://{}/mcp", listener.local_addr()?); + let slow_id = client + .send_thread_resume_request(ThreadResumeParams { + thread_id: slow, + config: Some(std::collections::HashMap::from([( + "mcp_servers.blocked".to_string(), + json!({"url": url, "required": true, "startup_timeout_sec": 30}), + )])), + exclude_turns: true, + ..Default::default() + }) + .await?; + let (blocked, _) = timeout(DEFAULT_READ_TIMEOUT, listener.accept()).await??; + let quick_id = client + .send_thread_resume_request(ThreadResumeParams { + thread_id: quick.clone(), + exclude_turns: true, + ..Default::default() + }) + .await?; + let response: ThreadResumeResponse = + timeout(DEFAULT_READ_TIMEOUT, client.read_response(quick_id)).await??; + assert_eq!(response.thread.id, quick); + drop((blocked, listener)); + timeout( + DEFAULT_READ_TIMEOUT, + client.read_stream_until_error_message(RequestId::Integer(slow_id)), + ) + .await??; + Ok(()) +} + #[tokio::test] async fn thread_resume_paginated_model_context_preserves_original_metadata() -> Result<()> { let server = create_mock_responses_server_repeating_assistant("Done").await;