Allow independent app-server thread resumes to overlap (#61934)

## Why

A slow thread resume, such as one waiting for MCP startup, can block unrelated resumes behind the exclusive thread-list permit.

## What changed

Allow resumes using canonical thread IDs to share thread-list access for independent root threads. Keep aliases, paths, supplied history, daemon recovery, and agent-owned or uncertain sources under exclusive ordering.

Check source identity and ownership during resume, upgrading to exclusive access when needed. Preserve probe order during upgrades and keep later list mutations behind earlier probes.

## Testing

Add coverage showing that an independent cold resume completes while another waits for MCP startup, that only canonical ID resumes share access, and that permit upgrades preserve probe order ahead of a queued writer.

GitOrigin-RevId: 23fa6ddd551689be478fcca8e8bac2f8210984e2
This commit is contained in:
Charlie Marsh
2026-11-17 01:43:38 +00:00
committed by copyberry
parent d32c2ba608
commit 9537934ed6
6 changed files with 462 additions and 16 deletions
@@ -1,26 +1,125 @@
//! Coordinates exclusive access to thread-list mutations and resumes.
//! Orders list writers behind earlier resume probes, including probes that need to upgrade.
use super::internal_error;
use codex_app_server_protocol::JSONRPCErrorError;
use futures::future::BoxFuture;
use futures::future::MaybeDone;
use futures::future::maybe_done;
use tokio::sync::Mutex;
use tokio::sync::MutexGuard;
use tokio::sync::RwLock;
use tokio::sync::RwLockReadGuard;
use tokio::sync::Semaphore;
use tokio::sync::SemaphorePermit;
const PERMITS: u32 = 1024;
type ProbeOrder<'a> = MaybeDone<BoxFuture<'a, MutexGuard<'a, ()>>>;
pub(crate) struct ThreadListState {
admission: RwLock<()>,
probe_order: Mutex<()>,
semaphore: Semaphore,
}
impl Default for ThreadListState {
fn default() -> Self {
Self {
semaphore: Semaphore::new(/*permits*/ 1),
admission: RwLock::new(()),
probe_order: Mutex::new(()),
semaphore: Semaphore::new(PERMITS as usize),
}
}
}
impl ThreadListState {
#[expect(
clippy::await_holding_invalid_type,
reason = "keep later probes behind the writer until its semaphore permit is acquired"
)]
pub(super) async fn acquire_exclusive(&self) -> Result<SemaphorePermit<'_>, JSONRPCErrorError> {
self.semaphore.acquire().await.map_err(|err| {
let _admission = self.admission.write().await;
self.acquire(PERMITS).await
}
#[expect(
clippy::await_holding_invalid_type,
reason = "keep later writers behind the probe until its shared permit is acquired"
)]
pub(super) async fn acquire_resume(
&self,
shared: bool,
) -> Result<ThreadListResumePermit<'_>, JSONRPCErrorError> {
if shared {
let admission = self.admission.read().await;
let permit = self.acquire(/*count*/ 1).await?;
let mut order: ProbeOrder<'_> = maybe_done(Box::pin(self.probe_order.lock()));
// Poll once to reserve this probe's FIFO position without blocking other probes.
let _ = futures::poll!(&mut order);
Ok(ThreadListResumePermit {
state: self,
permit,
admission: Some(admission),
order: Some(order),
})
} else {
let permit = self.acquire_exclusive().await?;
Ok(ThreadListResumePermit {
state: self,
permit,
admission: None,
order: None,
})
}
}
async fn acquire(&self, count: u32) -> Result<SemaphorePermit<'_>, JSONRPCErrorError> {
self.semaphore.acquire_many(count).await.map_err(|err| {
internal_error(format!("failed to acquire thread list state permit: {err}"))
})
}
}
pub(super) struct ThreadListResumePermit<'a> {
state: &'a ThreadListState,
permit: SemaphorePermit<'a>,
admission: Option<RwLockReadGuard<'a, ()>>,
order: Option<ProbeOrder<'a>>,
}
impl<'a> ThreadListResumePermit<'a> {
pub(super) fn is_shared(&self) -> bool {
self.permit.num_permits() == 1
}
pub(super) fn finish_probe(&mut self) {
self.order.take();
self.admission.take();
}
pub(super) async fn upgrade(self) -> Result<Self, JSONRPCErrorError> {
let Self {
state,
permit,
admission,
order,
} = self;
debug_assert!(admission.is_some());
drop(permit);
let mut order = order.ok_or_else(|| internal_error("missing probe reservation"))?;
(&mut order).await;
// The admission read guard keeps later writers out of the semaphore queue.
let permit = state.acquire(PERMITS).await?;
drop(order);
drop(admission);
Ok(Self {
state,
permit,
admission: None,
order: None,
})
}
}
#[cfg(test)]
#[path = "thread_list_state_tests.rs"]
mod tests;
@@ -0,0 +1,32 @@
#![cfg(test)]
use super::*;
use std::time::Duration;
#[tokio::test]
async fn upgrades_follow_probe_order_even_when_the_second_probe_finishes_first() {
let state = ThreadListState::default();
let first = state
.acquire_resume(/*shared*/ true)
.await
.expect("first probe");
let second = state
.acquire_resume(/*shared*/ true)
.await
.expect("second probe");
let writer = state.acquire_exclusive();
tokio::pin!(writer);
assert!(futures::poll!(&mut writer).is_pending());
let second_upgrade = second.upgrade();
tokio::pin!(second_upgrade);
assert!(futures::poll!(&mut second_upgrade).is_pending());
tokio::time::timeout(Duration::from_secs(1), async {
let first = first.upgrade().await.expect("first upgrade");
assert!(futures::poll!(&mut second_upgrade).is_pending());
drop(first);
drop(second_upgrade.await.expect("second upgrade"));
let _writer = writer.await.expect("writer proceeds");
})
.await
.expect("upgrades and writer must finish");
}
@@ -168,6 +168,120 @@ async fn late_command_cancellation_reaches_the_current_subscriber() -> Result<()
Ok(())
}
#[tokio::test]
async fn only_canonical_id_resumes_share_list_state() -> Result<()> {
let server = wiremock::MockServer::start().await;
let test = test_codex().build_with_auto_env(&server).await?;
let thread_id = test.session_configured.thread_id;
test.thread_store
.persist_thread(thread_id, PersistContext::Standard)
.await?;
let (processor, mut outgoing) = test_processor(&test).await?;
let connection_id = ConnectionId(1);
processor
.connection_initialized(connection_id, ConnectionCapabilities::default())
.await;
let held = processor
.thread_list_state_permit
.acquire_resume(/*shared*/ true)
.await
.map_err(|error| anyhow::anyhow!(error.message))?;
let resume_id = ConnectionRequestId {
connection_id,
request_id: RequestId::Integer(1),
};
let resume = tokio::time::timeout(
Duration::from_secs(10),
processor.thread_resume(
ThreadResumeTarget::Client(resume_id),
ThreadResumeParams {
thread_id: thread_id.to_string(),
exclude_turns: true,
..Default::default()
},
/*app_server_client_name*/ None,
/*app_server_client_version*/ None,
/*client_mcp_extensions*/ Default::default(),
),
)
.await
.context("loaded resume blocked by an unrelated held list-state permit")?
.map_err(|error| anyhow::anyhow!("resume failed: {}", error.message))?;
assert!(resume.is_none());
assert!(
std::iter::from_fn(|| outgoing.try_recv().ok()).any(|envelope| matches!(
envelope,
OutgoingEnvelope::ToConnection {
message: OutgoingMessage::Response(response),
..
} if response.id == RequestId::Integer(1)
))
);
// The alias uses a different request queue, so it must wait for shared resumes to finish.
let alias_resume = processor.thread_resume(
ThreadResumeTarget::Client(ConnectionRequestId {
connection_id,
request_id: RequestId::Integer(3),
}),
ThreadResumeParams {
thread_id: thread_id.to_string().replace('-', ""),
exclude_turns: true,
..Default::default()
},
/*app_server_client_name*/ None,
/*app_server_client_version*/ None,
/*client_mcp_extensions*/ Default::default(),
);
tokio::pin!(alias_resume);
assert!(futures::poll!(&mut alias_resume).is_pending());
drop(held);
let alias_response = tokio::time::timeout(Duration::from_secs(10), alias_resume)
.await
.context("aliased resume did not finish after shared resumes")?
.map_err(|error| anyhow::anyhow!("aliased resume failed: {}", error.message))?;
assert!(alias_response.is_none());
assert!(
std::iter::from_fn(|| outgoing.try_recv().ok()).any(|envelope| matches!(
envelope,
OutgoingEnvelope::ToConnection { message: OutgoingMessage::Response(response), .. }
if response.id == RequestId::Integer(3)
))
);
// A path may not exist until an earlier exclusive restore completes. Do not
// inspect it or return a path error ahead of that operation.
let writer = processor
.acquire_thread_list_state_permit()
.await
.expect("writer permit");
let path_resume = processor.thread_resume(
ThreadResumeTarget::Client(ConnectionRequestId {
connection_id,
request_id: RequestId::Integer(2),
}),
ThreadResumeParams {
thread_id: thread_id.to_string(),
path: Some(
test.config
.codex_home
.join("missing-rollout.jsonl")
.to_path_buf(),
),
exclude_turns: true,
..Default::default()
},
/*app_server_client_name*/ None,
/*app_server_client_version*/ None,
/*client_mcp_extensions*/ Default::default(),
);
tokio::pin!(path_resume);
assert!(futures::poll!(&mut path_resume).is_pending());
drop(writer);
let error = path_resume.await.expect_err("missing running-thread path");
assert!(error.message.contains("stale path"));
Ok(())
}
#[test_case(true; "listener attached before shutdown")]
#[test_case(false; "listener attached after shutdown")]
#[tokio::test]
@@ -7,7 +7,9 @@ mod daemon_snapshot;
#[path = "realtime_history.rs"]
mod realtime_history;
mod resume_permit;
mod workspace_switching;
use resume_permit::ResumePermit;
#[path = "thread_resume_account.rs"]
mod thread_resume_account;
@@ -122,6 +124,21 @@ struct ResumeConfigState {
persisted_settings: Option<PersistedResumeSettings>,
}
#[derive(Default)]
struct ResumeRetryState<'a> {
prepared_config: Option<PreparedResumeConfig>,
force_exclusive: bool,
held_permit: Option<ResumePermit<'a>>,
}
impl<'a> ResumeRetryState<'a> {
async fn upgrade(&mut self, permit: ResumePermit<'a>) -> Result<(), JSONRPCErrorError> {
self.force_exclusive = true;
self.held_permit = Some(permit.upgrade().await?);
Ok(())
}
}
struct PreparedResumeConfig {
state: ResumeConfigState,
config: Config,
@@ -540,6 +557,8 @@ enum RunningThreadResumeResult {
/// The optional stored thread contains the history-bearing probe that cold
/// resume can reuse instead of reading the rollout again.
NotRunning(Option<Box<StoredThread>>),
/// The source changed ownership and must use exclusive list ordering.
NeedsExclusive,
}
impl ThreadRequestProcessor {
@@ -626,7 +645,7 @@ impl ThreadRequestProcessor {
app_server_client_version: Option<String>,
client_mcp_extensions: ClientMcpExtensions,
) -> Result<Option<ClientResponsePayload>, JSONRPCErrorError> {
let mut prepared_config = None;
let mut retry = ResumeRetryState::default();
// Keep the resume future off the request handler's stack.
while Box::pin(self.thread_resume_inner(
&target,
@@ -634,7 +653,7 @@ impl ThreadRequestProcessor {
app_server_client_name.clone(),
app_server_client_version.clone(),
client_mcp_extensions.clone(),
&mut prepared_config,
&mut retry,
))
.await?
.is_continue()
@@ -3903,14 +3922,14 @@ impl ThreadRequestProcessor {
.await;
}
async fn thread_resume_inner(
&self,
async fn thread_resume_inner<'a>(
&'a self,
target: &ThreadResumeTarget,
params: &ThreadResumeParams,
app_server_client_name: Option<String>,
app_server_client_version: Option<String>,
client_mcp_extensions: ClientMcpExtensions,
prepared_config: &mut Option<PreparedResumeConfig>,
retry: &mut ResumeRetryState<'a>,
) -> Result<ControlFlow<()>, JSONRPCErrorError> {
if let Ok(thread_id) = ThreadId::from_string(&params.thread_id)
&& self
@@ -3930,12 +3949,18 @@ impl ThreadRequestProcessor {
let redact_resume_payloads =
should_redact_thread_resume_payloads(app_server_client_name.as_deref());
let _thread_list_state_permit = self.acquire_thread_list_state_permit().await?;
let exclusive =
retry.force_exclusive || matches!(target, ThreadResumeTarget::DaemonRecovery(_));
let mut permit = match retry.held_permit.take() {
Some(permit) => permit,
None => ResumePermit::acquire(self, params, exclusive).await?,
};
let stored_thread_from_running_probe = match target {
ThreadResumeTarget::Client(request_id) => match self
.resume_running_thread(
request_id,
params,
&mut permit,
app_server_client_name.clone(),
app_server_client_version.clone(),
/*cold_resume_history*/ None,
@@ -3944,7 +3969,7 @@ impl ThreadRequestProcessor {
{
RunningThreadResumeResult::Handled(completion) => {
// The listener may need this permit to finish the response.
drop(_thread_list_state_permit);
drop(permit);
// A stopped listener can discard an accepted command before responding.
completion
.await
@@ -3952,6 +3977,10 @@ impl ThreadRequestProcessor {
return Ok(ControlFlow::Break(()));
}
RunningThreadResumeResult::NotRunning(stored_thread) => stored_thread,
RunningThreadResumeResult::NeedsExclusive => {
retry.upgrade(permit).await?;
return Ok(ControlFlow::Continue(()));
}
},
ThreadResumeTarget::DaemonRecovery(saved) => {
let thread_id = ThreadId::from_string(&params.thread_id)
@@ -3990,7 +4019,8 @@ impl ThreadRequestProcessor {
} = params.clone();
let include_turns = !exclude_turns;
let prepared_model_context = prepared_config
let prepared_model_context = retry
.prepared_config
.as_mut()
.and_then(|prepared| prepared.model_context.take());
let (thread_history, resume_source_thread) = if let Some(history) = history {
@@ -4010,6 +4040,11 @@ impl ThreadRequestProcessor {
.await?
}
};
permit.check_id(stored_thread.thread_id)?;
if permit.is_shared() && !resume_permit::source_can_share(&stored_thread) {
retry.upgrade(permit).await?;
return Ok(ControlFlow::Continue(()));
}
thread_resume_account::check(
&self.config.features,
app_server_client_name.as_deref(),
@@ -4022,8 +4057,26 @@ impl ThreadRequestProcessor {
prepared_model_context,
)
.await?;
permit.check_id(stored_thread.thread_id)?;
if permit.is_shared() && !resume_permit::source_can_share(&stored_thread) {
retry.upgrade(permit).await?;
return Ok(ControlFlow::Continue(()));
}
(thread_history, Some(stored_thread))
};
if let InitialHistory::Resumed(resumed) = &thread_history {
permit.check_id(resumed.conversation_id)?;
}
if permit.is_shared()
&& let Some((source, _)) = thread_history.get_resumed_session_sources()
&& !resume_permit::source_kind_can_share(
&source,
thread_history.get_resumed_parent_thread_id(),
)
{
retry.upgrade(permit).await?;
return Ok(ControlFlow::Continue(()));
}
// Path-based resume can use an empty request thread ID. Coordinate once its real
// identity is known; unrelated loaded threads never wait for this cold startup.
let _goal_resume_guard = if let InitialHistory::Resumed(resumed) = &thread_history {
@@ -4059,7 +4112,7 @@ impl ThreadRequestProcessor {
let paginated_resume = paginated_thread_id.is_some();
if paginated_resume
&& include_turns
&& prepared_config.is_none()
&& retry.prepared_config.is_none()
&& let ThreadResumeTarget::Client(request_id) = target
{
self.send_deprecation_notice(
@@ -4104,6 +4157,7 @@ impl ThreadRequestProcessor {
.resume_running_thread(
request_id,
&attach_params,
&mut permit,
app_server_client_name,
app_server_client_version,
cold_resume_history,
@@ -4111,7 +4165,7 @@ impl ThreadRequestProcessor {
.await?
{
RunningThreadResumeResult::Handled(completion) => {
drop(_thread_list_state_permit);
drop(permit);
completion
.await
.map_err(|_| thread_closing_for_resume(child_thread_id))?;
@@ -4120,9 +4174,14 @@ impl ThreadRequestProcessor {
RunningThreadResumeResult::NotRunning(_) => Err(invalid_request(
"cannot resume an unloaded multi-agent v2 sub-agent through its parent; resume the parent first, or use thread/read to inspect it",
)),
RunningThreadResumeResult::NeedsExclusive => {
retry.upgrade(permit).await?;
Ok(ControlFlow::Continue(()))
}
};
}
permit.finish_probe();
// Copied or referenced history can contain another thread's settings. Only snapshots
// explicitly owned by this thread can restore its cwd and workspace folders.
let history_settings = if let InitialHistory::Resumed(resumed) = &thread_history {
@@ -4245,12 +4304,12 @@ impl ThreadRequestProcessor {
persisted_metadata,
persisted_settings,
};
let mut config = match prepared_config.take() {
let mut config = match retry.prepared_config.take() {
Some(prepared) if prepared.state == config_state => prepared.config,
_ => {
// Config loading can call back into Desktop; release both locks during host work.
drop(_goal_resume_guard);
drop(_thread_list_state_permit);
drop(permit);
let config = self
.config_manager
.load_for_resume(
@@ -4281,7 +4340,7 @@ impl ThreadRequestProcessor {
)
.await
.map_err(|err| config_load_error(&err))?;
*prepared_config = Some(PreparedResumeConfig {
retry.prepared_config = Some(PreparedResumeConfig {
state: config_state,
config,
model_context: if paginated_resume
@@ -4617,6 +4676,7 @@ impl ThreadRequestProcessor {
&self,
request_id: &ConnectionRequestId,
params: &ThreadResumeParams,
permit: &mut ResumePermit<'_>,
app_server_client_name: Option<String>,
app_server_client_version: Option<String>,
cold_resume_history: Option<&[RolloutItem]>,
@@ -4686,8 +4746,18 @@ impl ThreadRequestProcessor {
active_path.display()
)));
}
permit.check_id(existing_thread_id)?;
let (config_snapshot, selected_service_tier) =
existing_thread.resume_config_snapshot().await;
if permit.is_shared()
&& (!resume_permit::source_can_share(&source_thread)
|| !resume_permit::source_kind_can_share(
&config_snapshot.session_source,
/*parent_thread_id*/ None,
))
{
return Ok(RunningThreadResumeResult::NeedsExclusive);
}
let mismatch_details = collect_resume_override_mismatches(
params,
&config_snapshot,
@@ -4744,6 +4814,7 @@ impl ThreadRequestProcessor {
mismatch_details.join("; ")
);
}
permit.finish_probe();
self.config_manager
.check_thread_model_provider(existing_thread.config().await.as_ref())
.await
@@ -0,0 +1,76 @@
//! Coordinates independent root resumes while keeping list mutations and uncertain sources exclusive.
//!
//! The request queue serializes canonical thread IDs across listener replacement. UUID aliases,
//! paths, and uncertain sources use exclusive list ordering before resolving their identity.
use super::*;
use crate::request_processors::thread_list_state::ThreadListResumePermit;
use codex_protocol::protocol::SessionSource;
pub(super) struct ResumePermit<'a> {
list: ThreadListResumePermit<'a>,
thread_id: Option<ThreadId>,
}
impl<'a> ResumePermit<'a> {
/// Only canonical IDs can share: all requests for that spelling use the same queue.
/// Callers check source ownership under the permit and retry exclusively when needed.
pub(super) async fn acquire(
processor: &'a ThreadRequestProcessor,
params: &ThreadResumeParams,
force_exclusive: bool,
) -> Result<Self, JSONRPCErrorError> {
let thread_id = if !force_exclusive && params.history.is_none() && params.path.is_none() {
ThreadId::from_string(&params.thread_id).ok()
} else {
None
};
let shared = thread_id.is_some_and(|id| id.to_string() == params.thread_id);
let list = processor
.thread_list_state_permit
.acquire_resume(shared)
.await?;
Ok(Self { list, thread_id })
}
pub(super) fn is_shared(&self) -> bool {
self.list.is_shared()
}
pub(super) fn finish_probe(&mut self) {
self.list.finish_probe();
}
pub(super) async fn upgrade(self) -> Result<Self, JSONRPCErrorError> {
let Self { list, thread_id } = self;
Ok(Self {
list: list.upgrade().await?,
thread_id,
})
}
/// Never use one thread's shared permit for another.
pub(super) fn check_id(&self, actual: ThreadId) -> Result<(), JSONRPCErrorError> {
if self.thread_id.is_some_and(|requested| requested != actual) {
return Err(invalid_request(
"thread source changed during resume; retry with the current thread ID or path",
));
}
Ok(())
}
}
pub(super) fn source_can_share(source: &StoredThread) -> bool {
source.attached_to_thread_id.is_none()
&& source_kind_can_share(&source.source, source.parent_thread_id)
}
/// Agent-owned, unknown and parent-linked sources retain the original exclusive ordering.
pub(super) fn source_kind_can_share(
source: &SessionSource,
parent_thread_id: Option<ThreadId>,
) -> bool {
parent_thread_id.is_none()
&& !source.is_non_root_agent()
&& !matches!(source, SessionSource::Unknown)
}
@@ -170,6 +170,60 @@ const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs
#[cfg(not(windows))]
const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10);
#[tokio::test]
async fn independent_cold_resumes_overlap() -> Result<()> {
let responses = create_mock_responses_server_repeating_assistant("Done").await;
let codex_home = TempDir::new()?;
mock_responses_config(&responses.uri()).write(codex_home.path())?;
let make_rollout = |preview| {
create_fake_rollout(
codex_home.path(),
"2025-01-05T12-00-00",
"2025-01-05T12:00:00Z",
preview,
Some("mock_provider"),
/*git_info*/ None,
)
};
let slow = make_rollout("slow")?;
let quick = make_rollout("quick")?;
let mut client = TestAppServer::builder()
.with_codex_home(codex_home.path())
.build_initialized()
.await?;
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?;
let url = format!("http://{}/mcp", listener.local_addr()?);
let slow_id = client
.send_thread_resume_request(ThreadResumeParams {
thread_id: slow,
config: Some(std::collections::HashMap::from([(
"mcp_servers.blocked".to_string(),
json!({"url": url, "required": true, "startup_timeout_sec": 30}),
)])),
exclude_turns: true,
..Default::default()
})
.await?;
let (blocked, _) = timeout(DEFAULT_READ_TIMEOUT, listener.accept()).await??;
let quick_id = client
.send_thread_resume_request(ThreadResumeParams {
thread_id: quick.clone(),
exclude_turns: true,
..Default::default()
})
.await?;
let response: ThreadResumeResponse =
timeout(DEFAULT_READ_TIMEOUT, client.read_response(quick_id)).await??;
assert_eq!(response.thread.id, quick);
drop((blocked, listener));
timeout(
DEFAULT_READ_TIMEOUT,
client.read_stream_until_error_message(RequestId::Integer(slow_id)),
)
.await??;
Ok(())
}
#[tokio::test]
async fn thread_resume_paginated_model_context_preserves_original_metadata() -> Result<()> {
let server = create_mock_responses_server_repeating_assistant("Done").await;