Preserve process state across concurrent unified exec updates (#62557)

## Why

Process lifecycle handlers could overwrite each other's updates by reading
and replacing the entire state. For example, a rejected stdin write could
restore an older snapshot after the output collector recorded an exit code.

## What changed

Apply lifecycle transitions in place under the watch channel's write lock.
Rejected stdin writes and confirmed termination only mark the process as
exited, preserving any exit code already reported by the executor.

## Testing

Regression tests cover both closed stdin and unknown-process responses.
They verify that subsequent lifecycle updates preserve the exit code and
sandbox denial recovered from the final read, and that repeated calls to
`fail_and_terminate` preserve the first failure message.

GitOrigin-RevId: f090c0860c86b6e668c31d122d986ea4f4a119b6
This commit is contained in:
jif
2026-11-19 17:33:26 +00:00
committed by copyberry
parent 325123ae4a
commit 4d517cce4a
3 changed files with 133 additions and 57 deletions
+39 -38
View File
@@ -171,8 +171,7 @@ impl UnifiedExecProcess {
Ok(response) => match response.status {
WriteStatus::Accepted => Ok(()),
WriteStatus::UnknownProcess | WriteStatus::StdinClosed => {
let state = self.state_rx.borrow().clone();
let _ = self.state_tx.send_replace(state.exited(state.exit_code));
self.state_tx.send_modify(|state| state.has_exited = true);
self.output.cancellation_token.cancel();
Err(UnifiedExecError::WriteToStdin)
}
@@ -205,10 +204,10 @@ impl UnifiedExecProcess {
}
pub(super) fn has_exited(&self) -> bool {
let state = self.state_rx.borrow().clone();
let has_exited = self.state_rx.borrow().has_exited;
match &self.process_handle {
ProcessHandle::Local(process_handle) => state.has_exited || process_handle.has_exited(),
ProcessHandle::ExecServer(_) => state.has_exited,
ProcessHandle::Local(process_handle) => has_exited || process_handle.has_exited(),
ProcessHandle::ExecServer(_) => has_exited,
}
}
@@ -216,12 +215,12 @@ impl UnifiedExecProcess {
if self.timed_out() {
return Some(124);
}
let state = self.state_rx.borrow().clone();
let exit_code = self.state_rx.borrow().exit_code;
match &self.process_handle {
ProcessHandle::Local(process_handle) => {
state.exit_code.or_else(|| process_handle.exit_code())
exit_code.or_else(|| process_handle.exit_code())
}
ProcessHandle::ExecServer(_) => state.exit_code,
ProcessHandle::ExecServer(_) => exit_code,
}
}
@@ -264,7 +263,9 @@ impl UnifiedExecProcess {
.map_err(|err| UnifiedExecError::process_failed(err.to_string()))?;
}
}
self.signal_exit(self.exit_code());
// Termination confirms exit without reporting a new exit code. Keep any
// status the output collector published while the request was in flight.
self.state_tx.send_modify(|state| state.has_exited = true);
self.finish_termination();
Ok(())
}
@@ -282,10 +283,13 @@ impl UnifiedExecProcess {
}
pub(super) fn fail_and_terminate(&self, message: String) {
let state = self.state_rx.borrow().clone();
if state.failure_message.is_none() {
let _ = self.state_tx.send_replace(state.failed(message));
}
self.state_tx.send_if_modified(|state| {
if state.failure_message.is_some() {
return false;
}
state.failed(message);
true
});
self.terminate();
}
@@ -411,8 +415,7 @@ impl UnifiedExecProcess {
let cancellation_token = managed.output.cancellation_token.clone();
async move {
let exit_code = exit_rx.await.ok();
let state = state_tx.borrow().clone();
let _ = state_tx.send_replace(state.exited(exit_code));
state_tx.send_modify(|state| state.exited(exit_code));
cancellation_token.cancel();
}
});
@@ -442,8 +445,11 @@ impl UnifiedExecProcess {
let mut state_rx = managed.state_rx.clone();
if tokio::time::timeout(EARLY_EXIT_GRACE_PERIOD, async {
loop {
let state = state_rx.borrow().clone();
if state.has_exited || state.failure_message.is_some() {
let has_exited = {
let state = state_rx.borrow();
state.has_exited || state.failure_message.is_some()
};
if has_exited {
break;
}
if state_rx.changed().await.is_err() {
@@ -510,10 +516,9 @@ impl UnifiedExecProcess {
Some(Ok(event)) => Some(event),
Some(Err(broadcast::error::RecvError::Lagged(_))) | None => None,
Some(Err(broadcast::error::RecvError::Closed)) => {
let state = state_tx.borrow().clone();
let _ = state_tx.send_replace(
state.failed("exec-server process event stream closed".to_string()),
);
state_tx.send_modify(|state| {
state.failed("exec-server process event stream closed".to_string());
});
output_closed.store(true, Ordering::Release);
output_closed_notify.notify_waiters();
cancellation_token.cancel();
@@ -550,8 +555,7 @@ impl UnifiedExecProcess {
{
Ok(response) => response,
Err(err) => {
let state = state_tx.borrow().clone();
let _ = state_tx.send_replace(state.failed(err.to_string()));
state_tx.send_modify(|state| state.failed(err.to_string()));
output_closed.store(true, Ordering::Release);
output_closed_notify.notify_waiters();
cancellation_token.cancel();
@@ -577,20 +581,18 @@ impl UnifiedExecProcess {
}
last_seq = last_seq.max(next_seq.saturating_sub(1));
if let Some(message) = failure {
let state = state_tx.borrow().clone();
let _ = state_tx.send_replace(state.failed(message));
state_tx.send_modify(|state| state.failed(message));
output_closed.store(true, Ordering::Release);
output_closed_notify.notify_waiters();
cancellation_token.cancel();
break;
}
if sandbox_denied || exited {
let mut state = state_tx.borrow().clone();
state.sandbox_denied |= sandbox_denied;
let _ = state_tx.send_replace(if exited {
state.exited(exit_code)
} else {
state
state_tx.send_modify(|state| {
state.sandbox_denied |= sandbox_denied;
if exited {
state.exited(exit_code);
}
});
}
if exited && output_drain_policy == OutputDrainPolicy::BoundedAfterExit {
@@ -630,9 +632,10 @@ impl UnifiedExecProcess {
continue;
}
last_seq = seq;
let mut state = state_tx.borrow().clone();
state.sandbox_denied |= sandbox_denied.unwrap_or(false);
let _ = state_tx.send_replace(state.exited(Some(exit_code)));
state_tx.send_modify(|state| {
state.sandbox_denied |= sandbox_denied.unwrap_or(false);
state.exited(Some(exit_code));
});
if output_drain_policy == OutputDrainPolicy::BoundedAfterExit {
cancellation_token.cancel();
}
@@ -647,8 +650,7 @@ impl UnifiedExecProcess {
break;
}
ExecProcessEvent::Failed(message) => {
let state = state_tx.borrow().clone();
let _ = state_tx.send_replace(state.failed(message));
state_tx.send_modify(|state| state.failed(message));
output_closed.store(true, Ordering::Release);
output_closed_notify.notify_waiters();
cancellation_token.cancel();
@@ -704,8 +706,7 @@ impl UnifiedExecProcess {
}
fn signal_exit(&self, exit_code: Option<i32>) {
let state = self.state_rx.borrow().clone();
let _ = self.state_tx.send_replace(state.exited(exit_code));
self.state_tx.send_modify(|state| state.exited(exit_code));
self.output.cancellation_token.cancel();
}
}
+10 -15
View File
@@ -1,4 +1,7 @@
#[derive(Clone, Debug, Default, Eq, PartialEq)]
//! Exit, failure, and sandbox-denial facts shared by process lifecycle observers.
//! Apply transitions under the watch channel's write lock to preserve concurrent updates.
#[derive(Debug, Default, Eq, PartialEq)]
pub(crate) struct ProcessState {
pub(crate) has_exited: bool,
pub(crate) exit_code: Option<i32>,
@@ -7,21 +10,13 @@ pub(crate) struct ProcessState {
}
impl ProcessState {
pub(crate) fn exited(&self, exit_code: Option<i32>) -> Self {
Self {
has_exited: true,
exit_code,
failure_message: self.failure_message.clone(),
sandbox_denied: self.sandbox_denied,
}
pub(crate) fn exited(&mut self, exit_code: Option<i32>) {
self.has_exited = true;
self.exit_code = exit_code;
}
pub(crate) fn failed(&self, message: String) -> Self {
Self {
has_exited: true,
exit_code: self.exit_code,
failure_message: Some(message),
sandbox_denied: self.sandbox_denied,
}
pub(crate) fn failed(&mut self, message: String) {
self.has_exited = true;
self.failure_message = Some(message);
}
}
@@ -152,22 +152,102 @@ async fn remote_write_closed_stdin_marks_process_exited() {
#[tokio::test]
async fn fail_and_terminate_preserves_failure_message() {
let process = remote_process(
WriteStatus::Accepted,
WriteStatus::StdinClosed,
/*terminate_error*/ None,
codex_sandboxing::SandboxType::None,
)
.await;
process.fail_and_terminate("network denied".to_string());
assert!(matches!(
process.write(b"hello").await,
Err(UnifiedExecError::WriteToStdin)
));
process
.terminate_confirmed()
.await
.expect("terminate should succeed");
process.fail_and_terminate("second failure".to_string());
assert!(process.has_exited());
assert_eq!(
process.failure_message(),
Some("network denied".to_string())
(
process.has_exited(),
process.exit_code(),
process.failure_message()
),
(true, None, Some("network denied".to_string()))
);
}
#[test_case::test_case(WriteStatus::UnknownProcess; "unknown_process")]
#[test_case::test_case(WriteStatus::StdinClosed; "closed_stdin")]
#[tokio::test(start_paused = true)]
async fn remote_lifecycle_updates_preserve_recovered_exit_state(write_status: WriteStatus) {
let (wake_tx, _wake_rx) = watch::channel(0);
let process = UnifiedExecProcess::from_exec_server_started(
StartedExecProcess {
process: Arc::new(MockExecProcess {
process_id: "test-process".to_string().into(),
write_response: WriteResponse {
status: write_status,
},
read_responses: Mutex::new(VecDeque::from([ReadResponse {
chunks: Vec::new(),
next_seq: 1,
exited: true,
exit_code: Some(7),
closed: true,
failure: None,
sandbox_denied: true,
}])),
terminate_error: None,
wake_tx,
}),
sandbox_type: Some(codex_sandboxing::SandboxType::None),
},
OutputDrainPolicy::WaitForOutputClosure,
)
.await
.expect("remote process should start");
// A rejected write first reports exit without a status. The final read then
// recovers the executor's actual exit code and sandbox-denial classification.
let closed = process.output_handles().output_closed_notify.notified();
tokio::pin!(closed);
closed.as_mut().enable();
process.output_handles().collection_finished.cancel();
assert!(matches!(
process.write(b"hello").await,
Err(UnifiedExecError::WriteToStdin)
));
closed.await;
assert!(matches!(
process.write(b"hello").await,
Err(UnifiedExecError::WriteToStdin)
));
process.fail_and_terminate("network denied".to_string());
process
.terminate_confirmed()
.await
.expect("terminate should succeed");
assert_eq!(
(
process.has_exited(),
process.exit_code(),
process.failure_message()
),
(true, Some(7), Some("network denied".to_string()))
);
assert!(matches!(
process
.check_for_sandbox_denial_with_text("unrelated output")
.await,
Err(UnifiedExecError::SandboxDenied { .. })
));
}
#[tokio::test]
async fn remote_terminate_confirmed_updates_state_on_success_only() {
let process = remote_process(