mirror of
https://github.com/openai/codex.git
synced 2026-11-15 21:50:46 +00:00
feat: exec-server prep for unified exec (#15691)
This PR partially rebase `unified_exec` on the `exec-server` and adapt the `exec-server` accordingly. ## What changed in `exec-server` 1. Replaced the old "broadcast-driven; process-global" event model with process-scoped session events. The goal is to be able to have dedicated handler for each process. 2. Add to protocol contract to support explicit lifecycle status and stream ordering: - `WriteResponse` now returns `WriteStatus` (Accepted, UnknownProcess, StdinClosed, Starting) instead of a bool. - Added seq fields to output/exited notifications. - Added terminal process/closed notification. 3. Demultiplexed remote notifications into per-process channels. Same as for the event sys 4. Local and remote backends now both implement ExecBackend. 5. Local backend wraps internal process ID/operations into per-process ExecProcess objects. 6. Remote backend registers a session channel before launch and unregisters on failed launch. ## What changed in `unified_exec` 1. Added unified process-state model and backend-neutral process wrapper. This will probably disappear in the future, but it makes it easier to keep the work flowing on both side. - `UnifiedExecProcess` now handles both local PTY sessions and remote exec-server processes through a shared `ProcessHandle`. - Added `ProcessState` to track has_exited, exit_code, and terminal failure message consistently across backends. 2. Routed write and lifecycle handling through process-level methods. ## Some rationals 1. The change centralizes execution transport in exec-server while preserving policy and orchestration ownership in core, avoiding duplicated launch approval logic. This comes from internal discussion. 2. Session-scoped events remove coupling/cross-talk between processes and make stream ordering and terminal state explicit (seq, closed, failed). 3. The failure-path surfacing (remote launch failures, write failures, transport disconnects) makes command tool output and cleanup behavior deterministic ## Follow-ups: * Unify the concept of thread ID behind an obfuscated struct * FD handling * Full zsh-fork compatibility * Full network sandboxing compatibility * Handle ws disconnection
This commit is contained in:
@@ -6,8 +6,8 @@ use std::sync::atomic::Ordering;
|
||||
use tokio::sync::Mutex;
|
||||
use tokio::sync::Notify;
|
||||
use tokio::sync::broadcast;
|
||||
use tokio::sync::mpsc;
|
||||
use tokio::sync::oneshot::error::TryRecvError;
|
||||
use tokio::sync::watch;
|
||||
use tokio::task::JoinHandle;
|
||||
use tokio::time::Duration;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
@@ -15,8 +15,12 @@ use tokio_util::sync::CancellationToken;
|
||||
use crate::exec::ExecToolCallOutput;
|
||||
use crate::exec::StreamOutput;
|
||||
use crate::exec::is_likely_sandbox_denied;
|
||||
use codex_exec_server::ExecProcess;
|
||||
use codex_exec_server::ReadResponse as ExecReadResponse;
|
||||
use codex_exec_server::StartedExecProcess;
|
||||
use codex_exec_server::WriteStatus;
|
||||
use codex_protocol::protocol::TruncationPolicy;
|
||||
use codex_sandboxing::SandboxType;
|
||||
use codex_utils_output_truncation::TruncationPolicy;
|
||||
use codex_utils_output_truncation::formatted_truncate_text;
|
||||
use codex_utils_pty::ExecCommandSession;
|
||||
use codex_utils_pty::SpawnedPty;
|
||||
@@ -24,6 +28,9 @@ use codex_utils_pty::SpawnedPty;
|
||||
use super::UNIFIED_EXEC_OUTPUT_MAX_TOKENS;
|
||||
use super::UnifiedExecError;
|
||||
use super::head_tail_buffer::HeadTailBuffer;
|
||||
use super::process_state::ProcessState;
|
||||
|
||||
const EARLY_EXIT_GRACE_PERIOD: Duration = Duration::from_millis(150);
|
||||
|
||||
pub(crate) trait SpawnLifecycle: std::fmt::Debug + Send + Sync {
|
||||
/// Returns file descriptors that must stay open across the child `exec()`.
|
||||
@@ -41,11 +48,13 @@ pub(crate) trait SpawnLifecycle: std::fmt::Debug + Send + Sync {
|
||||
pub(crate) type SpawnLifecycleHandle = Box<dyn SpawnLifecycle>;
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
/// Spawn lifecycle that performs no extra setup around process launch.
|
||||
pub(crate) struct NoopSpawnLifecycle;
|
||||
|
||||
impl SpawnLifecycle for NoopSpawnLifecycle {}
|
||||
|
||||
pub(crate) type OutputBuffer = Arc<Mutex<HeadTailBuffer>>;
|
||||
/// Shared output state exposed to polling and streaming consumers.
|
||||
pub(crate) struct OutputHandles {
|
||||
pub(crate) output_buffer: OutputBuffer,
|
||||
pub(crate) output_notify: Arc<Notify>,
|
||||
@@ -54,27 +63,44 @@ pub(crate) struct OutputHandles {
|
||||
pub(crate) cancellation_token: CancellationToken,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
/// Transport-specific process handle used by unified exec.
|
||||
enum ProcessHandle {
|
||||
Local(Box<ExecCommandSession>),
|
||||
Remote(Arc<dyn ExecProcess>),
|
||||
}
|
||||
|
||||
/// Unified wrapper over local PTY sessions and exec-server-backed processes.
|
||||
pub(crate) struct UnifiedExecProcess {
|
||||
process_handle: ExecCommandSession,
|
||||
output_rx: broadcast::Receiver<Vec<u8>>,
|
||||
process_handle: ProcessHandle,
|
||||
output_tx: broadcast::Sender<Vec<u8>>,
|
||||
output_buffer: OutputBuffer,
|
||||
output_notify: Arc<Notify>,
|
||||
output_closed: Arc<AtomicBool>,
|
||||
output_closed_notify: Arc<Notify>,
|
||||
cancellation_token: CancellationToken,
|
||||
output_drained: Arc<Notify>,
|
||||
output_task: JoinHandle<()>,
|
||||
state_tx: watch::Sender<ProcessState>,
|
||||
state_rx: watch::Receiver<ProcessState>,
|
||||
output_task: Option<JoinHandle<()>>,
|
||||
sandbox_type: SandboxType,
|
||||
_spawn_lifecycle: SpawnLifecycleHandle,
|
||||
_spawn_lifecycle: Option<SpawnLifecycleHandle>,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for UnifiedExecProcess {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("UnifiedExecProcess")
|
||||
.field("has_exited", &self.has_exited())
|
||||
.field("exit_code", &self.exit_code())
|
||||
.field("sandbox_type", &self.sandbox_type)
|
||||
.finish_non_exhaustive()
|
||||
}
|
||||
}
|
||||
|
||||
impl UnifiedExecProcess {
|
||||
pub(super) fn new(
|
||||
process_handle: ExecCommandSession,
|
||||
initial_output_rx: tokio::sync::broadcast::Receiver<Vec<u8>>,
|
||||
fn new(
|
||||
process_handle: ProcessHandle,
|
||||
sandbox_type: SandboxType,
|
||||
spawn_lifecycle: SpawnLifecycleHandle,
|
||||
spawn_lifecycle: Option<SpawnLifecycleHandle>,
|
||||
) -> Self {
|
||||
let output_buffer = Arc::new(Mutex::new(HeadTailBuffer::default()));
|
||||
let output_notify = Arc::new(Notify::new());
|
||||
@@ -82,48 +108,49 @@ impl UnifiedExecProcess {
|
||||
let output_closed_notify = Arc::new(Notify::new());
|
||||
let cancellation_token = CancellationToken::new();
|
||||
let output_drained = Arc::new(Notify::new());
|
||||
let mut receiver = initial_output_rx;
|
||||
let output_rx = receiver.resubscribe();
|
||||
let buffer_clone = Arc::clone(&output_buffer);
|
||||
let notify_clone = Arc::clone(&output_notify);
|
||||
let output_closed_clone = Arc::clone(&output_closed);
|
||||
let output_closed_notify_clone = Arc::clone(&output_closed_notify);
|
||||
let output_task = tokio::spawn(async move {
|
||||
loop {
|
||||
match receiver.recv().await {
|
||||
Ok(chunk) => {
|
||||
let mut guard = buffer_clone.lock().await;
|
||||
guard.push_chunk(chunk);
|
||||
drop(guard);
|
||||
notify_clone.notify_waiters();
|
||||
}
|
||||
Err(tokio::sync::broadcast::error::RecvError::Lagged(_)) => continue,
|
||||
Err(tokio::sync::broadcast::error::RecvError::Closed) => {
|
||||
output_closed_clone.store(true, Ordering::Release);
|
||||
output_closed_notify_clone.notify_waiters();
|
||||
break;
|
||||
}
|
||||
};
|
||||
}
|
||||
});
|
||||
let (output_tx, _) = broadcast::channel(64);
|
||||
let (state_tx, state_rx) = watch::channel(ProcessState::default());
|
||||
|
||||
Self {
|
||||
process_handle,
|
||||
output_rx,
|
||||
output_tx,
|
||||
output_buffer,
|
||||
output_notify,
|
||||
output_closed,
|
||||
output_closed_notify,
|
||||
cancellation_token,
|
||||
output_drained,
|
||||
output_task,
|
||||
state_tx,
|
||||
state_rx,
|
||||
output_task: None,
|
||||
sandbox_type,
|
||||
_spawn_lifecycle: spawn_lifecycle,
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn writer_sender(&self) -> mpsc::Sender<Vec<u8>> {
|
||||
self.process_handle.writer_sender()
|
||||
pub(super) async fn write(&self, data: &[u8]) -> Result<(), UnifiedExecError> {
|
||||
match &self.process_handle {
|
||||
ProcessHandle::Local(process_handle) => process_handle
|
||||
.writer_sender()
|
||||
.send(data.to_vec())
|
||||
.await
|
||||
.map_err(|_| UnifiedExecError::WriteToStdin),
|
||||
ProcessHandle::Remote(process_handle) => {
|
||||
match process_handle.write(data.to_vec()).await {
|
||||
Ok(response) => match response.status {
|
||||
WriteStatus::Accepted => Ok(()),
|
||||
WriteStatus::UnknownProcess | WriteStatus::StdinClosed => {
|
||||
let state = self.state_rx.borrow().clone();
|
||||
let _ = self.state_tx.send_replace(state.exited(state.exit_code));
|
||||
self.cancellation_token.cancel();
|
||||
Err(UnifiedExecError::WriteToStdin)
|
||||
}
|
||||
WriteStatus::Starting => Err(UnifiedExecError::WriteToStdin),
|
||||
},
|
||||
Err(err) => Err(UnifiedExecError::process_failed(err.to_string())),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn output_handles(&self) -> OutputHandles {
|
||||
@@ -137,7 +164,7 @@ impl UnifiedExecProcess {
|
||||
}
|
||||
|
||||
pub(super) fn output_receiver(&self) -> tokio::sync::broadcast::Receiver<Vec<u8>> {
|
||||
self.output_rx.resubscribe()
|
||||
self.output_tx.subscribe()
|
||||
}
|
||||
|
||||
pub(super) fn cancellation_token(&self) -> CancellationToken {
|
||||
@@ -149,19 +176,39 @@ impl UnifiedExecProcess {
|
||||
}
|
||||
|
||||
pub(super) fn has_exited(&self) -> bool {
|
||||
self.process_handle.has_exited()
|
||||
let state = self.state_rx.borrow().clone();
|
||||
match &self.process_handle {
|
||||
ProcessHandle::Local(process_handle) => state.has_exited || process_handle.has_exited(),
|
||||
ProcessHandle::Remote(_) => state.has_exited,
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn exit_code(&self) -> Option<i32> {
|
||||
self.process_handle.exit_code()
|
||||
let state = self.state_rx.borrow().clone();
|
||||
match &self.process_handle {
|
||||
ProcessHandle::Local(process_handle) => {
|
||||
state.exit_code.or_else(|| process_handle.exit_code())
|
||||
}
|
||||
ProcessHandle::Remote(_) => state.exit_code,
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn terminate(&self) {
|
||||
self.output_closed.store(true, Ordering::Release);
|
||||
self.output_closed_notify.notify_waiters();
|
||||
self.process_handle.terminate();
|
||||
match &self.process_handle {
|
||||
ProcessHandle::Local(process_handle) => process_handle.terminate(),
|
||||
ProcessHandle::Remote(process_handle) => {
|
||||
let process_handle = Arc::clone(process_handle);
|
||||
tokio::spawn(async move {
|
||||
let _ = process_handle.terminate().await;
|
||||
});
|
||||
}
|
||||
}
|
||||
self.cancellation_token.cancel();
|
||||
self.output_task.abort();
|
||||
if let Some(output_task) = &self.output_task {
|
||||
output_task.abort();
|
||||
}
|
||||
}
|
||||
|
||||
async fn snapshot_output(&self) -> Vec<Vec<u8>> {
|
||||
@@ -173,6 +220,10 @@ impl UnifiedExecProcess {
|
||||
self.sandbox_type
|
||||
}
|
||||
|
||||
pub(super) fn failure_message(&self) -> Option<String> {
|
||||
self.state_rx.borrow().failure_message.clone()
|
||||
}
|
||||
|
||||
pub(super) async fn check_for_sandbox_denial(&self) -> Result<(), UnifiedExecError> {
|
||||
let _ =
|
||||
tokio::time::timeout(Duration::from_millis(20), self.output_notify.notified()).await;
|
||||
@@ -232,29 +283,47 @@ impl UnifiedExecProcess {
|
||||
mut exit_rx,
|
||||
} = spawned;
|
||||
let output_rx = codex_utils_pty::combine_output_receivers(stdout_rx, stderr_rx);
|
||||
let managed = Self::new(process_handle, output_rx, sandbox_type, spawn_lifecycle);
|
||||
let mut managed = Self::new(
|
||||
ProcessHandle::Local(Box::new(process_handle)),
|
||||
sandbox_type,
|
||||
Some(spawn_lifecycle),
|
||||
);
|
||||
managed.output_task = Some(Self::spawn_local_output_task(
|
||||
output_rx,
|
||||
Arc::clone(&managed.output_buffer),
|
||||
Arc::clone(&managed.output_notify),
|
||||
Arc::clone(&managed.output_closed),
|
||||
Arc::clone(&managed.output_closed_notify),
|
||||
managed.output_tx.clone(),
|
||||
));
|
||||
|
||||
let exit_ready = matches!(exit_rx.try_recv(), Ok(_) | Err(TryRecvError::Closed));
|
||||
|
||||
if exit_ready {
|
||||
managed.signal_exit();
|
||||
managed.check_for_sandbox_denial().await?;
|
||||
return Ok(managed);
|
||||
match exit_rx.try_recv() {
|
||||
Ok(exit_code) => {
|
||||
managed.signal_exit(Some(exit_code));
|
||||
managed.check_for_sandbox_denial().await?;
|
||||
return Ok(managed);
|
||||
}
|
||||
Err(TryRecvError::Closed) => {
|
||||
managed.signal_exit(/*exit_code*/ None);
|
||||
managed.check_for_sandbox_denial().await?;
|
||||
return Ok(managed);
|
||||
}
|
||||
Err(TryRecvError::Empty) => {}
|
||||
}
|
||||
|
||||
if tokio::time::timeout(Duration::from_millis(150), &mut exit_rx)
|
||||
.await
|
||||
.is_ok()
|
||||
{
|
||||
managed.signal_exit();
|
||||
if let Ok(exit_result) = tokio::time::timeout(EARLY_EXIT_GRACE_PERIOD, &mut exit_rx).await {
|
||||
managed.signal_exit(exit_result.ok());
|
||||
managed.check_for_sandbox_denial().await?;
|
||||
return Ok(managed);
|
||||
}
|
||||
|
||||
tokio::spawn({
|
||||
let state_tx = managed.state_tx.clone();
|
||||
let cancellation_token = managed.cancellation_token.clone();
|
||||
async move {
|
||||
let _ = exit_rx.await;
|
||||
let exit_code = exit_rx.await.ok();
|
||||
let state = state_tx.borrow().clone();
|
||||
let _ = state_tx.send_replace(state.exited(exit_code));
|
||||
cancellation_token.cancel();
|
||||
}
|
||||
});
|
||||
@@ -262,7 +331,162 @@ impl UnifiedExecProcess {
|
||||
Ok(managed)
|
||||
}
|
||||
|
||||
fn signal_exit(&self) {
|
||||
pub(super) async fn from_remote_started(
|
||||
started: StartedExecProcess,
|
||||
sandbox_type: SandboxType,
|
||||
) -> Result<Self, UnifiedExecError> {
|
||||
let process_handle = ProcessHandle::Remote(Arc::clone(&started.process));
|
||||
let mut managed = Self::new(process_handle, sandbox_type, /*spawn_lifecycle*/ None);
|
||||
let output_handles = managed.output_handles();
|
||||
managed.output_task = Some(Self::spawn_remote_output_task(
|
||||
started,
|
||||
output_handles,
|
||||
managed.output_tx.clone(),
|
||||
managed.state_tx.clone(),
|
||||
));
|
||||
|
||||
let mut state_rx = managed.state_rx.clone();
|
||||
if tokio::time::timeout(EARLY_EXIT_GRACE_PERIOD, async {
|
||||
loop {
|
||||
let state = state_rx.borrow().clone();
|
||||
if state.has_exited || state.failure_message.is_some() {
|
||||
break;
|
||||
}
|
||||
if state_rx.changed().await.is_err() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
})
|
||||
.await
|
||||
.is_ok()
|
||||
{
|
||||
managed.check_for_sandbox_denial().await?;
|
||||
}
|
||||
|
||||
Ok(managed)
|
||||
}
|
||||
|
||||
fn spawn_remote_output_task(
|
||||
started: StartedExecProcess,
|
||||
output_handles: OutputHandles,
|
||||
output_tx: broadcast::Sender<Vec<u8>>,
|
||||
state_tx: watch::Sender<ProcessState>,
|
||||
) -> JoinHandle<()> {
|
||||
let OutputHandles {
|
||||
output_buffer,
|
||||
output_notify,
|
||||
output_closed,
|
||||
output_closed_notify,
|
||||
cancellation_token,
|
||||
} = output_handles;
|
||||
let process = started.process;
|
||||
let mut wake_rx = process.subscribe_wake();
|
||||
tokio::spawn(async move {
|
||||
let mut after_seq = None;
|
||||
loop {
|
||||
match process
|
||||
.read(after_seq, /*max_bytes*/ None, /*wait_ms*/ Some(0))
|
||||
.await
|
||||
{
|
||||
Ok(response) => {
|
||||
let ExecReadResponse {
|
||||
chunks,
|
||||
next_seq,
|
||||
exited,
|
||||
exit_code,
|
||||
closed,
|
||||
failure,
|
||||
} = response;
|
||||
|
||||
for chunk in chunks {
|
||||
let bytes = chunk.chunk.into_inner();
|
||||
let mut guard = output_buffer.lock().await;
|
||||
guard.push_chunk(bytes.clone());
|
||||
drop(guard);
|
||||
let _ = output_tx.send(bytes);
|
||||
output_notify.notify_waiters();
|
||||
}
|
||||
|
||||
if let Some(message) = failure {
|
||||
let state = state_tx.borrow().clone();
|
||||
let _ = state_tx.send_replace(state.failed(message));
|
||||
output_closed.store(true, Ordering::Release);
|
||||
output_closed_notify.notify_waiters();
|
||||
cancellation_token.cancel();
|
||||
break;
|
||||
}
|
||||
|
||||
if exited {
|
||||
let state = state_tx.borrow().clone();
|
||||
let _ = state_tx.send_replace(state.exited(exit_code));
|
||||
}
|
||||
|
||||
if closed {
|
||||
output_closed.store(true, Ordering::Release);
|
||||
output_closed_notify.notify_waiters();
|
||||
cancellation_token.cancel();
|
||||
}
|
||||
|
||||
after_seq = next_seq.checked_sub(1);
|
||||
if output_closed.load(Ordering::Acquire) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
Err(err) => {
|
||||
let state = state_tx.borrow().clone();
|
||||
let _ = state_tx.send_replace(state.failed(err.to_string()));
|
||||
output_closed.store(true, Ordering::Release);
|
||||
output_closed_notify.notify_waiters();
|
||||
cancellation_token.cancel();
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if wake_rx.changed().await.is_err() {
|
||||
let state = state_tx.borrow().clone();
|
||||
let _ = state_tx
|
||||
.send_replace(state.failed("exec-server wake channel closed".to_string()));
|
||||
output_closed.store(true, Ordering::Release);
|
||||
output_closed_notify.notify_waiters();
|
||||
cancellation_token.cancel();
|
||||
break;
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
fn spawn_local_output_task(
|
||||
mut receiver: tokio::sync::broadcast::Receiver<Vec<u8>>,
|
||||
buffer: OutputBuffer,
|
||||
output_notify: Arc<Notify>,
|
||||
output_closed: Arc<AtomicBool>,
|
||||
output_closed_notify: Arc<Notify>,
|
||||
output_tx: broadcast::Sender<Vec<u8>>,
|
||||
) -> JoinHandle<()> {
|
||||
tokio::spawn(async move {
|
||||
loop {
|
||||
match receiver.recv().await {
|
||||
Ok(chunk) => {
|
||||
let mut guard = buffer.lock().await;
|
||||
guard.push_chunk(chunk.clone());
|
||||
drop(guard);
|
||||
let _ = output_tx.send(chunk);
|
||||
output_notify.notify_waiters();
|
||||
}
|
||||
Err(tokio::sync::broadcast::error::RecvError::Lagged(_)) => continue,
|
||||
Err(tokio::sync::broadcast::error::RecvError::Closed) => {
|
||||
output_closed.store(true, Ordering::Release);
|
||||
output_closed_notify.notify_waiters();
|
||||
break;
|
||||
}
|
||||
};
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
fn signal_exit(&self, exit_code: Option<i32>) {
|
||||
let state = self.state_rx.borrow().clone();
|
||||
let _ = self.state_tx.send_replace(state.exited(exit_code));
|
||||
self.cancellation_token.cancel();
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user