mirror of
https://github.com/openai/codex.git
synced 2026-10-21 17:48:44 +00:00
[codex] add code-mode host failure supervision hooks (#30110)
## Why A process host should be discarded and rebuilt after critical actor or V8 failure, while the existing in-process production path must keep its current cell-error semantics. This change establishes that failure boundary without adding the host process or remote client. ## What changed - add optional task-failure supervision to the transport-neutral code-mode session runtime - report Tokio cell-actor failures and V8 runtime-thread panics to a host-provided fail-stop handler - preserve the existing handler-less in-process behavior - make host-owned cell ID allocation fail before numeric wraparound ## Follow-up The V8 panic signal surfaced here should also be consumed by the `InProcessCodeModeSession` manager in a future change so it can fail the affected cell. This PR intentionally leaves the handler-less in-process behavior unchanged while putting the required panic tracking in place. ## Stack This is **2 of 4** in the process-owned code-mode session stack. - #30108 is merged into `main` - The next PR targets this branch ## Validation - `just test -p codex-code-mode` — 53 passed - `just argument-comment-lint -p codex-code-mode` - `just fix -p codex-code-mode`
This commit is contained in:
@@ -1,11 +1,14 @@
|
||||
use std::panic::AssertUnwindSafe;
|
||||
use std::sync::Arc;
|
||||
|
||||
use futures::FutureExt;
|
||||
use tokio::task::JoinSet;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tracing::warn;
|
||||
|
||||
use super::CellHost;
|
||||
use super::CellToolCall;
|
||||
use crate::TaskFailureHandler;
|
||||
use crate::runtime::RuntimeCommand;
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
@@ -20,10 +23,20 @@ pub(super) fn spawn_notification<H: CellHost>(
|
||||
call_id: String,
|
||||
text: String,
|
||||
cancellation_token: CancellationToken,
|
||||
task_failure_handler: Option<TaskFailureHandler>,
|
||||
) {
|
||||
tasks.spawn(async move {
|
||||
if let Err(err) = host.notify(call_id, text, cancellation_token).await {
|
||||
warn!("failed to deliver code mode notification: {err}");
|
||||
let callback =
|
||||
AssertUnwindSafe(async move { host.notify(call_id, text, cancellation_token).await })
|
||||
.catch_unwind()
|
||||
.await;
|
||||
match callback {
|
||||
Ok(Ok(())) => {}
|
||||
Ok(Err(err)) => warn!("failed to deliver code mode notification: {err}"),
|
||||
Err(_) => report_task_failure(
|
||||
task_failure_handler.as_ref(),
|
||||
"code mode notification task panicked".to_string(),
|
||||
),
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -34,14 +47,32 @@ pub(super) fn spawn_tool<H: CellHost>(
|
||||
invocation: CellToolCall,
|
||||
runtime_tx: std::sync::mpsc::Sender<RuntimeCommand>,
|
||||
cancellation_token: CancellationToken,
|
||||
task_failure_handler: Option<TaskFailureHandler>,
|
||||
) {
|
||||
tasks.spawn(async move {
|
||||
let id = invocation.id.clone();
|
||||
let command = match host.invoke_tool(invocation, cancellation_token).await {
|
||||
Ok(result) => RuntimeCommand::ToolResponse { id, result },
|
||||
Err(error_text) => RuntimeCommand::ToolError { id, error_text },
|
||||
let callback =
|
||||
AssertUnwindSafe(async move { host.invoke_tool(invocation, cancellation_token).await })
|
||||
.catch_unwind()
|
||||
.await;
|
||||
let (command, failure_reason) = match callback {
|
||||
Ok(Ok(result)) => (RuntimeCommand::ToolResponse { id, result }, None),
|
||||
Ok(Err(error_text)) => (RuntimeCommand::ToolError { id, error_text }, None),
|
||||
Err(_) => {
|
||||
let failure_reason = "code mode tool task panicked".to_string();
|
||||
(
|
||||
RuntimeCommand::ToolError {
|
||||
id,
|
||||
error_text: failure_reason.clone(),
|
||||
},
|
||||
Some(failure_reason),
|
||||
)
|
||||
}
|
||||
};
|
||||
let _ = runtime_tx.send(command);
|
||||
if let Some(failure_reason) = failure_reason {
|
||||
report_task_failure(task_failure_handler.as_ref(), failure_reason);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
@@ -50,28 +81,48 @@ pub(super) async fn finish_callbacks(
|
||||
notification_tasks: &mut JoinSet<()>,
|
||||
tool_tasks: &mut JoinSet<()>,
|
||||
completion: CallbackCompletion,
|
||||
task_failure_handler: Option<&TaskFailureHandler>,
|
||||
) {
|
||||
if matches!(completion, CallbackCompletion::Cancel) {
|
||||
cancellation_token.cancel();
|
||||
}
|
||||
drain_tasks(notification_tasks, "notification").await;
|
||||
drain_tasks(notification_tasks, "notification", task_failure_handler).await;
|
||||
cancellation_token.cancel();
|
||||
drain_tasks(tool_tasks, "tool").await;
|
||||
drain_tasks(tool_tasks, "tool", task_failure_handler).await;
|
||||
}
|
||||
|
||||
pub(super) fn log_task_result(
|
||||
pub(super) fn report_task_result(
|
||||
task_result: Option<Result<(), tokio::task::JoinError>>,
|
||||
description: &str,
|
||||
task_failure_handler: Option<&TaskFailureHandler>,
|
||||
) {
|
||||
if let Some(Err(err)) = task_result
|
||||
&& !err.is_cancelled()
|
||||
{
|
||||
warn!("code mode {description} task failed: {err}");
|
||||
report_task_failure(
|
||||
task_failure_handler,
|
||||
format!("code mode {description} task failed: {err}"),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async fn drain_tasks(tasks: &mut JoinSet<()>, description: &str) {
|
||||
while let Some(result) = tasks.join_next().await {
|
||||
log_task_result(Some(result), description);
|
||||
fn report_task_failure(task_failure_handler: Option<&TaskFailureHandler>, failure_reason: String) {
|
||||
warn!("{failure_reason}");
|
||||
if let Some(task_failure_handler) = task_failure_handler {
|
||||
task_failure_handler(failure_reason);
|
||||
}
|
||||
}
|
||||
|
||||
async fn drain_tasks(
|
||||
tasks: &mut JoinSet<()>,
|
||||
description: &str,
|
||||
task_failure_handler: Option<&TaskFailureHandler>,
|
||||
) {
|
||||
while let Some(result) = tasks.join_next().await {
|
||||
report_task_result(Some(result), description, task_failure_handler);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "callbacks_tests.rs"]
|
||||
mod tests;
|
||||
|
||||
Reference in New Issue
Block a user