mirror of
https://github.com/openai/codex.git
synced 2026-11-17 22:10:37 +00:00
code-mode: decouple cell creation from observation
This commit is contained in:
@@ -48,19 +48,10 @@ impl CellActor {
|
||||
request: CellRequest,
|
||||
stored_values: HashMap<String, JsonValue>,
|
||||
host: Arc<H>,
|
||||
initial_observe_mode: ObserveMode,
|
||||
cell_state: Arc<CellState>,
|
||||
) -> Result<
|
||||
(
|
||||
CellHandle,
|
||||
CellEventFuture,
|
||||
impl Future<Output = ()> + Send + 'static,
|
||||
),
|
||||
String,
|
||||
> {
|
||||
) -> Result<(CellHandle, impl Future<Output = ()> + Send + 'static), String> {
|
||||
let (event_tx, event_rx) = mpsc::unbounded_channel();
|
||||
let (command_tx, command_rx) = mpsc::unbounded_channel();
|
||||
let (initial_response_tx, initial_response_rx) = oneshot::channel();
|
||||
let (runtime_tx, runtime_control_tx, runtime_terminate_handle) = spawn_runtime(
|
||||
stored_values,
|
||||
runtime_request(request),
|
||||
@@ -78,14 +69,8 @@ impl CellActor {
|
||||
},
|
||||
event_rx,
|
||||
command_rx,
|
||||
Observer {
|
||||
mode: initial_observe_mode,
|
||||
response_tx: initial_response_tx,
|
||||
},
|
||||
);
|
||||
let initial_response =
|
||||
Box::pin(async move { initial_response_rx.await.unwrap_or(Err(CellError::Closed)) });
|
||||
Ok((handle, initial_response, task))
|
||||
Ok((handle, task))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -106,7 +91,6 @@ async fn run_cell<H: CellHost>(
|
||||
context: CellContext,
|
||||
mut event_rx: mpsc::UnboundedReceiver<RuntimeEvent>,
|
||||
command_rx: mpsc::UnboundedReceiver<CellCommand>,
|
||||
initial_observer: Observer,
|
||||
) {
|
||||
let CellContext {
|
||||
runtime_tx,
|
||||
@@ -118,8 +102,10 @@ async fn run_cell<H: CellHost>(
|
||||
let callback_cancellation_token = cancellation_token.child_token();
|
||||
let mut content_items = Vec::new();
|
||||
let mut pending_tool_call_ids = Vec::new();
|
||||
let mut pending_initial_yield_items: Option<Vec<OutputItem>> = None;
|
||||
let mut pending_frontier_ready = false;
|
||||
let mut observer = Some(initial_observer);
|
||||
let mut observer: Option<Observer> = None;
|
||||
let mut has_been_observed = false;
|
||||
let mut termination = false;
|
||||
let mut runtime_closed = false;
|
||||
let mut runtime_paused = false;
|
||||
@@ -136,6 +122,10 @@ async fn run_cell<H: CellHost>(
|
||||
_ = cancellation_token.cancelled(), if !termination => {
|
||||
termination = true;
|
||||
yield_timer = None;
|
||||
if let Some(mut yielded_items) = pending_initial_yield_items.take() {
|
||||
yielded_items.append(&mut content_items);
|
||||
content_items = yielded_items;
|
||||
}
|
||||
drop(command_rx.take());
|
||||
begin_termination(
|
||||
&runtime_tx,
|
||||
@@ -189,30 +179,47 @@ async fn run_cell<H: CellHost>(
|
||||
let _ = response_tx.send(Err(CellError::Busy));
|
||||
continue;
|
||||
}
|
||||
if matches!(mode, ObserveMode::PendingFrontier) && pending_frontier_ready {
|
||||
pending_frontier_ready = false;
|
||||
match send_cell_event(
|
||||
has_been_observed = true;
|
||||
if matches!(mode, ObserveMode::YieldAfter(_))
|
||||
&& let Some(yielded_items) = pending_initial_yield_items.take()
|
||||
{
|
||||
let delivered = match send_cell_event(
|
||||
response_tx,
|
||||
CellEvent::Pending {
|
||||
content_items: std::mem::take(&mut content_items),
|
||||
pending_tool_call_ids: std::mem::take(&mut pending_tool_call_ids),
|
||||
CellEvent::Yielded {
|
||||
content_items: yielded_items,
|
||||
},
|
||||
) {
|
||||
Ok(()) => {}
|
||||
Err(CellEvent::Pending {
|
||||
content_items: undelivered_items,
|
||||
pending_tool_call_ids: undelivered_tool_call_ids,
|
||||
}) => {
|
||||
content_items = undelivered_items;
|
||||
pending_tool_call_ids = undelivered_tool_call_ids;
|
||||
pending_frontier_ready = true;
|
||||
Ok(()) => true,
|
||||
Err(CellEvent::Yielded { content_items }) => {
|
||||
pending_initial_yield_items = Some(content_items);
|
||||
has_been_observed = false;
|
||||
false
|
||||
}
|
||||
Err(event) => {
|
||||
panic!("pending delivery returned an unexpected event: {event:?}")
|
||||
panic!("initial yield delivery returned an unexpected event: {event:?}")
|
||||
}
|
||||
};
|
||||
if delivered && runtime_paused {
|
||||
pending_frontier_ready = false;
|
||||
pending_tool_call_ids.clear();
|
||||
resume_for_observation(
|
||||
mode,
|
||||
&mut runtime_paused,
|
||||
&runtime_tx,
|
||||
&runtime_control_tx,
|
||||
);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if matches!(mode, ObserveMode::PendingFrontier) && pending_frontier_ready {
|
||||
pending_frontier_ready = !send_pending_event(
|
||||
response_tx,
|
||||
&mut pending_initial_yield_items,
|
||||
&mut content_items,
|
||||
&mut pending_tool_call_ids,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
observer = Some(Observer { mode, response_tx });
|
||||
yield_timer = observer.as_ref().and_then(observer_timer);
|
||||
if runtime_paused && matches!(mode, ObserveMode::YieldAfter(_)) {
|
||||
@@ -254,6 +261,10 @@ async fn run_cell<H: CellHost>(
|
||||
let Some(event) = maybe_event else {
|
||||
runtime_closed = true;
|
||||
if termination || cancellation_token.is_cancelled() {
|
||||
let termination_content_items = take_all_content(
|
||||
&mut pending_initial_yield_items,
|
||||
&mut content_items,
|
||||
);
|
||||
finish_callbacks(
|
||||
&callback_cancellation_token,
|
||||
&mut notification_tasks,
|
||||
@@ -264,7 +275,7 @@ async fn run_cell<H: CellHost>(
|
||||
&cell_state,
|
||||
observer.take().map(|observer| observer.response_tx),
|
||||
CellEvent::Terminated {
|
||||
content_items: std::mem::take(&mut content_items),
|
||||
content_items: termination_content_items,
|
||||
},
|
||||
);
|
||||
break;
|
||||
@@ -284,7 +295,7 @@ async fn run_cell<H: CellHost>(
|
||||
.commit_completion(
|
||||
HashMap::new(),
|
||||
event,
|
||||
/*pending_initial_yield_items*/ None,
|
||||
pending_initial_yield_items.take(),
|
||||
Arc::clone(&cell_state),
|
||||
)
|
||||
.await
|
||||
@@ -293,7 +304,9 @@ async fn run_cell<H: CellHost>(
|
||||
CompletionCommit::Rejected(event) => Some(event),
|
||||
};
|
||||
match cell_state.deliver_completion(
|
||||
observer.take().map(|observer| observer.response_tx),
|
||||
observer
|
||||
.take()
|
||||
.map(|observer| (observer.mode, observer.response_tx)),
|
||||
) {
|
||||
CompletionDelivery::Delivered => break,
|
||||
CompletionDelivery::Buffered => {}
|
||||
@@ -316,42 +329,32 @@ async fn run_cell<H: CellHost>(
|
||||
}
|
||||
RuntimeEvent::Pending => {
|
||||
runtime_paused = true;
|
||||
if matches!(
|
||||
observer.as_ref().map(|observer| observer.mode),
|
||||
Some(ObserveMode::PendingFrontier)
|
||||
) {
|
||||
if let Some(observer) = observer.take_if(|observer| {
|
||||
observer.mode == ObserveMode::PendingFrontier
|
||||
}) {
|
||||
yield_timer = None;
|
||||
pending_frontier_ready = !send_pending_event(
|
||||
observer.response_tx,
|
||||
&mut pending_initial_yield_items,
|
||||
&mut content_items,
|
||||
&mut pending_tool_call_ids,
|
||||
);
|
||||
} else if observer.is_some() || has_been_observed {
|
||||
pending_frontier_ready = false;
|
||||
match send_observer_event(
|
||||
observer.take(),
|
||||
CellEvent::Pending {
|
||||
content_items: std::mem::take(&mut content_items),
|
||||
pending_tool_call_ids: std::mem::take(
|
||||
&mut pending_tool_call_ids,
|
||||
),
|
||||
},
|
||||
) {
|
||||
Ok(()) => {}
|
||||
Err(CellEvent::Pending {
|
||||
content_items: undelivered_items,
|
||||
pending_tool_call_ids: undelivered_tool_call_ids,
|
||||
}) => {
|
||||
content_items = undelivered_items;
|
||||
pending_tool_call_ids = undelivered_tool_call_ids;
|
||||
pending_frontier_ready = true;
|
||||
}
|
||||
Err(event) => {
|
||||
panic!("pending delivery returned an unexpected event: {event:?}")
|
||||
}
|
||||
}
|
||||
} else {
|
||||
pending_tool_call_ids.clear();
|
||||
let _ = runtime_control_tx.send(RuntimeControlCommand::Continue);
|
||||
runtime_paused = false;
|
||||
} else {
|
||||
pending_frontier_ready = true;
|
||||
}
|
||||
}
|
||||
RuntimeEvent::ContentItem(item) => content_items.push(output_item(item)),
|
||||
RuntimeEvent::YieldRequested => {
|
||||
// An unattached yield is normally a no-op. Preserve only the first
|
||||
// pre-observation yield so create followed by its initial observe/wait
|
||||
// retains the former execute initial-response behavior. After an
|
||||
// observation attaches, later unattached yields do not affect a future
|
||||
// observation.
|
||||
let yield_observer = matches!(
|
||||
observer.as_ref().map(|observer| observer.mode),
|
||||
Some(ObserveMode::YieldAfter(_))
|
||||
@@ -367,6 +370,11 @@ async fn run_cell<H: CellHost>(
|
||||
),
|
||||
&mut content_items,
|
||||
);
|
||||
} else if observer.is_none()
|
||||
&& !has_been_observed
|
||||
&& pending_initial_yield_items.is_none()
|
||||
{
|
||||
pending_initial_yield_items = Some(std::mem::take(&mut content_items));
|
||||
}
|
||||
}
|
||||
RuntimeEvent::Notify { call_id, text } => {
|
||||
@@ -400,6 +408,10 @@ async fn run_cell<H: CellHost>(
|
||||
runtime_closed = true;
|
||||
yield_timer = None;
|
||||
if termination || cancellation_token.is_cancelled() {
|
||||
let termination_content_items = take_all_content(
|
||||
&mut pending_initial_yield_items,
|
||||
&mut content_items,
|
||||
);
|
||||
finish_callbacks(
|
||||
&callback_cancellation_token,
|
||||
&mut notification_tasks,
|
||||
@@ -410,7 +422,7 @@ async fn run_cell<H: CellHost>(
|
||||
&cell_state,
|
||||
observer.take().map(|observer| observer.response_tx),
|
||||
CellEvent::Terminated {
|
||||
content_items: std::mem::take(&mut content_items),
|
||||
content_items: termination_content_items,
|
||||
},
|
||||
);
|
||||
break;
|
||||
@@ -430,7 +442,7 @@ async fn run_cell<H: CellHost>(
|
||||
.commit_completion(
|
||||
stored_value_writes,
|
||||
event,
|
||||
/*pending_initial_yield_items*/ None,
|
||||
pending_initial_yield_items.take(),
|
||||
Arc::clone(&cell_state),
|
||||
)
|
||||
.await
|
||||
@@ -439,7 +451,9 @@ async fn run_cell<H: CellHost>(
|
||||
CompletionCommit::Rejected(event) => Some(event),
|
||||
};
|
||||
match cell_state.deliver_completion(
|
||||
observer.take().map(|observer| observer.response_tx),
|
||||
observer
|
||||
.take()
|
||||
.map(|observer| (observer.mode, observer.response_tx)),
|
||||
) {
|
||||
CompletionDelivery::Delivered => break,
|
||||
CompletionDelivery::Buffered => {}
|
||||
@@ -502,6 +516,40 @@ fn send_cell_event(
|
||||
}
|
||||
}
|
||||
|
||||
fn send_pending_event(
|
||||
response_tx: oneshot::Sender<Result<CellEvent, CellError>>,
|
||||
pending_initial_yield_items: &mut Option<Vec<OutputItem>>,
|
||||
content_items: &mut Vec<OutputItem>,
|
||||
pending_tool_call_ids: &mut Vec<String>,
|
||||
) -> bool {
|
||||
let had_initial_yield = pending_initial_yield_items.is_some();
|
||||
let mut delivered_items = pending_initial_yield_items.take().unwrap_or_default();
|
||||
let initial_yield_len = delivered_items.len();
|
||||
delivered_items.append(content_items);
|
||||
match send_cell_event(
|
||||
response_tx,
|
||||
CellEvent::Pending {
|
||||
content_items: delivered_items,
|
||||
pending_tool_call_ids: std::mem::take(pending_tool_call_ids),
|
||||
},
|
||||
) {
|
||||
Ok(()) => true,
|
||||
Err(CellEvent::Pending {
|
||||
content_items: mut undelivered_items,
|
||||
pending_tool_call_ids: undelivered_tool_call_ids,
|
||||
}) => {
|
||||
let following_items = undelivered_items.split_off(initial_yield_len);
|
||||
if had_initial_yield {
|
||||
*pending_initial_yield_items = Some(undelivered_items);
|
||||
}
|
||||
*content_items = following_items;
|
||||
*pending_tool_call_ids = undelivered_tool_call_ids;
|
||||
false
|
||||
}
|
||||
Err(event) => panic!("pending delivery returned an unexpected event: {event:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
fn restore_undelivered_yield(delivery: Result<(), CellEvent>, content_items: &mut Vec<OutputItem>) {
|
||||
match delivery {
|
||||
Ok(()) => {}
|
||||
@@ -523,6 +571,17 @@ fn rejected_completion_content(event: Option<CellEvent>) -> Vec<OutputItem> {
|
||||
}
|
||||
}
|
||||
|
||||
fn take_all_content(
|
||||
pending_initial_yield_items: &mut Option<Vec<OutputItem>>,
|
||||
content_items: &mut Vec<OutputItem>,
|
||||
) -> Vec<OutputItem> {
|
||||
let Some(mut yielded_items) = pending_initial_yield_items.take() else {
|
||||
return std::mem::take(content_items);
|
||||
};
|
||||
yielded_items.append(content_items);
|
||||
yielded_items
|
||||
}
|
||||
|
||||
fn finish_termination(
|
||||
cell_state: &CellState,
|
||||
observer_tx: Option<oneshot::Sender<Result<CellEvent, CellError>>>,
|
||||
|
||||
@@ -20,6 +20,7 @@ struct TestHost;
|
||||
|
||||
#[derive(Default)]
|
||||
struct RecordingHost {
|
||||
committed: AtomicBool,
|
||||
notified: AtomicBool,
|
||||
}
|
||||
|
||||
@@ -80,7 +81,9 @@ impl CellHost for RecordingHost {
|
||||
pending_initial_yield_items: Option<Vec<OutputItem>>,
|
||||
cell_state: Arc<CellState>,
|
||||
) -> CompletionCommit {
|
||||
cell_state.commit_completion(event, pending_initial_yield_items, || {})
|
||||
cell_state.commit_completion(event, pending_initial_yield_items, || {
|
||||
self.committed.store(true, Ordering::Release);
|
||||
})
|
||||
}
|
||||
|
||||
async fn closed(&self) {}
|
||||
@@ -89,25 +92,22 @@ impl CellHost for RecordingHost {
|
||||
struct CellActorHarness {
|
||||
event_tx: mpsc::UnboundedSender<RuntimeEvent>,
|
||||
handle: CellHandle,
|
||||
initial_event_rx: oneshot::Receiver<Result<CellEvent, CellError>>,
|
||||
task: tokio::task::JoinHandle<()>,
|
||||
runtime_tx: std_mpsc::Sender<RuntimeCommand>,
|
||||
runtime_control_rx: std_mpsc::Receiver<RuntimeControlCommand>,
|
||||
_runtime_pause_tx: std_mpsc::Sender<RuntimeControlCommand>,
|
||||
_runtime_event_rx: mpsc::UnboundedReceiver<RuntimeEvent>,
|
||||
}
|
||||
|
||||
fn spawn_cell_actor_harness(initial_observe_mode: ObserveMode) -> CellActorHarness {
|
||||
spawn_cell_actor_harness_with_host(initial_observe_mode, Arc::new(TestHost))
|
||||
fn spawn_cell_actor_harness() -> CellActorHarness {
|
||||
spawn_cell_actor_harness_with_host(Arc::new(TestHost))
|
||||
}
|
||||
|
||||
fn spawn_cell_actor_harness_with_host<H: CellHost>(
|
||||
initial_observe_mode: ObserveMode,
|
||||
host: Arc<H>,
|
||||
) -> CellActorHarness {
|
||||
fn spawn_cell_actor_harness_with_host<H: CellHost>(host: Arc<H>) -> CellActorHarness {
|
||||
let (event_tx, event_rx) = mpsc::unbounded_channel();
|
||||
let (command_tx, command_rx) = mpsc::unbounded_channel();
|
||||
let (initial_event_tx, initial_event_rx) = oneshot::channel();
|
||||
let (runtime_event_tx, runtime_event_rx) = mpsc::unbounded_channel();
|
||||
let (runtime_tx, _runtime_control_tx, runtime_terminate_handle) = spawn_runtime(
|
||||
let (runtime_tx, runtime_pause_tx, runtime_terminate_handle) = spawn_runtime(
|
||||
HashMap::new(),
|
||||
ExecuteRequest {
|
||||
tool_call_id: "call-1".to_string(),
|
||||
@@ -126,25 +126,22 @@ fn spawn_cell_actor_harness_with_host<H: CellHost>(
|
||||
let task = tokio::spawn(run_cell(
|
||||
host,
|
||||
CellContext {
|
||||
runtime_tx,
|
||||
runtime_tx: runtime_tx.clone(),
|
||||
runtime_control_tx,
|
||||
runtime_terminate_handle,
|
||||
cell_state,
|
||||
},
|
||||
event_rx,
|
||||
command_rx,
|
||||
Observer {
|
||||
mode: initial_observe_mode,
|
||||
response_tx: initial_event_tx,
|
||||
},
|
||||
));
|
||||
|
||||
CellActorHarness {
|
||||
event_tx,
|
||||
handle,
|
||||
initial_event_rx,
|
||||
task,
|
||||
runtime_tx,
|
||||
runtime_control_rx,
|
||||
_runtime_pause_tx: runtime_pause_tx,
|
||||
_runtime_event_rx: runtime_event_rx,
|
||||
}
|
||||
}
|
||||
@@ -159,9 +156,462 @@ async fn wait_for_notification(host: &RecordingHost) {
|
||||
.expect("notification barrier timed out");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn completion_and_output_are_buffered_until_the_first_observation() {
|
||||
let host = Arc::new(RecordingHost::default());
|
||||
let harness = spawn_cell_actor_harness_with_host(Arc::clone(&host));
|
||||
harness
|
||||
.event_tx
|
||||
.send(RuntimeEvent::ContentItem(
|
||||
FunctionCallOutputContentItem::InputText {
|
||||
text: "before observation".to_string(),
|
||||
},
|
||||
))
|
||||
.unwrap();
|
||||
harness
|
||||
.event_tx
|
||||
.send(RuntimeEvent::Result {
|
||||
stored_value_writes: HashMap::new(),
|
||||
error_text: None,
|
||||
})
|
||||
.unwrap();
|
||||
while !host.committed.load(Ordering::Acquire) {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
|
||||
assert_eq!(
|
||||
harness
|
||||
.handle
|
||||
.observe(ObserveMode::YieldAfter(Duration::ZERO))
|
||||
.await,
|
||||
Ok(CellEvent::Completed {
|
||||
content_items: vec![OutputItem::Text {
|
||||
text: "before observation".to_string(),
|
||||
}],
|
||||
error_text: None,
|
||||
})
|
||||
);
|
||||
harness.task.await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn pending_frontier_is_buffered_while_runtime_commands_are_queued() {
|
||||
let host = Arc::new(RecordingHost::default());
|
||||
let harness = spawn_cell_actor_harness_with_host(Arc::clone(&host));
|
||||
harness.event_tx.send(RuntimeEvent::Pending).unwrap();
|
||||
harness
|
||||
.event_tx
|
||||
.send(RuntimeEvent::Notify {
|
||||
call_id: "notify-1".to_string(),
|
||||
text: "pending processed".to_string(),
|
||||
})
|
||||
.unwrap();
|
||||
while !host.notified.load(Ordering::Acquire) {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
harness
|
||||
.runtime_tx
|
||||
.send(RuntimeCommand::ToolResponse {
|
||||
id: "tool-1".to_string(),
|
||||
result: JsonValue::Null,
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
assert!(matches!(
|
||||
harness.runtime_control_rx.try_recv(),
|
||||
Err(std_mpsc::TryRecvError::Empty)
|
||||
));
|
||||
assert_eq!(
|
||||
harness.handle.observe(ObserveMode::PendingFrontier).await,
|
||||
Ok(CellEvent::Pending {
|
||||
content_items: Vec::new(),
|
||||
pending_tool_call_ids: Vec::new(),
|
||||
})
|
||||
);
|
||||
assert!(matches!(
|
||||
harness.runtime_control_rx.try_recv(),
|
||||
Err(std_mpsc::TryRecvError::Empty)
|
||||
));
|
||||
|
||||
let termination = harness.handle.terminate();
|
||||
drop(harness.event_tx);
|
||||
assert_eq!(
|
||||
termination.await,
|
||||
Ok(CellEvent::Terminated {
|
||||
content_items: Vec::new(),
|
||||
})
|
||||
);
|
||||
harness.task.await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn buffered_yield_observation_resumes_an_unobserved_pending_frontier() {
|
||||
let host = Arc::new(RecordingHost::default());
|
||||
let harness = spawn_cell_actor_harness_with_host(Arc::clone(&host));
|
||||
harness.event_tx.send(RuntimeEvent::YieldRequested).unwrap();
|
||||
harness.event_tx.send(RuntimeEvent::Pending).unwrap();
|
||||
harness
|
||||
.event_tx
|
||||
.send(RuntimeEvent::Notify {
|
||||
call_id: "notify-1".to_string(),
|
||||
text: "pending processed".to_string(),
|
||||
})
|
||||
.unwrap();
|
||||
while !host.notified.load(Ordering::Acquire) {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
|
||||
assert_eq!(
|
||||
harness
|
||||
.handle
|
||||
.observe(ObserveMode::YieldAfter(Duration::from_secs(60)))
|
||||
.await,
|
||||
Ok(CellEvent::Yielded {
|
||||
content_items: Vec::new(),
|
||||
})
|
||||
);
|
||||
loop {
|
||||
match harness.runtime_control_rx.try_recv() {
|
||||
Ok(RuntimeControlCommand::Continue) => break,
|
||||
Ok(command) => panic!("expected continue, got {command:?}"),
|
||||
Err(std_mpsc::TryRecvError::Empty) => tokio::task::yield_now().await,
|
||||
Err(std_mpsc::TryRecvError::Disconnected) => {
|
||||
panic!("runtime control channel disconnected")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
host.notified.store(false, Ordering::Release);
|
||||
harness.event_tx.send(RuntimeEvent::Pending).unwrap();
|
||||
harness
|
||||
.event_tx
|
||||
.send(RuntimeEvent::Notify {
|
||||
call_id: "notify-2".to_string(),
|
||||
text: "later pending processed".to_string(),
|
||||
})
|
||||
.unwrap();
|
||||
while !host.notified.load(Ordering::Acquire) {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
assert!(matches!(
|
||||
harness.runtime_control_rx.try_recv(),
|
||||
Ok(RuntimeControlCommand::Continue)
|
||||
));
|
||||
|
||||
let termination = harness.handle.terminate();
|
||||
drop(harness.event_tx);
|
||||
assert_eq!(
|
||||
termination.await,
|
||||
Ok(CellEvent::Terminated {
|
||||
content_items: Vec::new(),
|
||||
})
|
||||
);
|
||||
harness.task.await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn first_observation_preserves_a_yield_that_raced_with_creation() {
|
||||
let host = Arc::new(RecordingHost::default());
|
||||
let harness = spawn_cell_actor_harness_with_host(Arc::clone(&host));
|
||||
harness
|
||||
.event_tx
|
||||
.send(RuntimeEvent::ContentItem(
|
||||
FunctionCallOutputContentItem::InputText {
|
||||
text: "before".to_string(),
|
||||
},
|
||||
))
|
||||
.unwrap();
|
||||
harness.event_tx.send(RuntimeEvent::YieldRequested).unwrap();
|
||||
harness
|
||||
.event_tx
|
||||
.send(RuntimeEvent::ContentItem(
|
||||
FunctionCallOutputContentItem::InputText {
|
||||
text: "after".to_string(),
|
||||
},
|
||||
))
|
||||
.unwrap();
|
||||
harness
|
||||
.event_tx
|
||||
.send(RuntimeEvent::Notify {
|
||||
call_id: "after-initial-yield".to_string(),
|
||||
text: "barrier".to_string(),
|
||||
})
|
||||
.unwrap();
|
||||
wait_for_notification(&host).await;
|
||||
|
||||
assert_eq!(
|
||||
tokio::time::timeout(
|
||||
Duration::from_secs(1),
|
||||
harness
|
||||
.handle
|
||||
.observe(ObserveMode::YieldAfter(Duration::from_secs(60))),
|
||||
)
|
||||
.await
|
||||
.expect("initial yield was not preserved"),
|
||||
Ok(CellEvent::Yielded {
|
||||
content_items: vec![OutputItem::Text {
|
||||
text: "before".to_string(),
|
||||
}],
|
||||
})
|
||||
);
|
||||
assert_eq!(
|
||||
harness
|
||||
.handle
|
||||
.observe(ObserveMode::YieldAfter(Duration::ZERO))
|
||||
.await,
|
||||
Ok(CellEvent::Yielded {
|
||||
content_items: vec![OutputItem::Text {
|
||||
text: "after".to_string(),
|
||||
}],
|
||||
})
|
||||
);
|
||||
|
||||
let termination = harness.handle.terminate();
|
||||
drop(harness.event_tx);
|
||||
assert_eq!(
|
||||
termination.await,
|
||||
Ok(CellEvent::Terminated {
|
||||
content_items: Vec::new(),
|
||||
})
|
||||
);
|
||||
harness.task.await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dropped_pending_observer_preserves_pre_observation_yield() {
|
||||
let host = Arc::new(RecordingHost::default());
|
||||
let harness = spawn_cell_actor_harness_with_host(Arc::clone(&host));
|
||||
harness
|
||||
.event_tx
|
||||
.send(RuntimeEvent::ContentItem(
|
||||
FunctionCallOutputContentItem::InputText {
|
||||
text: "before".to_string(),
|
||||
},
|
||||
))
|
||||
.unwrap();
|
||||
harness.event_tx.send(RuntimeEvent::YieldRequested).unwrap();
|
||||
harness
|
||||
.event_tx
|
||||
.send(RuntimeEvent::ContentItem(
|
||||
FunctionCallOutputContentItem::InputText {
|
||||
text: "after".to_string(),
|
||||
},
|
||||
))
|
||||
.unwrap();
|
||||
harness
|
||||
.event_tx
|
||||
.send(RuntimeEvent::Notify {
|
||||
call_id: "before-pending-observer".to_string(),
|
||||
text: "barrier".to_string(),
|
||||
})
|
||||
.unwrap();
|
||||
wait_for_notification(&host).await;
|
||||
host.notified.store(false, Ordering::Release);
|
||||
|
||||
let dropped_observation = harness.handle.observe(ObserveMode::PendingFrontier);
|
||||
assert_eq!(
|
||||
harness.handle.observe(ObserveMode::PendingFrontier).await,
|
||||
Err(CellError::Busy)
|
||||
);
|
||||
drop(dropped_observation);
|
||||
harness.event_tx.send(RuntimeEvent::Pending).unwrap();
|
||||
harness
|
||||
.event_tx
|
||||
.send(RuntimeEvent::Notify {
|
||||
call_id: "after-dropped-pending".to_string(),
|
||||
text: "barrier".to_string(),
|
||||
})
|
||||
.unwrap();
|
||||
wait_for_notification(&host).await;
|
||||
|
||||
assert_eq!(
|
||||
tokio::time::timeout(
|
||||
Duration::from_secs(1),
|
||||
harness
|
||||
.handle
|
||||
.observe(ObserveMode::YieldAfter(Duration::from_secs(60))),
|
||||
)
|
||||
.await
|
||||
.expect("initial yield was not preserved after failed pending delivery"),
|
||||
Ok(CellEvent::Yielded {
|
||||
content_items: vec![OutputItem::Text {
|
||||
text: "before".to_string(),
|
||||
}],
|
||||
})
|
||||
);
|
||||
|
||||
let termination = harness.handle.terminate();
|
||||
drop(harness.event_tx);
|
||||
assert_eq!(
|
||||
termination.await,
|
||||
Ok(CellEvent::Terminated {
|
||||
content_items: vec![OutputItem::Text {
|
||||
text: "after".to_string(),
|
||||
}],
|
||||
})
|
||||
);
|
||||
harness.task.await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dropped_pending_observer_preserves_pre_observation_yield_at_completion() {
|
||||
let host = Arc::new(RecordingHost::default());
|
||||
let harness = spawn_cell_actor_harness_with_host(Arc::clone(&host));
|
||||
harness
|
||||
.event_tx
|
||||
.send(RuntimeEvent::ContentItem(
|
||||
FunctionCallOutputContentItem::InputText {
|
||||
text: "before".to_string(),
|
||||
},
|
||||
))
|
||||
.unwrap();
|
||||
harness.event_tx.send(RuntimeEvent::YieldRequested).unwrap();
|
||||
harness
|
||||
.event_tx
|
||||
.send(RuntimeEvent::ContentItem(
|
||||
FunctionCallOutputContentItem::InputText {
|
||||
text: "after".to_string(),
|
||||
},
|
||||
))
|
||||
.unwrap();
|
||||
harness
|
||||
.event_tx
|
||||
.send(RuntimeEvent::Notify {
|
||||
call_id: "before-pending-observer".to_string(),
|
||||
text: "barrier".to_string(),
|
||||
})
|
||||
.unwrap();
|
||||
wait_for_notification(&host).await;
|
||||
|
||||
let dropped_observation = harness.handle.observe(ObserveMode::PendingFrontier);
|
||||
assert_eq!(
|
||||
harness.handle.observe(ObserveMode::PendingFrontier).await,
|
||||
Err(CellError::Busy)
|
||||
);
|
||||
drop(dropped_observation);
|
||||
harness
|
||||
.event_tx
|
||||
.send(RuntimeEvent::Result {
|
||||
stored_value_writes: HashMap::new(),
|
||||
error_text: None,
|
||||
})
|
||||
.unwrap();
|
||||
while !host.committed.load(Ordering::Acquire) {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
|
||||
assert_eq!(
|
||||
tokio::time::timeout(
|
||||
Duration::from_secs(1),
|
||||
harness
|
||||
.handle
|
||||
.observe(ObserveMode::YieldAfter(Duration::from_secs(60))),
|
||||
)
|
||||
.await
|
||||
.expect("initial yield was not preserved after failed completion delivery"),
|
||||
Ok(CellEvent::Yielded {
|
||||
content_items: vec![OutputItem::Text {
|
||||
text: "before".to_string(),
|
||||
}],
|
||||
})
|
||||
);
|
||||
assert_eq!(
|
||||
harness
|
||||
.handle
|
||||
.observe(ObserveMode::YieldAfter(Duration::ZERO))
|
||||
.await,
|
||||
Ok(CellEvent::Completed {
|
||||
content_items: vec![OutputItem::Text {
|
||||
text: "after".to_string(),
|
||||
}],
|
||||
error_text: None,
|
||||
})
|
||||
);
|
||||
harness.task.await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn unattached_yield_after_the_first_observation_is_a_no_op() {
|
||||
let host = Arc::new(RecordingHost::default());
|
||||
let harness = spawn_cell_actor_harness_with_host(Arc::clone(&host));
|
||||
assert_eq!(
|
||||
harness
|
||||
.handle
|
||||
.observe(ObserveMode::YieldAfter(Duration::ZERO))
|
||||
.await,
|
||||
Ok(CellEvent::Yielded {
|
||||
content_items: Vec::new(),
|
||||
})
|
||||
);
|
||||
|
||||
harness
|
||||
.event_tx
|
||||
.send(RuntimeEvent::ContentItem(
|
||||
FunctionCallOutputContentItem::InputText {
|
||||
text: "before ignored yield".to_string(),
|
||||
},
|
||||
))
|
||||
.unwrap();
|
||||
harness.event_tx.send(RuntimeEvent::YieldRequested).unwrap();
|
||||
harness
|
||||
.event_tx
|
||||
.send(RuntimeEvent::Notify {
|
||||
call_id: "after-ignored-yield".to_string(),
|
||||
text: "barrier".to_string(),
|
||||
})
|
||||
.unwrap();
|
||||
wait_for_notification(&host).await;
|
||||
|
||||
let observation = harness
|
||||
.handle
|
||||
.observe(ObserveMode::YieldAfter(Duration::from_secs(60)));
|
||||
tokio::pin!(observation);
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(/*millis*/ 20), &mut observation)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
|
||||
harness
|
||||
.event_tx
|
||||
.send(RuntimeEvent::ContentItem(
|
||||
FunctionCallOutputContentItem::InputText {
|
||||
text: "after observer attached".to_string(),
|
||||
},
|
||||
))
|
||||
.unwrap();
|
||||
harness.event_tx.send(RuntimeEvent::YieldRequested).unwrap();
|
||||
assert_eq!(
|
||||
observation.await,
|
||||
Ok(CellEvent::Yielded {
|
||||
content_items: vec![
|
||||
OutputItem::Text {
|
||||
text: "before ignored yield".to_string(),
|
||||
},
|
||||
OutputItem::Text {
|
||||
text: "after observer attached".to_string(),
|
||||
},
|
||||
],
|
||||
})
|
||||
);
|
||||
|
||||
let termination = harness.handle.terminate();
|
||||
drop(harness.event_tx);
|
||||
assert_eq!(
|
||||
termination.await,
|
||||
Ok(CellEvent::Terminated {
|
||||
content_items: Vec::new(),
|
||||
})
|
||||
);
|
||||
harness.task.await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn yield_timer_preempts_buffered_runtime_output() {
|
||||
let harness = spawn_cell_actor_harness(ObserveMode::YieldAfter(Duration::ZERO));
|
||||
let harness = spawn_cell_actor_harness();
|
||||
let initial_event = harness
|
||||
.handle
|
||||
.observe(ObserveMode::YieldAfter(Duration::ZERO));
|
||||
harness.event_tx.send(RuntimeEvent::Started).unwrap();
|
||||
harness
|
||||
.event_tx
|
||||
@@ -173,7 +623,7 @@ async fn yield_timer_preempts_buffered_runtime_output() {
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
harness.initial_event_rx.await.unwrap(),
|
||||
initial_event.await,
|
||||
Ok(CellEvent::Yielded {
|
||||
content_items: Vec::new(),
|
||||
})
|
||||
@@ -194,7 +644,7 @@ async fn yield_timer_preempts_buffered_runtime_output() {
|
||||
|
||||
#[tokio::test]
|
||||
async fn queued_termination_preempts_unobserved_runtime_completion() {
|
||||
let harness = spawn_cell_actor_harness(ObserveMode::YieldAfter(Duration::from_secs(60)));
|
||||
let harness = spawn_cell_actor_harness();
|
||||
harness
|
||||
.event_tx
|
||||
.send(RuntimeEvent::Result {
|
||||
@@ -207,20 +657,14 @@ async fn queued_termination_preempts_unobserved_runtime_completion() {
|
||||
let terminated = Ok(CellEvent::Terminated {
|
||||
content_items: Vec::new(),
|
||||
});
|
||||
assert_eq!(termination.await, terminated.clone());
|
||||
assert_eq!(harness.initial_event_rx.await.unwrap(), terminated);
|
||||
assert_eq!(termination.await, terminated);
|
||||
harness.task.await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn observation_dropped_before_dequeue_does_not_consume_output() {
|
||||
let host = Arc::new(RecordingHost::default());
|
||||
let harness = spawn_cell_actor_harness_with_host(
|
||||
ObserveMode::YieldAfter(Duration::from_secs(60)),
|
||||
Arc::clone(&host),
|
||||
);
|
||||
harness.event_tx.send(RuntimeEvent::YieldRequested).unwrap();
|
||||
assert!(harness.initial_event_rx.await.unwrap().is_ok());
|
||||
let harness = spawn_cell_actor_harness_with_host(Arc::clone(&host));
|
||||
|
||||
drop(
|
||||
harness
|
||||
@@ -246,10 +690,14 @@ async fn observation_dropped_before_dequeue_does_not_consume_output() {
|
||||
wait_for_notification(&host).await;
|
||||
|
||||
assert_eq!(
|
||||
harness
|
||||
.handle
|
||||
.observe(ObserveMode::YieldAfter(Duration::ZERO))
|
||||
.await,
|
||||
tokio::time::timeout(
|
||||
Duration::from_secs(1),
|
||||
harness
|
||||
.handle
|
||||
.observe(ObserveMode::YieldAfter(Duration::from_secs(60))),
|
||||
)
|
||||
.await
|
||||
.expect("initial yield was not preserved"),
|
||||
Ok(CellEvent::Yielded {
|
||||
content_items: vec![OutputItem::Text {
|
||||
text: "survives pre-dequeue cancellation".to_string(),
|
||||
@@ -271,12 +719,16 @@ async fn observation_dropped_before_dequeue_does_not_consume_output() {
|
||||
#[tokio::test]
|
||||
async fn dropped_yield_observer_preserves_output_for_the_next_observation() {
|
||||
let host = Arc::new(RecordingHost::default());
|
||||
let harness = spawn_cell_actor_harness_with_host(
|
||||
ObserveMode::YieldAfter(Duration::from_secs(60)),
|
||||
Arc::clone(&host),
|
||||
let harness = spawn_cell_actor_harness_with_host(Arc::clone(&host));
|
||||
assert_eq!(
|
||||
harness
|
||||
.handle
|
||||
.observe(ObserveMode::YieldAfter(Duration::ZERO))
|
||||
.await,
|
||||
Ok(CellEvent::Yielded {
|
||||
content_items: Vec::new(),
|
||||
})
|
||||
);
|
||||
harness.event_tx.send(RuntimeEvent::YieldRequested).unwrap();
|
||||
assert!(harness.initial_event_rx.await.unwrap().is_ok());
|
||||
|
||||
let dropped_observation = harness
|
||||
.handle
|
||||
@@ -333,12 +785,16 @@ async fn dropped_yield_observer_preserves_output_for_the_next_observation() {
|
||||
#[tokio::test]
|
||||
async fn dropped_pending_observer_preserves_the_frontier_for_the_next_observation() {
|
||||
let host = Arc::new(RecordingHost::default());
|
||||
let harness = spawn_cell_actor_harness_with_host(
|
||||
ObserveMode::YieldAfter(Duration::from_secs(60)),
|
||||
Arc::clone(&host),
|
||||
let harness = spawn_cell_actor_harness_with_host(Arc::clone(&host));
|
||||
assert_eq!(
|
||||
harness
|
||||
.handle
|
||||
.observe(ObserveMode::YieldAfter(Duration::ZERO))
|
||||
.await,
|
||||
Ok(CellEvent::Yielded {
|
||||
content_items: Vec::new(),
|
||||
})
|
||||
);
|
||||
harness.event_tx.send(RuntimeEvent::YieldRequested).unwrap();
|
||||
assert!(harness.initial_event_rx.await.unwrap().is_ok());
|
||||
|
||||
let dropped_observation = harness.handle.observe(ObserveMode::PendingFrontier);
|
||||
assert_eq!(
|
||||
@@ -407,7 +863,7 @@ async fn only_the_first_termination_claims_a_buffered_completion() {
|
||||
CompletionCommit::Committed
|
||||
);
|
||||
assert!(matches!(
|
||||
cell_state.deliver_completion(/*response_tx*/ None),
|
||||
cell_state.deliver_completion(/*observer*/ None),
|
||||
CompletionDelivery::Buffered
|
||||
));
|
||||
|
||||
@@ -431,17 +887,31 @@ async fn termination_claim_prevents_stored_value_commit() {
|
||||
let termination = cell_state.request_termination();
|
||||
let mut commit_ran = false;
|
||||
let completion = CellEvent::Completed {
|
||||
content_items: Vec::new(),
|
||||
content_items: vec![OutputItem::Text {
|
||||
text: "after yield".to_string(),
|
||||
}],
|
||||
error_text: None,
|
||||
};
|
||||
|
||||
assert_eq!(
|
||||
cell_state.commit_completion(
|
||||
completion.clone(),
|
||||
/*pending_initial_yield_items*/ None,
|
||||
completion,
|
||||
Some(vec![OutputItem::Text {
|
||||
text: "before yield".to_string(),
|
||||
}]),
|
||||
|| commit_ran = true
|
||||
),
|
||||
CompletionCommit::Rejected(completion)
|
||||
CompletionCommit::Rejected(CellEvent::Completed {
|
||||
content_items: vec![
|
||||
OutputItem::Text {
|
||||
text: "before yield".to_string(),
|
||||
},
|
||||
OutputItem::Text {
|
||||
text: "after yield".to_string(),
|
||||
},
|
||||
],
|
||||
error_text: None,
|
||||
})
|
||||
);
|
||||
assert!(!commit_ran);
|
||||
|
||||
@@ -473,7 +943,9 @@ fn failed_completion_delivery_rebuffers_the_event() {
|
||||
let (response_tx, response_rx) = oneshot::channel();
|
||||
drop(response_rx);
|
||||
assert!(matches!(
|
||||
cell_state.deliver_completion(Some(response_tx)),
|
||||
cell_state.deliver_completion(Some(
|
||||
(ObserveMode::YieldAfter(Duration::ZERO), response_tx,)
|
||||
)),
|
||||
CompletionDelivery::Buffered
|
||||
));
|
||||
assert!(cell_state.accepting_observations());
|
||||
@@ -506,7 +978,7 @@ fn buffered_initial_yield_precedes_buffered_completion_for_yield_observer() {
|
||||
CompletionCommit::Committed
|
||||
);
|
||||
assert!(matches!(
|
||||
cell_state.deliver_completion(/*response_tx*/ None),
|
||||
cell_state.deliver_completion(/*observer*/ None),
|
||||
CompletionDelivery::Buffered
|
||||
));
|
||||
|
||||
@@ -551,7 +1023,7 @@ fn pending_observer_merges_initial_yield_and_completion_output() {
|
||||
CompletionCommit::Committed
|
||||
);
|
||||
assert!(matches!(
|
||||
cell_state.deliver_completion(/*response_tx*/ None),
|
||||
cell_state.deliver_completion(/*observer*/ None),
|
||||
CompletionDelivery::Buffered
|
||||
));
|
||||
|
||||
@@ -596,7 +1068,7 @@ fn dropped_pending_observation_preserves_the_initial_yield_boundary() {
|
||||
CompletionCommit::Committed
|
||||
);
|
||||
assert!(matches!(
|
||||
cell_state.deliver_completion(/*response_tx*/ None),
|
||||
cell_state.deliver_completion(/*observer*/ None),
|
||||
CompletionDelivery::Buffered
|
||||
));
|
||||
|
||||
|
||||
@@ -137,6 +137,15 @@ pub(crate) enum CompletionCommit {
|
||||
Rejected(CellEvent),
|
||||
}
|
||||
|
||||
impl CompletionCommit {
|
||||
pub(crate) fn rejected(
|
||||
event: CellEvent,
|
||||
pending_initial_yield_items: Option<Vec<OutputItem>>,
|
||||
) -> Self {
|
||||
Self::Rejected(prepend_initial_yield(event, pending_initial_yield_items))
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) enum ObservationDelivery {
|
||||
Running(oneshot::Sender<Result<CellEvent, CellError>>),
|
||||
Delivered,
|
||||
@@ -207,7 +216,7 @@ impl CellState {
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
if !matches!(*phase, CellPhase::Running) || self.cancellation_token.is_cancelled() {
|
||||
return CompletionCommit::Rejected(event);
|
||||
return CompletionCommit::rejected(event, pending_initial_yield_items);
|
||||
}
|
||||
commit();
|
||||
*phase = CellPhase::Completed {
|
||||
@@ -219,7 +228,7 @@ impl CellState {
|
||||
|
||||
pub(crate) fn deliver_completion(
|
||||
&self,
|
||||
response_tx: Option<oneshot::Sender<Result<CellEvent, CellError>>>,
|
||||
observer: Option<(ObserveMode, oneshot::Sender<Result<CellEvent, CellError>>)>,
|
||||
) -> CompletionDelivery {
|
||||
let mut phase = self
|
||||
.phase
|
||||
@@ -233,16 +242,38 @@ impl CellState {
|
||||
} => (pending_initial_yield_items, event),
|
||||
previous => {
|
||||
*phase = previous;
|
||||
return CompletionDelivery::Rejected(response_tx);
|
||||
return CompletionDelivery::Rejected(
|
||||
observer.map(|(_mode, response_tx)| response_tx),
|
||||
);
|
||||
}
|
||||
};
|
||||
let Some(response_tx) = response_tx else {
|
||||
let Some((mode, response_tx)) = observer else {
|
||||
*phase = CellPhase::Completed {
|
||||
pending_initial_yield_items,
|
||||
event,
|
||||
};
|
||||
return CompletionDelivery::Buffered;
|
||||
};
|
||||
if matches!(mode, ObserveMode::PendingFrontier) && pending_initial_yield_items.is_some() {
|
||||
let delivered_event =
|
||||
prepend_initial_yield(event.clone(), pending_initial_yield_items.clone());
|
||||
return match response_tx.send(Ok(delivered_event)) {
|
||||
Ok(()) => {
|
||||
self.cancellation_token.cancel();
|
||||
CompletionDelivery::Delivered
|
||||
}
|
||||
Err(Ok(_)) => {
|
||||
*phase = CellPhase::Completed {
|
||||
pending_initial_yield_items,
|
||||
event,
|
||||
};
|
||||
CompletionDelivery::Buffered
|
||||
}
|
||||
Err(Err(error)) => {
|
||||
panic!("completion delivery unexpectedly carried an actor error: {error:?}")
|
||||
}
|
||||
};
|
||||
}
|
||||
match response_tx.send(Ok(event)) {
|
||||
Ok(()) => {
|
||||
self.cancellation_token.cancel();
|
||||
|
||||
@@ -89,20 +89,25 @@ impl CodeModeService {
|
||||
|
||||
pub async fn execute(&self, request: ExecuteRequest) -> Result<StartedCell, String> {
|
||||
let yield_time_ms = request.yield_time_ms.unwrap_or(DEFAULT_EXEC_YIELD_TIME_MS);
|
||||
let started = self
|
||||
let runtime_cell_id = self
|
||||
.runtime
|
||||
.execute(
|
||||
runtime_request(request),
|
||||
.create_cell(runtime_request(request))
|
||||
.await
|
||||
.map_err(|error| error.to_string())?;
|
||||
let pending_event = self
|
||||
.runtime
|
||||
.begin_observe(
|
||||
&runtime_cell_id,
|
||||
runtime::ObserveMode::YieldAfter(Duration::from_millis(yield_time_ms)),
|
||||
)
|
||||
.await
|
||||
.map_err(|error| error.to_string())?;
|
||||
let cell_id = protocol_cell_id(&started.cell_id);
|
||||
let cell_id = protocol_cell_id(&runtime_cell_id);
|
||||
let response_cell_id = cell_id.clone();
|
||||
let (response_tx, response_rx) = oneshot::channel();
|
||||
tokio::spawn(async move {
|
||||
let response = started
|
||||
.initial_event()
|
||||
let response = pending_event
|
||||
.event()
|
||||
.await
|
||||
.map_err(|error| error.to_string())
|
||||
.and_then(|event| runtime_response(&response_cell_id, event));
|
||||
@@ -115,17 +120,15 @@ impl CodeModeService {
|
||||
&self,
|
||||
request: ExecuteRequest,
|
||||
) -> Result<ExecuteToPendingOutcome, String> {
|
||||
let started = self
|
||||
let runtime_cell_id = self
|
||||
.runtime
|
||||
.execute(
|
||||
runtime_request(request),
|
||||
runtime::ObserveMode::PendingFrontier,
|
||||
)
|
||||
.create_cell(runtime_request(request))
|
||||
.await
|
||||
.map_err(|error| error.to_string())?;
|
||||
let cell_id = protocol_cell_id(&started.cell_id);
|
||||
let event = started
|
||||
.initial_event()
|
||||
let cell_id = protocol_cell_id(&runtime_cell_id);
|
||||
let event = self
|
||||
.runtime
|
||||
.observe(&runtime_cell_id, runtime::ObserveMode::PendingFrontier)
|
||||
.await
|
||||
.map_err(|error| error.to_string())?;
|
||||
pending_outcome(&cell_id, event)
|
||||
|
||||
@@ -365,6 +365,64 @@ async fn termination_cancels_pending_callbacks_before_responding() {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn termination_discards_stored_writes_before_the_next_cell_can_load_them() {
|
||||
let (delegate, mut events_rx) = BlockingDelegate::new();
|
||||
let service = CodeModeService::with_delegate(delegate.clone());
|
||||
let cell = service
|
||||
.execute(ExecuteRequest {
|
||||
enabled_tools: vec![blocking_tool()],
|
||||
source: r#"
|
||||
store("candidate", "leaked");
|
||||
await tools.block({});
|
||||
"#
|
||||
.to_string(),
|
||||
yield_time_ms: Some(60_000),
|
||||
..execute_request("")
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// Reaching the delegate proves that the store ran before execution became gated.
|
||||
assert_eq!(next_event(&mut events_rx).await, DelegateEvent::ToolStarted);
|
||||
assert_eq!(
|
||||
service.terminate(cell.cell_id).await.unwrap(),
|
||||
WaitOutcome::LiveCell(RuntimeResponse::Terminated {
|
||||
cell_id: cell_id("1"),
|
||||
content_items: Vec::new(),
|
||||
})
|
||||
);
|
||||
assert!(delegate.tool_finished.load(Ordering::Acquire));
|
||||
assert_eq!(
|
||||
next_event(&mut events_rx).await,
|
||||
DelegateEvent::ToolCancelled
|
||||
);
|
||||
assert_eq!(
|
||||
next_event(&mut events_rx).await,
|
||||
DelegateEvent::CellClosed(cell_id("1"))
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
service
|
||||
.execute(ExecuteRequest {
|
||||
yield_time_ms: Some(60_000),
|
||||
..execute_request(r#"text(String(load("candidate")));"#)
|
||||
})
|
||||
.await
|
||||
.unwrap()
|
||||
.initial_response()
|
||||
.await
|
||||
.unwrap(),
|
||||
RuntimeResponse::Result {
|
||||
cell_id: cell_id("2"),
|
||||
content_items: vec![FunctionCallOutputContentItem::InputText {
|
||||
text: "undefined".to_string(),
|
||||
}],
|
||||
error_text: None,
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn shutdown_cancels_notifications_while_natural_completion_is_draining() {
|
||||
let (delegate, mut events_rx) = HeldNotificationDelegate::new();
|
||||
|
||||
@@ -362,12 +362,26 @@ await Promise.all([
|
||||
|
||||
tokio::time::sleep(Duration::from_secs(2)).await;
|
||||
|
||||
let resumed_response = tokio::time::timeout(
|
||||
Duration::from_secs(1),
|
||||
service.wait_to_pending(WaitToPendingRequest {
|
||||
cell_id: cell_id("1"),
|
||||
}),
|
||||
)
|
||||
// Resuming can expose an empty quiescent frontier before the expired timer
|
||||
// callback is dequeued, so keep observing until its tool call is visible.
|
||||
let resumed_response = tokio::time::timeout(Duration::from_secs(5), async {
|
||||
loop {
|
||||
let response = service
|
||||
.wait_to_pending(WaitToPendingRequest {
|
||||
cell_id: cell_id("1"),
|
||||
})
|
||||
.await?;
|
||||
if !matches!(
|
||||
&response,
|
||||
WaitToPendingOutcome::LiveCell(ExecuteToPendingOutcome::Pending {
|
||||
pending_tool_call_ids,
|
||||
..
|
||||
}) if pending_tool_call_ids.is_empty()
|
||||
) {
|
||||
break Ok::<_, String>(response);
|
||||
}
|
||||
}
|
||||
})
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
|
||||
@@ -67,22 +67,13 @@ impl<D: SessionRuntimeDelegate> SessionRuntime<D> {
|
||||
!self.inner.shutdown_token.is_cancelled()
|
||||
}
|
||||
|
||||
pub(crate) async fn execute(
|
||||
&self,
|
||||
request: CreateCellRequest,
|
||||
initial_observe_mode: ObserveMode,
|
||||
) -> Result<StartedCell, Error> {
|
||||
pub(crate) async fn create_cell(&self, request: CreateCellRequest) -> Result<CellId, Error> {
|
||||
if self.inner.shutdown_token.is_cancelled() {
|
||||
return Err(Error::ShuttingDown);
|
||||
}
|
||||
let cell_id = self.allocate_cell_id();
|
||||
let initial_event = self
|
||||
.start_cell(cell_id.clone(), request, initial_observe_mode)
|
||||
.await?;
|
||||
Ok(StartedCell {
|
||||
cell_id,
|
||||
initial_event,
|
||||
})
|
||||
self.start_cell(cell_id.clone(), request).await?;
|
||||
Ok(cell_id)
|
||||
}
|
||||
|
||||
pub(crate) async fn observe(
|
||||
@@ -146,12 +137,7 @@ impl<D: SessionRuntimeDelegate> SessionRuntime<D> {
|
||||
)
|
||||
}
|
||||
|
||||
async fn start_cell(
|
||||
&self,
|
||||
cell_id: CellId,
|
||||
request: CreateCellRequest,
|
||||
initial_observe_mode: ObserveMode,
|
||||
) -> Result<RuntimeEventFuture, Error> {
|
||||
async fn start_cell(&self, cell_id: CellId, request: CreateCellRequest) -> Result<(), Error> {
|
||||
let stored_values = self.inner.stored_values.lock().await.clone();
|
||||
let host = Arc::new(RuntimeCellHost {
|
||||
cell_id: cell_id.clone(),
|
||||
@@ -165,18 +151,12 @@ impl<D: SessionRuntimeDelegate> SessionRuntime<D> {
|
||||
return Err(Error::DuplicateCell(cell_id));
|
||||
}
|
||||
let cell_state = Arc::new(CellState::new(self.inner.shutdown_token.child_token()));
|
||||
let (handle, initial_event, task) = CellActor::prepare(
|
||||
request,
|
||||
stored_values,
|
||||
host,
|
||||
initial_observe_mode,
|
||||
cell_state,
|
||||
)
|
||||
.map_err(Error::Runtime)?;
|
||||
let (handle, task) =
|
||||
CellActor::prepare(request, stored_values, host, cell_state).map_err(Error::Runtime)?;
|
||||
cells.insert(cell_id.clone(), handle);
|
||||
self.inner.cell_tasks.spawn(task);
|
||||
drop(cells);
|
||||
Ok(map_actor_event(cell_id, initial_event))
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn begin_shutdown(&self) {
|
||||
@@ -191,19 +171,7 @@ impl<D: SessionRuntimeDelegate> Drop for SessionRuntime<D> {
|
||||
}
|
||||
}
|
||||
|
||||
/// A cell admitted by [`SessionRuntime::execute`].
|
||||
pub(crate) struct StartedCell {
|
||||
pub(crate) cell_id: CellId,
|
||||
initial_event: RuntimeEventFuture,
|
||||
}
|
||||
|
||||
impl StartedCell {
|
||||
pub(crate) async fn initial_event(self) -> Result<CellEvent, Error> {
|
||||
self.initial_event.await
|
||||
}
|
||||
}
|
||||
|
||||
/// An admitted observation that has not reached its requested frontier yet.
|
||||
/// An admitted cell event that has not reached its requested frontier yet.
|
||||
pub(crate) struct PendingEvent {
|
||||
event: RuntimeEventFuture,
|
||||
}
|
||||
@@ -263,7 +231,7 @@ impl<D: SessionRuntimeDelegate> CellHost for RuntimeCellHost<D> {
|
||||
let mut stored_values = tokio::select! {
|
||||
biased;
|
||||
_ = cancellation_token.cancelled() => {
|
||||
return CompletionCommit::Rejected(event);
|
||||
return CompletionCommit::rejected(event, pending_initial_yield_items);
|
||||
}
|
||||
stored_values = self.inner.stored_values.lock() => stored_values,
|
||||
};
|
||||
|
||||
@@ -88,18 +88,17 @@ async fn termination_rejects_a_waiting_store_commit_before_the_next_cell_can_loa
|
||||
);
|
||||
|
||||
let reader = runtime
|
||||
.execute(
|
||||
CreateCellRequest {
|
||||
tool_call_id: "reader".to_string(),
|
||||
enabled_tools: Vec::new(),
|
||||
source: r#"text(String(load("candidate")));"#.to_string(),
|
||||
},
|
||||
ObserveMode::YieldAfter(Duration::from_secs(1)),
|
||||
)
|
||||
.create_cell(CreateCellRequest {
|
||||
tool_call_id: "reader".to_string(),
|
||||
enabled_tools: Vec::new(),
|
||||
source: r#"text(String(load("candidate")));"#.to_string(),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
reader.initial_event().await,
|
||||
runtime
|
||||
.observe(&reader, ObserveMode::YieldAfter(Duration::from_secs(1)))
|
||||
.await,
|
||||
Ok(CellEvent::Completed {
|
||||
content_items: vec![OutputItem::Text {
|
||||
text: "undefined".to_string(),
|
||||
@@ -127,16 +126,13 @@ async fn shutdown_rejects_cell_admission_queued_before_the_registry_lock() {
|
||||
let runtime = Arc::new(SessionRuntime::new(Arc::new(RecordingDelegate)));
|
||||
let cells = runtime.inner.cells.lock().await;
|
||||
|
||||
let execution = runtime.execute(
|
||||
execute_request("while (true) {}"),
|
||||
ObserveMode::YieldAfter(Duration::from_millis(/*millis*/ 1)),
|
||||
);
|
||||
tokio::pin!(execution);
|
||||
std::future::poll_fn(|context| match execution.as_mut().poll(context) {
|
||||
let creation = runtime.create_cell(execute_request("while (true) {}"));
|
||||
tokio::pin!(creation);
|
||||
std::future::poll_fn(|context| match creation.as_mut().poll(context) {
|
||||
Poll::Pending => Poll::Ready(()),
|
||||
Poll::Ready(Ok(_)) => panic!("execution completed before the registry lock was released"),
|
||||
Poll::Ready(Ok(_)) => panic!("creation completed before the registry lock was released"),
|
||||
Poll::Ready(Err(error)) => {
|
||||
panic!("execution failed before the registry lock was released: {error}")
|
||||
panic!("creation failed before the registry lock was released: {error}")
|
||||
}
|
||||
})
|
||||
.await;
|
||||
@@ -154,23 +150,25 @@ async fn shutdown_rejects_cell_admission_queued_before_the_registry_lock() {
|
||||
|
||||
assert!(!runtime.is_alive());
|
||||
drop(cells);
|
||||
assert!(matches!(execution.await, Err(Error::ShuttingDown)));
|
||||
assert!(matches!(creation.await, Err(Error::ShuttingDown)));
|
||||
assert_eq!(shutdown.await, Ok(()));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn drop_terminates_cells_when_the_registry_is_locked() {
|
||||
let runtime = SessionRuntime::new(Arc::new(RecordingDelegate));
|
||||
let started = runtime
|
||||
.execute(
|
||||
execute_request("while (true) {}"),
|
||||
ObserveMode::YieldAfter(Duration::from_millis(/*millis*/ 1)),
|
||||
)
|
||||
let cell_id = runtime
|
||||
.create_cell(execute_request("while (true) {}"))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(started.cell_id, CellId::new("1"));
|
||||
assert_eq!(cell_id, CellId::new("1"));
|
||||
assert_eq!(
|
||||
started.initial_event().await,
|
||||
runtime
|
||||
.observe(
|
||||
&cell_id,
|
||||
ObserveMode::YieldAfter(Duration::from_millis(/*millis*/ 1)),
|
||||
)
|
||||
.await,
|
||||
Ok(CellEvent::Yielded {
|
||||
content_items: Vec::new(),
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user