mirror of
https://github.com/openai/codex.git
synced 2026-11-21 22:50:37 +00:00
Preload watchdog helper control tools
This commit is contained in:
@@ -23,8 +23,11 @@ use codex_protocol::ThreadId;
|
||||
use codex_protocol::error::CodexErr;
|
||||
use codex_protocol::error::Result as CodexResult;
|
||||
use codex_protocol::models::ContentItem;
|
||||
use codex_protocol::models::FunctionCallOutputPayload;
|
||||
use codex_protocol::models::MessagePhase;
|
||||
use codex_protocol::models::ResponseItem;
|
||||
use codex_protocol::protocol::CollabCloseEndEvent;
|
||||
use codex_protocol::protocol::Event;
|
||||
use codex_protocol::protocol::InitialHistory;
|
||||
use codex_protocol::protocol::InterAgentCommunication;
|
||||
use codex_protocol::protocol::Op;
|
||||
@@ -37,11 +40,17 @@ use codex_protocol::user_input::UserInput;
|
||||
use codex_rollout::state_db;
|
||||
use codex_state::DirectionalThreadSpawnEdgeStatus;
|
||||
use codex_thread_store::ReadThreadParams;
|
||||
use codex_tools::create_compact_parent_context_tool;
|
||||
use codex_tools::create_watchdog_close_self_tool;
|
||||
use codex_tools::create_watchdog_snooze_tool;
|
||||
use codex_tools::create_watchdog_tools_namespace;
|
||||
use serde::Serialize;
|
||||
use std::collections::HashMap;
|
||||
use std::collections::VecDeque;
|
||||
use std::sync::Arc;
|
||||
use std::sync::Weak;
|
||||
use std::time::SystemTime;
|
||||
use std::time::UNIX_EPOCH;
|
||||
use tokio::sync::watch;
|
||||
use tracing::warn;
|
||||
|
||||
@@ -49,6 +58,8 @@ const AGENT_NAMES: &str = include_str!("agent_names.txt");
|
||||
const ROOT_LAST_TASK_MESSAGE: &str = "Main thread";
|
||||
const CODEX_EXPERIMENTAL_FORK_PREVIOUS_RESPONSE_ID_ENV: &str =
|
||||
"CODEX_EXPERIMENTAL_FORK_PREVIOUS_RESPONSE_ID";
|
||||
const WATCHDOG_BOOT_TOOL_SEARCH_CALL_ID: &str = "synthetic_watchdog_tool_search";
|
||||
const WATCHDOG_BOOT_LIST_AGENTS_CALL_ID: &str = "synthetic_watchdog_list_agents";
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub(crate) enum SpawnAgentForkMode {
|
||||
@@ -77,6 +88,18 @@ pub(crate) struct ListedAgent {
|
||||
pub(crate) last_task_message: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub(crate) enum WatchdogParentCompactionResult {
|
||||
NotWatchdogHelper,
|
||||
ParentBusy {
|
||||
parent_thread_id: ThreadId,
|
||||
},
|
||||
Submitted {
|
||||
parent_thread_id: ThreadId,
|
||||
submission_id: String,
|
||||
},
|
||||
}
|
||||
|
||||
fn default_agent_nickname_list() -> Vec<&'static str> {
|
||||
AGENT_NAMES
|
||||
.lines()
|
||||
@@ -141,6 +164,70 @@ fn is_watchdog_helper_source(session_source: &SessionSource) -> bool {
|
||||
)
|
||||
}
|
||||
|
||||
fn unix_timestamp_seconds() -> u64 {
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.map(|duration| duration.as_secs())
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
fn synthetic_watchdog_tool_search_items() -> Vec<RolloutItem> {
|
||||
let namespace = create_watchdog_tools_namespace(vec![
|
||||
create_compact_parent_context_tool(),
|
||||
create_watchdog_close_self_tool(),
|
||||
create_watchdog_snooze_tool(),
|
||||
]);
|
||||
let Ok(namespace) = serde_json::to_value(namespace) else {
|
||||
return Vec::new();
|
||||
};
|
||||
|
||||
vec![
|
||||
RolloutItem::ResponseItem(ResponseItem::ToolSearchCall {
|
||||
id: None,
|
||||
call_id: Some(WATCHDOG_BOOT_TOOL_SEARCH_CALL_ID.to_string()),
|
||||
status: Some("completed".to_string()),
|
||||
execution: "client".to_string(),
|
||||
arguments: serde_json::json!({
|
||||
"query": "watchdog namespace tools",
|
||||
}),
|
||||
}),
|
||||
RolloutItem::ResponseItem(ResponseItem::ToolSearchOutput {
|
||||
call_id: Some(WATCHDOG_BOOT_TOOL_SEARCH_CALL_ID.to_string()),
|
||||
status: "completed".to_string(),
|
||||
execution: "client".to_string(),
|
||||
tools: vec![namespace],
|
||||
}),
|
||||
]
|
||||
}
|
||||
|
||||
fn synthetic_watchdog_list_agents_items(
|
||||
owner_thread_id: ThreadId,
|
||||
agents: Vec<ListedAgent>,
|
||||
) -> Vec<RolloutItem> {
|
||||
let envelope = serde_json::json!({
|
||||
"source": "pre_injected_agents_list",
|
||||
"generated_at": unix_timestamp_seconds(),
|
||||
"owner_thread_id": owner_thread_id.to_string(),
|
||||
"agents": agents,
|
||||
});
|
||||
let mut output = FunctionCallOutputPayload::from_text(envelope.to_string());
|
||||
output.success = Some(true);
|
||||
|
||||
vec![
|
||||
RolloutItem::ResponseItem(ResponseItem::FunctionCall {
|
||||
id: None,
|
||||
name: "list_agents".to_string(),
|
||||
namespace: None,
|
||||
arguments: "{}".to_string(),
|
||||
call_id: WATCHDOG_BOOT_LIST_AGENTS_CALL_ID.to_string(),
|
||||
}),
|
||||
RolloutItem::ResponseItem(ResponseItem::FunctionCallOutput {
|
||||
call_id: WATCHDOG_BOOT_LIST_AGENTS_CALL_ID.to_string(),
|
||||
output,
|
||||
}),
|
||||
]
|
||||
}
|
||||
|
||||
/// Control-plane handle for multi-agent operations.
|
||||
/// `AgentControl` is held by each session (via `SessionServices`). It provides capability to
|
||||
/// spawn new agents and the inter-agent communication layer.
|
||||
@@ -497,6 +584,12 @@ impl AgentControl {
|
||||
keep_forked_rollout_item(item)
|
||||
});
|
||||
}
|
||||
if is_watchdog_helper_source(&session_source) {
|
||||
forked_rollout_items.extend(
|
||||
self.watchdog_boot_context_items(state, parent_thread_id)
|
||||
.await,
|
||||
);
|
||||
}
|
||||
|
||||
state
|
||||
.fork_thread_with_source(
|
||||
@@ -906,6 +999,36 @@ impl AgentControl {
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn finish_watchdog_helper(&self, helper_thread_id: ThreadId) -> bool {
|
||||
let Some(watchdogs) = self.watchdogs.as_ref() else {
|
||||
return false;
|
||||
};
|
||||
watchdogs.finish_active_helper(helper_thread_id).await
|
||||
}
|
||||
|
||||
pub(crate) async fn finish_watchdog_helper_thread(
|
||||
&self,
|
||||
agent_id: ThreadId,
|
||||
) -> CodexResult<()> {
|
||||
let state = self.upgrade()?;
|
||||
if let Ok(thread) = state.get_thread(agent_id).await {
|
||||
if let Some(state_db_ctx) = thread.state_db()
|
||||
&& let Err(err) = state_db_ctx
|
||||
.set_thread_spawn_edge_status(
|
||||
agent_id,
|
||||
DirectionalThreadSpawnEdgeStatus::Closed,
|
||||
)
|
||||
.await
|
||||
{
|
||||
warn!("failed to persist thread-spawn edge status for {agent_id}: {err}");
|
||||
}
|
||||
thread.codex.session.flush_rollout().await?;
|
||||
}
|
||||
let _ = state.remove_thread(&agent_id).await;
|
||||
self.state.release_spawned_thread(agent_id);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn watchdog_manager(&self) -> CodexResult<&Arc<WatchdogManager>> {
|
||||
self.watchdogs.as_ref().ok_or_else(|| {
|
||||
CodexErr::UnsupportedOperation("watchdog manager unavailable".to_string())
|
||||
@@ -954,7 +1077,8 @@ impl AgentControl {
|
||||
let Ok(thread) = state.get_thread(agent_id).await else {
|
||||
return AgentStatus::NotFound;
|
||||
};
|
||||
thread.agent_status().await
|
||||
self.reported_agent_status(agent_id, thread.agent_status().await)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) fn register_session_root(
|
||||
@@ -1086,7 +1210,9 @@ impl AgentControl {
|
||||
{
|
||||
agents.push(ListedAgent {
|
||||
agent_name: root_path.to_string(),
|
||||
agent_status: root_thread.agent_status().await,
|
||||
agent_status: self
|
||||
.reported_agent_status(root_thread_id, root_thread.agent_status().await)
|
||||
.await,
|
||||
last_task_message: Some(ROOT_LAST_TASK_MESSAGE.to_string()),
|
||||
});
|
||||
}
|
||||
@@ -1095,6 +1221,14 @@ impl AgentControl {
|
||||
let Some(thread_id) = metadata.agent_id else {
|
||||
continue;
|
||||
};
|
||||
if let Some(watchdogs) = self.watchdogs.as_ref()
|
||||
&& watchdogs
|
||||
.target_for_active_helper(thread_id)
|
||||
.await
|
||||
.is_some()
|
||||
{
|
||||
continue;
|
||||
}
|
||||
if resolved_prefix
|
||||
.as_ref()
|
||||
.is_some_and(|prefix| !agent_matches_prefix(metadata.agent_path.as_ref(), prefix))
|
||||
@@ -1113,7 +1247,9 @@ impl AgentControl {
|
||||
let last_task_message = metadata.last_task_message.clone();
|
||||
agents.push(ListedAgent {
|
||||
agent_name,
|
||||
agent_status: thread.agent_status().await,
|
||||
agent_status: self
|
||||
.reported_agent_status(thread_id, thread.agent_status().await)
|
||||
.await,
|
||||
last_task_message,
|
||||
});
|
||||
}
|
||||
@@ -1121,6 +1257,108 @@ impl AgentControl {
|
||||
Ok(agents)
|
||||
}
|
||||
|
||||
async fn reported_agent_status(&self, agent_id: ThreadId, status: AgentStatus) -> AgentStatus {
|
||||
if matches!(status, AgentStatus::PendingInit)
|
||||
&& let Some(watchdogs) = self.watchdogs.as_ref()
|
||||
&& watchdogs.is_watchdog_handle(agent_id).await
|
||||
{
|
||||
return AgentStatus::Running;
|
||||
}
|
||||
status
|
||||
}
|
||||
|
||||
pub(crate) async fn send_watchdog_close_event(
|
||||
&self,
|
||||
owner_thread_id: ThreadId,
|
||||
target_thread_id: ThreadId,
|
||||
receiver_agent_nickname: Option<String>,
|
||||
receiver_agent_role: Option<String>,
|
||||
status: AgentStatus,
|
||||
) -> CodexResult<()> {
|
||||
let state = self.upgrade()?;
|
||||
let owner_thread = state.get_thread(owner_thread_id).await?;
|
||||
owner_thread
|
||||
.codex
|
||||
.session
|
||||
.send_event_raw(Event {
|
||||
id: format!("watchdog-close-{target_thread_id}"),
|
||||
msg: CollabCloseEndEvent {
|
||||
call_id: format!("watchdog-close-{target_thread_id}"),
|
||||
sender_thread_id: owner_thread_id,
|
||||
receiver_thread_id: target_thread_id,
|
||||
receiver_agent_nickname,
|
||||
receiver_agent_role,
|
||||
status,
|
||||
}
|
||||
.into(),
|
||||
})
|
||||
.await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) async fn compact_parent_for_watchdog_helper(
|
||||
&self,
|
||||
helper_thread_id: ThreadId,
|
||||
) -> CodexResult<WatchdogParentCompactionResult> {
|
||||
let Some(watchdogs) = self.watchdogs.as_ref() else {
|
||||
return Ok(WatchdogParentCompactionResult::NotWatchdogHelper);
|
||||
};
|
||||
let Some(parent_thread_id) = watchdogs.owner_for_active_helper(helper_thread_id).await
|
||||
else {
|
||||
return Ok(WatchdogParentCompactionResult::NotWatchdogHelper);
|
||||
};
|
||||
let state = self.upgrade()?;
|
||||
let parent_thread = state.get_thread(parent_thread_id).await?;
|
||||
if parent_thread
|
||||
.codex
|
||||
.session
|
||||
.active_turn
|
||||
.lock()
|
||||
.await
|
||||
.is_some()
|
||||
{
|
||||
return Ok(WatchdogParentCompactionResult::ParentBusy { parent_thread_id });
|
||||
}
|
||||
|
||||
state
|
||||
.send_op(parent_thread_id, Op::Compact)
|
||||
.await
|
||||
.map(|submission_id| WatchdogParentCompactionResult::Submitted {
|
||||
parent_thread_id,
|
||||
submission_id,
|
||||
})
|
||||
}
|
||||
|
||||
async fn watchdog_boot_context_items(
|
||||
&self,
|
||||
state: &Arc<ThreadManagerState>,
|
||||
owner_thread_id: ThreadId,
|
||||
) -> Vec<RolloutItem> {
|
||||
let owner_source = match state.get_thread(owner_thread_id).await {
|
||||
Ok(owner_thread) => {
|
||||
owner_thread
|
||||
.codex
|
||||
.thread_config_snapshot()
|
||||
.await
|
||||
.session_source
|
||||
}
|
||||
Err(_) => SessionSource::Cli,
|
||||
};
|
||||
self.register_session_root(owner_thread_id, &owner_source);
|
||||
let agents = self
|
||||
.list_agents(&owner_source, /*path_prefix*/ None)
|
||||
.await
|
||||
.unwrap_or_default();
|
||||
|
||||
synthetic_watchdog_tool_search_items()
|
||||
.into_iter()
|
||||
.chain(synthetic_watchdog_list_agents_items(
|
||||
owner_thread_id,
|
||||
agents,
|
||||
))
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Starts a detached watcher for sub-agents spawned from another thread.
|
||||
///
|
||||
/// This is only enabled for `SubAgentSource::ThreadSpawn`, where a parent thread exists and
|
||||
|
||||
@@ -589,6 +589,29 @@ async fn watchdog_helper_forks_owner_history() {
|
||||
&history_items,
|
||||
"previous owner response: pong 81 (118)"
|
||||
));
|
||||
assert!(history_items.iter().any(|item| matches!(
|
||||
item,
|
||||
ResponseItem::ToolSearchCall { call_id: Some(call_id), .. }
|
||||
if call_id == "synthetic_watchdog_tool_search"
|
||||
)));
|
||||
assert!(history_items.iter().any(|item| match item {
|
||||
ResponseItem::ToolSearchOutput {
|
||||
call_id: Some(call_id),
|
||||
tools,
|
||||
..
|
||||
} if call_id == "synthetic_watchdog_tool_search" => {
|
||||
let rendered = serde_json::to_string(tools).expect("tools should serialize");
|
||||
rendered.contains("compact_parent_context")
|
||||
&& rendered.contains("close_self")
|
||||
&& rendered.contains("snooze")
|
||||
}
|
||||
_ => false,
|
||||
}));
|
||||
assert!(history_items.iter().any(|item| matches!(
|
||||
item,
|
||||
ResponseItem::FunctionCall { name, call_id, .. }
|
||||
if name == "list_agents" && call_id == "synthetic_watchdog_list_agents"
|
||||
)));
|
||||
assert!(
|
||||
!helper_thread
|
||||
.codex
|
||||
@@ -715,6 +738,226 @@ async fn watchdog_forwards_completed_helper_without_waiting_for_interval() {
|
||||
.expect("watchdog should forward completed helper output without interval delay");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn watchdog_snooze_delays_next_helper_and_resumes_after_delay() {
|
||||
let harness = AgentControlHarness::new().await;
|
||||
let (owner_thread_id, owner_thread) = harness.start_thread().await;
|
||||
let (target_thread_id, _) = harness.start_thread().await;
|
||||
let mut config = harness.config.clone();
|
||||
config
|
||||
.features
|
||||
.enable(Feature::AgentWatchdog)
|
||||
.expect("test config should allow feature update");
|
||||
|
||||
harness
|
||||
.control
|
||||
.register_watchdog(WatchdogRegistration {
|
||||
owner_thread_id,
|
||||
target_thread_id,
|
||||
child_depth: 0,
|
||||
interval_s: 1,
|
||||
prompt: "snooze scheduling check".to_string(),
|
||||
config,
|
||||
})
|
||||
.await
|
||||
.expect("watchdog registration should succeed");
|
||||
|
||||
let owner_turn = owner_thread.codex.session.new_default_turn().await;
|
||||
owner_thread
|
||||
.codex
|
||||
.session
|
||||
.send_event(
|
||||
owner_turn.as_ref(),
|
||||
EventMsg::TurnComplete(TurnCompleteEvent {
|
||||
turn_id: owner_turn.sub_id.clone(),
|
||||
last_agent_message: Some("root done".to_string()),
|
||||
completed_at: None,
|
||||
duration_ms: None,
|
||||
time_to_first_token_ms: None,
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
|
||||
let first_helper_id = timeout(Duration::from_secs(5), async {
|
||||
loop {
|
||||
if let Some((thread_id, _)) = harness.manager.captured_ops().into_iter().find(
|
||||
|(thread_id, op)| {
|
||||
*thread_id != owner_thread_id
|
||||
&& *thread_id != target_thread_id
|
||||
&& matches!(op, Op::UserInput { items, .. } if items.iter().any(|item| match item {
|
||||
UserInput::Text { text, .. } => text.contains("snooze scheduling check"),
|
||||
UserInput::Image { .. }
|
||||
| UserInput::LocalImage { .. }
|
||||
| UserInput::Skill { .. }
|
||||
| UserInput::Mention { .. } => false,
|
||||
_ => false,
|
||||
}))
|
||||
},
|
||||
) {
|
||||
break thread_id;
|
||||
}
|
||||
sleep(Duration::from_millis(50)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("watchdog should spawn a helper before snooze");
|
||||
|
||||
let result = harness
|
||||
.control
|
||||
.snooze_watchdog_helper(first_helper_id, /*delay_seconds*/ None)
|
||||
.await
|
||||
.expect("active helper should snooze its watchdog");
|
||||
assert_eq!(result.target_thread_id, target_thread_id);
|
||||
assert_eq!(result.delay_seconds, 1);
|
||||
harness
|
||||
.control
|
||||
.finish_watchdog_helper_thread(first_helper_id)
|
||||
.await
|
||||
.expect("snoozed helper should finish");
|
||||
|
||||
sleep(Duration::from_millis(300)).await;
|
||||
assert!(
|
||||
!harness
|
||||
.manager
|
||||
.captured_ops()
|
||||
.into_iter()
|
||||
.any(|(thread_id, op)| {
|
||||
thread_id != owner_thread_id
|
||||
&& thread_id != target_thread_id
|
||||
&& thread_id != first_helper_id
|
||||
&& matches!(op, Op::UserInput { items, .. } if items.iter().any(|item| match item {
|
||||
UserInput::Text { text, .. } => text.contains("snooze scheduling check"),
|
||||
UserInput::Image { .. }
|
||||
| UserInput::LocalImage { .. }
|
||||
| UserInput::Skill { .. }
|
||||
| UserInput::Mention { .. } => false,
|
||||
_ => false,
|
||||
}))
|
||||
}),
|
||||
"watchdog should not spawn another helper before the snooze delay elapses"
|
||||
);
|
||||
|
||||
timeout(Duration::from_secs(5), async {
|
||||
loop {
|
||||
if harness.manager.captured_ops().into_iter().any(|(thread_id, op)| {
|
||||
thread_id != owner_thread_id
|
||||
&& thread_id != target_thread_id
|
||||
&& thread_id != first_helper_id
|
||||
&& matches!(op, Op::UserInput { items, .. } if items.iter().any(|item| match item {
|
||||
UserInput::Text { text, .. } => text.contains("snooze scheduling check"),
|
||||
UserInput::Image { .. }
|
||||
| UserInput::LocalImage { .. }
|
||||
| UserInput::Skill { .. }
|
||||
| UserInput::Mention { .. } => false,
|
||||
_ => false,
|
||||
}))
|
||||
}) {
|
||||
break;
|
||||
}
|
||||
sleep(Duration::from_millis(50)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("watchdog should resume spawning helpers after the snooze delay");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn watchdog_plain_goodbye_final_message_closes_handle() {
|
||||
let harness = AgentControlHarness::new().await;
|
||||
let (owner_thread_id, owner_thread) = harness.start_thread().await;
|
||||
let (target_thread_id, _) = harness.start_thread().await;
|
||||
let mut config = harness.config.clone();
|
||||
config
|
||||
.features
|
||||
.enable(Feature::AgentWatchdog)
|
||||
.expect("test config should allow feature update");
|
||||
|
||||
harness
|
||||
.control
|
||||
.register_watchdog(WatchdogRegistration {
|
||||
owner_thread_id,
|
||||
target_thread_id,
|
||||
child_depth: 0,
|
||||
interval_s: 60,
|
||||
prompt: "check in".to_string(),
|
||||
config,
|
||||
})
|
||||
.await
|
||||
.expect("watchdog registration should succeed");
|
||||
|
||||
let owner_turn = owner_thread.codex.session.new_default_turn().await;
|
||||
owner_thread
|
||||
.codex
|
||||
.session
|
||||
.send_event(
|
||||
owner_turn.as_ref(),
|
||||
EventMsg::TurnComplete(TurnCompleteEvent {
|
||||
turn_id: owner_turn.sub_id.clone(),
|
||||
last_agent_message: Some("root done".to_string()),
|
||||
completed_at: None,
|
||||
duration_ms: None,
|
||||
time_to_first_token_ms: None,
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
|
||||
let helper_thread_id = timeout(Duration::from_secs(5), async {
|
||||
loop {
|
||||
if let Some((thread_id, _)) = harness.manager.captured_ops().into_iter().find(
|
||||
|(thread_id, op)| {
|
||||
*thread_id != owner_thread_id
|
||||
&& *thread_id != target_thread_id
|
||||
&& matches!(op, Op::UserInput { items, .. } if items.iter().any(|item| match item {
|
||||
UserInput::Text { text, .. } => text.contains("check in"),
|
||||
UserInput::Image { .. }
|
||||
| UserInput::LocalImage { .. }
|
||||
| UserInput::Skill { .. }
|
||||
| UserInput::Mention { .. } => false,
|
||||
_ => false,
|
||||
}))
|
||||
},
|
||||
) {
|
||||
break thread_id;
|
||||
}
|
||||
sleep(Duration::from_millis(50)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("watchdog should spawn a helper");
|
||||
|
||||
let helper_thread = harness
|
||||
.manager
|
||||
.get_thread(helper_thread_id)
|
||||
.await
|
||||
.expect("helper thread should be registered");
|
||||
let helper_turn = helper_thread.codex.session.new_default_turn().await;
|
||||
helper_thread
|
||||
.codex
|
||||
.session
|
||||
.send_event(
|
||||
helper_turn.as_ref(),
|
||||
EventMsg::TurnComplete(TurnCompleteEvent {
|
||||
turn_id: helper_turn.sub_id.clone(),
|
||||
last_agent_message: Some("goodbye".to_string()),
|
||||
completed_at: None,
|
||||
duration_ms: None,
|
||||
time_to_first_token_ms: None,
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
|
||||
timeout(Duration::from_secs(5), async {
|
||||
loop {
|
||||
if !harness.control.is_watchdog_handle(target_thread_id).await {
|
||||
break;
|
||||
}
|
||||
sleep(Duration::from_millis(50)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("plain goodbye final message should close the watchdog handle");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn send_input_errors_when_thread_missing() {
|
||||
let harness = AgentControlHarness::new().await;
|
||||
@@ -1065,6 +1308,37 @@ async fn spawn_agent_creates_thread_and_sends_prompt() {
|
||||
assert_eq!(captured, Some(expected));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn spawn_agent_fork_rejects_missing_parent_spawn_call_id_for_non_watchdogs() {
|
||||
let harness = AgentControlHarness::new().await;
|
||||
let (parent_thread_id, _) = harness.start_thread().await;
|
||||
|
||||
let err = harness
|
||||
.control
|
||||
.spawn_agent_with_metadata(
|
||||
harness.config.clone(),
|
||||
text_input("child task"),
|
||||
Some(SessionSource::SubAgent(SubAgentSource::ThreadSpawn {
|
||||
parent_thread_id,
|
||||
depth: 1,
|
||||
agent_path: None,
|
||||
agent_nickname: None,
|
||||
agent_role: None,
|
||||
})),
|
||||
SpawnAgentOptions {
|
||||
fork_mode: Some(SpawnAgentForkMode::FullHistory),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect_err("forked worker spawns should require the parent spawn call id");
|
||||
|
||||
assert_eq!(
|
||||
err.to_string(),
|
||||
"Fatal error: spawn_agent fork requires a parent spawn call id"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn spawn_agent_can_fork_parent_thread_history_with_sanitized_items() {
|
||||
let harness = AgentControlHarness::new().await;
|
||||
@@ -1160,6 +1434,7 @@ async fn spawn_agent_can_fork_parent_thread_history_with_sanitized_items() {
|
||||
agent_role: None,
|
||||
})),
|
||||
SpawnAgentOptions {
|
||||
fork_parent_spawn_call_id: Some(parent_spawn_call_id.clone()),
|
||||
fork_mode: Some(SpawnAgentForkMode::FullHistory),
|
||||
fork_parent_spawn_call_id: Some(parent_spawn_call_id),
|
||||
..Default::default()
|
||||
@@ -1480,6 +1755,7 @@ async fn spawn_agent_fork_flushes_parent_rollout_before_loading_history() {
|
||||
agent_role: None,
|
||||
})),
|
||||
SpawnAgentOptions {
|
||||
fork_parent_spawn_call_id: Some(parent_spawn_call_id.clone()),
|
||||
fork_mode: Some(SpawnAgentForkMode::FullHistory),
|
||||
fork_parent_spawn_call_id: Some(parent_spawn_call_id),
|
||||
..Default::default()
|
||||
@@ -1590,6 +1866,7 @@ async fn spawn_agent_fork_last_n_turns_keeps_only_recent_turns() {
|
||||
agent_role: None,
|
||||
})),
|
||||
SpawnAgentOptions {
|
||||
fork_parent_spawn_call_id: Some(parent_spawn_call_id.clone()),
|
||||
fork_mode: Some(SpawnAgentForkMode::LastNTurns(2)),
|
||||
fork_parent_spawn_call_id: Some(parent_spawn_call_id),
|
||||
..Default::default()
|
||||
|
||||
@@ -8,6 +8,7 @@ pub(crate) mod watchdog;
|
||||
|
||||
pub(crate) use codex_protocol::protocol::AgentStatus;
|
||||
pub(crate) use control::AgentControl;
|
||||
pub(crate) use control::WatchdogParentCompactionResult;
|
||||
pub(crate) use mailbox::Mailbox;
|
||||
pub(crate) use mailbox::MailboxReceiver;
|
||||
pub(crate) use registry::exceeds_thread_spawn_depth_limit;
|
||||
|
||||
@@ -307,19 +307,32 @@ impl WatchdogManager {
|
||||
return;
|
||||
}
|
||||
let helper_suppressed = self.take_suppressed_helper(helper_id).await;
|
||||
let mut close_watchdog_handle = false;
|
||||
if let AgentStatus::Completed(Some(message)) = helper_status
|
||||
&& !helper_suppressed
|
||||
&& let Err(err) = control_for_spawn
|
||||
{
|
||||
close_watchdog_handle = final_message_requests_watchdog_close(&message);
|
||||
if let Err(err) = control_for_spawn
|
||||
.send_watchdog_wakeup(snapshot.owner_thread_id, message)
|
||||
.await
|
||||
{
|
||||
warn!(
|
||||
helper_id = %helper_id,
|
||||
owner_thread_id = %snapshot.owner_thread_id,
|
||||
"watchdog helper forward failed: {err}"
|
||||
);
|
||||
{
|
||||
warn!(
|
||||
helper_id = %helper_id,
|
||||
owner_thread_id = %snapshot.owner_thread_id,
|
||||
"watchdog helper forward failed: {err}"
|
||||
);
|
||||
}
|
||||
}
|
||||
let _ = control_for_spawn.shutdown_live_agent(helper_id).await;
|
||||
if close_watchdog_handle {
|
||||
let _ = control_for_spawn
|
||||
.unregister_watchdog_handle(target_thread_id)
|
||||
.await;
|
||||
let _ = control_for_spawn
|
||||
.shutdown_live_agent(target_thread_id)
|
||||
.await;
|
||||
return;
|
||||
}
|
||||
self.update_after_spawn(
|
||||
target_thread_id,
|
||||
generation,
|
||||
@@ -534,6 +547,25 @@ impl WatchdogManager {
|
||||
Some(result)
|
||||
}
|
||||
|
||||
pub(crate) async fn finish_active_helper(&self, helper_thread_id: ThreadId) -> bool {
|
||||
let found = {
|
||||
let mut registrations = self.registrations.lock().await;
|
||||
let Some(entry) = registrations
|
||||
.values_mut()
|
||||
.find(|entry| entry.active_helper_id == Some(helper_thread_id))
|
||||
else {
|
||||
return false;
|
||||
};
|
||||
entry.active_helper_id = None;
|
||||
true
|
||||
};
|
||||
self.suppressed_helpers
|
||||
.lock()
|
||||
.await
|
||||
.insert(helper_thread_id);
|
||||
found
|
||||
}
|
||||
|
||||
async fn update_after_spawn(
|
||||
&self,
|
||||
target_thread_id: ThreadId,
|
||||
@@ -582,6 +614,10 @@ fn is_watchdog_terminated(status: &AgentStatus) -> bool {
|
||||
matches!(status, AgentStatus::Shutdown | AgentStatus::NotFound)
|
||||
}
|
||||
|
||||
fn final_message_requests_watchdog_close(message: &str) -> bool {
|
||||
message.trim().eq_ignore_ascii_case("goodbye")
|
||||
}
|
||||
|
||||
async fn get_status(manager_state: &Arc<ThreadManagerState>, thread_id: ThreadId) -> AgentStatus {
|
||||
match manager_state.get_thread(thread_id).await {
|
||||
Ok(thread) => thread.agent_status().await,
|
||||
|
||||
@@ -6158,6 +6158,51 @@ async fn load_config_rejects_unsafe_agent_role_nickname_candidates() -> std::io:
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn load_config_reads_top_level_watchdog_interval() -> std::io::Result<()> {
|
||||
let codex_home = TempDir::new()?;
|
||||
let cfg = ConfigToml {
|
||||
watchdog_interval_s: Some(3),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let config = Config::load_from_base_config_with_overrides(
|
||||
cfg,
|
||||
ConfigOverrides::default(),
|
||||
codex_home.abs(),
|
||||
)
|
||||
.await?;
|
||||
|
||||
assert_eq!(config.watchdog_interval_s, 3);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn load_config_rejects_nonpositive_top_level_watchdog_interval() -> std::io::Result<()> {
|
||||
let codex_home = TempDir::new()?;
|
||||
let cfg = ConfigToml {
|
||||
watchdog_interval_s: Some(0),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let err = Config::load_from_base_config_with_overrides(
|
||||
cfg,
|
||||
ConfigOverrides::default(),
|
||||
codex_home.abs(),
|
||||
)
|
||||
.await
|
||||
.expect_err("nonpositive watchdog interval should fail");
|
||||
|
||||
assert_eq!(err.kind(), std::io::ErrorKind::InvalidInput);
|
||||
assert_eq!(
|
||||
err.to_string(),
|
||||
"watchdog_interval_s must be greater than zero"
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn model_catalog_json_loads_from_path() -> std::io::Result<()> {
|
||||
let codex_home = TempDir::new()?;
|
||||
|
||||
@@ -58,6 +58,7 @@ pub(crate) fn parse_agent_id_targets(
|
||||
}
|
||||
|
||||
pub(crate) use close_agent::Handler as CloseAgentHandler;
|
||||
pub(crate) use compact_parent_context::Handler as CompactParentContextHandler;
|
||||
pub(crate) use resume_agent::Handler as ResumeAgentHandler;
|
||||
pub(crate) use send_input::Handler as SendInputHandler;
|
||||
pub(crate) use spawn::Handler as SpawnAgentHandler;
|
||||
@@ -66,6 +67,7 @@ pub(crate) use watchdog_self_close::Handler as WatchdogSelfCloseHandler;
|
||||
pub(crate) use watchdog_snooze::Handler as WatchdogSnoozeHandler;
|
||||
|
||||
pub(crate) mod close_agent;
|
||||
mod compact_parent_context;
|
||||
mod resume_agent;
|
||||
mod send_input;
|
||||
mod spawn;
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
use super::*;
|
||||
use crate::agent::WatchdogParentCompactionResult;
|
||||
|
||||
pub(crate) struct Handler;
|
||||
|
||||
impl ToolHandler for Handler {
|
||||
type Output = CompactParentContextResult;
|
||||
|
||||
fn kind(&self) -> ToolKind {
|
||||
ToolKind::Function
|
||||
}
|
||||
|
||||
fn matches_kind(&self, payload: &ToolPayload) -> bool {
|
||||
matches!(payload, ToolPayload::Function { .. })
|
||||
}
|
||||
|
||||
async fn handle(&self, invocation: ToolInvocation) -> Result<Self::Output, FunctionCallError> {
|
||||
let ToolInvocation {
|
||||
session, payload, ..
|
||||
} = invocation;
|
||||
let arguments = function_arguments(payload)?;
|
||||
let args: CompactParentContextArgs = parse_arguments(&arguments)?;
|
||||
let _ = (args.reason, args.evidence);
|
||||
let result = session
|
||||
.services
|
||||
.agent_control
|
||||
.compact_parent_for_watchdog_helper(session.conversation_id)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
FunctionCallError::RespondToModel(format!("compact_parent_context failed: {err}"))
|
||||
})?;
|
||||
Ok(CompactParentContextResult::from(result))
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct CompactParentContextArgs {
|
||||
reason: Option<String>,
|
||||
evidence: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub(crate) struct CompactParentContextResult {
|
||||
kind: &'static str,
|
||||
parent_thread_id: Option<String>,
|
||||
submission_id: Option<String>,
|
||||
}
|
||||
|
||||
impl From<WatchdogParentCompactionResult> for CompactParentContextResult {
|
||||
fn from(value: WatchdogParentCompactionResult) -> Self {
|
||||
match value {
|
||||
WatchdogParentCompactionResult::NotWatchdogHelper => Self {
|
||||
kind: "not_watchdog_helper",
|
||||
parent_thread_id: None,
|
||||
submission_id: None,
|
||||
},
|
||||
WatchdogParentCompactionResult::ParentBusy { parent_thread_id } => Self {
|
||||
kind: "parent_busy",
|
||||
parent_thread_id: Some(parent_thread_id.to_string()),
|
||||
submission_id: None,
|
||||
},
|
||||
WatchdogParentCompactionResult::Submitted {
|
||||
parent_thread_id,
|
||||
submission_id,
|
||||
} => Self {
|
||||
kind: "submitted",
|
||||
parent_thread_id: Some(parent_thread_id.to_string()),
|
||||
submission_id: Some(submission_id),
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl ToolOutput for CompactParentContextResult {
|
||||
fn log_preview(&self) -> String {
|
||||
tool_output_json_text(self, "compact_parent_context")
|
||||
}
|
||||
|
||||
fn success_for_logging(&self) -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
fn to_response_item(&self, call_id: &str, payload: &ToolPayload) -> ResponseInputItem {
|
||||
tool_output_response_item(call_id, payload, self, Some(true), "compact_parent_context")
|
||||
}
|
||||
|
||||
fn code_mode_result(&self, _payload: &ToolPayload) -> JsonValue {
|
||||
tool_output_code_mode_result(self, "compact_parent_context")
|
||||
}
|
||||
}
|
||||
@@ -27,7 +27,7 @@ impl ToolHandler for Handler {
|
||||
.await
|
||||
else {
|
||||
return Err(FunctionCallError::RespondToModel(
|
||||
"watchdog_self_close is only available in watchdog check-in threads.".to_string(),
|
||||
"watchdog.close_self is only available in watchdog check-in threads.".to_string(),
|
||||
));
|
||||
};
|
||||
let Some(target_thread_id) = session
|
||||
@@ -37,7 +37,7 @@ impl ToolHandler for Handler {
|
||||
.await
|
||||
else {
|
||||
return Err(FunctionCallError::RespondToModel(
|
||||
"watchdog_self_close is only available in watchdog check-in threads.".to_string(),
|
||||
"watchdog.close_self is only available in watchdog check-in threads.".to_string(),
|
||||
));
|
||||
};
|
||||
|
||||
@@ -46,6 +46,23 @@ impl ToolHandler for Handler {
|
||||
.agent_control
|
||||
.get_status(target_thread_id)
|
||||
.await;
|
||||
let receiver_agent = session
|
||||
.services
|
||||
.agent_control
|
||||
.get_agent_metadata(target_thread_id)
|
||||
.unwrap_or_default();
|
||||
|
||||
let _ = session
|
||||
.services
|
||||
.agent_control
|
||||
.send_watchdog_close_event(
|
||||
owner_thread_id,
|
||||
target_thread_id,
|
||||
receiver_agent.agent_nickname,
|
||||
receiver_agent.agent_role,
|
||||
status.clone(),
|
||||
)
|
||||
.await;
|
||||
|
||||
if let Some(message) = args.message
|
||||
&& !message.trim().is_empty()
|
||||
@@ -82,7 +99,7 @@ pub(crate) struct WatchdogSelfCloseResult {
|
||||
|
||||
impl ToolOutput for WatchdogSelfCloseResult {
|
||||
fn log_preview(&self) -> String {
|
||||
tool_output_json_text(self, "watchdog_self_close")
|
||||
tool_output_json_text(self, "close_self")
|
||||
}
|
||||
|
||||
fn success_for_logging(&self) -> bool {
|
||||
@@ -90,10 +107,10 @@ impl ToolOutput for WatchdogSelfCloseResult {
|
||||
}
|
||||
|
||||
fn to_response_item(&self, call_id: &str, payload: &ToolPayload) -> ResponseInputItem {
|
||||
tool_output_response_item(call_id, payload, self, Some(true), "watchdog_self_close")
|
||||
tool_output_response_item(call_id, payload, self, Some(true), "close_self")
|
||||
}
|
||||
|
||||
fn code_mode_result(&self, _payload: &ToolPayload) -> JsonValue {
|
||||
tool_output_code_mode_result(self, "watchdog_self_close")
|
||||
tool_output_code_mode_result(self, "close_self")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -443,6 +443,61 @@ async fn multi_agent_v2_spawn_defaults_to_full_fork_and_rejects_child_model_over
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multi_agent_v2_spawn_watchdog_role_returns_inert_handle_and_ignores_fork_turns() {
|
||||
let (mut session, mut turn) = make_session_and_context().await;
|
||||
let manager = thread_manager();
|
||||
let root = manager
|
||||
.start_thread((*turn.config).clone())
|
||||
.await
|
||||
.expect("root thread should start");
|
||||
let agent_control = manager.agent_control();
|
||||
session.services.agent_control = agent_control.clone();
|
||||
session.conversation_id = root.thread_id;
|
||||
let mut config = (*turn.config).clone();
|
||||
config
|
||||
.features
|
||||
.enable(Feature::AgentWatchdog)
|
||||
.expect("test config should allow feature update");
|
||||
config
|
||||
.features
|
||||
.enable(Feature::MultiAgentV2)
|
||||
.expect("test config should allow feature update");
|
||||
turn.config = Arc::new(config);
|
||||
|
||||
let output = SpawnAgentHandlerV2
|
||||
.handle(invocation(
|
||||
Arc::new(session),
|
||||
Arc::new(turn),
|
||||
"spawn_agent",
|
||||
function_payload(json!({
|
||||
"message": "check in later",
|
||||
"task_name": "watchdog",
|
||||
"agent_type": "watchdog",
|
||||
"fork_turns": "ignored for watchdogs"
|
||||
})),
|
||||
))
|
||||
.await
|
||||
.expect("watchdog spawn should ignore fork_turns and succeed");
|
||||
let (content, success) = expect_text_output(output);
|
||||
let result: serde_json::Value =
|
||||
serde_json::from_str(&content).expect("spawn_agent result should be json");
|
||||
assert_eq!(result["task_name"], "/root/watchdog");
|
||||
|
||||
let watchdog_id = manager
|
||||
.captured_ops()
|
||||
.into_iter()
|
||||
.filter_map(|(thread_id, op)| (thread_id != root.thread_id).then_some((thread_id, op)))
|
||||
.find_map(|(thread_id, op)| (op == Op::Interrupt).then_some(thread_id))
|
||||
.expect("watchdog handle should receive only an inert interrupt");
|
||||
assert_eq!(success, Some(true));
|
||||
assert_eq!(
|
||||
agent_control.get_status(watchdog_id).await,
|
||||
AgentStatus::Running
|
||||
);
|
||||
assert!(agent_control.is_watchdog_handle(watchdog_id).await);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multi_agent_v2_spawn_partial_fork_turns_allows_agent_type_override() {
|
||||
let (mut session, mut turn) = make_session_and_context().await;
|
||||
@@ -564,7 +619,7 @@ async fn spawn_agent_watchdog_role_returns_inert_handle() {
|
||||
assert_eq!(success, Some(true));
|
||||
assert_eq!(
|
||||
agent_control.get_status(agent_id).await,
|
||||
AgentStatus::PendingInit
|
||||
AgentStatus::Running
|
||||
);
|
||||
let ops_for_agent = manager
|
||||
.captured_ops()
|
||||
@@ -575,6 +630,71 @@ async fn spawn_agent_watchdog_role_returns_inert_handle() {
|
||||
assert!(agent_control.is_watchdog_handle(agent_id).await);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn compact_parent_context_submits_compaction_for_idle_parent() {
|
||||
let (mut session, turn) = make_session_and_context().await;
|
||||
let manager = thread_manager();
|
||||
let agent_control = manager.agent_control();
|
||||
session.services.agent_control = agent_control.clone();
|
||||
let owner = manager
|
||||
.start_thread((*turn.config).clone())
|
||||
.await
|
||||
.expect("owner thread should start");
|
||||
let target = manager
|
||||
.start_thread((*turn.config).clone())
|
||||
.await
|
||||
.expect("watchdog handle should start");
|
||||
let helper_thread_id = ThreadId::new();
|
||||
agent_control
|
||||
.register_watchdog(WatchdogRegistration {
|
||||
owner_thread_id: owner.thread_id,
|
||||
target_thread_id: target.thread_id,
|
||||
child_depth: 1,
|
||||
interval_s: 60,
|
||||
prompt: "check in".to_string(),
|
||||
config: (*turn.config).clone(),
|
||||
})
|
||||
.await
|
||||
.expect("watchdog registration should succeed");
|
||||
agent_control
|
||||
.set_watchdog_active_helper_for_tests(target.thread_id, helper_thread_id)
|
||||
.await;
|
||||
session.conversation_id = helper_thread_id;
|
||||
|
||||
let output = CompactParentContextHandler
|
||||
.handle(invocation(
|
||||
Arc::new(session),
|
||||
Arc::new(turn),
|
||||
"compact_parent_context",
|
||||
function_payload(json!({"reason": "root is idle"})),
|
||||
))
|
||||
.await
|
||||
.expect("watchdog helper should request parent compaction");
|
||||
let (content, success) = expect_text_output(output);
|
||||
|
||||
assert_eq!(success, Some(true));
|
||||
let result: serde_json::Value =
|
||||
serde_json::from_str(&content).expect("compact_parent_context result should be json");
|
||||
let submission_id = result
|
||||
.get("submission_id")
|
||||
.and_then(|value| value.as_str())
|
||||
.expect("submitted compaction should include a submission id");
|
||||
assert!(!submission_id.is_empty());
|
||||
assert_eq!(
|
||||
result,
|
||||
json!({
|
||||
"kind": "submitted",
|
||||
"parent_thread_id": owner.thread_id.to_string(),
|
||||
"submission_id": submission_id,
|
||||
})
|
||||
);
|
||||
let captured = manager
|
||||
.captured_ops()
|
||||
.into_iter()
|
||||
.find(|(thread_id, op)| *thread_id == owner.thread_id && matches!(op, Op::Compact));
|
||||
assert_eq!(captured, Some((owner.thread_id, Op::Compact)));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn watchdog_snooze_rejects_non_watchdog_thread() {
|
||||
let (session, turn) = make_session_and_context().await;
|
||||
@@ -661,17 +781,198 @@ async fn watchdog_snooze_suppresses_helper_and_clears_active_helper() {
|
||||
.watchdog_helper_is_suppressed_for_tests(helper_thread_id)
|
||||
.await
|
||||
);
|
||||
assert_eq!(
|
||||
agent_control.get_status(helper_thread_id).await,
|
||||
AgentStatus::NotFound
|
||||
);
|
||||
assert!(
|
||||
!manager
|
||||
.captured_ops()
|
||||
.iter()
|
||||
.any(|(thread_id, op)| *thread_id == helper_thread_id && matches!(op, Op::Shutdown)),
|
||||
"snooze should finish the helper turn without a shutdown op"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn watchdog_self_close_rejects_non_watchdog_thread() {
|
||||
async fn multi_agent_v2_watchdog_followup_task_parent_wakes_owner_and_finishes_helper() {
|
||||
let (mut session, mut turn) = make_session_and_context().await;
|
||||
let manager = thread_manager();
|
||||
let agent_control = manager.agent_control();
|
||||
let owner = manager
|
||||
.start_thread((*turn.config).clone())
|
||||
.await
|
||||
.expect("owner thread should start");
|
||||
let target = manager
|
||||
.start_thread((*turn.config).clone())
|
||||
.await
|
||||
.expect("watchdog handle should start");
|
||||
let helper_thread_id = session.conversation_id;
|
||||
session.services.agent_control = agent_control.clone();
|
||||
let mut config = (*turn.config).clone();
|
||||
config
|
||||
.features
|
||||
.enable(Feature::AgentWatchdog)
|
||||
.expect("test config should allow watchdog feature update");
|
||||
config
|
||||
.features
|
||||
.enable(Feature::MultiAgentV2)
|
||||
.expect("test config should allow multi-agent v2 feature update");
|
||||
turn.config = Arc::new(config.clone());
|
||||
turn.session_source = SessionSource::SubAgent(SubAgentSource::ThreadSpawn {
|
||||
parent_thread_id: owner.thread_id,
|
||||
depth: 1,
|
||||
agent_path: Some(AgentPath::try_from("/root/watchdog").expect("watchdog path")),
|
||||
agent_nickname: None,
|
||||
agent_role: Some("watchdog".to_string()),
|
||||
});
|
||||
|
||||
agent_control
|
||||
.register_watchdog(WatchdogRegistration {
|
||||
owner_thread_id: owner.thread_id,
|
||||
target_thread_id: target.thread_id,
|
||||
child_depth: 0,
|
||||
interval_s: 60,
|
||||
prompt: "check in".to_string(),
|
||||
config,
|
||||
})
|
||||
.await
|
||||
.expect("watchdog registration should succeed");
|
||||
agent_control
|
||||
.set_watchdog_active_helper_for_tests(target.thread_id, helper_thread_id)
|
||||
.await;
|
||||
|
||||
let output = FollowupTaskHandlerV2
|
||||
.handle(invocation(
|
||||
Arc::new(session),
|
||||
Arc::new(turn),
|
||||
"followup_task",
|
||||
function_payload(json!({
|
||||
"target": "parent",
|
||||
"message": "continue the user task"
|
||||
})),
|
||||
))
|
||||
.await
|
||||
.expect("watchdog helper should wake its parent");
|
||||
let (_, success) = expect_text_output(output);
|
||||
|
||||
assert_eq!(success, Some(true));
|
||||
assert_eq!(
|
||||
agent_control
|
||||
.watchdog_target_for_active_helper(helper_thread_id)
|
||||
.await,
|
||||
None
|
||||
);
|
||||
assert!(
|
||||
agent_control
|
||||
.watchdog_helper_is_suppressed_for_tests(helper_thread_id)
|
||||
.await
|
||||
);
|
||||
assert_eq!(
|
||||
agent_control.get_status(helper_thread_id).await,
|
||||
AgentStatus::NotFound
|
||||
);
|
||||
assert!(manager.captured_ops().iter().any(|(thread_id, op)| {
|
||||
*thread_id == owner.thread_id
|
||||
&& matches!(
|
||||
op,
|
||||
Op::InterAgentCommunication { communication }
|
||||
if communication.author.as_str() == "/root/watchdog"
|
||||
&& communication.recipient == AgentPath::root()
|
||||
&& communication.other_recipients.is_empty()
|
||||
&& communication.content == "continue the user task"
|
||||
&& communication.trigger_turn
|
||||
)
|
||||
}));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multi_agent_v2_watchdog_send_message_parent_is_rejected() {
|
||||
let (mut session, mut turn) = make_session_and_context().await;
|
||||
let manager = thread_manager();
|
||||
let agent_control = manager.agent_control();
|
||||
let owner = manager
|
||||
.start_thread((*turn.config).clone())
|
||||
.await
|
||||
.expect("owner thread should start");
|
||||
let target = manager
|
||||
.start_thread((*turn.config).clone())
|
||||
.await
|
||||
.expect("watchdog handle should start");
|
||||
let helper_thread_id = session.conversation_id;
|
||||
session.services.agent_control = agent_control.clone();
|
||||
let mut config = (*turn.config).clone();
|
||||
config
|
||||
.features
|
||||
.enable(Feature::AgentWatchdog)
|
||||
.expect("test config should allow watchdog feature update");
|
||||
config
|
||||
.features
|
||||
.enable(Feature::MultiAgentV2)
|
||||
.expect("test config should allow multi-agent v2 feature update");
|
||||
turn.config = Arc::new(config.clone());
|
||||
turn.session_source = SessionSource::SubAgent(SubAgentSource::ThreadSpawn {
|
||||
parent_thread_id: owner.thread_id,
|
||||
depth: 1,
|
||||
agent_path: Some(AgentPath::try_from("/root/watchdog").expect("watchdog path")),
|
||||
agent_nickname: None,
|
||||
agent_role: Some("watchdog".to_string()),
|
||||
});
|
||||
|
||||
agent_control
|
||||
.register_watchdog(WatchdogRegistration {
|
||||
owner_thread_id: owner.thread_id,
|
||||
target_thread_id: target.thread_id,
|
||||
child_depth: 0,
|
||||
interval_s: 60,
|
||||
prompt: "check in".to_string(),
|
||||
config,
|
||||
})
|
||||
.await
|
||||
.expect("watchdog registration should succeed");
|
||||
agent_control
|
||||
.set_watchdog_active_helper_for_tests(target.thread_id, helper_thread_id)
|
||||
.await;
|
||||
|
||||
let Err(err) = SendMessageHandlerV2
|
||||
.handle(invocation(
|
||||
Arc::new(session),
|
||||
Arc::new(turn),
|
||||
"send_message",
|
||||
function_payload(json!({
|
||||
"target": "parent",
|
||||
"message": "queued watchdog update"
|
||||
})),
|
||||
))
|
||||
.await
|
||||
else {
|
||||
panic!("watchdog helper send_message to parent should be rejected");
|
||||
};
|
||||
|
||||
assert_eq!(
|
||||
err,
|
||||
FunctionCallError::RespondToModel(
|
||||
"watchdog check-in threads must use followup_task with target `parent` to message their parent."
|
||||
.to_string()
|
||||
)
|
||||
);
|
||||
assert_eq!(
|
||||
agent_control
|
||||
.watchdog_target_for_active_helper(helper_thread_id)
|
||||
.await,
|
||||
Some(target.thread_id)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn watchdog_close_self_rejects_non_watchdog_thread() {
|
||||
let (session, turn) = make_session_and_context().await;
|
||||
|
||||
let err = WatchdogSelfCloseHandler
|
||||
.handle(invocation(
|
||||
Arc::new(session),
|
||||
Arc::new(turn),
|
||||
"watchdog_self_close",
|
||||
"close_self",
|
||||
function_payload(json!({})),
|
||||
))
|
||||
.await
|
||||
@@ -680,13 +981,13 @@ async fn watchdog_self_close_rejects_non_watchdog_thread() {
|
||||
assert_eq!(
|
||||
err,
|
||||
FunctionCallError::RespondToModel(
|
||||
"watchdog_self_close is only available in watchdog check-in threads.".to_string(),
|
||||
"watchdog.close_self is only available in watchdog check-in threads.".to_string(),
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn watchdog_self_close_notifies_owner_and_unregisters_handle() {
|
||||
async fn watchdog_close_self_notifies_owner_and_unregisters_handle() {
|
||||
let (mut session, mut turn) = make_session_and_context().await;
|
||||
let manager = thread_manager();
|
||||
let agent_control = manager.agent_control();
|
||||
@@ -726,7 +1027,7 @@ async fn watchdog_self_close_notifies_owner_and_unregisters_handle() {
|
||||
.handle(invocation(
|
||||
Arc::new(session),
|
||||
Arc::new(turn),
|
||||
"watchdog_self_close",
|
||||
"close_self",
|
||||
function_payload(json!({"message": "watchdog done"})),
|
||||
))
|
||||
.await
|
||||
@@ -736,7 +1037,7 @@ async fn watchdog_self_close_notifies_owner_and_unregisters_handle() {
|
||||
serde_json::from_str(&content).expect("self-close result should be json");
|
||||
|
||||
assert_eq!(success, Some(true));
|
||||
assert_eq!(result["previous_status"], json!("pending_init"));
|
||||
assert_eq!(result["previous_status"], json!("running"));
|
||||
assert!(!agent_control.is_watchdog_handle(target.thread_id).await);
|
||||
assert_eq!(
|
||||
agent_control.get_status(target.thread_id).await,
|
||||
@@ -753,6 +1054,23 @@ async fn watchdog_self_close_notifies_owner_and_unregisters_handle() {
|
||||
thread_id == owner.thread_id
|
||||
&& matches!(op, Op::InterAgentCommunication { communication } if communication == expected)
|
||||
}));
|
||||
let close_event = timeout(Duration::from_secs(1), async {
|
||||
loop {
|
||||
let event = owner
|
||||
.thread
|
||||
.next_event()
|
||||
.await
|
||||
.expect("owner event channel should stay open");
|
||||
if let EventMsg::CollabCloseEnd(close) = event.msg {
|
||||
break close;
|
||||
}
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("watchdog self-close should publish a close event for the handle");
|
||||
assert_eq!(close_event.sender_thread_id, owner.thread_id);
|
||||
assert_eq!(close_event.receiver_thread_id, target.thread_id);
|
||||
assert_eq!(close_event.status, AgentStatus::Running);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -1220,6 +1538,79 @@ async fn multi_agent_v2_followup_task_rejects_root_target_from_child() {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multi_agent_v2_followup_task_rejects_parent_target_from_non_watchdog_child() {
|
||||
let (mut session, mut turn) = make_session_and_context().await;
|
||||
let manager = thread_manager();
|
||||
let root = manager
|
||||
.start_thread((*turn.config).clone())
|
||||
.await
|
||||
.expect("root thread should start");
|
||||
session.services.agent_control = manager.agent_control();
|
||||
session.conversation_id = root.thread_id;
|
||||
let mut config = (*turn.config).clone();
|
||||
config
|
||||
.features
|
||||
.enable(Feature::MultiAgentV2)
|
||||
.expect("test config should allow feature update");
|
||||
turn.config = Arc::new(config);
|
||||
|
||||
let child_path = AgentPath::try_from("/root/worker").expect("agent path");
|
||||
let child_thread_id = session
|
||||
.services
|
||||
.agent_control
|
||||
.spawn_agent_with_metadata(
|
||||
(*turn.config).clone(),
|
||||
vec![UserInput::Text {
|
||||
text: "inspect this repo".to_string(),
|
||||
text_elements: Vec::new(),
|
||||
}]
|
||||
.into(),
|
||||
Some(SessionSource::SubAgent(SubAgentSource::ThreadSpawn {
|
||||
parent_thread_id: root.thread_id,
|
||||
depth: 1,
|
||||
agent_path: Some(child_path.clone()),
|
||||
agent_nickname: None,
|
||||
agent_role: None,
|
||||
})),
|
||||
crate::agent::control::SpawnAgentOptions::default(),
|
||||
)
|
||||
.await
|
||||
.expect("worker spawn should succeed")
|
||||
.thread_id;
|
||||
session.conversation_id = child_thread_id;
|
||||
turn.session_source = SessionSource::SubAgent(SubAgentSource::ThreadSpawn {
|
||||
parent_thread_id: root.thread_id,
|
||||
depth: 1,
|
||||
agent_path: Some(child_path),
|
||||
agent_nickname: None,
|
||||
agent_role: None,
|
||||
});
|
||||
|
||||
let Err(err) = FollowupTaskHandlerV2
|
||||
.handle(invocation(
|
||||
Arc::new(session),
|
||||
Arc::new(turn),
|
||||
"followup_task",
|
||||
function_payload(json!({
|
||||
"target": "parent",
|
||||
"message": "wake up"
|
||||
})),
|
||||
))
|
||||
.await
|
||||
else {
|
||||
panic!("non-watchdog followup_task should reject the direct parent target");
|
||||
};
|
||||
|
||||
assert_eq!(
|
||||
err,
|
||||
FunctionCallError::RespondToModel(
|
||||
"Only watchdog check-in threads can use followup_task with target `parent`; use send_message for parent updates."
|
||||
.to_string()
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multi_agent_v2_list_agents_returns_completed_status_and_last_task_message() {
|
||||
let (mut session, mut turn) = make_session_and_context().await;
|
||||
@@ -1465,6 +1856,153 @@ async fn multi_agent_v2_list_agents_omits_closed_agents() {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn watchdog_handle_is_listed_and_close_agent_removes_it() {
|
||||
let (mut session, mut turn) = make_session_and_context().await;
|
||||
let manager = thread_manager();
|
||||
let root = manager
|
||||
.start_thread((*turn.config).clone())
|
||||
.await
|
||||
.expect("root thread should start");
|
||||
let agent_control = manager.agent_control();
|
||||
session.services.agent_control = agent_control.clone();
|
||||
session.conversation_id = root.thread_id;
|
||||
let mut config = (*turn.config).clone();
|
||||
config
|
||||
.features
|
||||
.enable(Feature::AgentWatchdog)
|
||||
.expect("test config should allow feature update");
|
||||
config
|
||||
.features
|
||||
.enable(Feature::MultiAgentV2)
|
||||
.expect("test config should allow feature update");
|
||||
let enabled_config = config.clone();
|
||||
turn.config = Arc::new(config);
|
||||
|
||||
let session = Arc::new(session);
|
||||
let turn = Arc::new(turn);
|
||||
let spawn_output = SpawnAgentHandler
|
||||
.handle(invocation(
|
||||
session.clone(),
|
||||
turn.clone(),
|
||||
"spawn_agent",
|
||||
function_payload(json!({
|
||||
"message": "check this branch periodically",
|
||||
"agent_type": "watchdog"
|
||||
})),
|
||||
))
|
||||
.await
|
||||
.expect("watchdog spawn should succeed");
|
||||
let (spawn_content, spawn_success) = expect_text_output(spawn_output);
|
||||
let spawn_result: serde_json::Value =
|
||||
serde_json::from_str(&spawn_content).expect("watchdog spawn result should be json");
|
||||
let watchdog_id = parse_agent_id(
|
||||
spawn_result["agent_id"]
|
||||
.as_str()
|
||||
.expect("watchdog spawn result should include agent_id"),
|
||||
);
|
||||
assert_eq!(spawn_success, Some(true));
|
||||
assert!(agent_control.is_watchdog_handle(watchdog_id).await);
|
||||
|
||||
let helper_id = agent_control
|
||||
.spawn_agent(
|
||||
enabled_config,
|
||||
vec![UserInput::Text {
|
||||
text: "watchdog helper implementation detail".to_string(),
|
||||
text_elements: Vec::new(),
|
||||
}]
|
||||
.into(),
|
||||
Some(SessionSource::SubAgent(SubAgentSource::ThreadSpawn {
|
||||
parent_thread_id: root.thread_id,
|
||||
depth: 1,
|
||||
agent_path: None,
|
||||
agent_nickname: None,
|
||||
agent_role: Some("watchdog".to_string()),
|
||||
})),
|
||||
)
|
||||
.await
|
||||
.expect("watchdog helper should start");
|
||||
agent_control
|
||||
.set_watchdog_active_helper_for_tests(watchdog_id, helper_id)
|
||||
.await;
|
||||
|
||||
let list_output = ListAgentsHandlerV2
|
||||
.handle(invocation(
|
||||
session.clone(),
|
||||
turn.clone(),
|
||||
"list_agents",
|
||||
function_payload(json!({})),
|
||||
))
|
||||
.await
|
||||
.expect("list_agents should include the watchdog handle");
|
||||
let (list_content, list_success) = expect_text_output(list_output);
|
||||
let list_result: ListAgentsResult =
|
||||
serde_json::from_str(&list_content).expect("list_agents result should be json");
|
||||
assert_eq!(list_success, Some(true));
|
||||
let watchdog_listing = list_result
|
||||
.agents
|
||||
.iter()
|
||||
.find(|agent| agent.agent_name == watchdog_id.to_string())
|
||||
.expect("list_agents should include the watchdog handle");
|
||||
assert_eq!(watchdog_listing.agent_status, json!("running"));
|
||||
assert!(
|
||||
!list_result
|
||||
.agents
|
||||
.iter()
|
||||
.any(|agent| agent.agent_name == helper_id.to_string()),
|
||||
"active watchdog helpers should not be exposed as targetable list_agents entries"
|
||||
);
|
||||
|
||||
let close_output = CloseAgentHandlerV2
|
||||
.handle(invocation(
|
||||
session.clone(),
|
||||
turn.clone(),
|
||||
"close_agent",
|
||||
function_payload(json!({"target": watchdog_id.to_string()})),
|
||||
))
|
||||
.await
|
||||
.expect("close_agent should close the watchdog handle");
|
||||
let (close_content, close_success) = expect_text_output(close_output);
|
||||
let close_result: close_agent::CloseAgentResult =
|
||||
serde_json::from_str(&close_content).expect("close_agent result should be json");
|
||||
assert_eq!(close_success, Some(true));
|
||||
assert_eq!(close_result.previous_status, AgentStatus::PendingInit);
|
||||
assert!(!agent_control.is_watchdog_handle(watchdog_id).await);
|
||||
assert_eq!(
|
||||
agent_control.get_status(watchdog_id).await,
|
||||
AgentStatus::NotFound
|
||||
);
|
||||
assert_eq!(
|
||||
agent_control.get_status(helper_id).await,
|
||||
AgentStatus::NotFound
|
||||
);
|
||||
assert!(
|
||||
manager
|
||||
.captured_ops()
|
||||
.iter()
|
||||
.any(|(thread_id, op)| *thread_id == helper_id && matches!(op, Op::Shutdown))
|
||||
);
|
||||
|
||||
let list_after_close_output = ListAgentsHandlerV2
|
||||
.handle(invocation(
|
||||
session,
|
||||
turn,
|
||||
"list_agents",
|
||||
function_payload(json!({})),
|
||||
))
|
||||
.await
|
||||
.expect("list_agents should omit the closed watchdog handle");
|
||||
let (list_after_close_content, _) = expect_text_output(list_after_close_output);
|
||||
let list_after_close_result: ListAgentsResult = serde_json::from_str(&list_after_close_content)
|
||||
.expect("list_agents result after close should be json");
|
||||
assert!(
|
||||
!list_after_close_result
|
||||
.agents
|
||||
.iter()
|
||||
.any(|agent| agent.agent_name == watchdog_id.to_string())
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multi_agent_v2_send_message_rejects_legacy_items_field() {
|
||||
let (mut session, mut turn) = make_session_and_context().await;
|
||||
@@ -2807,7 +3345,7 @@ async fn wait_agent_rejects_only_watchdog_handles() {
|
||||
panic!("expected model-facing error");
|
||||
};
|
||||
assert!(message.contains("watchdog handle ids"));
|
||||
assert!(message.contains("pending_init"));
|
||||
assert!(message.contains("running"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -25,6 +25,7 @@ impl ToolHandler for Handler {
|
||||
MessageDeliveryMode::TriggerTurn,
|
||||
args.target,
|
||||
args.message,
|
||||
args.interrupt,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
@@ -4,8 +4,13 @@
|
||||
//! resulting `InterAgentCommunication` should wake the target immediately.
|
||||
|
||||
use super::*;
|
||||
use crate::session::turn_context::TurnContext;
|
||||
use crate::tools::context::FunctionToolOutput;
|
||||
use codex_protocol::ThreadId;
|
||||
use codex_protocol::protocol::InterAgentCommunication;
|
||||
use codex_protocol::protocol::SessionSource;
|
||||
use codex_protocol::protocol::SubAgentSource;
|
||||
use std::sync::Arc;
|
||||
|
||||
#[derive(Clone, Copy, PartialEq, Eq)]
|
||||
pub(crate) enum MessageDeliveryMode {
|
||||
@@ -43,6 +48,8 @@ pub(crate) struct SendMessageArgs {
|
||||
pub(crate) struct FollowupTaskArgs {
|
||||
pub(crate) target: String,
|
||||
pub(crate) message: String,
|
||||
#[serde(default)]
|
||||
pub(crate) interrupt: bool,
|
||||
}
|
||||
|
||||
fn message_content(message: String) -> Result<String, FunctionCallError> {
|
||||
@@ -60,8 +67,16 @@ pub(crate) async fn handle_message_string_tool(
|
||||
mode: MessageDeliveryMode,
|
||||
target: String,
|
||||
message: String,
|
||||
interrupt: bool,
|
||||
) -> Result<FunctionToolOutput, FunctionCallError> {
|
||||
handle_message_submission(invocation, mode, target, message_content(message)?).await
|
||||
handle_message_submission(
|
||||
invocation,
|
||||
mode,
|
||||
target,
|
||||
message_content(message)?,
|
||||
interrupt,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn handle_message_submission(
|
||||
@@ -69,6 +84,7 @@ async fn handle_message_submission(
|
||||
mode: MessageDeliveryMode,
|
||||
target: String,
|
||||
prompt: String,
|
||||
interrupt: bool,
|
||||
) -> Result<FunctionToolOutput, FunctionCallError> {
|
||||
let ToolInvocation {
|
||||
session,
|
||||
@@ -76,22 +92,62 @@ async fn handle_message_submission(
|
||||
call_id,
|
||||
..
|
||||
} = invocation;
|
||||
let receiver_thread_id = resolve_agent_target(&session, &turn, &target).await?;
|
||||
let target_is_parent = target == "parent";
|
||||
let receiver_thread_id = resolve_message_target(&session, &turn, &target).await?;
|
||||
let direct_parent_thread_id = direct_parent_thread_id(&turn.session_source);
|
||||
let is_direct_parent = direct_parent_thread_id == Some(receiver_thread_id);
|
||||
let watchdog_owner_thread_id = session
|
||||
.services
|
||||
.agent_control
|
||||
.watchdog_owner_for_active_helper(session.conversation_id)
|
||||
.await;
|
||||
let is_watchdog_parent = watchdog_owner_thread_id == Some(receiver_thread_id);
|
||||
let receiver_agent = session
|
||||
.services
|
||||
.agent_control
|
||||
.get_agent_metadata(receiver_thread_id)
|
||||
.unwrap_or_default();
|
||||
if mode == MessageDeliveryMode::QueueOnly && is_watchdog_parent {
|
||||
return Err(FunctionCallError::RespondToModel(
|
||||
"watchdog check-in threads must use followup_task with target `parent` to message their parent."
|
||||
.to_string(),
|
||||
));
|
||||
}
|
||||
if mode == MessageDeliveryMode::TriggerTurn
|
||||
&& is_direct_parent
|
||||
&& !is_watchdog_parent
|
||||
&& target_is_parent
|
||||
{
|
||||
return Err(FunctionCallError::RespondToModel(
|
||||
"Only watchdog check-in threads can use followup_task with target `parent`; use send_message for parent updates."
|
||||
.to_string(),
|
||||
));
|
||||
}
|
||||
if mode == MessageDeliveryMode::TriggerTurn
|
||||
&& receiver_agent
|
||||
.agent_path
|
||||
.as_ref()
|
||||
.is_some_and(AgentPath::is_root)
|
||||
&& !is_watchdog_parent
|
||||
{
|
||||
return Err(FunctionCallError::RespondToModel(
|
||||
"Tasks can't be assigned to the root agent".to_string(),
|
||||
));
|
||||
}
|
||||
if mode == MessageDeliveryMode::TriggerTurn && is_direct_parent && !is_watchdog_parent {
|
||||
return Err(FunctionCallError::RespondToModel(
|
||||
"Only watchdog check-in threads can use followup_task with target `parent`; use send_message for parent updates."
|
||||
.to_string(),
|
||||
));
|
||||
}
|
||||
if interrupt {
|
||||
session
|
||||
.services
|
||||
.agent_control
|
||||
.interrupt_agent(receiver_thread_id)
|
||||
.await
|
||||
.map_err(|err| collab_agent_error(receiver_thread_id, err))?;
|
||||
}
|
||||
session
|
||||
.send_event(
|
||||
&turn,
|
||||
@@ -104,9 +160,17 @@ async fn handle_message_submission(
|
||||
.into(),
|
||||
)
|
||||
.await;
|
||||
let receiver_agent_path = receiver_agent.agent_path.clone().ok_or_else(|| {
|
||||
FunctionCallError::RespondToModel("target agent is missing an agent_path".to_string())
|
||||
})?;
|
||||
let receiver_agent_path = receiver_agent
|
||||
.agent_path
|
||||
.clone()
|
||||
.or_else(|| {
|
||||
is_direct_parent
|
||||
.then(|| direct_parent_path(&turn.session_source))
|
||||
.flatten()
|
||||
})
|
||||
.ok_or_else(|| {
|
||||
FunctionCallError::RespondToModel("target agent is missing an agent_path".to_string())
|
||||
})?;
|
||||
let communication = InterAgentCommunication::new(
|
||||
turn.session_source
|
||||
.get_agent_path()
|
||||
@@ -143,6 +207,72 @@ async fn handle_message_submission(
|
||||
)
|
||||
.await;
|
||||
result?;
|
||||
if mode == MessageDeliveryMode::TriggerTurn && is_watchdog_parent {
|
||||
let _ = session
|
||||
.services
|
||||
.agent_control
|
||||
.finish_watchdog_helper(session.conversation_id)
|
||||
.await;
|
||||
session
|
||||
.services
|
||||
.agent_control
|
||||
.finish_watchdog_helper_thread(session.conversation_id)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
FunctionCallError::RespondToModel(format!(
|
||||
"failed to finish watchdog helper after followup_task: {err}"
|
||||
))
|
||||
})?;
|
||||
}
|
||||
|
||||
Ok(FunctionToolOutput::from_text(String::new(), Some(true)))
|
||||
}
|
||||
|
||||
async fn resolve_message_target(
|
||||
session: &Arc<crate::session::session::Session>,
|
||||
turn: &Arc<TurnContext>,
|
||||
target: &str,
|
||||
) -> Result<ThreadId, FunctionCallError> {
|
||||
if target == "parent" {
|
||||
return direct_parent_thread_id(&turn.session_source).ok_or_else(|| {
|
||||
FunctionCallError::RespondToModel(
|
||||
"target `parent` is only available from a spawned agent.".to_string(),
|
||||
)
|
||||
});
|
||||
}
|
||||
resolve_agent_target(session, turn, target).await
|
||||
}
|
||||
|
||||
fn direct_parent_thread_id(session_source: &SessionSource) -> Option<ThreadId> {
|
||||
match session_source {
|
||||
SessionSource::SubAgent(SubAgentSource::ThreadSpawn {
|
||||
parent_thread_id, ..
|
||||
}) => Some(*parent_thread_id),
|
||||
SessionSource::Cli
|
||||
| SessionSource::VSCode
|
||||
| SessionSource::Exec
|
||||
| SessionSource::Mcp
|
||||
| SessionSource::Custom(_)
|
||||
| SessionSource::Internal(_)
|
||||
| SessionSource::SubAgent(SubAgentSource::Review)
|
||||
| SessionSource::SubAgent(SubAgentSource::Compact)
|
||||
| SessionSource::SubAgent(SubAgentSource::MemoryConsolidation)
|
||||
| SessionSource::SubAgent(SubAgentSource::Other(_))
|
||||
| SessionSource::Unknown => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn direct_parent_path(session_source: &SessionSource) -> Option<AgentPath> {
|
||||
parent_path(session_source.get_agent_path()?.as_str())
|
||||
}
|
||||
|
||||
fn parent_path(agent_path: &str) -> Option<AgentPath> {
|
||||
if agent_path == AgentPath::ROOT {
|
||||
return None;
|
||||
}
|
||||
let parent = agent_path.rsplit_once('/')?.0;
|
||||
if parent.is_empty() {
|
||||
return None;
|
||||
}
|
||||
AgentPath::try_from(parent).ok()
|
||||
}
|
||||
|
||||
@@ -25,6 +25,7 @@ impl ToolHandler for Handler {
|
||||
MessageDeliveryMode::QueueOnly,
|
||||
args.target,
|
||||
args.message,
|
||||
/*interrupt*/ false,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
@@ -1,14 +1,24 @@
|
||||
use super::*;
|
||||
use crate::agent::RemovedWatchdog;
|
||||
use crate::agent::WatchdogRegistration;
|
||||
use crate::agent::control::SpawnAgentForkMode;
|
||||
use crate::agent::control::SpawnAgentOptions;
|
||||
use crate::agent::control::render_input_preview;
|
||||
use crate::agent::exceeds_thread_spawn_depth_limit;
|
||||
use crate::agent::next_thread_spawn_depth;
|
||||
use crate::agent::role::DEFAULT_ROLE_NAME;
|
||||
use crate::agent::role::apply_role_to_config;
|
||||
use crate::agent::role::watchdog_interval_for_role;
|
||||
use crate::config::Config;
|
||||
use crate::session::turn_context::TurnEnvironment;
|
||||
use codex_features::Feature;
|
||||
use codex_protocol::AgentPath;
|
||||
use codex_protocol::ThreadId;
|
||||
use codex_protocol::protocol::InterAgentCommunication;
|
||||
use codex_protocol::protocol::Op;
|
||||
use codex_protocol::protocol::SessionSource;
|
||||
use std::collections::HashMap;
|
||||
use std::collections::HashSet;
|
||||
|
||||
pub(crate) struct Handler;
|
||||
|
||||
@@ -33,18 +43,35 @@ impl ToolHandler for Handler {
|
||||
} = invocation;
|
||||
let arguments = function_arguments(payload)?;
|
||||
let args: SpawnAgentArgs = parse_arguments(&arguments)?;
|
||||
let fork_mode = args.fork_mode()?;
|
||||
let role_name = args
|
||||
.agent_type
|
||||
.as_deref()
|
||||
.map(str::trim)
|
||||
.filter(|role| !role.is_empty());
|
||||
|
||||
let initial_operation = parse_collab_input(Some(args.message), /*items*/ None)?;
|
||||
let prompt = render_input_preview(&initial_operation);
|
||||
|
||||
let session_source = turn.session_source.clone();
|
||||
let child_depth = next_thread_spawn_depth(&session_source);
|
||||
let max_depth = turn.config.agent_max_depth;
|
||||
let watchdog_interval_s = watchdog_interval_for_role(&turn.config, role_name);
|
||||
let is_watchdog = watchdog_interval_s.is_some();
|
||||
if is_watchdog && !turn.config.features.enabled(Feature::AgentWatchdog) {
|
||||
return Err(FunctionCallError::RespondToModel(
|
||||
"watchdogs are disabled".to_string(),
|
||||
));
|
||||
}
|
||||
if is_watchdog && matches!(session_source, SessionSource::SubAgent(_)) {
|
||||
return Err(FunctionCallError::RespondToModel(
|
||||
"watchdogs can only be spawned by root agents".to_string(),
|
||||
));
|
||||
}
|
||||
if exceeds_thread_spawn_depth_limit(child_depth, max_depth) {
|
||||
return Err(FunctionCallError::RespondToModel(
|
||||
"Agent depth limit reached. Solve the task yourself.".to_string(),
|
||||
));
|
||||
}
|
||||
let fork_mode = if is_watchdog { None } else { args.fork_mode()? };
|
||||
let initial_operation = parse_collab_input(Some(args.message), /*items*/ None)?;
|
||||
let prompt = render_input_preview(&initial_operation);
|
||||
session
|
||||
.send_event(
|
||||
&turn,
|
||||
@@ -89,45 +116,68 @@ impl ToolHandler for Handler {
|
||||
role_name,
|
||||
Some(args.task_name.clone()),
|
||||
)?;
|
||||
let result = session
|
||||
.services
|
||||
.agent_control
|
||||
.spawn_agent_with_metadata(
|
||||
let result = if let Some(watchdog_interval_s) = watchdog_interval_s {
|
||||
let thread_id = spawn_watchdog(
|
||||
&session.services.agent_control,
|
||||
config,
|
||||
match (spawn_source.get_agent_path(), initial_operation) {
|
||||
(Some(recipient), Op::UserInput { items, .. })
|
||||
if items
|
||||
.iter()
|
||||
.all(|item| matches!(item, UserInput::Text { .. })) =>
|
||||
{
|
||||
Op::InterAgentCommunication {
|
||||
communication: InterAgentCommunication::new(
|
||||
turn.session_source
|
||||
.get_agent_path()
|
||||
.unwrap_or_else(AgentPath::root),
|
||||
recipient,
|
||||
Vec::new(),
|
||||
prompt.clone(),
|
||||
/*trigger_turn*/ true,
|
||||
),
|
||||
}
|
||||
}
|
||||
(_, initial_operation) => initial_operation,
|
||||
},
|
||||
Some(spawn_source),
|
||||
SpawnAgentOptions {
|
||||
fork_parent_spawn_call_id: fork_mode.as_ref().map(|_| call_id.clone()),
|
||||
fork_mode,
|
||||
environments: Some(
|
||||
turn.environments
|
||||
.iter()
|
||||
.map(TurnEnvironment::selection)
|
||||
.collect(),
|
||||
),
|
||||
},
|
||||
prompt.clone(),
|
||||
session.conversation_id,
|
||||
child_depth,
|
||||
watchdog_interval_s,
|
||||
spawn_source,
|
||||
)
|
||||
.await
|
||||
.map_err(collab_spawn_error);
|
||||
.map_err(collab_spawn_error)?;
|
||||
Ok(crate::agent::control::LiveAgent {
|
||||
thread_id,
|
||||
metadata: session
|
||||
.services
|
||||
.agent_control
|
||||
.get_agent_metadata(thread_id)
|
||||
.unwrap_or_default(),
|
||||
status: session.services.agent_control.get_status(thread_id).await,
|
||||
})
|
||||
} else {
|
||||
session
|
||||
.services
|
||||
.agent_control
|
||||
.spawn_agent_with_metadata(
|
||||
config,
|
||||
match (spawn_source.get_agent_path(), initial_operation) {
|
||||
(Some(recipient), Op::UserInput { items, .. })
|
||||
if items
|
||||
.iter()
|
||||
.all(|item| matches!(item, UserInput::Text { .. })) =>
|
||||
{
|
||||
Op::InterAgentCommunication {
|
||||
communication: InterAgentCommunication::new(
|
||||
turn.session_source
|
||||
.get_agent_path()
|
||||
.unwrap_or_else(AgentPath::root),
|
||||
recipient,
|
||||
Vec::new(),
|
||||
prompt.clone(),
|
||||
/*trigger_turn*/ true,
|
||||
),
|
||||
}
|
||||
}
|
||||
(_, initial_operation) => initial_operation,
|
||||
},
|
||||
Some(spawn_source),
|
||||
SpawnAgentOptions {
|
||||
fork_parent_spawn_call_id: fork_mode.as_ref().map(|_| call_id.clone()),
|
||||
fork_mode,
|
||||
environments: Some(
|
||||
turn.environments
|
||||
.iter()
|
||||
.map(TurnEnvironment::selection)
|
||||
.collect(),
|
||||
),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.map_err(collab_spawn_error)
|
||||
};
|
||||
let (new_thread_id, new_agent_metadata, status) = match &result {
|
||||
Ok(spawned_agent) => (
|
||||
Some(spawned_agent.thread_id),
|
||||
@@ -211,6 +261,68 @@ impl ToolHandler for Handler {
|
||||
}
|
||||
}
|
||||
|
||||
async fn spawn_watchdog(
|
||||
agent_control: &crate::agent::AgentControl,
|
||||
config: Config,
|
||||
prompt: String,
|
||||
owner_thread_id: ThreadId,
|
||||
child_depth: i32,
|
||||
interval_s: i64,
|
||||
spawn_source: SessionSource,
|
||||
) -> codex_protocol::error::Result<ThreadId> {
|
||||
let mut handle_config = config.clone();
|
||||
handle_config
|
||||
.mcp_servers
|
||||
.set(HashMap::new())
|
||||
.map_err(|err| {
|
||||
codex_protocol::error::CodexErr::UnsupportedOperation(format!(
|
||||
"failed to clear watchdog MCP servers: {err}"
|
||||
))
|
||||
})?;
|
||||
let target_thread_id = agent_control
|
||||
.spawn_agent(handle_config, Op::Interrupt, Some(spawn_source))
|
||||
.await?;
|
||||
let superseded_before_register = agent_control
|
||||
.unregister_watchdogs_for_owner(owner_thread_id)
|
||||
.await;
|
||||
shutdown_removed_watchdogs(agent_control, superseded_before_register).await;
|
||||
let registration = WatchdogRegistration {
|
||||
owner_thread_id,
|
||||
target_thread_id,
|
||||
child_depth,
|
||||
interval_s,
|
||||
prompt,
|
||||
config,
|
||||
};
|
||||
let superseded_after_register = match agent_control.register_watchdog(registration).await {
|
||||
Ok(removed) => removed,
|
||||
Err(err) => {
|
||||
let _ = agent_control.close_agent(target_thread_id).await;
|
||||
return Err(err);
|
||||
}
|
||||
};
|
||||
shutdown_removed_watchdogs(agent_control, superseded_after_register).await;
|
||||
Ok(target_thread_id)
|
||||
}
|
||||
|
||||
async fn shutdown_removed_watchdogs(
|
||||
agent_control: &crate::agent::AgentControl,
|
||||
removed_watchdogs: Vec<RemovedWatchdog>,
|
||||
) {
|
||||
let mut thread_ids = HashSet::new();
|
||||
for removed in removed_watchdogs {
|
||||
thread_ids.insert(removed.target_thread_id);
|
||||
if let Some(helper_id) = removed.active_helper_id {
|
||||
thread_ids.insert(helper_id);
|
||||
}
|
||||
}
|
||||
let mut thread_ids = thread_ids.into_iter().collect::<Vec<_>>();
|
||||
thread_ids.sort_by_key(ToString::to_string);
|
||||
for thread_id in thread_ids {
|
||||
let _ = agent_control.close_agent(thread_id).await;
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct SpawnAgentArgs {
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
use crate::shell::Shell;
|
||||
use crate::shell::ShellType;
|
||||
use crate::tools::handlers::agent_jobs::BatchJobHandler;
|
||||
use crate::tools::handlers::multi_agents::CompactParentContextHandler;
|
||||
use crate::tools::handlers::multi_agents::WatchdogSelfCloseHandler;
|
||||
use crate::tools::handlers::multi_agents::WatchdogSnoozeHandler;
|
||||
use crate::tools::handlers::multi_agents_common::DEFAULT_WAIT_TIMEOUT_MS;
|
||||
@@ -216,6 +217,9 @@ pub(crate) fn build_specs_with_discoverable_tools(
|
||||
ToolHandlerKind::CodeModeWait => {
|
||||
builder.register_handler(handler.name, code_mode_wait_handler.clone());
|
||||
}
|
||||
ToolHandlerKind::CompactParentContext => {
|
||||
builder.register_handler(handler.name, Arc::new(CompactParentContextHandler));
|
||||
}
|
||||
ToolHandlerKind::DynamicTool => {
|
||||
builder.register_handler(handler.name, dynamic_tool_handler.clone());
|
||||
}
|
||||
|
||||
@@ -128,7 +128,8 @@ pub fn create_send_message_tool() -> ToolSpec {
|
||||
(
|
||||
"target".to_string(),
|
||||
JsonSchema::string(Some(
|
||||
"Relative or canonical task name to message (from spawn_agent).".to_string(),
|
||||
"Relative or canonical task name to message (from spawn_agent), or `parent` from a spawned non-watchdog agent."
|
||||
.to_string(),
|
||||
)),
|
||||
),
|
||||
(
|
||||
@@ -159,7 +160,8 @@ pub fn create_followup_task_tool() -> ToolSpec {
|
||||
(
|
||||
"target".to_string(),
|
||||
JsonSchema::string(Some(
|
||||
"Agent id or canonical task name to message (from spawn_agent).".to_string(),
|
||||
"Agent id or canonical task name to message (from spawn_agent), or `parent` from a watchdog check-in."
|
||||
.to_string(),
|
||||
)),
|
||||
),
|
||||
(
|
||||
@@ -172,7 +174,7 @@ pub fn create_followup_task_tool() -> ToolSpec {
|
||||
|
||||
ToolSpec::Function(ResponsesApiTool {
|
||||
name: "followup_task".to_string(),
|
||||
description: "Send a message to an existing non-root target agent and trigger a turn in that target. If the target is currently mid-turn, the message is queued and will be used to start the target's next turn, after the current turn completes."
|
||||
description: "Send a string message to an existing non-root agent and trigger a turn in the target. Watchdog check-ins may use target `parent`. Use interrupt=true to redirect work immediately. If interrupt=false and the target's turn has not completed, the message is queued and starts the target's next turn after the current turn completes."
|
||||
.to_string(),
|
||||
strict: false,
|
||||
defer_loading: None,
|
||||
@@ -260,18 +262,18 @@ pub fn create_close_agent_tool_v1() -> ToolSpec {
|
||||
})
|
||||
}
|
||||
|
||||
pub fn create_watchdog_self_close_tool() -> ToolSpec {
|
||||
pub fn create_watchdog_close_self_tool() -> ToolSpec {
|
||||
let properties = BTreeMap::from([(
|
||||
"message".to_string(),
|
||||
JsonSchema::string(Some(
|
||||
"Optional final message to send to the parent/root thread before closing this watchdog handle and ending this check-in immediately."
|
||||
"Optional final message sent to the parent agent before closing this watchdog."
|
||||
.to_string(),
|
||||
)),
|
||||
)]);
|
||||
|
||||
ToolSpec::Function(ResponsesApiTool {
|
||||
name: "watchdog_self_close".to_string(),
|
||||
description: "Watchdog-only: send an optional final message to the parent/root thread, close this watchdog's persistent handle, and end this check-in immediately."
|
||||
name: "close_self".to_string(),
|
||||
description: "Watchdog-only: send an optional final message to the parent agent, stop future wakeups for this watchdog, and end the current check-in immediately. Use this tool, not a final assistant message, when the watchdog must shut down."
|
||||
.to_string(),
|
||||
strict: false,
|
||||
defer_loading: Some(true),
|
||||
@@ -280,6 +282,33 @@ pub fn create_watchdog_self_close_tool() -> ToolSpec {
|
||||
})
|
||||
}
|
||||
|
||||
pub fn create_compact_parent_context_tool() -> ToolSpec {
|
||||
let properties = BTreeMap::from([
|
||||
(
|
||||
"reason".to_string(),
|
||||
JsonSchema::string(Some(
|
||||
"Short reason why the parent/root thread should be compacted.".to_string(),
|
||||
)),
|
||||
),
|
||||
(
|
||||
"evidence".to_string(),
|
||||
JsonSchema::string(Some(
|
||||
"Specific observation that the parent/root thread is idle or stuck.".to_string(),
|
||||
)),
|
||||
),
|
||||
]);
|
||||
|
||||
ToolSpec::Function(ResponsesApiTool {
|
||||
name: "compact_parent_context".to_string(),
|
||||
description: "Watchdog-only: request compaction for this watchdog helper's parent/root thread when it is idle and appears stuck."
|
||||
.to_string(),
|
||||
strict: false,
|
||||
defer_loading: Some(true),
|
||||
parameters: JsonSchema::object(properties, /*required*/ None, Some(false.into())),
|
||||
output_schema: None,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn create_watchdog_snooze_tool() -> ToolSpec {
|
||||
let properties = BTreeMap::from([
|
||||
(
|
||||
|
||||
@@ -31,6 +31,7 @@ pub use agent_tool::SpawnAgentToolOptions;
|
||||
pub use agent_tool::WaitAgentTimeoutOptions;
|
||||
pub use agent_tool::create_close_agent_tool_v1;
|
||||
pub use agent_tool::create_close_agent_tool_v2;
|
||||
pub use agent_tool::create_compact_parent_context_tool;
|
||||
pub use agent_tool::create_followup_task_tool;
|
||||
pub use agent_tool::create_list_agents_tool;
|
||||
pub use agent_tool::create_resume_agent_tool;
|
||||
@@ -40,7 +41,7 @@ pub use agent_tool::create_spawn_agent_tool_v1;
|
||||
pub use agent_tool::create_spawn_agent_tool_v2;
|
||||
pub use agent_tool::create_wait_agent_tool_v1;
|
||||
pub use agent_tool::create_wait_agent_tool_v2;
|
||||
pub use agent_tool::create_watchdog_self_close_tool;
|
||||
pub use agent_tool::create_watchdog_close_self_tool;
|
||||
pub use agent_tool::create_watchdog_snooze_tool;
|
||||
pub use agent_tool::create_watchdog_tools_namespace;
|
||||
pub use apply_patch_tool::ApplyPatchToolArgs;
|
||||
|
||||
@@ -57,7 +57,7 @@ use crate::create_view_image_tool;
|
||||
use crate::create_wait_agent_tool_v1;
|
||||
use crate::create_wait_agent_tool_v2;
|
||||
use crate::create_wait_tool;
|
||||
use crate::create_watchdog_self_close_tool;
|
||||
use crate::create_watchdog_close_self_tool;
|
||||
use crate::create_watchdog_snooze_tool;
|
||||
use crate::create_watchdog_tools_namespace;
|
||||
use crate::create_web_search_tool;
|
||||
@@ -500,14 +500,14 @@ pub fn build_tool_registry_plan(
|
||||
if config.agent_watchdog {
|
||||
plan.push_spec(
|
||||
create_watchdog_tools_namespace(vec![
|
||||
create_watchdog_self_close_tool(),
|
||||
create_watchdog_close_self_tool(),
|
||||
create_watchdog_snooze_tool(),
|
||||
]),
|
||||
/*supports_parallel_tool_calls*/ false,
|
||||
config.code_mode_enabled,
|
||||
);
|
||||
plan.register_handler(
|
||||
crate::ToolName::namespaced("watchdog", "watchdog_self_close"),
|
||||
crate::ToolName::namespaced("watchdog", "close_self"),
|
||||
ToolHandlerKind::WatchdogSelfClose,
|
||||
);
|
||||
plan.register_handler(
|
||||
|
||||
@@ -16,6 +16,7 @@ pub enum ToolHandlerKind {
|
||||
CloseAgentV2,
|
||||
CodeModeExecute,
|
||||
CodeModeWait,
|
||||
CompactParentContext,
|
||||
DynamicTool,
|
||||
FollowupTaskV2,
|
||||
Goal,
|
||||
|
||||
Reference in New Issue
Block a user