diff --git a/codex-rs/core/src/tools/handlers/agent_jobs.rs b/codex-rs/core/src/tools/handlers/agent_jobs.rs index a3ca96b2cb..ff196c6f67 100644 --- a/codex-rs/core/src/tools/handlers/agent_jobs.rs +++ b/codex-rs/core/src/tools/handlers/agent_jobs.rs @@ -33,6 +33,8 @@ use uuid::Uuid; mod report_agent_job_result; mod spawn_agents_on_csv; +#[path = "agent_jobs_slots.rs"] +mod slots; #[path = "agent_jobs_startup.rs"] mod startup; @@ -204,6 +206,24 @@ async fn run_agent_job_loop( progressed = true; } + let scheduler_progress = db.get_agent_job_progress(job_id.as_str()).await?; + if slots::reclaim_inactive_active_items( + session.clone(), + db.clone(), + job_id.as_str(), + &mut active_items, + scheduler_progress.running_items, + ) + .await? + { + progressed = true; + } + + if !cancel_requested && db.is_agent_job_cancelled(job_id.as_str()).await? { + cancel_requested = true; + progressed = true; + } + if !cancel_requested && active_items.len() + starting_items.len() < options.max_concurrency && startup::launch_pending_items( @@ -212,7 +232,10 @@ async fn run_agent_job_loop( &job, job_id.as_str(), &options, - active_items.len(), + startup::SchedulerOccupancy { + active_items: active_items.len(), + db_running_items: scheduler_progress.running_items, + }, &mut starting_items, ) .await? diff --git a/codex-rs/core/src/tools/handlers/agent_jobs/report_agent_job_result.rs b/codex-rs/core/src/tools/handlers/agent_jobs/report_agent_job_result.rs index 4641e793c8..aa0836a18d 100644 --- a/codex-rs/core/src/tools/handlers/agent_jobs/report_agent_job_result.rs +++ b/codex-rs/core/src/tools/handlers/agent_jobs/report_agent_job_result.rs @@ -61,12 +61,13 @@ pub async fn handle( )); } let db = required_state_db(&session)?; - let reporting_thread_id = session.conversation_id.to_string(); + let reporting_thread_id = session.conversation_id; + let reporting_thread_id_str = reporting_thread_id.to_string(); let accepted = db .report_agent_job_item_result( args.job_id.as_str(), args.item_id.as_str(), - reporting_thread_id.as_str(), + reporting_thread_id_str.as_str(), &args.result, ) .await @@ -83,6 +84,19 @@ pub async fn handle( .mark_agent_job_cancelled(args.job_id.as_str(), message) .await; } + if accepted { + let agent_control = session.services.agent_control.clone(); + tracing::debug!( + job_id = args.job_id, + item_id = args.item_id, + thread_id = %reporting_thread_id, + "agent job accepted worker result; scheduling worker shutdown" + ); + tokio::spawn(async move { + tokio::task::yield_now().await; + let _ = agent_control.shutdown_live_agent(reporting_thread_id).await; + }); + } let content = serde_json::to_string(&ReportAgentJobResultToolResult { accepted }).map_err(|err| { FunctionCallError::Fatal(format!( diff --git a/codex-rs/core/src/tools/handlers/agent_jobs_slots.rs b/codex-rs/core/src/tools/handlers/agent_jobs_slots.rs new file mode 100644 index 0000000000..1552d6c473 --- /dev/null +++ b/codex-rs/core/src/tools/handlers/agent_jobs_slots.rs @@ -0,0 +1,79 @@ +use super::*; +use std::collections::HashMap; + +#[derive(Debug)] +struct InactiveActiveItem { + thread_id: ThreadId, + item_id: String, + db_status: Option, + db_assigned_thread_id: Option, +} + +pub(super) async fn reclaim_inactive_active_items( + session: Arc, + db: Arc, + job_id: &str, + active_items: &mut HashMap, + db_running_items: usize, +) -> anyhow::Result { + let mut inactive_items = Vec::new(); + for (thread_id, item) in active_items.iter() { + let thread_id_str = thread_id.to_string(); + let db_item = db.get_agent_job_item(job_id, item.item_id.as_str()).await?; + let Some(db_item) = db_item else { + inactive_items.push(InactiveActiveItem { + thread_id: *thread_id, + item_id: item.item_id.clone(), + db_status: None, + db_assigned_thread_id: None, + }); + continue; + }; + let still_running = matches!(db_item.status, codex_state::AgentJobItemStatus::Running) + && db_item.assigned_thread_id.as_deref() == Some(thread_id_str.as_str()); + if still_running { + continue; + } + inactive_items.push(InactiveActiveItem { + thread_id: *thread_id, + item_id: item.item_id.clone(), + db_status: Some(db_item.status), + db_assigned_thread_id: db_item.assigned_thread_id, + }); + } + if inactive_items.is_empty() { + return Ok(false); + } + + tracing::info!( + job_id, + db_running_items, + active_items = active_items.len(), + reclaimed_items = inactive_items.len(), + "agent job reclaiming scheduler slots for items that are no longer running in state" + ); + + for inactive_item in inactive_items { + let _ = session + .services + .agent_control + .shutdown_live_agent(inactive_item.thread_id) + .await; + active_items.remove(&inactive_item.thread_id); + tracing::debug!( + job_id, + item_id = inactive_item.item_id, + thread_id = %inactive_item.thread_id, + db_status = inactive_item + .db_status + .as_ref() + .map(|status| status.as_str()) + .unwrap_or("missing"), + db_assigned_thread_id = inactive_item.db_assigned_thread_id.as_deref().unwrap_or(""), + active_items = active_items.len(), + "agent job reclaimed scheduler slot" + ); + } + + Ok(true) +} diff --git a/codex-rs/core/src/tools/handlers/agent_jobs_startup.rs b/codex-rs/core/src/tools/handlers/agent_jobs_startup.rs index 13def93f97..1fb1445232 100644 --- a/codex-rs/core/src/tools/handlers/agent_jobs_startup.rs +++ b/codex-rs/core/src/tools/handlers/agent_jobs_startup.rs @@ -27,6 +27,12 @@ pub(super) struct StartupTasks { launching_items: HashMap, } +#[derive(Clone, Copy, Debug)] +pub(super) struct SchedulerOccupancy { + pub(super) active_items: usize, + pub(super) db_running_items: usize, +} + impl StartupTasks { pub(super) fn len(&self) -> usize { self.starting_items.len() @@ -62,12 +68,12 @@ pub(super) async fn launch_pending_items( job: &codex_state::AgentJob, job_id: &str, options: &JobRunnerOptions, - active_items_len: usize, + occupancy: SchedulerOccupancy, startup_tasks: &mut StartupTasks, ) -> anyhow::Result { let slots = options .max_concurrency - .saturating_sub(active_items_len + startup_tasks.len()); + .saturating_sub(occupancy.active_items + startup_tasks.len()); if slots == 0 { return Ok(false); } @@ -146,7 +152,8 @@ pub(super) async fn launch_pending_items( tracing::info!( job_id, launched, - active_items = active_items_len, + db_running_items = occupancy.db_running_items, + active_items = occupancy.active_items, starting_items = startup_tasks.len(), target_concurrency = options.max_concurrency, "agent job queued worker startups" diff --git a/codex-rs/core/tests/suite/agent_jobs.rs b/codex-rs/core/tests/suite/agent_jobs.rs index b275b1878a..04008b4025 100644 --- a/codex-rs/core/tests/suite/agent_jobs.rs +++ b/codex-rs/core/tests/suite/agent_jobs.rs @@ -15,6 +15,9 @@ use std::sync::Arc; use std::sync::atomic::AtomicBool; use std::sync::atomic::AtomicUsize; use std::sync::atomic::Ordering; +use std::time::Duration; +use tokio::time::sleep; +use tokio::time::timeout; use wiremock::Mock; use wiremock::Respond; use wiremock::ResponseTemplate; @@ -53,6 +56,78 @@ impl StopAfterFirstResponder { } } +struct DelayedWorkerAfterReportResponder { + spawn_args_json: String, + seen_main: AtomicBool, + worker_calls: Arc, + delayed_worker_output: AtomicBool, +} + +impl DelayedWorkerAfterReportResponder { + fn new(spawn_args_json: String, worker_calls: Arc) -> Self { + Self { + spawn_args_json, + seen_main: AtomicBool::new(false), + worker_calls, + delayed_worker_output: AtomicBool::new(false), + } + } +} + +impl Respond for DelayedWorkerAfterReportResponder { + fn respond(&self, request: &wiremock::Request) -> ResponseTemplate { + let body_bytes = decode_body_bytes(request); + let body: Value = serde_json::from_slice(&body_bytes).unwrap_or(Value::Null); + + let call_output_ids = function_call_output_call_ids(&body); + if !call_output_ids.is_empty() { + let response = sse_response(sse(vec![ + ev_response_created("resp-tool"), + ev_completed("resp-tool"), + ])); + if call_output_ids + .iter() + .any(|call_id| call_id == "call-worker-0") + && !self.delayed_worker_output.swap(true, Ordering::SeqCst) + { + return response.set_delay(Duration::from_secs(2)); + } + return response; + } + + if let Some((job_id, item_id)) = extract_job_and_item(&body) { + let call_index = self.worker_calls.fetch_add(1, Ordering::SeqCst); + let call_id = format!("call-worker-{call_index}"); + let args = json!({ + "job_id": job_id, + "item_id": item_id, + "result": { "item_id": item_id } + }); + let args_json = serde_json::to_string(&args).unwrap_or_else(|err| { + panic!("worker args serialize: {err}"); + }); + return sse_response(sse(vec![ + ev_response_created("resp-worker"), + ev_function_call(&call_id, "report_agent_job_result", &args_json), + ev_completed("resp-worker"), + ])); + } + + if !self.seen_main.swap(true, Ordering::SeqCst) { + return sse_response(sse(vec![ + ev_response_created("resp-main"), + ev_function_call("call-spawn", "spawn_agents_on_csv", &self.spawn_args_json), + ev_completed("resp-main"), + ])); + } + + sse_response(sse(vec![ + ev_response_created("resp-default"), + ev_completed("resp-default"), + ])) + } +} + impl Respond for StopAfterFirstResponder { fn respond(&self, request: &wiremock::Request) -> ResponseTemplate { let body_bytes = decode_body_bytes(request); @@ -176,6 +251,17 @@ fn has_function_call_output(body: &Value) -> bool { }) } +fn function_call_output_call_ids(body: &Value) -> Vec { + body.get("input") + .and_then(Value::as_array) + .into_iter() + .flatten() + .filter(|item| item.get("type").and_then(Value::as_str) == Some("function_call_output")) + .filter_map(|item| item.get("call_id").and_then(Value::as_str)) + .map(str::to_string) + .collect() +} + fn extract_job_and_item(body: &Value) -> Option<(String, String)> { let texts = message_input_texts(body); let mut combined = texts.join("\n"); @@ -446,3 +532,75 @@ async fn spawn_agents_on_csv_stop_halts_future_items() -> Result<()> { assert_eq!(worker_calls.load(Ordering::SeqCst), 1); Ok(()) } + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn spawn_agents_on_csv_reclaims_slot_after_report_before_worker_completes() -> Result<()> { + let server = start_mock_server().await; + let mut builder = test_codex().with_config(|config| { + config + .features + .enable(Feature::SpawnCsv) + .expect("test config should allow feature update"); + config + .features + .enable(Feature::Sqlite) + .expect("test config should allow feature update"); + }); + let test = Arc::new(builder.build(&server).await?); + + let input_path = test.cwd_path().join("agent_jobs_reclaim.csv"); + let output_path = test.cwd_path().join("agent_jobs_reclaim_out.csv"); + fs::write(&input_path, "path\nfile-1\nfile-2\n")?; + + let args = json!({ + "csv_path": input_path.display().to_string(), + "instruction": "Return {path}", + "output_csv_path": output_path.display().to_string(), + "max_concurrency": 1, + }); + let args_json = serde_json::to_string(&args)?; + + let worker_calls = Arc::new(AtomicUsize::new(0)); + let responder = DelayedWorkerAfterReportResponder::new(args_json, worker_calls.clone()); + Mock::given(method("POST")) + .and(path_regex(".*/responses$")) + .respond_with(responder) + .mount(&server) + .await; + + let submit = tokio::spawn({ + let test = Arc::clone(&test); + async move { test.submit_turn("run batch job").await } + }); + + timeout(Duration::from_secs(1), async { + while worker_calls.load(Ordering::SeqCst) < 2 { + sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("second worker should start before first worker's delayed post-report response"); + + submit.await??; + + let output = fs::read_to_string(&output_path)?; + assert_eq!(output.lines().skip(1).count(), 2); + + let requests = worker_calls.load(Ordering::SeqCst); + assert_eq!(requests, 2); + + let root_requests = server + .received_requests() + .await + .expect("wiremock should capture requests"); + let saw_spawn_output = root_requests.iter().any(|request| { + let body_bytes = decode_body_bytes(request); + let body: Value = serde_json::from_slice(&body_bytes).unwrap_or(Value::Null); + function_call_output_call_ids(&body) + .iter() + .any(|call_id| call_id == "call-spawn") + }); + assert!(saw_spawn_output); + + Ok(()) +}