mirror of
https://github.com/openai/codex.git
synced 2026-11-18 22:20:48 +00:00
Reclaim agent-job fanout slots after report
This commit is contained in:
committed by
michael mcgrew
parent
73d19173b9
commit
2c0e53171c
@@ -33,6 +33,8 @@ use uuid::Uuid;
|
||||
|
||||
mod report_agent_job_result;
|
||||
mod spawn_agents_on_csv;
|
||||
#[path = "agent_jobs_slots.rs"]
|
||||
mod slots;
|
||||
#[path = "agent_jobs_startup.rs"]
|
||||
mod startup;
|
||||
|
||||
@@ -204,6 +206,24 @@ async fn run_agent_job_loop(
|
||||
progressed = true;
|
||||
}
|
||||
|
||||
let scheduler_progress = db.get_agent_job_progress(job_id.as_str()).await?;
|
||||
if slots::reclaim_inactive_active_items(
|
||||
session.clone(),
|
||||
db.clone(),
|
||||
job_id.as_str(),
|
||||
&mut active_items,
|
||||
scheduler_progress.running_items,
|
||||
)
|
||||
.await?
|
||||
{
|
||||
progressed = true;
|
||||
}
|
||||
|
||||
if !cancel_requested && db.is_agent_job_cancelled(job_id.as_str()).await? {
|
||||
cancel_requested = true;
|
||||
progressed = true;
|
||||
}
|
||||
|
||||
if !cancel_requested
|
||||
&& active_items.len() + starting_items.len() < options.max_concurrency
|
||||
&& startup::launch_pending_items(
|
||||
@@ -212,7 +232,10 @@ async fn run_agent_job_loop(
|
||||
&job,
|
||||
job_id.as_str(),
|
||||
&options,
|
||||
active_items.len(),
|
||||
startup::SchedulerOccupancy {
|
||||
active_items: active_items.len(),
|
||||
db_running_items: scheduler_progress.running_items,
|
||||
},
|
||||
&mut starting_items,
|
||||
)
|
||||
.await?
|
||||
|
||||
@@ -61,12 +61,13 @@ pub async fn handle(
|
||||
));
|
||||
}
|
||||
let db = required_state_db(&session)?;
|
||||
let reporting_thread_id = session.conversation_id.to_string();
|
||||
let reporting_thread_id = session.conversation_id;
|
||||
let reporting_thread_id_str = reporting_thread_id.to_string();
|
||||
let accepted = db
|
||||
.report_agent_job_item_result(
|
||||
args.job_id.as_str(),
|
||||
args.item_id.as_str(),
|
||||
reporting_thread_id.as_str(),
|
||||
reporting_thread_id_str.as_str(),
|
||||
&args.result,
|
||||
)
|
||||
.await
|
||||
@@ -83,6 +84,19 @@ pub async fn handle(
|
||||
.mark_agent_job_cancelled(args.job_id.as_str(), message)
|
||||
.await;
|
||||
}
|
||||
if accepted {
|
||||
let agent_control = session.services.agent_control.clone();
|
||||
tracing::debug!(
|
||||
job_id = args.job_id,
|
||||
item_id = args.item_id,
|
||||
thread_id = %reporting_thread_id,
|
||||
"agent job accepted worker result; scheduling worker shutdown"
|
||||
);
|
||||
tokio::spawn(async move {
|
||||
tokio::task::yield_now().await;
|
||||
let _ = agent_control.shutdown_live_agent(reporting_thread_id).await;
|
||||
});
|
||||
}
|
||||
let content =
|
||||
serde_json::to_string(&ReportAgentJobResultToolResult { accepted }).map_err(|err| {
|
||||
FunctionCallError::Fatal(format!(
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
use super::*;
|
||||
use std::collections::HashMap;
|
||||
|
||||
#[derive(Debug)]
|
||||
struct InactiveActiveItem {
|
||||
thread_id: ThreadId,
|
||||
item_id: String,
|
||||
db_status: Option<codex_state::AgentJobItemStatus>,
|
||||
db_assigned_thread_id: Option<String>,
|
||||
}
|
||||
|
||||
pub(super) async fn reclaim_inactive_active_items(
|
||||
session: Arc<Session>,
|
||||
db: Arc<codex_state::StateRuntime>,
|
||||
job_id: &str,
|
||||
active_items: &mut HashMap<ThreadId, ActiveJobItem>,
|
||||
db_running_items: usize,
|
||||
) -> anyhow::Result<bool> {
|
||||
let mut inactive_items = Vec::new();
|
||||
for (thread_id, item) in active_items.iter() {
|
||||
let thread_id_str = thread_id.to_string();
|
||||
let db_item = db.get_agent_job_item(job_id, item.item_id.as_str()).await?;
|
||||
let Some(db_item) = db_item else {
|
||||
inactive_items.push(InactiveActiveItem {
|
||||
thread_id: *thread_id,
|
||||
item_id: item.item_id.clone(),
|
||||
db_status: None,
|
||||
db_assigned_thread_id: None,
|
||||
});
|
||||
continue;
|
||||
};
|
||||
let still_running = matches!(db_item.status, codex_state::AgentJobItemStatus::Running)
|
||||
&& db_item.assigned_thread_id.as_deref() == Some(thread_id_str.as_str());
|
||||
if still_running {
|
||||
continue;
|
||||
}
|
||||
inactive_items.push(InactiveActiveItem {
|
||||
thread_id: *thread_id,
|
||||
item_id: item.item_id.clone(),
|
||||
db_status: Some(db_item.status),
|
||||
db_assigned_thread_id: db_item.assigned_thread_id,
|
||||
});
|
||||
}
|
||||
if inactive_items.is_empty() {
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
tracing::info!(
|
||||
job_id,
|
||||
db_running_items,
|
||||
active_items = active_items.len(),
|
||||
reclaimed_items = inactive_items.len(),
|
||||
"agent job reclaiming scheduler slots for items that are no longer running in state"
|
||||
);
|
||||
|
||||
for inactive_item in inactive_items {
|
||||
let _ = session
|
||||
.services
|
||||
.agent_control
|
||||
.shutdown_live_agent(inactive_item.thread_id)
|
||||
.await;
|
||||
active_items.remove(&inactive_item.thread_id);
|
||||
tracing::debug!(
|
||||
job_id,
|
||||
item_id = inactive_item.item_id,
|
||||
thread_id = %inactive_item.thread_id,
|
||||
db_status = inactive_item
|
||||
.db_status
|
||||
.as_ref()
|
||||
.map(|status| status.as_str())
|
||||
.unwrap_or("missing"),
|
||||
db_assigned_thread_id = inactive_item.db_assigned_thread_id.as_deref().unwrap_or(""),
|
||||
active_items = active_items.len(),
|
||||
"agent job reclaimed scheduler slot"
|
||||
);
|
||||
}
|
||||
|
||||
Ok(true)
|
||||
}
|
||||
@@ -27,6 +27,12 @@ pub(super) struct StartupTasks {
|
||||
launching_items: HashMap<TaskId, LaunchingJobItem>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub(super) struct SchedulerOccupancy {
|
||||
pub(super) active_items: usize,
|
||||
pub(super) db_running_items: usize,
|
||||
}
|
||||
|
||||
impl StartupTasks {
|
||||
pub(super) fn len(&self) -> usize {
|
||||
self.starting_items.len()
|
||||
@@ -62,12 +68,12 @@ pub(super) async fn launch_pending_items(
|
||||
job: &codex_state::AgentJob,
|
||||
job_id: &str,
|
||||
options: &JobRunnerOptions,
|
||||
active_items_len: usize,
|
||||
occupancy: SchedulerOccupancy,
|
||||
startup_tasks: &mut StartupTasks,
|
||||
) -> anyhow::Result<bool> {
|
||||
let slots = options
|
||||
.max_concurrency
|
||||
.saturating_sub(active_items_len + startup_tasks.len());
|
||||
.saturating_sub(occupancy.active_items + startup_tasks.len());
|
||||
if slots == 0 {
|
||||
return Ok(false);
|
||||
}
|
||||
@@ -146,7 +152,8 @@ pub(super) async fn launch_pending_items(
|
||||
tracing::info!(
|
||||
job_id,
|
||||
launched,
|
||||
active_items = active_items_len,
|
||||
db_running_items = occupancy.db_running_items,
|
||||
active_items = occupancy.active_items,
|
||||
starting_items = startup_tasks.len(),
|
||||
target_concurrency = options.max_concurrency,
|
||||
"agent job queued worker startups"
|
||||
|
||||
@@ -15,6 +15,9 @@ use std::sync::Arc;
|
||||
use std::sync::atomic::AtomicBool;
|
||||
use std::sync::atomic::AtomicUsize;
|
||||
use std::sync::atomic::Ordering;
|
||||
use std::time::Duration;
|
||||
use tokio::time::sleep;
|
||||
use tokio::time::timeout;
|
||||
use wiremock::Mock;
|
||||
use wiremock::Respond;
|
||||
use wiremock::ResponseTemplate;
|
||||
@@ -53,6 +56,78 @@ impl StopAfterFirstResponder {
|
||||
}
|
||||
}
|
||||
|
||||
struct DelayedWorkerAfterReportResponder {
|
||||
spawn_args_json: String,
|
||||
seen_main: AtomicBool,
|
||||
worker_calls: Arc<AtomicUsize>,
|
||||
delayed_worker_output: AtomicBool,
|
||||
}
|
||||
|
||||
impl DelayedWorkerAfterReportResponder {
|
||||
fn new(spawn_args_json: String, worker_calls: Arc<AtomicUsize>) -> Self {
|
||||
Self {
|
||||
spawn_args_json,
|
||||
seen_main: AtomicBool::new(false),
|
||||
worker_calls,
|
||||
delayed_worker_output: AtomicBool::new(false),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Respond for DelayedWorkerAfterReportResponder {
|
||||
fn respond(&self, request: &wiremock::Request) -> ResponseTemplate {
|
||||
let body_bytes = decode_body_bytes(request);
|
||||
let body: Value = serde_json::from_slice(&body_bytes).unwrap_or(Value::Null);
|
||||
|
||||
let call_output_ids = function_call_output_call_ids(&body);
|
||||
if !call_output_ids.is_empty() {
|
||||
let response = sse_response(sse(vec![
|
||||
ev_response_created("resp-tool"),
|
||||
ev_completed("resp-tool"),
|
||||
]));
|
||||
if call_output_ids
|
||||
.iter()
|
||||
.any(|call_id| call_id == "call-worker-0")
|
||||
&& !self.delayed_worker_output.swap(true, Ordering::SeqCst)
|
||||
{
|
||||
return response.set_delay(Duration::from_secs(2));
|
||||
}
|
||||
return response;
|
||||
}
|
||||
|
||||
if let Some((job_id, item_id)) = extract_job_and_item(&body) {
|
||||
let call_index = self.worker_calls.fetch_add(1, Ordering::SeqCst);
|
||||
let call_id = format!("call-worker-{call_index}");
|
||||
let args = json!({
|
||||
"job_id": job_id,
|
||||
"item_id": item_id,
|
||||
"result": { "item_id": item_id }
|
||||
});
|
||||
let args_json = serde_json::to_string(&args).unwrap_or_else(|err| {
|
||||
panic!("worker args serialize: {err}");
|
||||
});
|
||||
return sse_response(sse(vec![
|
||||
ev_response_created("resp-worker"),
|
||||
ev_function_call(&call_id, "report_agent_job_result", &args_json),
|
||||
ev_completed("resp-worker"),
|
||||
]));
|
||||
}
|
||||
|
||||
if !self.seen_main.swap(true, Ordering::SeqCst) {
|
||||
return sse_response(sse(vec![
|
||||
ev_response_created("resp-main"),
|
||||
ev_function_call("call-spawn", "spawn_agents_on_csv", &self.spawn_args_json),
|
||||
ev_completed("resp-main"),
|
||||
]));
|
||||
}
|
||||
|
||||
sse_response(sse(vec![
|
||||
ev_response_created("resp-default"),
|
||||
ev_completed("resp-default"),
|
||||
]))
|
||||
}
|
||||
}
|
||||
|
||||
impl Respond for StopAfterFirstResponder {
|
||||
fn respond(&self, request: &wiremock::Request) -> ResponseTemplate {
|
||||
let body_bytes = decode_body_bytes(request);
|
||||
@@ -176,6 +251,17 @@ fn has_function_call_output(body: &Value) -> bool {
|
||||
})
|
||||
}
|
||||
|
||||
fn function_call_output_call_ids(body: &Value) -> Vec<String> {
|
||||
body.get("input")
|
||||
.and_then(Value::as_array)
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.filter(|item| item.get("type").and_then(Value::as_str) == Some("function_call_output"))
|
||||
.filter_map(|item| item.get("call_id").and_then(Value::as_str))
|
||||
.map(str::to_string)
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn extract_job_and_item(body: &Value) -> Option<(String, String)> {
|
||||
let texts = message_input_texts(body);
|
||||
let mut combined = texts.join("\n");
|
||||
@@ -446,3 +532,75 @@ async fn spawn_agents_on_csv_stop_halts_future_items() -> Result<()> {
|
||||
assert_eq!(worker_calls.load(Ordering::SeqCst), 1);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn spawn_agents_on_csv_reclaims_slot_after_report_before_worker_completes() -> Result<()> {
|
||||
let server = start_mock_server().await;
|
||||
let mut builder = test_codex().with_config(|config| {
|
||||
config
|
||||
.features
|
||||
.enable(Feature::SpawnCsv)
|
||||
.expect("test config should allow feature update");
|
||||
config
|
||||
.features
|
||||
.enable(Feature::Sqlite)
|
||||
.expect("test config should allow feature update");
|
||||
});
|
||||
let test = Arc::new(builder.build(&server).await?);
|
||||
|
||||
let input_path = test.cwd_path().join("agent_jobs_reclaim.csv");
|
||||
let output_path = test.cwd_path().join("agent_jobs_reclaim_out.csv");
|
||||
fs::write(&input_path, "path\nfile-1\nfile-2\n")?;
|
||||
|
||||
let args = json!({
|
||||
"csv_path": input_path.display().to_string(),
|
||||
"instruction": "Return {path}",
|
||||
"output_csv_path": output_path.display().to_string(),
|
||||
"max_concurrency": 1,
|
||||
});
|
||||
let args_json = serde_json::to_string(&args)?;
|
||||
|
||||
let worker_calls = Arc::new(AtomicUsize::new(0));
|
||||
let responder = DelayedWorkerAfterReportResponder::new(args_json, worker_calls.clone());
|
||||
Mock::given(method("POST"))
|
||||
.and(path_regex(".*/responses$"))
|
||||
.respond_with(responder)
|
||||
.mount(&server)
|
||||
.await;
|
||||
|
||||
let submit = tokio::spawn({
|
||||
let test = Arc::clone(&test);
|
||||
async move { test.submit_turn("run batch job").await }
|
||||
});
|
||||
|
||||
timeout(Duration::from_secs(1), async {
|
||||
while worker_calls.load(Ordering::SeqCst) < 2 {
|
||||
sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("second worker should start before first worker's delayed post-report response");
|
||||
|
||||
submit.await??;
|
||||
|
||||
let output = fs::read_to_string(&output_path)?;
|
||||
assert_eq!(output.lines().skip(1).count(), 2);
|
||||
|
||||
let requests = worker_calls.load(Ordering::SeqCst);
|
||||
assert_eq!(requests, 2);
|
||||
|
||||
let root_requests = server
|
||||
.received_requests()
|
||||
.await
|
||||
.expect("wiremock should capture requests");
|
||||
let saw_spawn_output = root_requests.iter().any(|request| {
|
||||
let body_bytes = decode_body_bytes(request);
|
||||
let body: Value = serde_json::from_slice(&body_bytes).unwrap_or(Value::Null);
|
||||
function_call_output_call_ids(&body)
|
||||
.iter()
|
||||
.any(|call_id| call_id == "call-spawn")
|
||||
});
|
||||
assert!(saw_spawn_output);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user