mirror of
https://github.com/openai/codex.git
synced 2026-09-14 11:57:03 +00:00
Drop verifier
This commit is contained in:
jif-oai
@@ -77,7 +77,7 @@ impl TerminalProgressReporter {
|
||||
|
||||
impl ProgressReporter for TerminalProgressReporter {
|
||||
fn objective_posted(&self, objective: &str) {
|
||||
let objective_line = format!("{}", format!("→ objective: {objective}"));
|
||||
let objective_line = format!("{}", format!("→ objective: {objective}").dim());
|
||||
self.print_exchange("user", "solver", vec![objective_line], true);
|
||||
}
|
||||
|
||||
@@ -169,19 +169,18 @@ impl ProgressReporter for TerminalProgressReporter {
|
||||
fn final_delivery(&self, deliverable_path: &Path, summary: Option<&str>) {
|
||||
let delivery_line = format!(
|
||||
"{}",
|
||||
format!(
|
||||
"✓ solver reported final delivery at {}",
|
||||
deliverable_path.display()
|
||||
)
|
||||
.green()
|
||||
.bold()
|
||||
format!("→ path: {}", deliverable_path.display()).dim()
|
||||
);
|
||||
let summary_line = format!(
|
||||
"{}",
|
||||
format!("→ summary: {}", summary.unwrap_or("<none>")).dim()
|
||||
);
|
||||
self.print_exchange(
|
||||
"solver",
|
||||
"verifier",
|
||||
vec![delivery_line, summary_line],
|
||||
true,
|
||||
);
|
||||
let mut lines = vec![delivery_line];
|
||||
if summary.is_some_and(|summary| !summary.is_empty()) {
|
||||
let hint = " (final summary will be shown below)".to_string();
|
||||
lines.push(format!("{}", hint.dim()));
|
||||
}
|
||||
self.print_exchange("solver", "verifier", lines, true);
|
||||
}
|
||||
|
||||
fn run_interrupted(&self) {
|
||||
|
||||
@@ -14,14 +14,13 @@ The crate is designed to be embedded (via the library API) and also powers the `
|
||||
```
|
||||
objective → Solver
|
||||
Solver → direction_request → Director → directive → Solver
|
||||
Solver → verification_request → Verifier(s) → aggregated summary → Solver
|
||||
… (iterate) …
|
||||
Solver → final_delivery → Orchestrator returns RunOutcome
|
||||
```
|
||||
|
||||
- The Solver always speaks structured JSON. The orchestrator parses those messages and decides the next hop.
|
||||
- The Director provides crisp guidance (also JSON) that is forwarded back to the Solver.
|
||||
- One or more Verifiers assess claims and return verdicts; the orchestrator aggregates results and reports a summary to the Solver.
|
||||
- One or more Verifiers may assess the final deliverable; the orchestrator aggregates results and reports a summary to the Solver.
|
||||
- On final_delivery, the orchestrator resolves and validates the deliverable path and returns the `RunOutcome`.
|
||||
|
||||
## Directory Layout (Run Store)
|
||||
@@ -55,7 +54,7 @@ You can provide your own instructions by pre‑populating `Config.base_instructi
|
||||
|
||||
## Solver Signal Contract
|
||||
|
||||
The Solver communicates intent using JSON messages (possibly wrapped in a fenced block). The orchestrator accepts three shapes:
|
||||
The Solver communicates intent using JSON messages (possibly wrapped in a fenced block). The orchestrator accepts two shapes:
|
||||
|
||||
- Direction request (sent to Director):
|
||||
|
||||
@@ -63,12 +62,6 @@ The Solver communicates intent using JSON messages (possibly wrapped in a fenced
|
||||
{"type":"direction_request","prompt":"<question or decision>"}
|
||||
```
|
||||
|
||||
- Verification request (sent to Verifier(s)):
|
||||
|
||||
```json
|
||||
{"type":"verification_request","claim_path":"memory/claims/<file>.json","notes":null}
|
||||
```
|
||||
|
||||
- Final delivery (completes the run):
|
||||
|
||||
```json
|
||||
@@ -107,15 +100,11 @@ JSON may be fenced as ```json … ```; the orchestrator will strip the fence.
|
||||
3. On `direction_request`:
|
||||
- Post a structured request to the Director and await the first assistant message.
|
||||
- Parse it into a `DirectiveResponse` and forward the normalized JSON to the Solver.
|
||||
4. On `verification_request`:
|
||||
- Send structured requests to each Verifier and await each first assistant message.
|
||||
- Aggregate verdicts into an `AggregatedVerifierVerdict` and post the summary back to the Solver.
|
||||
5. On `final_delivery`:
|
||||
4. On `final_delivery`:
|
||||
- Canonicalize and validate that `deliverable_path` stays within the run directory.
|
||||
- Optionally run a verification pass using configured Verifier(s), aggregate results, and post a summary back to the Solver.
|
||||
- Notify the progress reporter, touch the run store, and return `RunOutcome`.
|
||||
|
||||
Note: The orchestrator does not re‑run verification after `final_delivery`. Verification is performed whenever the Solver issues a `verification_request` during the run.
|
||||
|
||||
## Library Usage
|
||||
|
||||
```rust
|
||||
|
||||
@@ -248,27 +248,6 @@ impl InftyOrchestrator {
|
||||
sessions.store.touch()?;
|
||||
state.pending_solver_turn_completion = true;
|
||||
}
|
||||
SolverSignal::VerificationRequest { claim_path, notes } => {
|
||||
let claim_path = crate::utils::required_trimmed(
|
||||
claim_path,
|
||||
"solver verification_request missing claim_path",
|
||||
)?;
|
||||
if let Some(p) = self.progress_ref() { p.verification_request(&claim_path, notes.as_deref()); }
|
||||
let verified = self
|
||||
.handle_verification_request(
|
||||
sessions,
|
||||
&mut verifier_pool,
|
||||
&claim_path,
|
||||
notes.as_deref(),
|
||||
options,
|
||||
&solver_role,
|
||||
)
|
||||
.await?;
|
||||
sessions.store.touch()?;
|
||||
if verified {
|
||||
state.pending_solver_turn_completion = true;
|
||||
}
|
||||
}
|
||||
SolverSignal::FinalDelivery {
|
||||
deliverable_path,
|
||||
summary,
|
||||
@@ -391,32 +370,6 @@ impl InftyOrchestrator {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn handle_verification_request(
|
||||
&self,
|
||||
sessions: &mut RunSessions,
|
||||
verifier_pool: &mut VerifierPool,
|
||||
claim_path: &str,
|
||||
notes: Option<&str>,
|
||||
options: &RunExecutionOptions,
|
||||
solver_role: &SolverRole,
|
||||
) -> Result<bool> {
|
||||
let objective = crate::utils::objective_as_str(options);
|
||||
|
||||
let request = VerificationRequestPayload::new(claim_path, notes, objective);
|
||||
if verifier_pool.is_empty() {
|
||||
return Ok(true);
|
||||
}
|
||||
let round = verifier_pool.collect_round(&request).await?;
|
||||
verifier_pool
|
||||
.rotate_passing(sessions, &self.conversation_manager, &round.passing_roles)
|
||||
.await?;
|
||||
let summary = round.summary;
|
||||
self.emit_verification_summary(&summary);
|
||||
let req = SolverRequest::from(&summary);
|
||||
solver_role.call(&req).await?;
|
||||
Ok(summary.overall.is_pass())
|
||||
}
|
||||
|
||||
async fn run_final_verification(
|
||||
&self,
|
||||
sessions: &mut RunSessions,
|
||||
|
||||
@@ -17,17 +17,13 @@ Responsibilities:
|
||||
Available Codex tools mirror standard Codex sessions (e.g. `shell`, `apply_patch`). Assume all filesystem paths are relative to the current run store directory unless stated otherwise.
|
||||
|
||||
## Communication contract
|
||||
The orchestrator routes your structured messages to the Director or Verifier roles. Respond with **JSON only**—no leading prose or trailing commentary. Wrap JSON in a fenced block only if the agent policy forces it.
|
||||
The orchestrator routes your structured messages to the Director. Respond with **JSON only**—no leading prose or trailing commentary. Wrap JSON in a fenced block only if the agent policy forces it.
|
||||
|
||||
- Every reply must populate the full schema, even when a field does not apply. Set unused string fields to `null`.
|
||||
- Direction request (send to Director):
|
||||
```json
|
||||
{"type":"direction_request","prompt":"<concise question or decision>","claim_path":null,"notes":null,"deliverable_path":null,"summary":null}
|
||||
```
|
||||
- Verification request (send to Verifier). Do not ask for verification before having the final answer. The Verifier is not made for intermediate verification:
|
||||
```json
|
||||
{"type":"verification_request","prompt":null,"claim_path":"memory/claims/<file>.json","notes":null,"deliverable_path":null,"summary":null}
|
||||
```
|
||||
- Final delivery (after receiving the finalization instruction):
|
||||
```json
|
||||
{"type":"final_delivery","prompt":null,"claim_path":null,"notes":null,"deliverable_path":"deliverable/summary.txt","summary":"<answer plus supporting context>"}
|
||||
@@ -39,6 +35,6 @@ The orchestrator routes your structured messages to the Director or Verifier rol
|
||||
- When uncertainty remains, prioritise experiments or reasoning steps that move you closer to a finished proof rather than cataloguing known results.
|
||||
- Keep the run resilient to restarts: document intent, intermediate results, and follow-up tasks in `memory/`.
|
||||
- Prefer concrete evidence (tests, diffs, logs). Link every claim to artifacts or durable notes so the Verifier can reproduce your reasoning.
|
||||
- On failure feedback from a Verifier, update artifacts/notes/tests, then issue a new verification request referencing the superseding claim.
|
||||
- On failure feedback from a Verifier, update artifacts/notes/tests, and then iterate (ask the Director if needed) before attempting final delivery again.
|
||||
- When the orchestrator instructs you to finalize, build the `deliverable/` directory exactly as requested, summarise the outcome, and respond with the `final_delivery` JSON.
|
||||
- Only a final solution to the objective is an acceptable result to be sent to the verifier. If you do not find any solution, try to create a new one on your own.
|
||||
|
||||
@@ -41,27 +41,16 @@ impl SolverRole {
|
||||
}
|
||||
|
||||
pub fn solver_signal_schema() -> Value {
|
||||
// Only allow asking the director or sending the final result.
|
||||
serde_json::json!({
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"type": {
|
||||
"type": "string",
|
||||
"enum": ["direction_request", "verification_request", "final_delivery"]
|
||||
},
|
||||
"type": { "type": "string", "enum": ["direction_request", "final_delivery"] },
|
||||
"prompt": { "type": ["string", "null"] },
|
||||
"claim_path": { "type": ["string", "null"] },
|
||||
"notes": { "type": ["string", "null"] },
|
||||
"deliverable_path": { "type": ["string", "null"] },
|
||||
"summary": { "type": ["string", "null"] }
|
||||
},
|
||||
"required": [
|
||||
"type",
|
||||
"prompt",
|
||||
"claim_path",
|
||||
"notes",
|
||||
"deliverable_path",
|
||||
"summary"
|
||||
],
|
||||
"required": ["type", "prompt", "deliverable_path", "summary"],
|
||||
"additionalProperties": false
|
||||
})
|
||||
}
|
||||
@@ -187,12 +176,6 @@ pub enum SolverSignal {
|
||||
#[serde(default)]
|
||||
prompt: Option<String>,
|
||||
},
|
||||
VerificationRequest {
|
||||
#[serde(default)]
|
||||
claim_path: Option<String>,
|
||||
#[serde(default)]
|
||||
notes: Option<String>,
|
||||
},
|
||||
FinalDelivery {
|
||||
#[serde(default)]
|
||||
deliverable_path: Option<String>,
|
||||
|
||||
@@ -133,36 +133,9 @@ async fn execute_new_run_drives_to_completion() -> anyhow::Result<()> {
|
||||
]),
|
||||
responses::sse(vec![
|
||||
responses::ev_response_created("solver-resp-2"),
|
||||
responses::ev_assistant_message(
|
||||
"solver-msg-2",
|
||||
r#"{"type":"verification_request","prompt":null,"claim_path":"memory/claims/attempt1.json","notes":null,"deliverable_path":null,"summary":null}"#,
|
||||
),
|
||||
responses::ev_assistant_message("solver-msg-2", "Acknowledged"),
|
||||
responses::ev_completed("solver-resp-2"),
|
||||
]),
|
||||
responses::sse(vec![
|
||||
responses::ev_response_created("verifier-resp-1"),
|
||||
responses::ev_assistant_message(
|
||||
"verifier-msg-1",
|
||||
r#"{"verdict":"fail","reasons":["Missing tests"],"suggestions":["Add regression tests"]}"#,
|
||||
),
|
||||
responses::ev_completed("verifier-resp-1"),
|
||||
]),
|
||||
responses::sse(vec![
|
||||
responses::ev_response_created("solver-resp-3"),
|
||||
responses::ev_assistant_message(
|
||||
"solver-msg-3",
|
||||
r#"{"type":"verification_request","prompt":null,"claim_path":"memory/claims/attempt2.json","notes":null,"deliverable_path":null,"summary":null}"#,
|
||||
),
|
||||
responses::ev_completed("solver-resp-3"),
|
||||
]),
|
||||
responses::sse(vec![
|
||||
responses::ev_response_created("verifier-resp-2"),
|
||||
responses::ev_assistant_message(
|
||||
"verifier-msg-2",
|
||||
r#"{"verdict":"pass","reasons":[],"suggestions":[]}"#,
|
||||
),
|
||||
responses::ev_completed("verifier-resp-2"),
|
||||
]),
|
||||
responses::sse(vec![
|
||||
responses::ev_response_created("solver-resp-4"),
|
||||
responses::ev_assistant_message(
|
||||
|
||||
@@ -208,12 +208,12 @@ async fn verifier_request_includes_output_schema() -> anyhow::Result<()> {
|
||||
|
||||
let server = responses::start_mock_server().await;
|
||||
|
||||
// 1) Solver: emit a verification_request so the orchestrator calls the Verifier.
|
||||
// 1) Solver: issue a final_delivery which triggers verifier requests.
|
||||
let body_solver = responses::sse(vec![
|
||||
responses::ev_response_created("solver-resp-1"),
|
||||
responses::ev_assistant_message(
|
||||
"solver-msg-1",
|
||||
r#"{"type":"verification_request","prompt":null,"claim_path":"memory/claims/baseline-computation.json","notes":null,"deliverable_path":null,"summary":null}"#,
|
||||
r#"{"type":"final_delivery","deliverable_path":"deliverable/summary.txt","summary":null}"#,
|
||||
),
|
||||
responses::ev_completed("solver-resp-1"),
|
||||
]);
|
||||
|
||||
Reference in New Issue
Block a user