openai/codex
1
0
Fork 0
mirror of https://github.com/openai/codex.git synced 2026-11-01 19:33:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

Make harness more automatic and add a README

Browse Source
This commit is contained in:
jimmyfraiture
2025-09-22 10:07:12 +01:00
parent 15b40f9ae0
commit 5c229c58e8
8 changed files with 457 additions and 108 deletions
Download Patch File Download Diff File Expand all files Collapse all files
codex-rs/core/tests/common/agent_harness.rs
+102 -76
View File
@@ -2,6 +2,7 @@ use std::collections::VecDeque;
use std::fs::File;
use std::mem;
use std::path::Path;
use std::path::PathBuf;
use std::sync::Arc;
use std::sync::Mutex;
use std::time::Duration;
@@ -15,6 +16,7 @@ use codex_core::WireApi;
use codex_core::protocol::Event;
use codex_core::protocol::EventMsg;
use codex_core::protocol::Op;
use serde::de::DeserializeOwned;
use serde_json::Value;
use serde_json::json;
use tempfile::TempDir;
@@ -41,57 +43,12 @@ pub struct HarnessData {
}
pub async fn run_fixture(dir: impl AsRef<Path>) -> Result<HarnessData> {
let dir = dir.as_ref();
let sse_path = dir.join("response_events.json");
let sse_sequences = load_sse_sequences(&sse_path)
.with_context(|| format!("load SSE fixture from {}", sse_path.display()))?;
let sequence_count = sse_sequences.len();
let prompts_path = dir.join("user_prompts.json");
let prompts_file = File::open(&prompts_path)
.with_context(|| format!("open prompts fixture {}", prompts_path.display()))?;
let prompt_ops: Vec<Op> = serde_json::from_reader(prompts_file)
.with_context(|| format!("parse prompts fixture {}", prompts_path.display()))?;
let expected_request_path = dir.join("expected_request.json");
let expected_request_file = File::open(&expected_request_path).with_context(|| {
format!(
"open expected request fixture {}",
expected_request_path.display()
)
})?;
let expected_request: Value =
serde_json::from_reader(expected_request_file).with_context(|| {
format!(
"parse expected request fixture {}",
expected_request_path.display()
)
})?;
let expected_events_path = dir.join("expected_events.json");
let expected_events_file = File::open(&expected_events_path).with_context(|| {
format!(
"open expected events fixture {}",
expected_events_path.display()
)
})?;
let expected_events: Vec<Value> =
serde_json::from_reader(expected_events_file).with_context(|| {
format!(
"parse expected events fixture {}",
expected_events_path.display()
)
})?;
let expected = HarnessOutputs {
request: expected_request,
events: expected_events,
};
let fixture = HarnessFixture::load(dir.as_ref())?;
let sequence_count = fixture.sse_sequences.len();
let server = MockServer::start().await;
let responder_state = Arc::new(Mutex::new(VecDeque::from(sse_sequences)));
let responder_state = Arc::new(Mutex::new(VecDeque::from(fixture.sse_sequences)));
let responder = SequentialSseResponder {
bodies: responder_state.clone(),
};
@@ -139,14 +96,14 @@ pub async fn run_fixture(dir: impl AsRef<Path>) -> Result<HarnessData> {
}];
let codex = new_conversation.conversation;
for op in &prompt_ops {
for op in &fixture.prompt_ops {
codex
.submit(op.clone())
.await
.with_context(|| format!("submit op {op:?}"))?;
}
let expected_event_count = expected.events.len();
let expected_event_count = fixture.expected.events.len();
anyhow::ensure!(
expected_event_count >= events.len(),
"expected events fixture must include at least the session configured event"
@@ -206,7 +163,76 @@ pub async fn run_fixture(dir: impl AsRef<Path>) -> Result<HarnessData> {
events: sanitized_events,
};
Ok(HarnessData { actual, expected })
Ok(HarnessData {
actual,
expected: fixture.expected,
})
}
struct HarnessFixture {
prompt_ops: Vec<Op>,
expected: HarnessOutputs,
sse_sequences: Vec<String>,
}
impl HarnessFixture {
fn load(dir: &Path) -> Result<Self> {
let paths = HarnessFixturePaths::new(dir.to_path_buf());
let sse_sequences = load_sse_sequences(&paths.response_events)
.with_context(|| paths.context_message("load SSE fixture"))?;
let prompt_ops: Vec<Op> = load_json(&paths.user_prompts, "user prompts")?;
let expected_request: Value = load_json(&paths.expected_request, "expected request")?;
let expected_events: Vec<Value> = load_json(&paths.expected_events, "expected events")?;
let expected = HarnessOutputs {
request: expected_request,
events: expected_events,
};
Ok(Self {
prompt_ops,
expected,
sse_sequences,
})
}
}
struct HarnessFixturePaths {
dir: PathBuf,
response_events: PathBuf,
user_prompts: PathBuf,
expected_request: PathBuf,
expected_events: PathBuf,
}
impl HarnessFixturePaths {
fn new(dir: PathBuf) -> Self {
let response_events = dir.join("response_events.json");
let user_prompts = dir.join("user_prompts.json");
let expected_request = dir.join("expected_request.json");
let expected_events = dir.join("expected_events.json");
Self {
dir,
response_events,
user_prompts,
expected_request,
expected_events,
}
}
fn context_message(&self, action: &str) -> String {
format!("{action} for fixture {}", self.dir.display())
}
}
fn load_json<T>(path: &Path, description: &str) -> Result<T>
where
T: DeserializeOwned,
{
let file = File::open(path)
.with_context(|| format!("open {description} fixture {}", path.display()))?;
serde_json::from_reader(file)
.with_context(|| format!("parse {description} fixture {}", path.display()))
}
struct SequentialSseResponder {
@@ -302,26 +328,26 @@ fn sanitize_events(events: Vec<Event>, replacements: &[(String, &'static str)])
let mut value = serde_json::to_value(event).expect("serialize event");
sanitize_value(&mut value, replacements);
if let Some(msg) = value.get_mut("msg") {
if let Some(msg_obj) = msg.as_object_mut() {
let msg_type = msg_obj
.get("type")
.and_then(Value::as_str)
.unwrap_or_default();
if msg_type == "session_configured" {
if let Some(msg) = value.get_mut("msg")
&& let Some(msg_obj) = msg.as_object_mut()
{
let msg_type = msg_obj
.get("type")
.and_then(Value::as_str)
.unwrap_or_default();
if msg_type == "session_configured" {
msg_obj.insert(
"session_id".to_string(),
Value::String("<session>".to_string()),
);
if msg_obj.contains_key("history_log_id") {
msg_obj.insert("history_log_id".to_string(), json!(0));
}
if msg_obj.contains_key("rollout_path") {
msg_obj.insert(
"session_id".to_string(),
Value::String("<session>".to_string()),
"rollout_path".to_string(),
Value::String("<rollout>".to_string()),
);
if msg_obj.contains_key("history_log_id") {
msg_obj.insert("history_log_id".to_string(), json!(0));
}
if msg_obj.contains_key("rollout_path") {
msg_obj.insert(
"rollout_path".to_string(),
Value::String("<rollout>".to_string()),
);
}
}
}
}
@@ -333,13 +359,13 @@ fn sanitize_events(events: Vec<Event>, replacements: &[(String, &'static str)])
fn sanitize_request(mut value: Value, replacements: &[(String, &'static str)]) -> Value {
sanitize_value(&mut value, replacements);
if let Some(obj) = value.as_object_mut() {
if obj.contains_key("prompt_cache_key") {
obj.insert(
"prompt_cache_key".to_string(),
Value::String("<prompt_cache_key>".to_string()),
);
}
if let Some(obj) = value.as_object_mut()
&& obj.contains_key("prompt_cache_key")
{
obj.insert(
"prompt_cache_key".to_string(),
Value::String("<prompt_cache_key>".to_string()),
);
}
value
}
codex-rs/core/tests/fixtures/harness/README.md
Vendored
+32
View File
@@ -0,0 +1,32 @@
# Agent Harness Fixtures
These fixtures drive the integration tests in `core/tests/suite/agent_harness.rs`.
Each subdirectory under this folder corresponds to a single end-to-end scenario.
## Adding a New Fixture
1. Create a new directory inside `tests/fixtures/harness/` and give it a
descriptive name (e.g. `multi_tool_call`).
2. Add the following JSON files inside the directory:
- `user_prompts.json`: the list of `Op` objects that will be submitted to the
harness.
- `response_events.json`: the SSE payloads that the mock Responses API will
replay. A top-level array can contain objects (single request) or arrays
(multiple sequential requests).
- `expected_request.json`: the sanitized request body we expect the harness
to send. This can be either a single object or an array when multiple
requests are issued.
- `expected_events.json`: the sanitized Codex events we expect to observe.
3. Run `cargo test -p codex-core suite::agent_harness::<fixture_name>` to verify
the scenario passes.
The test macro in `core/tests/suite/agent_harness.rs` will automatically pick up
the new directory once it exists.
## Expected JSON Files Are Partial
The comparison helpers only assert that the fields present in the expected JSON
match the actual values. Any keys omitted from `expected_request.json` or
`expected_events.json` are treated as "don't care" and are ignored during the
assertion. This keeps the fixtures stable even when Codex introduces new fields;
only include the pieces that matter for the scenario you are describing.
codex-rs/core/tests/fixtures/harness/reasoning_turn/expected_events.json
Vendored
+85
View File
@@ -0,0 +1,85 @@
[
{
"id": "",
"msg": {
"history_entry_count": 0,
"history_log_id": 0,
"model": "gpt-5",
"rollout_path": "<rollout>",
"session_id": "<session>",
"type": "session_configured"
}
},
{
"id": "0",
"msg": {
"model_context_window": 272000,
"type": "task_started"
}
},
{
"id": "0",
"msg": {
"delta": "Considering Earth's rotation.",
"type": "agent_reasoning_delta"
}
},
{
"id": "0",
"msg": {
"delta": " Sunlight scatters differently.",
"type": "agent_reasoning_delta"
}
},
{
"id": "0",
"msg": {
"delta": "Sunrise happens when your location rotates into sunlight.",
"type": "agent_message_delta"
}
},
{
"id": "0",
"msg": {
"delta": " Sunset is the reverse, heading into night.",
"type": "agent_message_delta"
}
},
{
"id": "0",
"msg": {
"message": "Sunrise is when your spot on Earth rotates into daylight; sunset is the same rotation carrying you back into the planet's shadow. The colors differ because morning light travels through cooler, cleaner air, while evenings pass through dustier skies.",
"type": "agent_message"
}
},
{
"id": "0",
"msg": {
"info": {
"last_token_usage": {
"cached_input_tokens": 0,
"input_tokens": 18,
"output_tokens": 32,
"reasoning_output_tokens": 6,
"total_tokens": 50
},
"model_context_window": 272000,
"total_token_usage": {
"cached_input_tokens": 0,
"input_tokens": 18,
"output_tokens": 32,
"reasoning_output_tokens": 6,
"total_tokens": 50
}
},
"type": "token_count"
}
},
{
"id": "0",
"msg": {
"last_agent_message": "Sunrise is when your spot on Earth rotates into daylight; sunset is the same rotation carrying you back into the planet's shadow. The colors differ because morning light travels through cooler, cleaner air, while evenings pass through dustier skies.",
"type": "task_complete"
}
}
]
codex-rs/core/tests/fixtures/harness/reasoning_turn/expected_request.json
Vendored
+106
View File
File diff suppressed because one or more lines are too long
codex-rs/core/tests/fixtures/harness/reasoning_turn/response_events.json
Vendored
+56
View File
@@ -0,0 +1,56 @@
[
{
"type": "response.created",
"response": {
"id": "resp_reason"
}
},
{
"type": "response.reasoning_summary_part.added"
},
{
"type": "response.reasoning_summary_text.delta",
"delta": "Considering Earth's rotation."
},
{
"type": "response.reasoning_summary_text.delta",
"delta": " Sunlight scatters differently."
},
{
"type": "response.output_text.delta",
"delta": "Sunrise happens when your location rotates into sunlight."
},
{
"type": "response.output_text.delta",
"delta": " Sunset is the reverse, heading into night."
},
{
"type": "response.output_item.done",
"item": {
"type": "message",
"role": "assistant",
"content": [
{
"type": "output_text",
"text": "Sunrise is when your spot on Earth rotates into daylight; sunset is the same rotation carrying you back into the planet's shadow. The colors differ because morning light travels through cooler, cleaner air, while evenings pass through dustier skies."
}
]
}
},
{
"type": "response.completed",
"response": {
"id": "resp_reason",
"usage": {
"input_tokens": 18,
"input_tokens_details": null,
"output_tokens": 32,
"output_tokens_details": {
"reasoning_tokens": 6
},
"total_tokens": 50,
"reasoning_tokens": 10
}
}
}
]
codex-rs/core/tests/fixtures/harness/reasoning_turn/user_prompts.json
Vendored
+11
View File
@@ -0,0 +1,11 @@
[
{
"type": "user_input",
"items": [
{
"type": "text",
"text": "Explain the difference between sunrise and sunset."
}
]
}
]
codex-rs/core/tests/fixtures/harness/single_turn/expected_events.json
Vendored
-1
View File
@@ -42,7 +42,6 @@
"reasoning_output_tokens": 0,
"total_tokens": 15
},
"model_context_window": 272000,
"total_token_usage": {
"cached_input_tokens": 0,
"input_tokens": 10,
codex-rs/core/tests/suite/agent_harness.rs
+65 -31
View File
@@ -5,23 +5,30 @@ use core_test_support::agent_harness::HarnessOutputs;
use pretty_assertions::assert_eq;
use serde_json::Value;
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn single_turn_fixture_matches_expectations() {
let fixture_dir =
PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/harness/single_turn");
let harness = agent_harness::run_fixture(fixture_dir)
.await
.expect("run agent harness");
const HARNESS_FIXTURE_ROOT: &str = "tests/fixtures/harness";
assert_harness_outputs_match(&harness.actual, &harness.expected);
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn harness_fixtures_match_expectations() {
for fixture in fixture_names() {
run_fixture_test(&fixture).await;
}
}
fn assert_harness_outputs_match(actual: &HarnessOutputs, expected: &HarnessOutputs) {
assert_value_matches(&actual.request, &expected.request, "request");
async fn run_fixture_test(fixture: &str) {
let harness = agent_harness::run_fixture(fixture_dir(fixture))
.await
.unwrap_or_else(|err| panic!("run agent harness for fixture {fixture}: {err}"));
assert_harness_outputs_match(fixture, &harness.actual, &harness.expected);
}
#[track_caller]
fn assert_harness_outputs_match(fixture: &str, actual: &HarnessOutputs, expected: &HarnessOutputs) {
assert_value_matches(fixture, &actual.request, &expected.request, "request");
assert_eq!(
actual.events.len(),
expected.events.len(),
"event count mismatch: expected {} events but got {}",
"event count mismatch in fixture {fixture}: expected {} events but got {}",
expected.events.len(),
actual.events.len()
);
@@ -30,38 +37,38 @@ fn assert_harness_outputs_match(actual: &HarnessOutputs, expected: &HarnessOutpu
let actual_event = actual
.events
.get(index)
.unwrap_or_else(|| panic!("missing actual event at {path}"));
assert_value_matches(actual_event, expected_event, &path);
.unwrap_or_else(|| panic!("missing actual event at {path} for fixture {fixture}"));
assert_value_matches(fixture, actual_event, expected_event, &path);
}
}
fn assert_value_matches(actual: &Value, expected: &Value, path: &str) {
#[track_caller]
fn assert_value_matches(fixture: &str, actual: &Value, expected: &Value, path: &str) {
match expected {
Value::Object(expected_map) => {
let actual_map = actual
.as_object()
.unwrap_or_else(|| panic!("expected object at {path}, got {actual:?}"));
let actual_map = actual.as_object().unwrap_or_else(|| {
panic!("expected object at {path} in fixture {fixture}, got {actual:?}")
});
for (key, expected_value) in expected_map {
let next_path = if path.is_empty() {
key.to_string()
} else {
format!("{path}.{key}")
};
let actual_value = actual_map
.get(key)
.unwrap_or_else(|| panic!("missing field {next_path} in actual value"));
assert_value_matches(actual_value, expected_value, &next_path);
let actual_value = actual_map.get(key).unwrap_or_else(|| {
panic!("missing field {next_path} in actual value for fixture {fixture}")
});
assert_value_matches(fixture, actual_value, expected_value, &next_path);
}
}
Value::Array(expected_items) => {
let actual_items = actual
.as_array()
.unwrap_or_else(|| panic!("expected array at {path}, got {actual:?}"));
let actual_items = actual.as_array().unwrap_or_else(|| {
panic!("expected array at {path} in fixture {fixture}, got {actual:?}")
});
assert_eq!(
actual_items.len(),
expected_items.len(),
"array length mismatch at {}",
path
"array length mismatch at {path} in fixture {fixture}"
);
for (index, expected_value) in expected_items.iter().enumerate() {
let next_path = if path.is_empty() {
@@ -69,14 +76,41 @@ fn assert_value_matches(actual: &Value, expected: &Value, path: &str) {
} else {
format!("{path}[{index}]")
};
let actual_value = actual_items
.get(index)
.unwrap_or_else(|| panic!("missing array element at {next_path}"));
assert_value_matches(actual_value, expected_value, &next_path);
let actual_value = actual_items.get(index).unwrap_or_else(|| {
panic!("missing array element at {next_path} for fixture {fixture}")
});
assert_value_matches(fixture, actual_value, expected_value, &next_path);
}
}
_ => {
assert_eq!(actual, expected, "mismatch at {path}");
assert_eq!(actual, expected, "mismatch at {path} in fixture {fixture}");
}
}
}
fn fixtures_root() -> PathBuf {
PathBuf::from(env!("CARGO_MANIFEST_DIR")).join(HARNESS_FIXTURE_ROOT)
}
fn fixture_dir(fixture: &str) -> PathBuf {
fixtures_root().join(fixture)
}
fn fixture_names() -> Vec<String> {
let dir = fixtures_root();
let mut names: Vec<String> = std::fs::read_dir(&dir)
.unwrap_or_else(|err| panic!("read fixture directory {}: {err}", dir.display()))
.filter_map(|entry| {
entry.ok().and_then(|e| {
let path = e.path();
(path.is_dir()).then(|| {
path.file_name()
.and_then(|name| name.to_str())
.map(|name| name.to_string())
})
})
})
.flatten()
.collect();
names.sort();
names
}