diff --git a/codex-rs/core/tests/common/agent_harness.rs b/codex-rs/core/tests/common/agent_harness.rs index 786815aed0..0979dfe076 100644 --- a/codex-rs/core/tests/common/agent_harness.rs +++ b/codex-rs/core/tests/common/agent_harness.rs @@ -2,6 +2,7 @@ use std::collections::VecDeque; use std::fs::File; use std::mem; use std::path::Path; +use std::path::PathBuf; use std::sync::Arc; use std::sync::Mutex; use std::time::Duration; @@ -15,6 +16,7 @@ use codex_core::WireApi; use codex_core::protocol::Event; use codex_core::protocol::EventMsg; use codex_core::protocol::Op; +use serde::de::DeserializeOwned; use serde_json::Value; use serde_json::json; use tempfile::TempDir; @@ -41,57 +43,12 @@ pub struct HarnessData { } pub async fn run_fixture(dir: impl AsRef) -> Result { - let dir = dir.as_ref(); - - let sse_path = dir.join("response_events.json"); - let sse_sequences = load_sse_sequences(&sse_path) - .with_context(|| format!("load SSE fixture from {}", sse_path.display()))?; - let sequence_count = sse_sequences.len(); - - let prompts_path = dir.join("user_prompts.json"); - let prompts_file = File::open(&prompts_path) - .with_context(|| format!("open prompts fixture {}", prompts_path.display()))?; - let prompt_ops: Vec = serde_json::from_reader(prompts_file) - .with_context(|| format!("parse prompts fixture {}", prompts_path.display()))?; - - let expected_request_path = dir.join("expected_request.json"); - let expected_request_file = File::open(&expected_request_path).with_context(|| { - format!( - "open expected request fixture {}", - expected_request_path.display() - ) - })?; - let expected_request: Value = - serde_json::from_reader(expected_request_file).with_context(|| { - format!( - "parse expected request fixture {}", - expected_request_path.display() - ) - })?; - - let expected_events_path = dir.join("expected_events.json"); - let expected_events_file = File::open(&expected_events_path).with_context(|| { - format!( - "open expected events fixture {}", - expected_events_path.display() - ) - })?; - let expected_events: Vec = - serde_json::from_reader(expected_events_file).with_context(|| { - format!( - "parse expected events fixture {}", - expected_events_path.display() - ) - })?; - - let expected = HarnessOutputs { - request: expected_request, - events: expected_events, - }; + let fixture = HarnessFixture::load(dir.as_ref())?; + let sequence_count = fixture.sse_sequences.len(); let server = MockServer::start().await; - let responder_state = Arc::new(Mutex::new(VecDeque::from(sse_sequences))); + let responder_state = Arc::new(Mutex::new(VecDeque::from(fixture.sse_sequences))); let responder = SequentialSseResponder { bodies: responder_state.clone(), }; @@ -139,14 +96,14 @@ pub async fn run_fixture(dir: impl AsRef) -> Result { }]; let codex = new_conversation.conversation; - for op in &prompt_ops { + for op in &fixture.prompt_ops { codex .submit(op.clone()) .await .with_context(|| format!("submit op {op:?}"))?; } - let expected_event_count = expected.events.len(); + let expected_event_count = fixture.expected.events.len(); anyhow::ensure!( expected_event_count >= events.len(), "expected events fixture must include at least the session configured event" @@ -206,7 +163,76 @@ pub async fn run_fixture(dir: impl AsRef) -> Result { events: sanitized_events, }; - Ok(HarnessData { actual, expected }) + Ok(HarnessData { + actual, + expected: fixture.expected, + }) +} + +struct HarnessFixture { + prompt_ops: Vec, + expected: HarnessOutputs, + sse_sequences: Vec, +} + +impl HarnessFixture { + fn load(dir: &Path) -> Result { + let paths = HarnessFixturePaths::new(dir.to_path_buf()); + let sse_sequences = load_sse_sequences(&paths.response_events) + .with_context(|| paths.context_message("load SSE fixture"))?; + let prompt_ops: Vec = load_json(&paths.user_prompts, "user prompts")?; + let expected_request: Value = load_json(&paths.expected_request, "expected request")?; + let expected_events: Vec = load_json(&paths.expected_events, "expected events")?; + + let expected = HarnessOutputs { + request: expected_request, + events: expected_events, + }; + + Ok(Self { + prompt_ops, + expected, + sse_sequences, + }) + } +} + +struct HarnessFixturePaths { + dir: PathBuf, + response_events: PathBuf, + user_prompts: PathBuf, + expected_request: PathBuf, + expected_events: PathBuf, +} + +impl HarnessFixturePaths { + fn new(dir: PathBuf) -> Self { + let response_events = dir.join("response_events.json"); + let user_prompts = dir.join("user_prompts.json"); + let expected_request = dir.join("expected_request.json"); + let expected_events = dir.join("expected_events.json"); + Self { + dir, + response_events, + user_prompts, + expected_request, + expected_events, + } + } + + fn context_message(&self, action: &str) -> String { + format!("{action} for fixture {}", self.dir.display()) + } +} + +fn load_json(path: &Path, description: &str) -> Result +where + T: DeserializeOwned, +{ + let file = File::open(path) + .with_context(|| format!("open {description} fixture {}", path.display()))?; + serde_json::from_reader(file) + .with_context(|| format!("parse {description} fixture {}", path.display())) } struct SequentialSseResponder { @@ -302,26 +328,26 @@ fn sanitize_events(events: Vec, replacements: &[(String, &'static str)]) let mut value = serde_json::to_value(event).expect("serialize event"); sanitize_value(&mut value, replacements); - if let Some(msg) = value.get_mut("msg") { - if let Some(msg_obj) = msg.as_object_mut() { - let msg_type = msg_obj - .get("type") - .and_then(Value::as_str) - .unwrap_or_default(); - if msg_type == "session_configured" { + if let Some(msg) = value.get_mut("msg") + && let Some(msg_obj) = msg.as_object_mut() + { + let msg_type = msg_obj + .get("type") + .and_then(Value::as_str) + .unwrap_or_default(); + if msg_type == "session_configured" { + msg_obj.insert( + "session_id".to_string(), + Value::String("".to_string()), + ); + if msg_obj.contains_key("history_log_id") { + msg_obj.insert("history_log_id".to_string(), json!(0)); + } + if msg_obj.contains_key("rollout_path") { msg_obj.insert( - "session_id".to_string(), - Value::String("".to_string()), + "rollout_path".to_string(), + Value::String("".to_string()), ); - if msg_obj.contains_key("history_log_id") { - msg_obj.insert("history_log_id".to_string(), json!(0)); - } - if msg_obj.contains_key("rollout_path") { - msg_obj.insert( - "rollout_path".to_string(), - Value::String("".to_string()), - ); - } } } } @@ -333,13 +359,13 @@ fn sanitize_events(events: Vec, replacements: &[(String, &'static str)]) fn sanitize_request(mut value: Value, replacements: &[(String, &'static str)]) -> Value { sanitize_value(&mut value, replacements); - if let Some(obj) = value.as_object_mut() { - if obj.contains_key("prompt_cache_key") { - obj.insert( - "prompt_cache_key".to_string(), - Value::String("".to_string()), - ); - } + if let Some(obj) = value.as_object_mut() + && obj.contains_key("prompt_cache_key") + { + obj.insert( + "prompt_cache_key".to_string(), + Value::String("".to_string()), + ); } value } diff --git a/codex-rs/core/tests/fixtures/harness/README.md b/codex-rs/core/tests/fixtures/harness/README.md new file mode 100644 index 0000000000..919363d4e6 --- /dev/null +++ b/codex-rs/core/tests/fixtures/harness/README.md @@ -0,0 +1,32 @@ +# Agent Harness Fixtures + +These fixtures drive the integration tests in `core/tests/suite/agent_harness.rs`. +Each subdirectory under this folder corresponds to a single end-to-end scenario. + +## Adding a New Fixture + +1. Create a new directory inside `tests/fixtures/harness/` and give it a + descriptive name (e.g. `multi_tool_call`). +2. Add the following JSON files inside the directory: + - `user_prompts.json`: the list of `Op` objects that will be submitted to the + harness. + - `response_events.json`: the SSE payloads that the mock Responses API will + replay. A top-level array can contain objects (single request) or arrays + (multiple sequential requests). + - `expected_request.json`: the sanitized request body we expect the harness + to send. This can be either a single object or an array when multiple + requests are issued. + - `expected_events.json`: the sanitized Codex events we expect to observe. +3. Run `cargo test -p codex-core suite::agent_harness::` to verify + the scenario passes. + +The test macro in `core/tests/suite/agent_harness.rs` will automatically pick up +the new directory once it exists. + +## Expected JSON Files Are Partial + +The comparison helpers only assert that the fields present in the expected JSON +match the actual values. Any keys omitted from `expected_request.json` or +`expected_events.json` are treated as "don't care" and are ignored during the +assertion. This keeps the fixtures stable even when Codex introduces new fields; +only include the pieces that matter for the scenario you are describing. diff --git a/codex-rs/core/tests/fixtures/harness/reasoning_turn/expected_events.json b/codex-rs/core/tests/fixtures/harness/reasoning_turn/expected_events.json new file mode 100644 index 0000000000..812e3d610e --- /dev/null +++ b/codex-rs/core/tests/fixtures/harness/reasoning_turn/expected_events.json @@ -0,0 +1,85 @@ +[ + { + "id": "", + "msg": { + "history_entry_count": 0, + "history_log_id": 0, + "model": "gpt-5", + "rollout_path": "", + "session_id": "", + "type": "session_configured" + } + }, + { + "id": "0", + "msg": { + "model_context_window": 272000, + "type": "task_started" + } + }, + { + "id": "0", + "msg": { + "delta": "Considering Earth's rotation.", + "type": "agent_reasoning_delta" + } + }, + { + "id": "0", + "msg": { + "delta": " Sunlight scatters differently.", + "type": "agent_reasoning_delta" + } + }, + { + "id": "0", + "msg": { + "delta": "Sunrise happens when your location rotates into sunlight.", + "type": "agent_message_delta" + } + }, + { + "id": "0", + "msg": { + "delta": " Sunset is the reverse, heading into night.", + "type": "agent_message_delta" + } + }, + { + "id": "0", + "msg": { + "message": "Sunrise is when your spot on Earth rotates into daylight; sunset is the same rotation carrying you back into the planet's shadow. The colors differ because morning light travels through cooler, cleaner air, while evenings pass through dustier skies.", + "type": "agent_message" + } + }, + { + "id": "0", + "msg": { + "info": { + "last_token_usage": { + "cached_input_tokens": 0, + "input_tokens": 18, + "output_tokens": 32, + "reasoning_output_tokens": 6, + "total_tokens": 50 + }, + "model_context_window": 272000, + "total_token_usage": { + "cached_input_tokens": 0, + "input_tokens": 18, + "output_tokens": 32, + "reasoning_output_tokens": 6, + "total_tokens": 50 + } + }, + "type": "token_count" + } + }, + { + "id": "0", + "msg": { + "last_agent_message": "Sunrise is when your spot on Earth rotates into daylight; sunset is the same rotation carrying you back into the planet's shadow. The colors differ because morning light travels through cooler, cleaner air, while evenings pass through dustier skies.", + "type": "task_complete" + } + } +] diff --git a/codex-rs/core/tests/fixtures/harness/reasoning_turn/expected_request.json b/codex-rs/core/tests/fixtures/harness/reasoning_turn/expected_request.json new file mode 100644 index 0000000000..9ab70bc148 --- /dev/null +++ b/codex-rs/core/tests/fixtures/harness/reasoning_turn/expected_request.json @@ -0,0 +1,106 @@ +{ + "include": [ + "reasoning.encrypted_content" + ], + "input": [ + { + "content": [ + { + "text": "\n\n# Rust/codex-rs\n\nIn the codex-rs folder where the rust code lives:\n\n- Crate names are prefixed with `codex-`. For example, the `core` folder's crate is named `codex-core`\n- When using format! and you can inline variables into {}, always do that.\n- Install any commands the repo relies on (for example `just`, `rg`, or `cargo-insta`) if they aren't already available before running instructions here.\n- Never add or modify any code related to `CODEX_SANDBOX_NETWORK_DISABLED_ENV_VAR` or `CODEX_SANDBOX_ENV_VAR`.\n - You operate in a sandbox where `CODEX_SANDBOX_NETWORK_DISABLED=1` will be set whenever you use the `shell` tool. Any existing code that uses `CODEX_SANDBOX_NETWORK_DISABLED_ENV_VAR` was authored with this fact in mind. It is often used to early exit out of tests that the author knew you would not be able to run given your sandbox limitations.\n - Similarly, when you spawn a process using Seatbelt (`/usr/bin/sandbox-exec`), `CODEX_SANDBOX=seatbelt` will be set on the child process. Integration tests that want to run Seatbelt themselves cannot be run under Seatbelt, so checks for `CODEX_SANDBOX=seatbelt` are also often used to early exit out of tests, as appropriate.\n\nRun `just fmt` (in `codex-rs` directory) automatically after making Rust code changes; do not ask for approval to run it. Before finalizing a change to `codex-rs`, run `just fix -p ` (in `codex-rs` directory) to fix any linter issues in the code. Prefer scoping with `-p` to avoid slow workspace\u2011wide Clippy builds; only run `just fix` without `-p` if you changed shared crates. Additionally, run the tests:\n1. Run the test for the specific project that was changed. For example, if changes were made in `codex-rs/tui`, run `cargo test -p codex-tui`.\n2. Once those pass, if any changes were made in common, core, or protocol, run the complete test suite with `cargo test --all-features`.\nWhen running interactively, ask the user before running `just fix` to finalize. `just fmt` does not require approval. project-specific or individual tests can be run without asking the user, but do ask the user before running the complete test suite.\n\n## TUI style conventions\n\nSee `codex-rs/tui/styles.md`.\n\n## TUI code conventions\n\n- Use concise styling helpers from ratatui\u2019s Stylize trait.\n - Basic spans: use \"text\".into()\n - Styled spans: use \"text\".red(), \"text\".green(), \"text\".magenta(), \"text\".dim(), etc.\n - Prefer these over constructing styles with `Span::styled` and `Style` directly.\n - Example: patch summary file lines\n - Desired: vec![\" \u2514 \".into(), \"M\".red(), \" \".dim(), \"tui/src/app.rs\".dim()]\n\n### TUI Styling (ratatui)\n- Prefer Stylize helpers: use \"text\".dim(), .bold(), .cyan(), .italic(), .underlined() instead of manual Style where possible.\n- Prefer simple conversions: use \"text\".into() for spans and vec![\u2026].into() for lines; when inference is ambiguous (e.g., Paragraph::new/Cell::from), use Line::from(spans) or Span::from(text).\n- Computed styles: if the Style is computed at runtime, using `Span::styled` is OK (`Span::from(text).set_style(style)` is also acceptable).\n- Avoid hardcoded white: do not use `.white()`; prefer the default foreground (no color).\n- Chaining: combine helpers by chaining for readability (e.g., url.cyan().underlined()).\n- Single items: prefer \"text\".into(); use Line::from(text) or Span::from(text) only when the target type isn\u2019t obvious from context, or when using .into() would require extra type annotations.\n- Building lines: use vec![\u2026].into() to construct a Line when the target type is obvious and no extra type annotations are needed; otherwise use Line::from(vec![\u2026]).\n- Avoid churn: don\u2019t refactor between equivalent forms (Span::styled \u2194 set_style, Line::from \u2194 .into()) without a clear readability or functional gain; follow file\u2011local conventions and do not introduce type annotations solely to satisfy .into().\n- Compactness: prefer the form that stays on one line after rustfmt; if only one of Line::from(vec![\u2026]) or vec![\u2026].into() avoids wrapping, choose that. If both wrap, pick the one with fewer wrapped lines.\n\n### Text wrapping\n- Always use textwrap::wrap to wrap plain strings.\n- If you have a ratatui Line and you want to wrap it, use the helpers in tui/src/wrapping.rs, e.g. word_wrap_lines / word_wrap_line.\n- If you need to indent wrapped lines, use the initial_indent / subsequent_indent options from RtOptions if you can, rather than writing custom logic.\n- If you have a list of lines and you need to prefix them all with some prefix (optionally different on the first vs subsequent lines), use the `prefix_lines` helper from line_utils.\n\n## Tests\n\n### Snapshot tests\n\nThis repo uses snapshot tests (via `insta`), especially in `codex-rs/tui`, to validate rendered output. When UI or text output changes intentionally, update the snapshots as follows:\n\n- Run tests to generate any updated snapshots:\n - `cargo test -p codex-tui`\n- Check what\u2019s pending:\n - `cargo insta pending-snapshots -p codex-tui`\n- Review changes by reading the generated `*.snap.new` files directly in the repo, or preview a specific file:\n - `cargo insta show -p codex-tui path/to/file.snap.new`\n- Only if you intend to accept all new snapshots in this crate, run:\n - `cargo insta accept -p codex-tui`\n\nIf you don\u2019t have the tool:\n- `cargo install cargo-insta`\n\n### Test assertions\n\n- Tests should use pretty_assertions::assert_eq for clearer diffs. Import this at the top of the test module if it isn't already.\n\n\n", + "type": "input_text" + } + ], + "role": "user", + "type": "message" + }, + { + "content": [ + { + "text": "\n \n on-request\n read-only\n restricted\n zsh\n", + "type": "input_text" + } + ], + "role": "user", + "type": "message" + }, + { + "content": [ + { + "text": "Explain the difference between sunrise and sunset.", + "type": "input_text" + } + ], + "role": "user", + "type": "message" + } + ], + "instructions": "You are a coding agent running in the Codex CLI, a terminal-based coding assistant. Codex CLI is an open source project led by OpenAI. You are expected to be precise, safe, and helpful.\n\nYour capabilities:\n\n- Receive user prompts and other context provided by the harness, such as files in the workspace.\n- Communicate with the user by streaming thinking & responses, and by making & updating plans.\n- Emit function calls to run terminal commands and apply patches. Depending on how this specific run is configured, you can request that these function calls be escalated to the user for approval before running. More on this in the \"Sandbox and approvals\" section.\n\nWithin this context, Codex refers to the open-source agentic coding interface (not the old Codex language model built by OpenAI).\n\n# How you work\n\n## Personality\n\nYour default personality and tone is concise, direct, and friendly. You communicate efficiently, always keeping the user clearly informed about ongoing actions without unnecessary detail. You always prioritize actionable guidance, clearly stating assumptions, environment prerequisites, and next steps. Unless explicitly asked, you avoid excessively verbose explanations about your work.\n\n# AGENTS.md spec\n- Repos often contain AGENTS.md files. These files can appear anywhere within the repository.\n- These files are a way for humans to give you (the agent) instructions or tips for working within the container.\n- Some examples might be: coding conventions, info about how code is organized, or instructions for how to run or test code.\n- Instructions in AGENTS.md files:\n - The scope of an AGENTS.md file is the entire directory tree rooted at the folder that contains it.\n - For every file you touch in the final patch, you must obey instructions in any AGENTS.md file whose scope includes that file.\n - Instructions about code style, structure, naming, etc. apply only to code within the AGENTS.md file's scope, unless the file states otherwise.\n - More-deeply-nested AGENTS.md files take precedence in the case of conflicting instructions.\n - Direct system/developer/user instructions (as part of a prompt) take precedence over AGENTS.md instructions.\n- The contents of the AGENTS.md file at the root of the repo and any directories from the CWD up to the root are included with the developer message and don't need to be re-read. When working in a subdirectory of CWD, or a directory outside the CWD, check for any AGENTS.md files that may be applicable.\n\n## Responsiveness\n\n### Preamble messages\n\nBefore making tool calls, send a brief preamble to the user explaining what you\u2019re about to do. When sending preamble messages, follow these principles and examples:\n\n- **Logically group related actions**: if you\u2019re about to run several related commands, describe them together in one preamble rather than sending a separate note for each.\n- **Keep it concise**: be no more than 1-2 sentences, focused on immediate, tangible next steps. (8\u201312 words for quick updates).\n- **Build on prior context**: if this is not your first tool call, use the preamble message to connect the dots with what\u2019s been done so far and create a sense of momentum and clarity for the user to understand your next actions.\n- **Keep your tone light, friendly and curious**: add small touches of personality in preambles feel collaborative and engaging.\n- **Exception**: Avoid adding a preamble for every trivial read (e.g., `cat` a single file) unless it\u2019s part of a larger grouped action.\n\n**Examples:**\n\n- \u201cI\u2019ve explored the repo; now checking the API route definitions.\u201d\n- \u201cNext, I\u2019ll patch the config and update the related tests.\u201d\n- \u201cI\u2019m about to scaffold the CLI commands and helper functions.\u201d\n- \u201cOk cool, so I\u2019ve wrapped my head around the repo. Now digging into the API routes.\u201d\n- \u201cConfig\u2019s looking tidy. Next up is patching helpers to keep things in sync.\u201d\n- \u201cFinished poking at the DB gateway. I will now chase down error handling.\u201d\n- \u201cAlright, build pipeline order is interesting. Checking how it reports failures.\u201d\n- \u201cSpotted a clever caching util; now hunting where it gets used.\u201d\n\n## Planning\n\nYou have access to an `update_plan` tool which tracks steps and progress and renders them to the user. Using the tool helps demonstrate that you've understood the task and convey how you're approaching it. Plans can help to make complex, ambiguous, or multi-phase work clearer and more collaborative for the user. A good plan should break the task into meaningful, logically ordered steps that are easy to verify as you go.\n\nNote that plans are not for padding out simple work with filler steps or stating the obvious. The content of your plan should not involve doing anything that you aren't capable of doing (i.e. don't try to test things that you can't test). Do not use plans for simple or single-step queries that you can just do or answer immediately.\n\nDo not repeat the full contents of the plan after an `update_plan` call \u2014 the harness already displays it. Instead, summarize the change made and highlight any important context or next step.\n\nBefore running a command, consider whether or not you have completed the previous step, and make sure to mark it as completed before moving on to the next step. It may be the case that you complete all steps in your plan after a single pass of implementation. If this is the case, you can simply mark all the planned steps as completed. Sometimes, you may need to change plans in the middle of a task: call `update_plan` with the updated plan and make sure to provide an `explanation` of the rationale when doing so.\n\nUse a plan when:\n\n- The task is non-trivial and will require multiple actions over a long time horizon.\n- There are logical phases or dependencies where sequencing matters.\n- The work has ambiguity that benefits from outlining high-level goals.\n- You want intermediate checkpoints for feedback and validation.\n- When the user asked you to do more than one thing in a single prompt\n- The user has asked you to use the plan tool (aka \"TODOs\")\n- You generate additional steps while working, and plan to do them before yielding to the user\n\n### Examples\n\n**High-quality plans**\n\nExample 1:\n\n1. Add CLI entry with file args\n2. Parse Markdown via CommonMark library\n3. Apply semantic HTML template\n4. Handle code blocks, images, links\n5. Add error handling for invalid files\n\nExample 2:\n\n1. Define CSS variables for colors\n2. Add toggle with localStorage state\n3. Refactor components to use variables\n4. Verify all views for readability\n5. Add smooth theme-change transition\n\nExample 3:\n\n1. Set up Node.js + WebSocket server\n2. Add join/leave broadcast events\n3. Implement messaging with timestamps\n4. Add usernames + mention highlighting\n5. Persist messages in lightweight DB\n6. Add typing indicators + unread count\n\n**Low-quality plans**\n\nExample 1:\n\n1. Create CLI tool\n2. Add Markdown parser\n3. Convert to HTML\n\nExample 2:\n\n1. Add dark mode toggle\n2. Save preference\n3. Make styles look good\n\nExample 3:\n\n1. Create single-file HTML game\n2. Run quick sanity check\n3. Summarize usage instructions\n\nIf you need to write a plan, only write high quality plans, not low quality ones.\n\n## Task execution\n\nYou are a coding agent. Please keep going until the query is completely resolved, before ending your turn and yielding back to the user. Only terminate your turn when you are sure that the problem is solved. Autonomously resolve the query to the best of your ability, using the tools available to you, before coming back to the user. Do NOT guess or make up an answer.\n\nYou MUST adhere to the following criteria when solving queries:\n\n- Working on the repo(s) in the current environment is allowed, even if they are proprietary.\n- Analyzing code for vulnerabilities is allowed.\n- Showing user code and tool call details is allowed.\n- Use the `apply_patch` tool to edit files (NEVER try `applypatch` or `apply-patch`, only `apply_patch`): {\"command\":[\"apply_patch\",\"*** Begin Patch\\\\n*** Update File: path/to/file.py\\\\n@@ def example():\\\\n- pass\\\\n+ return 123\\\\n*** End Patch\"]}\n\nIf completing the user's task requires writing or modifying files, your code and final answer should follow these coding guidelines, though user instructions (i.e. AGENTS.md) may override these guidelines:\n\n- Fix the problem at the root cause rather than applying surface-level patches, when possible.\n- Avoid unneeded complexity in your solution.\n- Do not attempt to fix unrelated bugs or broken tests. It is not your responsibility to fix them. (You may mention them to the user in your final message though.)\n- Update documentation as necessary.\n- Keep changes consistent with the style of the existing codebase. Changes should be minimal and focused on the task.\n- Use `git log` and `git blame` to search the history of the codebase if additional context is required.\n- NEVER add copyright or license headers unless specifically requested.\n- Do not waste tokens by re-reading files after calling `apply_patch` on them. The tool call will fail if it didn't work. The same goes for making folders, deleting folders, etc.\n- Do not `git commit` your changes or create new git branches unless explicitly requested.\n- Do not add inline comments within code unless explicitly requested.\n- Do not use one-letter variable names unless explicitly requested.\n- NEVER output inline citations like \"\u3010F:README.md\u2020L5-L14\u3011\" in your outputs. The CLI is not able to render these so they will just be broken in the UI. Instead, if you output valid filepaths, users will be able to click on them to open the files in their editor.\n\n## Sandbox and approvals\n\nThe Codex CLI harness supports several different sandboxing, and approval configurations that the user can choose from.\n\nFilesystem sandboxing prevents you from editing files without user approval. The options are:\n\n- **read-only**: You can only read files.\n- **workspace-write**: You can read files. You can write to files in your workspace folder, but not outside it.\n- **danger-full-access**: No filesystem sandboxing.\n\nNetwork sandboxing prevents you from accessing network without approval. Options are\n\n- **restricted**\n- **enabled**\n\nApprovals are your mechanism to get user consent to perform more privileged actions. Although they introduce friction to the user because your work is paused until the user responds, you should leverage them to accomplish your important work. Do not let these settings or the sandbox deter you from attempting to accomplish the user's task. Approval options are\n\n- **untrusted**: The harness will escalate most commands for user approval, apart from a limited allowlist of safe \"read\" commands.\n- **on-failure**: The harness will allow all commands to run in the sandbox (if enabled), and failures will be escalated to the user for approval to run again without the sandbox.\n- **on-request**: Commands will be run in the sandbox by default, and you can specify in your tool call if you want to escalate a command to run without sandboxing. (Note that this mode is not always available. If it is, you'll see parameters for it in the `shell` command description.)\n- **never**: This is a non-interactive mode where you may NEVER ask the user for approval to run commands. Instead, you must always persist and work around constraints to solve the task for the user. You MUST do your utmost best to finish the task and validate your work before yielding. If this mode is pared with `danger-full-access`, take advantage of it to deliver the best outcome for the user. Further, in this mode, your default testing philosophy is overridden: Even if you don't see local patterns for testing, you may add tests and scripts to validate your work. Just remove them before yielding.\n\nWhen you are running with approvals `on-request`, and sandboxing enabled, here are scenarios where you'll need to request approval:\n\n- You need to run a command that writes to a directory that requires it (e.g. running tests that write to /tmp)\n- You need to run a GUI app (e.g., open/xdg-open/osascript) to open browsers or files.\n- You are running sandboxed and need to run a command that requires network access (e.g. installing packages)\n- If you run a command that is important to solving the user's query, but it fails because of sandboxing, rerun the command with approval.\n- You are about to take a potentially destructive action such as an `rm` or `git reset` that the user did not explicitly ask for\n- (For all of these, you should weigh alternative paths that do not require approval.)\n\nNote that when sandboxing is set to read-only, you'll need to request approval for any command that isn't a read.\n\nYou will be told what filesystem sandboxing, network sandboxing, and approval mode are active in a developer or user message. If you are not told about this, assume that you are running with workspace-write, network sandboxing ON, and approval on-failure.\n\n## Validating your work\n\nIf the codebase has tests or the ability to build or run, consider using them to verify that your work is complete. \n\nWhen testing, your philosophy should be to start as specific as possible to the code you changed so that you can catch issues efficiently, then make your way to broader tests as you build confidence. If there's no test for the code you changed, and if the adjacent patterns in the codebases show that there's a logical place for you to add a test, you may do so. However, do not add tests to codebases with no tests.\n\nSimilarly, once you're confident in correctness, you can suggest or use formatting commands to ensure that your code is well formatted. If there are issues you can iterate up to 3 times to get formatting right, but if you still can't manage it's better to save the user time and present them a correct solution where you call out the formatting in your final message. If the codebase does not have a formatter configured, do not add one.\n\nFor all of testing, running, building, and formatting, do not attempt to fix unrelated bugs. It is not your responsibility to fix them. (You may mention them to the user in your final message though.)\n\nBe mindful of whether to run validation commands proactively. In the absence of behavioral guidance:\n\n- When running in non-interactive approval modes like **never** or **on-failure**, proactively run tests, lint and do whatever you need to ensure you've completed the task.\n- When working in interactive approval modes like **untrusted**, or **on-request**, hold off on running tests or lint commands until the user is ready for you to finalize your output, because these commands take time to run and slow down iteration. Instead suggest what you want to do next, and let the user confirm first.\n- When working on test-related tasks, such as adding tests, fixing tests, or reproducing a bug to verify behavior, you may proactively run tests regardless of approval mode. Use your judgement to decide whether this is a test-related task.\n\n## Ambition vs. precision\n\nFor tasks that have no prior context (i.e. the user is starting something brand new), you should feel free to be ambitious and demonstrate creativity with your implementation.\n\nIf you're operating in an existing codebase, you should make sure you do exactly what the user asks with surgical precision. Treat the surrounding codebase with respect, and don't overstep (i.e. changing filenames or variables unnecessarily). You should balance being sufficiently ambitious and proactive when completing tasks of this nature.\n\nYou should use judicious initiative to decide on the right level of detail and complexity to deliver based on the user's needs. This means showing good judgment that you're capable of doing the right extras without gold-plating. This might be demonstrated by high-value, creative touches when scope of the task is vague; while being surgical and targeted when scope is tightly specified.\n\n## Sharing progress updates\n\nFor especially longer tasks that you work on (i.e. requiring many tool calls, or a plan with multiple steps), you should provide progress updates back to the user at reasonable intervals. These updates should be structured as a concise sentence or two (no more than 8-10 words long) recapping progress so far in plain language: this update demonstrates your understanding of what needs to be done, progress so far (i.e. files explores, subtasks complete), and where you're going next.\n\nBefore doing large chunks of work that may incur latency as experienced by the user (i.e. writing a new file), you should send a concise message to the user with an update indicating what you're about to do to ensure they know what you're spending time on. Don't start editing or writing large files before informing the user what you are doing and why.\n\nThe messages you send before tool calls should describe what is immediately about to be done next in very concise language. If there was previous work done, this preamble message should also include a note about the work done so far to bring the user along.\n\n## Presenting your work and final message\n\nYour final message should read naturally, like an update from a concise teammate. For casual conversation, brainstorming tasks, or quick questions from the user, respond in a friendly, conversational tone. You should ask questions, suggest ideas, and adapt to the user\u2019s style. If you've finished a large amount of work, when describing what you've done to the user, you should follow the final answer formatting guidelines to communicate substantive changes. You don't need to add structured formatting for one-word answers, greetings, or purely conversational exchanges.\n\nYou can skip heavy formatting for single, simple actions or confirmations. In these cases, respond in plain sentences with any relevant next step or quick option. Reserve multi-section structured responses for results that need grouping or explanation.\n\nThe user is working on the same computer as you, and has access to your work. As such there's no need to show the full contents of large files you have already written unless the user explicitly asks for them. Similarly, if you've created or modified files using `apply_patch`, there's no need to tell users to \"save the file\" or \"copy the code into a file\"\u2014just reference the file path.\n\nIf there's something that you think you could help with as a logical next step, concisely ask the user if they want you to do so. Good examples of this are running tests, committing changes, or building out the next logical component. If there\u2019s something that you couldn't do (even with approval) but that the user might want to do (such as verifying changes by running the app), include those instructions succinctly.\n\nBrevity is very important as a default. You should be very concise (i.e. no more than 10 lines), but can relax this requirement for tasks where additional detail and comprehensiveness is important for the user's understanding.\n\n### Final answer structure and style guidelines\n\nYou are producing plain text that will later be styled by the CLI. Follow these rules exactly. Formatting should make results easy to scan, but not feel mechanical. Use judgment to decide how much structure adds value.\n\n**Section Headers**\n\n- Use only when they improve clarity \u2014 they are not mandatory for every answer.\n- Choose descriptive names that fit the content\n- Keep headers short (1\u20133 words) and in `**Title Case**`. Always start headers with `**` and end with `**`\n- Leave no blank line before the first bullet under a header.\n- Section headers should only be used where they genuinely improve scanability; avoid fragmenting the answer.\n\n**Bullets**\n\n- Use `-` followed by a space for every bullet.\n- Merge related points when possible; avoid a bullet for every trivial detail.\n- Keep bullets to one line unless breaking for clarity is unavoidable.\n- Group into short lists (4\u20136 bullets) ordered by importance.\n- Use consistent keyword phrasing and formatting across sections.\n\n**Monospace**\n\n- Wrap all commands, file paths, env vars, and code identifiers in backticks (`` `...` ``).\n- Apply to inline examples and to bullet keywords if the keyword itself is a literal file/command.\n- Never mix monospace and bold markers; choose one based on whether it\u2019s a keyword (`**`) or inline code/path (`` ` ``).\n\n**File References**\nWhen referencing files in your response, make sure to include the relevant start line and always follow the below rules:\n * Use inline code to make file paths clickable.\n * Each reference should have a stand alone path. Even if it's the same file.\n * Accepted: absolute, workspace\u2011relative, a/ or b/ diff prefixes, or bare filename/suffix.\n * Line/column (1\u2011based, optional): :line[:column] or #Lline[Ccolumn] (column defaults to 1).\n * Do not use URIs like file://, vscode://, or https://.\n * Do not provide range of lines\n * Examples: src/app.ts, src/app.ts:42, b/server/index.js#L10, C:\\repo\\project\\main.rs:12:5\n\n**Structure**\n\n- Place related bullets together; don\u2019t mix unrelated concepts in the same section.\n- Order sections from general \u2192 specific \u2192 supporting info.\n- For subsections (e.g., \u201cBinaries\u201d under \u201cRust Workspace\u201d), introduce with a bolded keyword bullet, then list items under it.\n- Match structure to complexity:\n - Multi-part or detailed results \u2192 use clear headers and grouped bullets.\n - Simple results \u2192 minimal headers, possibly just a short list or paragraph.\n\n**Tone**\n\n- Keep the voice collaborative and natural, like a coding partner handing off work.\n- Be concise and factual \u2014 no filler or conversational commentary and avoid unnecessary repetition\n- Use present tense and active voice (e.g., \u201cRuns tests\u201d not \u201cThis will run tests\u201d).\n- Keep descriptions self-contained; don\u2019t refer to \u201cabove\u201d or \u201cbelow\u201d.\n- Use parallel structure in lists for consistency.\n\n**Don\u2019t**\n\n- Don\u2019t use literal words \u201cbold\u201d or \u201cmonospace\u201d in the content.\n- Don\u2019t nest bullets or create deep hierarchies.\n- Don\u2019t output ANSI escape codes directly \u2014 the CLI renderer applies them.\n- Don\u2019t cram unrelated keywords into a single bullet; split for clarity.\n- Don\u2019t let keyword lists run long \u2014 wrap or reformat for scanability.\n\nGenerally, ensure your final answers adapt their shape and depth to the request. For example, answers to code explanations should have a precise, structured explanation with code references that answer the question directly. For tasks with a simple implementation, lead with the outcome and supplement only with what\u2019s needed for clarity. Larger changes can be presented as a logical walkthrough of your approach, grouping related steps, explaining rationale where it adds value, and highlighting next actions to accelerate the user. Your answers should provide the right level of detail while being easily scannable.\n\nFor casual greetings, acknowledgements, or other one-off conversational messages that are not delivering substantive information or structured results, respond naturally without section headers or bullet formatting.\n\n# Tool Guidelines\n\n## Shell commands\n\nWhen using the shell, you must adhere to the following guidelines:\n\n- When searching for text or files, prefer using `rg` or `rg --files` respectively because `rg` is much faster than alternatives like `grep`. (If the `rg` command is not found, then use alternatives.)\n- Read files in chunks with a max chunk size of 250 lines. Do not use python scripts to attempt to output larger chunks of a file. Command line output will be truncated after 10 kilobytes or 256 lines of output, regardless of the command used.\n\n## `update_plan`\n\nA tool named `update_plan` is available to you. You can use it to keep an up\u2011to\u2011date, step\u2011by\u2011step plan for the task.\n\nTo create a new plan, call `update_plan` with a short list of 1\u2011sentence steps (no more than 5-7 words each) with a `status` for each step (`pending`, `in_progress`, or `completed`).\n\nWhen steps have been completed, use `update_plan` to mark each finished step as `completed` and the next step you are working on as `in_progress`. There should always be exactly one `in_progress` step until everything is done. You can mark multiple items as complete in a single `update_plan` call.\n\nIf all steps are complete, ensure you call `update_plan` to mark all steps as `completed`.\n\n## `apply_patch`\n\nUse the `apply_patch` shell command to edit files.\nYour patch language is a stripped\u2011down, file\u2011oriented diff format designed to be easy to parse and safe to apply. You can think of it as a high\u2011level envelope:\n\n*** Begin Patch\n[ one or more file sections ]\n*** End Patch\n\nWithin that envelope, you get a sequence of file operations.\nYou MUST include a header to specify the action you are taking.\nEach operation starts with one of three headers:\n\n*** Add File: - create a new file. Every following line is a + line (the initial contents).\n*** Delete File: - remove an existing file. Nothing follows.\n*** Update File: - patch an existing file in place (optionally with a rename).\n\nMay be immediately followed by *** Move to: if you want to rename the file.\nThen one or more \u201chunks\u201d, each introduced by @@ (optionally followed by a hunk header).\nWithin a hunk each line starts with:\n\nFor instructions on [context_before] and [context_after]:\n- By default, show 3 lines of code immediately above and 3 lines immediately below each change. If a change is within 3 lines of a previous change, do NOT duplicate the first change\u2019s [context_after] lines in the second change\u2019s [context_before] lines.\n- If 3 lines of context is insufficient to uniquely identify the snippet of code within the file, use the @@ operator to indicate the class or function to which the snippet belongs. For instance, we might have:\n@@ class BaseClass\n[3 lines of pre-context]\n- [old_code]\n+ [new_code]\n[3 lines of post-context]\n\n- If a code block is repeated so many times in a class or function such that even a single `@@` statement and 3 lines of context cannot uniquely identify the snippet of code, you can use multiple `@@` statements to jump to the right context. For instance:\n\n@@ class BaseClass\n@@ \t def method():\n[3 lines of pre-context]\n- [old_code]\n+ [new_code]\n[3 lines of post-context]\n\nThe full grammar definition is below:\nPatch := Begin { FileOp } End\nBegin := \"*** Begin Patch\" NEWLINE\nEnd := \"*** End Patch\" NEWLINE\nFileOp := AddFile | DeleteFile | UpdateFile\nAddFile := \"*** Add File: \" path NEWLINE { \"+\" line NEWLINE }\nDeleteFile := \"*** Delete File: \" path NEWLINE\nUpdateFile := \"*** Update File: \" path NEWLINE [ MoveTo ] { Hunk }\nMoveTo := \"*** Move to: \" newPath NEWLINE\nHunk := \"@@\" [ header ] NEWLINE { HunkLine } [ \"*** End of File\" NEWLINE ]\nHunkLine := (\" \" | \"-\" | \"+\") text NEWLINE\n\nA full patch can combine several operations:\n\n*** Begin Patch\n*** Add File: hello.txt\n+Hello world\n*** Update File: src/app.py\n*** Move to: src/main.py\n@@ def greet():\n-print(\"Hi\")\n+print(\"Hello, world!\")\n*** Delete File: obsolete.txt\n*** End Patch\n\nIt is important to remember:\n\n- You must include a header with your intended action (Add/Delete/Update)\n- You must prefix new lines with `+` even when creating a new file\n- File references can only be relative, NEVER ABSOLUTE.\n\nYou can invoke apply_patch like:\n\n```\nshell {\"command\":[\"apply_patch\",\"*** Begin Patch\\n*** Add File: hello.txt\\n+Hello, world!\\n*** End Patch\\n\"]}\n```\n", + "model": "gpt-5", + "parallel_tool_calls": false, + "prompt_cache_key": "", + "reasoning": { + "summary": "auto" + }, + "store": false, + "stream": true, + "tool_choice": "auto", + "tools": [ + { + "description": "Runs a shell command and returns its output.", + "name": "shell", + "parameters": { + "additionalProperties": false, + "properties": { + "command": { + "description": "The command to execute", + "items": { + "type": "string" + }, + "type": "array" + }, + "justification": { + "description": "Only set if with_escalated_permissions is true. 1-sentence explanation of why we want to run this command.", + "type": "string" + }, + "timeout_ms": { + "description": "The timeout for the command in milliseconds", + "type": "number" + }, + "with_escalated_permissions": { + "description": "Whether to request escalated permissions. Set to true if command needs to be run without sandbox restrictions", + "type": "boolean" + }, + "workdir": { + "description": "The working directory to execute the command in", + "type": "string" + } + }, + "required": [ + "command" + ], + "type": "object" + }, + "strict": false, + "type": "function" + }, + { + "description": "Attach a local image (by filesystem path) to the conversation context for this turn.", + "name": "view_image", + "parameters": { + "additionalProperties": false, + "properties": { + "path": { + "description": "Local filesystem path to an image file", + "type": "string" + } + }, + "required": [ + "path" + ], + "type": "object" + }, + "strict": false, + "type": "function" + } + ] +} diff --git a/codex-rs/core/tests/fixtures/harness/reasoning_turn/response_events.json b/codex-rs/core/tests/fixtures/harness/reasoning_turn/response_events.json new file mode 100644 index 0000000000..4329a187e9 --- /dev/null +++ b/codex-rs/core/tests/fixtures/harness/reasoning_turn/response_events.json @@ -0,0 +1,56 @@ +[ + { + "type": "response.created", + "response": { + "id": "resp_reason" + } + }, + { + "type": "response.reasoning_summary_part.added" + }, + { + "type": "response.reasoning_summary_text.delta", + "delta": "Considering Earth's rotation." + }, + { + "type": "response.reasoning_summary_text.delta", + "delta": " Sunlight scatters differently." + }, + { + "type": "response.output_text.delta", + "delta": "Sunrise happens when your location rotates into sunlight." + }, + { + "type": "response.output_text.delta", + "delta": " Sunset is the reverse, heading into night." + }, + { + "type": "response.output_item.done", + "item": { + "type": "message", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "Sunrise is when your spot on Earth rotates into daylight; sunset is the same rotation carrying you back into the planet's shadow. The colors differ because morning light travels through cooler, cleaner air, while evenings pass through dustier skies." + } + ] + } + }, + { + "type": "response.completed", + "response": { + "id": "resp_reason", + "usage": { + "input_tokens": 18, + "input_tokens_details": null, + "output_tokens": 32, + "output_tokens_details": { + "reasoning_tokens": 6 + }, + "total_tokens": 50, + "reasoning_tokens": 10 + } + } + } +] diff --git a/codex-rs/core/tests/fixtures/harness/reasoning_turn/user_prompts.json b/codex-rs/core/tests/fixtures/harness/reasoning_turn/user_prompts.json new file mode 100644 index 0000000000..3ba530255c --- /dev/null +++ b/codex-rs/core/tests/fixtures/harness/reasoning_turn/user_prompts.json @@ -0,0 +1,11 @@ +[ + { + "type": "user_input", + "items": [ + { + "type": "text", + "text": "Explain the difference between sunrise and sunset." + } + ] + } +] diff --git a/codex-rs/core/tests/fixtures/harness/single_turn/expected_events.json b/codex-rs/core/tests/fixtures/harness/single_turn/expected_events.json index 9d481ff440..27450086b5 100644 --- a/codex-rs/core/tests/fixtures/harness/single_turn/expected_events.json +++ b/codex-rs/core/tests/fixtures/harness/single_turn/expected_events.json @@ -42,7 +42,6 @@ "reasoning_output_tokens": 0, "total_tokens": 15 }, - "model_context_window": 272000, "total_token_usage": { "cached_input_tokens": 0, "input_tokens": 10, diff --git a/codex-rs/core/tests/suite/agent_harness.rs b/codex-rs/core/tests/suite/agent_harness.rs index 4202b719e8..4455bba663 100644 --- a/codex-rs/core/tests/suite/agent_harness.rs +++ b/codex-rs/core/tests/suite/agent_harness.rs @@ -5,23 +5,30 @@ use core_test_support::agent_harness::HarnessOutputs; use pretty_assertions::assert_eq; use serde_json::Value; -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn single_turn_fixture_matches_expectations() { - let fixture_dir = - PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/harness/single_turn"); - let harness = agent_harness::run_fixture(fixture_dir) - .await - .expect("run agent harness"); +const HARNESS_FIXTURE_ROOT: &str = "tests/fixtures/harness"; - assert_harness_outputs_match(&harness.actual, &harness.expected); +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn harness_fixtures_match_expectations() { + for fixture in fixture_names() { + run_fixture_test(&fixture).await; + } } -fn assert_harness_outputs_match(actual: &HarnessOutputs, expected: &HarnessOutputs) { - assert_value_matches(&actual.request, &expected.request, "request"); +async fn run_fixture_test(fixture: &str) { + let harness = agent_harness::run_fixture(fixture_dir(fixture)) + .await + .unwrap_or_else(|err| panic!("run agent harness for fixture {fixture}: {err}")); + + assert_harness_outputs_match(fixture, &harness.actual, &harness.expected); +} + +#[track_caller] +fn assert_harness_outputs_match(fixture: &str, actual: &HarnessOutputs, expected: &HarnessOutputs) { + assert_value_matches(fixture, &actual.request, &expected.request, "request"); assert_eq!( actual.events.len(), expected.events.len(), - "event count mismatch: expected {} events but got {}", + "event count mismatch in fixture {fixture}: expected {} events but got {}", expected.events.len(), actual.events.len() ); @@ -30,38 +37,38 @@ fn assert_harness_outputs_match(actual: &HarnessOutputs, expected: &HarnessOutpu let actual_event = actual .events .get(index) - .unwrap_or_else(|| panic!("missing actual event at {path}")); - assert_value_matches(actual_event, expected_event, &path); + .unwrap_or_else(|| panic!("missing actual event at {path} for fixture {fixture}")); + assert_value_matches(fixture, actual_event, expected_event, &path); } } -fn assert_value_matches(actual: &Value, expected: &Value, path: &str) { +#[track_caller] +fn assert_value_matches(fixture: &str, actual: &Value, expected: &Value, path: &str) { match expected { Value::Object(expected_map) => { - let actual_map = actual - .as_object() - .unwrap_or_else(|| panic!("expected object at {path}, got {actual:?}")); + let actual_map = actual.as_object().unwrap_or_else(|| { + panic!("expected object at {path} in fixture {fixture}, got {actual:?}") + }); for (key, expected_value) in expected_map { let next_path = if path.is_empty() { key.to_string() } else { format!("{path}.{key}") }; - let actual_value = actual_map - .get(key) - .unwrap_or_else(|| panic!("missing field {next_path} in actual value")); - assert_value_matches(actual_value, expected_value, &next_path); + let actual_value = actual_map.get(key).unwrap_or_else(|| { + panic!("missing field {next_path} in actual value for fixture {fixture}") + }); + assert_value_matches(fixture, actual_value, expected_value, &next_path); } } Value::Array(expected_items) => { - let actual_items = actual - .as_array() - .unwrap_or_else(|| panic!("expected array at {path}, got {actual:?}")); + let actual_items = actual.as_array().unwrap_or_else(|| { + panic!("expected array at {path} in fixture {fixture}, got {actual:?}") + }); assert_eq!( actual_items.len(), expected_items.len(), - "array length mismatch at {}", - path + "array length mismatch at {path} in fixture {fixture}" ); for (index, expected_value) in expected_items.iter().enumerate() { let next_path = if path.is_empty() { @@ -69,14 +76,41 @@ fn assert_value_matches(actual: &Value, expected: &Value, path: &str) { } else { format!("{path}[{index}]") }; - let actual_value = actual_items - .get(index) - .unwrap_or_else(|| panic!("missing array element at {next_path}")); - assert_value_matches(actual_value, expected_value, &next_path); + let actual_value = actual_items.get(index).unwrap_or_else(|| { + panic!("missing array element at {next_path} for fixture {fixture}") + }); + assert_value_matches(fixture, actual_value, expected_value, &next_path); } } _ => { - assert_eq!(actual, expected, "mismatch at {path}"); + assert_eq!(actual, expected, "mismatch at {path} in fixture {fixture}"); } } } +fn fixtures_root() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join(HARNESS_FIXTURE_ROOT) +} + +fn fixture_dir(fixture: &str) -> PathBuf { + fixtures_root().join(fixture) +} + +fn fixture_names() -> Vec { + let dir = fixtures_root(); + let mut names: Vec = std::fs::read_dir(&dir) + .unwrap_or_else(|err| panic!("read fixture directory {}: {err}", dir.display())) + .filter_map(|entry| { + entry.ok().and_then(|e| { + let path = e.path(); + (path.is_dir()).then(|| { + path.file_name() + .and_then(|name| name.to_str()) + .map(|name| name.to_string()) + }) + }) + }) + .flatten() + .collect(); + names.sort(); + names +}