diff --git a/.codespellrc b/.codespellrc index f01272c61d..eefde42a4e 100644 --- a/.codespellrc +++ b/.codespellrc @@ -1,6 +1,6 @@ [codespell] # Ref: https://github.com/codespell-project/codespell#using-a-config-file -skip = .git*,vendor,*-lock.yaml,*.lock,.codespellrc,*test.ts +skip = .git*,vendor,*-lock.yaml,*.lock,.codespellrc,*test.ts,*.jsonl check-hidden = true ignore-regex = ^\s*"image/\S+": ".*|\b(afterAll)\b ignore-words-list = ratatui,ser diff --git a/.github/ISSUE_TEMPLATE/4-feature-request.yml b/.github/ISSUE_TEMPLATE/4-feature-request.yml new file mode 100644 index 0000000000..fc95a67ec2 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/4-feature-request.yml @@ -0,0 +1,31 @@ +name: 🎁 Feature Request +description: Propose a new feature for Codex +labels: + - enhancement + - needs triage +body: + - type: markdown + attributes: + value: | + Is Codex missing a feature that you'd like to see? Feel free to propose it here. + + Before you submit a feature: + 1. Search existing issues for similar features. If you find one, 👍 it rather than opening a new one. + 2. The Codex team will try to balance the varying needs of the community when prioritizing or rejecting new features. Not all features will be accepted. See [Contributing](https://github.com/openai/codex#contributing) for more details. + + - type: textarea + id: feature + attributes: + label: What feature would you like to see? + validations: + required: true + - type: textarea + id: author + attributes: + label: Are you interested in implementing this feature? + description: Please wait for acknowledgement before implementing or opening a PR. + - type: textarea + id: notes + attributes: + label: Additional information + description: Is there anything else you think we should know? diff --git a/.github/actions/codex/bun.lock b/.github/actions/codex/bun.lock index 82e12cc4b6..cd2c52143c 100644 --- a/.github/actions/codex/bun.lock +++ b/.github/actions/codex/bun.lock @@ -8,8 +8,8 @@ "@actions/github": "^6.0.1", }, "devDependencies": { - "@types/bun": "^1.2.19", - "@types/node": "^24.1.0", + "@types/bun": "^1.2.20", + "@types/node": "^24.3.0", "prettier": "^3.6.2", "typescript": "^5.9.2", }, @@ -48,15 +48,15 @@ "@octokit/types": ["@octokit/types@13.10.0", "", { "dependencies": { "@octokit/openapi-types": "^24.2.0" } }, "sha512-ifLaO34EbbPj0Xgro4G5lP5asESjwHracYJvVaPIyXMuiuXLlhic3S47cBdTb+jfODkTE5YtGCLt3Ay3+J97sA=="], - "@types/bun": ["@types/bun@1.2.19", "", { "dependencies": { "bun-types": "1.2.19" } }, "sha512-d9ZCmrH3CJ2uYKXQIUuZ/pUnTqIvLDS0SK7pFmbx8ma+ziH/FRMoAq5bYpRG7y+w1gl+HgyNZbtqgMq4W4e2Lg=="], + "@types/bun": ["@types/bun@1.2.20", "", { "dependencies": { "bun-types": "1.2.20" } }, "sha512-dX3RGzQ8+KgmMw7CsW4xT5ITBSCrSbfHc36SNT31EOUg/LA9JWq0VDdEXDRSe1InVWpd2yLUM1FUF/kEOyTzYA=="], - "@types/node": ["@types/node@24.1.0", "", { "dependencies": { "undici-types": "~7.8.0" } }, "sha512-ut5FthK5moxFKH2T1CUOC6ctR67rQRvvHdFLCD2Ql6KXmMuCrjsSsRI9UsLCm9M18BMwClv4pn327UvB7eeO1w=="], + "@types/node": ["@types/node@24.3.0", "", { "dependencies": { "undici-types": "~7.10.0" } }, "sha512-aPTXCrfwnDLj4VvXrm+UUCQjNEvJgNA8s5F1cvwQU+3KNltTOkBm1j30uNLyqqPNe7gE3KFzImYoZEfLhp4Yow=="], "@types/react": ["@types/react@19.1.8", "", { "dependencies": { "csstype": "^3.0.2" } }, "sha512-AwAfQ2Wa5bCx9WP8nZL2uMZWod7J7/JSplxbTmBQ5ms6QpqNYm672H0Vu9ZVKVngQ+ii4R/byguVEUZQyeg44g=="], "before-after-hook": ["before-after-hook@2.2.3", "", {}, "sha512-NzUnlZexiaH/46WDhANlyR2bXRopNg4F/zuSA3OpZnllCUgRaOF2znDioDWrmbNVsuZk6l9pMquQB38cfBZwkQ=="], - "bun-types": ["bun-types@1.2.19", "", { "dependencies": { "@types/node": "*" }, "peerDependencies": { "@types/react": "^19" } }, "sha512-uAOTaZSPuYsWIXRpj7o56Let0g/wjihKCkeRqUBhlLVM/Bt+Fj9xTo+LhC1OV1XDaGkz4hNC80et5xgy+9KTHQ=="], + "bun-types": ["bun-types@1.2.20", "", { "dependencies": { "@types/node": "*" }, "peerDependencies": { "@types/react": "^19" } }, "sha512-pxTnQYOrKvdOwyiyd/7sMt9yFOenN004Y6O4lCcCUoKVej48FS5cvTw9geRaEcB9TsDZaJKAxPTVvi8tFsVuXA=="], "csstype": ["csstype@3.1.3", "", {}, "sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw=="], @@ -72,7 +72,7 @@ "undici": ["undici@5.29.0", "", { "dependencies": { "@fastify/busboy": "^2.0.0" } }, "sha512-raqeBD6NQK4SkWhQzeYKd1KmIG6dllBOTt55Rmkt4HtI9mwdWtJljnrXjAFUBLTSN67HWrOIZ3EPF4kjUw80Bg=="], - "undici-types": ["undici-types@7.8.0", "", {}, "sha512-9UJ2xGDvQ43tYyVMpuHlsgApydB8ZKfVYTsLDhXkFL/6gfkp+U8xTGdh8pMJv1SpZna0zxG1DwsKZsreLbXBxw=="], + "undici-types": ["undici-types@7.10.0", "", {}, "sha512-t5Fy/nfn+14LuOc2KNYg75vZqClpAiqscVvMygNnlsHBFpSXdJaYtXMcdNLpl/Qvc3P2cB3s6lOV51nqsFq4ag=="], "universal-user-agent": ["universal-user-agent@6.0.1", "", {}, "sha512-yCzhz6FN2wU1NiiQRogkTQszlQSlpWaw8SvVegAc+bDxbzHgh1vX8uIe8OYyMH6DwH+sdTJsgMl36+mSMdRJIQ=="], @@ -82,7 +82,7 @@ "@octokit/plugin-rest-endpoint-methods/@octokit/types": ["@octokit/types@12.6.0", "", { "dependencies": { "@octokit/openapi-types": "^20.0.0" } }, "sha512-1rhSOfRa6H9w4YwK0yrf5faDaDTb+yLyBUKOCV4xtCDB5VmIPqd/v9yr9o6SAzOAlRxMiRiCic6JVM1/kunVkw=="], - "bun-types/@types/node": ["@types/node@24.0.13", "", { "dependencies": { "undici-types": "~7.8.0" } }, "sha512-Qm9OYVOFHFYg3wJoTSrz80hoec5Lia/dPp84do3X7dZvLikQvM1YpmvTBEdIr/e+U8HTkFjLHLnl78K/qjf+jQ=="], + "bun-types/@types/node": ["@types/node@24.2.1", "", { "dependencies": { "undici-types": "~7.10.0" } }, "sha512-DRh5K+ka5eJic8CjH7td8QpYEV6Zo10gfRkjHCO3weqZHWDtAaSTFtl4+VMqOJ4N5jcuhZ9/l+yy8rVgw7BQeQ=="], "@octokit/plugin-paginate-rest/@octokit/types/@octokit/openapi-types": ["@octokit/openapi-types@20.0.0", "", {}, "sha512-EtqRBEjp1dL/15V7WiX5LJMIxxkdiGJnabzYx5Apx4FkQIFgAfKumXeYAqqJCj1s+BMX4cPFIFC4OLCR6stlnA=="], diff --git a/.github/actions/codex/package.json b/.github/actions/codex/package.json index 6c7ae9002b..8e0e540314 100644 --- a/.github/actions/codex/package.json +++ b/.github/actions/codex/package.json @@ -13,8 +13,8 @@ "@actions/github": "^6.0.1" }, "devDependencies": { - "@types/bun": "^1.2.19", - "@types/node": "^24.1.0", + "@types/bun": "^1.2.20", + "@types/node": "^24.3.0", "prettier": "^3.6.2", "typescript": "^5.9.2" } diff --git a/.github/codex/home/config.toml b/.github/codex/home/config.toml index bb1b362bb6..8e800ca068 100644 --- a/.github/codex/home/config.toml +++ b/.github/codex/home/config.toml @@ -1,3 +1,3 @@ -model = "o3" +model = "gpt-5" # Consider setting [mcp_servers] here! diff --git a/.github/dotslash-config.json b/.github/dotslash-config.json index 82b9eb93b3..f3b85b2b4b 100644 --- a/.github/dotslash-config.json +++ b/.github/dotslash-config.json @@ -17,6 +17,10 @@ "linux-aarch64": { "regex": "^codex-aarch64-unknown-linux-musl\\.zst$", "path": "codex" + }, + "windows-x86_64": { + "regex": "^codex-x86_64-pc-windows-msvc\\.exe\\.zst$", + "path": "codex.exe" } } } diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000000..03cedab29c --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,6 @@ +# External (non-OpenAI) Pull Request Requirements + +Before opening this Pull Request, please read the "Contributing" section of the README or your PR may be closed: +https://github.com/openai/codex#contributing + +If your PR conforms to our contribution guidelines, replace this text with a detailed and high quality description of your changes. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 32fb070924..e62d7bca56 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,7 +12,7 @@ jobs: NODE_OPTIONS: --max-old-space-size=4096 steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: Setup Node.js uses: actions/setup-node@v4 diff --git a/.github/workflows/codespell.yml b/.github/workflows/codespell.yml index 5737a6bca8..ce03a430fc 100644 --- a/.github/workflows/codespell.yml +++ b/.github/workflows/codespell.yml @@ -18,7 +18,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: Annotate locations with typos uses: codespell-project/codespell-problem-matcher@b80729f885d32f78a716c2f107b4db1025001c42 # v1 - name: Codespell diff --git a/.github/workflows/codex.yml b/.github/workflows/codex.yml index 6ca0d57f46..cb31275534 100644 --- a/.github/workflows/codex.yml +++ b/.github/workflows/codex.yml @@ -37,7 +37,7 @@ jobs: # Codex is not going to run. - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v5 - uses: dtolnay/rust-toolchain@1.88 with: diff --git a/.github/workflows/rust-ci.yml b/.github/workflows/rust-ci.yml index 97075a1f07..29733a46f6 100644 --- a/.github/workflows/rust-ci.yml +++ b/.github/workflows/rust-ci.yml @@ -1,42 +1,76 @@ name: rust-ci on: - pull_request: - branches: - - main - paths: - - "codex-rs/**" - - ".github/**" + pull_request: {} push: branches: - main - workflow_dispatch: -# For CI, we build in debug (`--profile dev`) rather than release mode so we -# get signal faster. +# CI builds in debug (dev) for faster signal. jobs: - # CI that don't need specific targets + # --- Detect what changed (always runs) ------------------------------------- + changed: + name: Detect changed areas + runs-on: ubuntu-24.04 + outputs: + codex: ${{ steps.detect.outputs.codex }} + workflows: ${{ steps.detect.outputs.workflows }} + steps: + - uses: actions/checkout@v5 + with: + fetch-depth: 0 + - name: Detect changed paths (no external action) + id: detect + shell: bash + run: | + set -euo pipefail + + if [[ "${{ github.event_name }}" == "pull_request" ]]; then + BASE_SHA='${{ github.event.pull_request.base.sha }}' + echo "Base SHA: $BASE_SHA" + # List files changed between base and current HEAD (merge-base aware) + mapfile -t files < <(git diff --name-only --no-renames "$BASE_SHA"...HEAD) + else + # On push / manual runs, default to running everything + files=("codex-rs/force" ".github/force") + fi + + codex=false + workflows=false + for f in "${files[@]}"; do + [[ $f == codex-rs/* ]] && codex=true + [[ $f == .github/* ]] && workflows=true + done + + echo "codex=$codex" >> "$GITHUB_OUTPUT" + echo "workflows=$workflows" >> "$GITHUB_OUTPUT" + + # --- CI that doesn't need specific targets --------------------------------- general: name: Format / etc runs-on: ubuntu-24.04 + needs: changed + if: ${{ needs.changed.outputs.codex == 'true' || needs.changed.outputs.workflows == 'true' || github.event_name == 'push' }} defaults: run: working-directory: codex-rs - steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - uses: dtolnay/rust-toolchain@1.88 with: components: rustfmt - name: cargo fmt run: cargo fmt -- --config imports_granularity=Item --check - # CI to validate on different os/targets + # --- CI to validate on different os/targets -------------------------------- lint_build_test: - name: ${{ matrix.runner }} - ${{ matrix.target }} + name: ${{ matrix.runner }} - ${{ matrix.target }}${{ matrix.profile == 'release' && ' (release)' || '' }} runs-on: ${{ matrix.runner }} timeout-minutes: 30 + needs: changed + # Keep job-level if to avoid spinning up runners when not needed + if: ${{ needs.changed.outputs.codex == 'true' || needs.changed.outputs.workflows == 'true' || github.event_name == 'push' }} defaults: run: working-directory: codex-rs @@ -44,26 +78,40 @@ jobs: strategy: fail-fast: false matrix: - # Note: While Codex CLI does not support Windows today, we include - # Windows in CI to ensure the code at least builds there. include: - runner: macos-14 target: aarch64-apple-darwin + profile: dev - runner: macos-14 target: x86_64-apple-darwin + profile: dev - runner: ubuntu-24.04 target: x86_64-unknown-linux-musl + profile: dev - runner: ubuntu-24.04 target: x86_64-unknown-linux-gnu + profile: dev - runner: ubuntu-24.04-arm target: aarch64-unknown-linux-musl + profile: dev - runner: ubuntu-24.04-arm target: aarch64-unknown-linux-gnu + profile: dev - runner: windows-latest target: x86_64-pc-windows-msvc + profile: dev + + # Also run representative release builds on Mac and Linux because + # there could be release-only build errors we want to catch. + - runner: macos-14 + target: aarch64-apple-darwin + profile: release + - runner: ubuntu-24.04 + target: x86_64-unknown-linux-musl + profile: release steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - uses: dtolnay/rust-toolchain@1.88 with: targets: ${{ matrix.target }} @@ -77,7 +125,7 @@ jobs: ~/.cargo/registry/cache/ ~/.cargo/git/db/ ${{ github.workspace }}/codex-rs/target/ - key: cargo-${{ matrix.runner }}-${{ matrix.target }}-${{ hashFiles('**/Cargo.lock') }} + key: cargo-${{ matrix.runner }}-${{ matrix.target }}-${{ matrix.profile }}-${{ hashFiles('**/Cargo.lock') }} - if: ${{ matrix.target == 'x86_64-unknown-linux-musl' || matrix.target == 'aarch64-unknown-linux-musl'}} name: Install musl build tools @@ -86,7 +134,6 @@ jobs: - name: cargo clippy id: clippy - continue-on-error: true run: cargo clippy --target ${{ matrix.target }} --all-features --tests -- -D warnings # Running `cargo build` from the workspace root builds the workspace using @@ -96,14 +143,18 @@ jobs: # slower, we only do this for the x86_64-unknown-linux-gnu target. - name: cargo build individual crates id: build - if: ${{ matrix.target == 'x86_64-unknown-linux-gnu' }} + if: ${{ matrix.target == 'x86_64-unknown-linux-gnu' && matrix.profile != 'release' }} continue-on-error: true - run: find . -name Cargo.toml -mindepth 2 -maxdepth 2 -print0 | xargs -0 -n1 -I{} bash -c 'cd "$(dirname "{}")" && cargo build' + run: | + find . -name Cargo.toml -mindepth 2 -maxdepth 2 -print0 \ + | xargs -0 -n1 -I{} bash -c 'cd "$(dirname "{}")" && cargo build --profile ${{ matrix.profile }}' - name: cargo test id: test + # `cargo test` takes too long for release builds to run them on every PR + if: ${{ matrix.profile != 'release' }} continue-on-error: true - run: cargo test --all-features --target ${{ matrix.target }} + run: cargo test --all-features --target ${{ matrix.target }} --profile ${{ matrix.profile }} env: RUST_BACKTRACE: 1 @@ -116,3 +167,27 @@ jobs: run: | echo "One or more checks failed (clippy, build, or test). See logs for details." exit 1 + + # --- Gatherer job that you mark as the ONLY required status ----------------- + results: + name: CI results (required) + needs: [changed, general, lint_build_test] + if: always() + runs-on: ubuntu-24.04 + steps: + - name: Summarize + shell: bash + run: | + echo "general: ${{ needs.general.result }}" + echo "matrix : ${{ needs.lint_build_test.result }}" + + # If nothing relevant changed (PR touching only root README, etc.), + # declare success regardless of other jobs. + if [[ '${{ needs.changed.outputs.codex }}' != 'true' && '${{ needs.changed.outputs.workflows }}' != 'true' && '${{ github.event_name }}' != 'push' ]]; then + echo 'No relevant changes -> CI not required.' + exit 0 + fi + + # Otherwise require the jobs to have succeeded + [[ '${{ needs.general.result }}' == 'success' ]] || { echo 'general failed'; exit 1; } + [[ '${{ needs.lint_build_test.result }}' == 'success' ]] || { echo 'matrix failed'; exit 1; } diff --git a/.github/workflows/rust-release.yml b/.github/workflows/rust-release.yml index 8bf7361da9..c00a7a5e03 100644 --- a/.github/workflows/rust-release.yml +++ b/.github/workflows/rust-release.yml @@ -19,7 +19,7 @@ jobs: tag-check: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Validate tag matches Cargo.toml version shell: bash @@ -74,7 +74,7 @@ jobs: target: x86_64-pc-windows-msvc steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - uses: dtolnay/rust-toolchain@1.88 with: targets: ${{ matrix.target }} @@ -87,7 +87,7 @@ jobs: ~/.cargo/registry/cache/ ~/.cargo/git/db/ ${{ github.workspace }}/codex-rs/target/ - key: cargo-release-${{ matrix.runner }}-${{ matrix.target }}-${{ hashFiles('**/Cargo.lock') }} + key: cargo-release-${{ matrix.runner }}-${{ matrix.target }}-release-${{ hashFiles('**/Cargo.lock') }} - if: ${{ matrix.target == 'x86_64-unknown-linux-musl' || matrix.target == 'aarch64-unknown-linux-musl'}} name: Install musl build tools @@ -117,10 +117,11 @@ jobs: dest="dist/${{ matrix.target }}" # For compatibility with environments that lack the `zstd` tool we - # additionally create a `.tar.gz` alongside every single binary that - # we publish. The end result is: + # additionally create a `.tar.gz` for all platforms and `.zip` for + # Windows alongside every single binary that we publish. The end result is: # codex-.zst (existing) # codex-.tar.gz (new) + # codex-.zip (only for Windows) # 1. Produce a .tar.gz for every file in the directory *before* we # run `zstd --rm`, because that flag deletes the original files. @@ -128,13 +129,20 @@ jobs: base="$(basename "$f")" # Skip files that are already archives (shouldn't happen, but be # safe). - if [[ "$base" == *.tar.gz ]]; then + if [[ "$base" == *.tar.gz || "$base" == *.zip ]]; then continue fi # Create per-binary tar.gz tar -C "$dest" -czf "$dest/${base}.tar.gz" "$base" + # Create zip archive for Windows binaries + # Must run from inside the dest dir so 7z won't + # embed the directory path inside the zip. + if [[ "${{ matrix.runner }}" == windows* ]]; then + (cd "$dest" && 7z a "${base}.zip" "$base") + fi + # Also create .zst (existing behaviour) *and* remove the original # uncompressed binary to keep the directory small. zstd -T0 -19 --rm "$dest/$base" @@ -155,7 +163,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v5 - uses: actions/download-artifact@v4 with: diff --git a/.vscode/launch.json b/.vscode/launch.json index 618207f301..d87ce482e4 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -1,18 +1,22 @@ { - "version": "0.2.0", - "configurations": [ - { - "type": "lldb", - "request": "launch", - "name": "Cargo launch", - "cargo": { - "cwd": "${workspaceFolder}/codex-rs", - "args": [ - "build", - "--bin=codex-tui" - ] - }, - "args": [] - } - ] + "version": "0.2.0", + "configurations": [ + { + "type": "lldb", + "request": "launch", + "name": "Cargo launch", + "cargo": { + "cwd": "${workspaceFolder}/codex-rs", + "args": ["build", "--bin=codex-tui"] + }, + "args": [] + }, + { + "type": "lldb", + "request": "attach", + "name": "Attach to running codex CLI", + "pid": "${command:pickProcess}", + "sourceLanguages": ["rust"] + } + ] } diff --git a/AGENTS.md b/AGENTS.md index 5c3f659c35..f1744a87e5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -2,10 +2,41 @@ In the codex-rs folder where the rust code lives: +- Crate names are prefixed with `codex-`. For examole, the `core` folder's crate is named `codex-core` +- When using format! and you can inline variables into {}, always do that. - Never add or modify any code related to `CODEX_SANDBOX_NETWORK_DISABLED_ENV_VAR` or `CODEX_SANDBOX_ENV_VAR`. - You operate in a sandbox where `CODEX_SANDBOX_NETWORK_DISABLED=1` will be set whenever you use the `shell` tool. Any existing code that uses `CODEX_SANDBOX_NETWORK_DISABLED_ENV_VAR` was authored with this fact in mind. It is often used to early exit out of tests that the author knew you would not be able to run given your sandbox limitations. - Similarly, when you spawn a process using Seatbelt (`/usr/bin/sandbox-exec`), `CODEX_SANDBOX=seatbelt` will be set on the child process. Integration tests that want to run Seatbelt themselves cannot be run under Seatbelt, so checks for `CODEX_SANDBOX=seatbelt` are also often used to early exit out of tests, as appropriate. -Before creating a pull request with changes to `codex-rs`, run `just fmt` (in `codex-rs` directory) to format the code and `just fix` (in `codex-rs` directory) to fix any linter issues in the code, ensure the test suite passes by running `cargo test --all-features` in the `codex-rs` directory. +Before finalizing a change to `codex-rs`, run `just fmt` (in `codex-rs` directory) to format the code and `just fix` (in `codex-rs` directory) to fix any linter issues in the code. Additionally, run the tests: +1. Run the test for the specific project that was changed. For example, if changes were made in `codex-rs/tui`, run `cargo test -p codex-tui`. +2. Once those pass, if any changes were made in common, core, or protocol, run the complete test suite with `cargo test --all-features`. -When making individual changes prefer running tests on individual files or projects first. +## TUI style conventions + +See `codex-rs/tui/styles.md`. + +## TUI code conventions + +- Use concise styling helpers from ratatui’s Stylize trait. + - Basic spans: use "text".into() + - Styled spans: use "text".red(), "text".green(), "text".magenta(), "text".dim(), etc. + - Prefer these over constructing styles with `Span::styled` and `Style` directly. + - Example: patch summary file lines + - Desired: vec![" └ ".into(), "M".red(), " ".dim(), "tui/src/app.rs".dim()] + +## Snapshot tests + +This repo uses snapshot tests (via `insta`), especially in `codex-rs/tui`, to validate rendered output. When UI or text output changes intentionally, update the snapshots as follows: + +- Run tests to generate any updated snapshots: + - `cargo test -p codex-tui` +- Check what’s pending: + - `cargo insta pending-snapshots -p codex-tui` +- Review changes by reading the generated `*.snap.new` files directly in the repo, or preview a specific file: + - `cargo insta show -p codex-tui path/to/file.snap.new` +- Only if you intend to accept all new snapshots in this crate, run: + - `cargo insta accept -p codex-tui` + +If you don’t have the tool: +- `cargo install cargo-insta` diff --git a/README.md b/README.md index 89063b7e7b..f7428dc35e 100644 --- a/README.md +++ b/README.md @@ -18,8 +18,11 @@ - [Quickstart](#quickstart) - [Installing and running Codex CLI](#installing-and-running-codex-cli) - [Using Codex with your ChatGPT plan](#using-codex-with-your-chatgpt-plan) - - [Connecting through VPS or remote](#connecting-through-vps-or-remote) + - [Connecting on a "Headless" Machine](#connecting-on-a-headless-machine) + - [Authenticate locally and copy your credentials to the "headless" machine](#authenticate-locally-and-copy-your-credentials-to-the-headless-machine) + - [Connecting through VPS or remote](#connecting-through-vps-or-remote) - [Usage-based billing alternative: Use an OpenAI API key](#usage-based-billing-alternative-use-an-openai-api-key) + - [Forcing a specific auth method (advanced)](#forcing-a-specific-auth-method-advanced) - [Choosing Codex's level of autonomy](#choosing-codexs-level-of-autonomy) - [**1. Read/write**](#1-readwrite) - [**2. Read-only**](#2-read-only) @@ -99,17 +102,47 @@ Each archive contains a single entry with the platform baked into the name (e.g. Codex CLI login

-After you run `codex` select Sign in with ChatGPT. You'll need a Plus, Pro, or Team ChatGPT account, and will get access to our latest models, including `gpt-5`, at no extra cost to your plan. (Enterprise is coming soon.) +Run `codex` and select **Sign in with ChatGPT**. You'll need a Plus, Pro, or Team ChatGPT account, and will get access to our latest models, including `gpt-5`, at no extra cost to your plan. (Enterprise is coming soon.) -> Important: If you've used the Codex CLI before, you'll need to follow these steps to migrate from usage-based billing with your API key: +> Important: If you've used the Codex CLI before, follow these steps to migrate from usage-based billing with your API key: > -> 1. Update the CLI with `codex update` and ensure `codex --version` is greater than 0.13 -> 2. Ensure that there is no `OPENAI_API_KEY` environment variable set. (Check that `env | grep 'OPENAI_API_KEY'` returns empty) +> 1. Update the CLI and ensure `codex --version` is `0.20.0` or later +> 2. Delete `~/.codex/auth.json` (this should be `C:\Users\USERNAME\.codex\auth.json` on Windows) > 3. Run `codex login` again If you encounter problems with the login flow, please comment on [this issue](https://github.com/openai/codex/issues/1243). -### Connecting through VPS or remote +### Connecting on a "Headless" Machine + +Today, the login process entails running a server on `localhost:1455`. If you are on a "headless" server, such as a Docker container or are `ssh`'d into a remote machine, loading `localhost:1455` in the browser on your local machine will not automatically connect to the webserver running on the _headless_ machine, so you must use one of the following workarounds: + +#### Authenticate locally and copy your credentials to the "headless" machine + +The easiest solution is likely to run through the `codex login` process on your local machine such that `localhost:1455` _is_ accessible in your web browser. When you complete the authentication process, an `auth.json` file should be available at `$CODEX_HOME/auth.json` (on Mac/Linux, `$CODEX_HOME` defaults to `~/.codex` whereas on Windows, it defaults to `%USERPROFILE%\.codex`). + +Because the `auth.json` file is not tied to a specific host, once you complete the authentication flow locally, you can copy the `$CODEX_HOME/auth.json` file to the headless machine and then `codex` should "just work" on that machine. Note to copy a file to a Docker container, you can do: + +```shell +# substitute MY_CONTAINER with the name or id of your Docker container: +CONTAINER_HOME=$(docker exec MY_CONTAINER printenv HOME) +docker exec MY_CONTAINER mkdir -p "$CONTAINER_HOME/.codex" +docker cp auth.json MY_CONTAINER:"$CONTAINER_HOME/.codex/auth.json" +``` + +whereas if you are `ssh`'d into a remote machine, you likely want to use [`scp`](https://en.wikipedia.org/wiki/Secure_copy_protocol): + +```shell +ssh user@remote 'mkdir -p ~/.codex' +scp ~/.codex/auth.json user@remote:~/.codex/auth.json +``` + +or try this one-liner: + +```shell +ssh user@remote 'mkdir -p ~/.codex && cat > ~/.codex/auth.json' < ~/.codex/auth.json +``` + +#### Connecting through VPS or remote If you run Codex on a remote machine (VPS/server) without a local browser, the login helper starts a server on `localhost:1455` on the remote host. To complete login in your local browser, forward that port to your machine before starting the login flow: @@ -133,6 +166,35 @@ Notes: - This command only sets the key for your current terminal session, which we recommend. To set it for all future sessions, you can also add the `export` line to your shell's configuration file (e.g., `~/.zshrc`). - If you have signed in with ChatGPT, Codex will default to using your ChatGPT credits. If you wish to use your API key, use the `/logout` command to clear your ChatGPT authentication. +#### Forcing a specific auth method (advanced) + +You can explicitly choose which authentication Codex should prefer when both are available. + +- To always use your API key (even when ChatGPT auth exists), set: + +```toml +# ~/.codex/config.toml +preferred_auth_method = "apikey" +``` + +Or override ad-hoc via CLI: + +```bash +codex --config preferred_auth_method="apikey" +``` + +- To prefer ChatGPT auth (default), set: + +```toml +# ~/.codex/config.toml +preferred_auth_method = "chatgpt" +``` + +Notes: + +- When `preferred_auth_method = "apikey"` and an API key is available, the login screen is skipped. +- When `preferred_auth_method = "chatgpt"` (default), Codex prefers ChatGPT auth if present; if only an API key is present, it will use the API key. Certain account types may also require API-key mode. + ### Choosing Codex's level of autonomy We always recommend running Codex in its default sandbox that gives you strong guardrails around what the agent can do. The default sandbox prevents it from editing files outside its workspace, or from accessing the network. @@ -534,9 +596,13 @@ We're excited to launch a **$1 million initiative** supporting open source proje ## Contributing -This project is under active development and the code will likely change pretty significantly. We'll update this message once that's complete! +This project is under active development and the code will likely change pretty significantly. -More broadly we welcome contributions - whether you are opening your very first pull request or you're a seasoned maintainer. At the same time we care about reliability and long-term maintainability, so the bar for merging code is intentionally **high**. The guidelines below spell out what "high-quality" means in practice and should make the whole process transparent and friendly. +**At the moment, we only plan to prioritize reviewing external contributions for bugs or security fixes.** + +If you want to add a new feature or change the behavior of an existing one, please open an issue proposing the feature and get approval from an OpenAI team member before spending time building it. + +**New contributions that don't go through this process may be closed** if they aren't aligned with our current roadmap or conflict with other priorities/upcoming features. ### Development workflow @@ -561,8 +627,9 @@ More broadly we welcome contributions - whether you are opening your very first ### Review process 1. One maintainer will be assigned as a primary reviewer. -2. We may ask for changes - please do not take this personally. We value the work, we just also value consistency and long-term maintainability. -3. When there is consensus that the PR meets the bar, a maintainer will squash-and-merge. +2. If your PR adds a new feature that was not previously discussed and approved, we may choose to close your PR (see [Contributing](#contributing)). +3. We may ask for changes - please do not take this personally. We value the work, but we also value consistency and long-term maintainability. +5. When there is consensus that the PR meets the bar, a maintainer will squash-and-merge. ### Community values diff --git a/codex-cli/bin/codex.js b/codex-cli/bin/codex.js index d92d8f2f4f..b22c5180c0 100755 --- a/codex-cli/bin/codex.js +++ b/codex-cli/bin/codex.js @@ -43,7 +43,7 @@ switch (platform) { targetTriple = "x86_64-pc-windows-msvc.exe"; break; case "arm64": - // We do not build this today, fall through... + // We do not build this today, fall through... default: break; } @@ -65,9 +65,43 @@ const binaryPath = path.join(__dirname, "..", "bin", `codex-${targetTriple}`); // receives a fatal signal, both processes exit in a predictable manner. const { spawn } = await import("child_process"); +async function tryImport(moduleName) { + try { + // eslint-disable-next-line node/no-unsupported-features/es-syntax + return await import(moduleName); + } catch (err) { + return null; + } +} + +async function resolveRgDir() { + const ripgrep = await tryImport("@vscode/ripgrep"); + if (!ripgrep?.rgPath) { + return null; + } + return path.dirname(ripgrep.rgPath); +} + +function getUpdatedPath(newDirs) { + const pathSep = process.platform === "win32" ? ";" : ":"; + const existingPath = process.env.PATH || ""; + const updatedPath = [ + ...newDirs, + ...existingPath.split(pathSep).filter(Boolean), + ].join(pathSep); + return updatedPath; +} + +const additionalDirs = []; +const rgDir = await resolveRgDir(); +if (rgDir) { + additionalDirs.push(rgDir); +} +const updatedPath = getUpdatedPath(additionalDirs); + const child = spawn(binaryPath, process.argv.slice(2), { stdio: "inherit", - env: { ...process.env, CODEX_MANAGED_BY_NPM: "1" }, + env: { ...process.env, PATH: updatedPath, CODEX_MANAGED_BY_NPM: "1" }, }); child.on("error", (err) => { @@ -120,4 +154,3 @@ if (childResult.type === "signal") { } else { process.exit(childResult.exitCode); } - diff --git a/codex-cli/package-lock.json b/codex-cli/package-lock.json new file mode 100644 index 0000000000..a1c840ade0 --- /dev/null +++ b/codex-cli/package-lock.json @@ -0,0 +1,119 @@ +{ + "name": "@openai/codex", + "version": "0.0.0-dev", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "@openai/codex", + "version": "0.0.0-dev", + "license": "Apache-2.0", + "dependencies": { + "@vscode/ripgrep": "^1.15.14" + }, + "bin": { + "codex": "bin/codex.js" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/@vscode/ripgrep": { + "version": "1.15.14", + "resolved": "https://registry.npmjs.org/@vscode/ripgrep/-/ripgrep-1.15.14.tgz", + "integrity": "sha512-/G1UJPYlm+trBWQ6cMO3sv6b8D1+G16WaJH1/DSqw32JOVlzgZbLkDxRyzIpTpv30AcYGMkCf5tUqGlW6HbDWw==", + "hasInstallScript": true, + "license": "MIT", + "dependencies": { + "https-proxy-agent": "^7.0.2", + "proxy-from-env": "^1.1.0", + "yauzl": "^2.9.2" + } + }, + "node_modules/agent-base": { + "version": "7.1.4", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", + "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==", + "license": "MIT", + "engines": { + "node": ">= 14" + } + }, + "node_modules/buffer-crc32": { + "version": "0.2.13", + "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", + "integrity": "sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==", + "license": "MIT", + "engines": { + "node": "*" + } + }, + "node_modules/debug": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.1.tgz", + "integrity": "sha512-KcKCqiftBJcZr++7ykoDIEwSa3XWowTfNPo92BYxjXiyYEVrUQh2aLyhxBCwww+heortUFxEJYcRzosstTEBYQ==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/fd-slicer": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.1.0.tgz", + "integrity": "sha512-cE1qsB/VwyQozZ+q1dGxR8LBYNZeofhEdUNGSMbQD3Gw2lAzX9Zb3uIU6Ebc/Fmyjo9AWWfnn0AUCHqtevs/8g==", + "license": "MIT", + "dependencies": { + "pend": "~1.2.0" + } + }, + "node_modules/https-proxy-agent": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", + "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/pend": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", + "integrity": "sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==", + "license": "MIT" + }, + "node_modules/proxy-from-env": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==", + "license": "MIT" + }, + "node_modules/yauzl": { + "version": "2.10.0", + "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-2.10.0.tgz", + "integrity": "sha512-p4a9I6X6nu6IhoGmBqAcbJy1mlC4j27vEPZX9F4L4/vZT3Lyq1VkFHw/V/PUcB9Buo+DG3iHkT0x3Qya58zc3g==", + "license": "MIT", + "dependencies": { + "buffer-crc32": "~0.2.3", + "fd-slicer": "~1.1.0" + } + } + } +} diff --git a/codex-cli/package.json b/codex-cli/package.json index c5464beae5..614ca1a832 100644 --- a/codex-cli/package.json +++ b/codex-cli/package.json @@ -16,5 +16,11 @@ "repository": { "type": "git", "url": "git+https://github.com/openai/codex.git" + }, + "dependencies": { + "@vscode/ripgrep": "^1.15.14" + }, + "devDependencies": { + "prettier": "^3.3.3" } } diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index ac3de56218..265857533f 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -176,9 +176,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.98" +version = "1.0.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e16d2d3311acee920a9eb8d33b8cbc1787ce4a264e85f964c2404b969bdcd487" +checksum = "b0674a1ddeecb70197781e945de4b3b8ffb61fa939a5597bcf48503737663100" [[package]] name = "arbitrary" @@ -186,26 +186,6 @@ version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dde20b3d026af13f561bdd0f15edf01fc734f0dafcedbaf42bba506a9517f223" -[[package]] -name = "arboard" -version = "3.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55f533f8e0af236ffe5eb979b99381df3258853f00ba2e44b6e1955292c75227" -dependencies = [ - "clipboard-win", - "image", - "log", - "objc2", - "objc2-app-kit", - "objc2-core-foundation", - "objc2-core-graphics", - "objc2-foundation", - "parking_lot", - "percent-encoding", - "windows-sys 0.59.0", - "x11rb", -] - [[package]] name = "arg_enum_proc_macro" version = "0.3.4" @@ -223,6 +203,12 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" +[[package]] +name = "ascii" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d92bec98840b8f03a5ff5413de5293bfcd8bf96467cf5452609f939ec6f5de16" + [[package]] name = "ascii-canvas" version = "3.0.0" @@ -501,6 +487,12 @@ dependencies = [ "shlex", ] +[[package]] +name = "cesu8" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" + [[package]] name = "cfg-expr" version = "0.15.8" @@ -539,10 +531,16 @@ dependencies = [ ] [[package]] -name = "clap" -version = "4.5.41" +name = "chunked_transfer" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be92d32e80243a54711e5d7ce823c35c41c9d929dc4ab58e1276f625841aadf9" +checksum = "6e4de3bc4ea267985becf712dc6d9eed8b04c953b3fcfb339ebc87acd9804901" + +[[package]] +name = "clap" +version = "4.5.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fc0e74a703892159f5ae7d3aac52c8e6c392f5ae5f359c70b5881d60aaac318" dependencies = [ "clap_builder", "clap_derive", @@ -550,9 +548,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.41" +version = "4.5.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "707eab41e9622f9139419d573eca0900137718000c517d47da73045f54331c3d" +checksum = "b3e7f4214277f3c7aa526a59dd3fbe306a370daee1f8b7b8c987069cd8e888a8" dependencies = [ "anstream", "anstyle", @@ -563,18 +561,18 @@ dependencies = [ [[package]] name = "clap_complete" -version = "4.5.55" +version = "4.5.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5abde44486daf70c5be8b8f8f1b66c49f86236edf6fa2abadb4d961c4c6229a" +checksum = "4d9501bd3f5f09f7bbee01da9a511073ed30a80cd7a509f1214bb74eadea71ad" dependencies = [ "clap", ] [[package]] name = "clap_derive" -version = "4.5.41" +version = "4.5.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef4f52386a59ca4c860f7393bcf8abd8dfd91ecccc0f774635ff68e92eeef491" +checksum = "14cb31bb0a7d536caef2639baa7fad459e15c3144efefa6dbd1c84562c4739f6" dependencies = [ "heck", "proc-macro2", @@ -667,6 +665,8 @@ dependencies = [ "codex-exec", "codex-login", "codex-mcp-server", + "codex-protocol", + "codex-protocol-ts", "codex-tui", "serde_json", "tokio", @@ -680,8 +680,9 @@ version = "0.0.0" dependencies = [ "clap", "codex-core", + "codex-protocol", "serde", - "toml 0.9.4", + "toml 0.9.5", ] [[package]] @@ -697,6 +698,7 @@ dependencies = [ "codex-apply-patch", "codex-login", "codex-mcp-client", + "codex-protocol", "core_test_support", "dirs", "env-flags", @@ -709,9 +711,11 @@ dependencies = [ "mcp-types", "mime_guess", "openssl-sys", + "os_info", "predicates", "pretty_assertions", "rand 0.9.2", + "regex-lite", "reqwest", "seccompiler", "serde", @@ -727,7 +731,7 @@ dependencies = [ "tokio", "tokio-test", "tokio-util", - "toml 0.9.4", + "toml 0.9.5", "toml_edit 0.23.3", "tracing", "tree-sitter", @@ -751,6 +755,9 @@ dependencies = [ "codex-common", "codex-core", "codex-ollama", + "codex-protocol", + "core_test_support", + "libc", "owo-colors", "predicates", "serde_json", @@ -759,6 +766,7 @@ dependencies = [ "tokio", "tracing", "tracing-subscriber", + "wiremock", ] [[package]] @@ -816,12 +824,18 @@ dependencies = [ "base64 0.22.1", "chrono", "pretty_assertions", + "rand 0.8.5", "reqwest", "serde", "serde_json", + "sha2", "tempfile", "thiserror 2.0.12", + "tiny_http", "tokio", + "url", + "urlencoding", + "webbrowser", ] [[package]] @@ -845,6 +859,8 @@ dependencies = [ "assert_cmd", "codex-arg0", "codex-core", + "codex-login", + "codex-protocol", "mcp-types", "mcp_test_support", "pretty_assertions", @@ -856,7 +872,7 @@ dependencies = [ "tempfile", "tokio", "tokio-test", - "toml 0.9.4", + "toml 0.9.5", "tracing", "tracing-subscriber", "uuid", @@ -875,17 +891,41 @@ dependencies = [ "serde_json", "tempfile", "tokio", - "toml 0.9.4", + "toml 0.9.5", "tracing", "wiremock", ] +[[package]] +name = "codex-protocol" +version = "0.0.0" +dependencies = [ + "mcp-types", + "pretty_assertions", + "serde", + "serde_bytes", + "serde_json", + "strum 0.27.2", + "strum_macros 0.27.2", + "ts-rs", + "uuid", +] + +[[package]] +name = "codex-protocol-ts" +version = "0.0.0" +dependencies = [ + "anyhow", + "clap", + "codex-protocol", + "ts-rs", +] + [[package]] name = "codex-tui" version = "0.0.0" dependencies = [ "anyhow", - "arboard", "base64 0.22.1", "chrono", "clap", @@ -896,16 +936,19 @@ dependencies = [ "codex-file-search", "codex-login", "codex-ollama", + "codex-protocol", "color-eyre", "crossterm", "diffy", "image", "insta", "lazy_static", + "libc", "mcp-types", + "once_cell", "path-clean", "pretty_assertions", - "rand 0.8.5", + "rand 0.9.2", "ratatui", "ratatui-image", "regex-lite", @@ -968,6 +1011,16 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" +[[package]] +name = "combine" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +dependencies = [ + "bytes", + "memchr", +] + [[package]] name = "compact_str" version = "0.8.1" @@ -1022,6 +1075,16 @@ dependencies = [ "libc", ] +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation-sys" version = "0.8.7" @@ -1342,16 +1405,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "dispatch2" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89a09f22a6c6069a18470eb92d2298acf25463f14256d24778e1230d789a2aec" -dependencies = [ - "bitflags 2.9.1", - "objc2", -] - [[package]] name = "display_container" version = "0.9.0" @@ -1815,16 +1868,6 @@ dependencies = [ "version_check", ] -[[package]] -name = "gethostname" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0176e0459c2e4a1fe232f984bca6890e681076abb9934f6cea7c326f3fc47818" -dependencies = [ - "libc", - "windows-targets 0.48.5", -] - [[package]] name = "getopts" version = "0.2.23" @@ -2492,6 +2535,28 @@ dependencies = [ "syn 2.0.104", ] +[[package]] +name = "jni" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a87aa2bb7d2af34197c04845522473242e1aa17c12f4935d5856491a7fb8c97" +dependencies = [ + "cesu8", + "cfg-if", + "combine", + "jni-sys", + "log", + "thiserror 1.0.69", + "walkdir", + "windows-sys 0.45.0", +] + +[[package]] +name = "jni-sys" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8eaf4bc02d17cbdd7ff4c7438cafcdf7fb9a4613313ad11b4f8fefe7d3fa0130" + [[package]] name = "jobserver" version = "0.1.33" @@ -2574,9 +2639,9 @@ checksum = "03087c2bad5e1034e8cace5926dec053fb3790248370865f5117a7d0213354c8" [[package]] name = "libc" -version = "0.2.174" +version = "0.2.175" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1171693293099992e19cddea4e8b849964e9846f4acee11b3948bcc337be8776" +checksum = "6a82ae493e598baaea5209805c49bbf2ea7de956d50d7da0da1164f9c6d28543" [[package]] name = "libfuzzer-sys" @@ -2723,6 +2788,7 @@ version = "0.0.0" dependencies = [ "serde", "serde_json", + "ts-rs", ] [[package]] @@ -2733,8 +2799,10 @@ dependencies = [ "assert_cmd", "codex-core", "codex-mcp-server", + "codex-protocol", "mcp-types", "pretty_assertions", + "serde", "serde_json", "shlex", "tempfile", @@ -2828,6 +2896,12 @@ dependencies = [ "tempfile", ] +[[package]] +name = "ndk-context" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27b02d87554356db9e9a873add8782d4ea6e3e58ea071a9adb9a2e8ddb884a8b" + [[package]] name = "new_debug_unreachable" version = "1.0.6" @@ -2983,49 +3057,13 @@ dependencies = [ [[package]] name = "objc2" -version = "0.6.1" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88c6597e14493ab2e44ce58f2fdecf095a51f12ca57bec060a11c57332520551" +checksum = "561f357ba7f3a2a61563a186a163d0a3a5247e1089524a3981d49adb775078bc" dependencies = [ "objc2-encode", ] -[[package]] -name = "objc2-app-kit" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6f29f568bec459b0ddff777cec4fe3fd8666d82d5a40ebd0ff7e66134f89bcc" -dependencies = [ - "bitflags 2.9.1", - "objc2", - "objc2-core-graphics", - "objc2-foundation", -] - -[[package]] -name = "objc2-core-foundation" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c10c2894a6fed806ade6027bcd50662746363a9589d3ec9d9bef30a4e4bc166" -dependencies = [ - "bitflags 2.9.1", - "dispatch2", - "objc2", -] - -[[package]] -name = "objc2-core-graphics" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "989c6c68c13021b5c2d6b71456ebb0f9dc78d752e86a98da7c716f4f9470f5a4" -dependencies = [ - "bitflags 2.9.1", - "dispatch2", - "objc2", - "objc2-core-foundation", - "objc2-io-surface", -] - [[package]] name = "objc2-encode" version = "4.1.0" @@ -3040,18 +3078,6 @@ checksum = "900831247d2fe1a09a683278e5384cfb8c80c79fe6b166f9d14bfdde0ea1b03c" dependencies = [ "bitflags 2.9.1", "objc2", - "objc2-core-foundation", -] - -[[package]] -name = "objc2-io-surface" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7282e9ac92529fa3457ce90ebb15f4ecbc383e8338060960760fa2cf75420c3c" -dependencies = [ - "bitflags 2.9.1", - "objc2", - "objc2-core-foundation", ] [[package]] @@ -3157,6 +3183,18 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" +[[package]] +name = "os_info" +version = "3.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0e1ac5fde8d43c34139135df8ea9ee9465394b2d8d20f032d38998f64afffc3" +dependencies = [ + "log", + "plist", + "serde", + "windows-sys 0.52.0", +] + [[package]] name = "overload" version = "0.1.1" @@ -3768,6 +3806,7 @@ dependencies = [ "base64 0.22.1", "bytes", "encoding_rs", + "futures-channel", "futures-core", "futures-util", "h2", @@ -4090,7 +4129,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" dependencies = [ "bitflags 2.9.1", - "core-foundation", + "core-foundation 0.9.4", "core-foundation-sys", "libc", "security-framework-sys", @@ -4249,6 +4288,17 @@ dependencies = [ "digest", ] +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "sharded-slab" version = "0.1.7" @@ -4613,7 +4663,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" dependencies = [ "bitflags 2.9.1", - "core-foundation", + "core-foundation 0.9.4", "system-configuration-sys", ] @@ -4670,6 +4720,15 @@ dependencies = [ "winapi", ] +[[package]] +name = "termcolor" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" +dependencies = [ + "winapi-util", +] + [[package]] name = "terminal_size" version = "0.4.2" @@ -4808,6 +4867,18 @@ dependencies = [ "crunchy", ] +[[package]] +name = "tiny_http" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389915df6413a2e74fb181895f933386023c71110878cd0825588928e64cdc82" +dependencies = [ + "ascii", + "chunked_transfer", + "httpdate", + "log", +] + [[package]] name = "tinystr" version = "0.8.1" @@ -4895,9 +4966,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.15" +version = "0.7.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66a539a9ad6d5d281510d5bd368c973d636c02dbf8a67300bfb6b950696ad7df" +checksum = "14307c986784f72ef81c89db7d9e28d6ac26d16213b109ea501696195e6e3ce5" dependencies = [ "bytes", "futures-core", @@ -4920,9 +4991,9 @@ dependencies = [ [[package]] name = "toml" -version = "0.9.4" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41ae868b5a0f67631c14589f7e250c1ea2c574ee5ba21c6c8dd4b1485705a5a1" +checksum = "75129e1dc5000bfbaa9fee9d1b21f974f9fbad9daec557a521ee6e080825f6e8" dependencies = [ "indexmap 2.10.0", "serde", @@ -5157,6 +5228,30 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "ts-rs" +version = "11.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ef1b7a6d914a34127ed8e1fa927eb7088903787bcded4fa3eef8f85ee1568be" +dependencies = [ + "serde_json", + "thiserror 2.0.12", + "ts-rs-macros", + "uuid", +] + +[[package]] +name = "ts-rs-macros" +version = "11.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9d4ed7b4c18cc150a6a0a1e9ea1ecfa688791220781af6e119f9599a8502a0a" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.104", + "termcolor", +] + [[package]] name = "tui-input" version = "0.14.0" @@ -5260,6 +5355,12 @@ dependencies = [ "serde", ] +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + [[package]] name = "utf8_iter" version = "1.0.4" @@ -5483,6 +5584,22 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webbrowser" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aaf4f3c0ba838e82b4e5ccc4157003fb8c324ee24c058470ffb82820becbde98" +dependencies = [ + "core-foundation 0.10.1", + "jni", + "log", + "ndk-context", + "objc2", + "objc2-foundation", + "url", + "web-sys", +] + [[package]] name = "weezl" version = "0.1.10" @@ -5671,6 +5788,15 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets 0.42.2", +] + [[package]] name = "windows-sys" version = "0.52.0" @@ -5700,17 +5826,17 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.48.5" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" dependencies = [ - "windows_aarch64_gnullvm 0.48.5", - "windows_aarch64_msvc 0.48.5", - "windows_i686_gnu 0.48.5", - "windows_i686_msvc 0.48.5", - "windows_x86_64_gnu 0.48.5", - "windows_x86_64_gnullvm 0.48.5", - "windows_x86_64_msvc 0.48.5", + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", ] [[package]] @@ -5747,9 +5873,9 @@ dependencies = [ [[package]] name = "windows_aarch64_gnullvm" -version = "0.48.5" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" [[package]] name = "windows_aarch64_gnullvm" @@ -5765,9 +5891,9 @@ checksum = "86b8d5f90ddd19cb4a147a5fa63ca848db3df085e25fee3cc10b39b6eebae764" [[package]] name = "windows_aarch64_msvc" -version = "0.48.5" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" [[package]] name = "windows_aarch64_msvc" @@ -5783,9 +5909,9 @@ checksum = "c7651a1f62a11b8cbd5e0d42526e55f2c99886c77e007179efff86c2b137e66c" [[package]] name = "windows_i686_gnu" -version = "0.48.5" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" [[package]] name = "windows_i686_gnu" @@ -5813,9 +5939,9 @@ checksum = "9ce6ccbdedbf6d6354471319e781c0dfef054c81fbc7cf83f338a4296c0cae11" [[package]] name = "windows_i686_msvc" -version = "0.48.5" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" [[package]] name = "windows_i686_msvc" @@ -5831,9 +5957,9 @@ checksum = "581fee95406bb13382d2f65cd4a908ca7b1e4c2f1917f143ba16efe98a589b5d" [[package]] name = "windows_x86_64_gnu" -version = "0.48.5" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" [[package]] name = "windows_x86_64_gnu" @@ -5849,9 +5975,9 @@ checksum = "2e55b5ac9ea33f2fc1716d1742db15574fd6fc8dadc51caab1c16a3d3b4190ba" [[package]] name = "windows_x86_64_gnullvm" -version = "0.48.5" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" [[package]] name = "windows_x86_64_gnullvm" @@ -5867,9 +5993,9 @@ checksum = "0a6e035dd0599267ce1ee132e51c27dd29437f63325753051e71dd9e42406c57" [[package]] name = "windows_x86_64_msvc" -version = "0.48.5" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" [[package]] name = "windows_x86_64_msvc" @@ -5931,23 +6057,6 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ea2f10b9bb0928dfb1b42b65e1f9e36f7f54dbdf08457afefb38afcdec4fa2bb" -[[package]] -name = "x11rb" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d91ffca73ee7f68ce055750bf9f6eca0780b8c85eff9bc046a3b0da41755e12" -dependencies = [ - "gethostname", - "rustix 0.38.44", - "x11rb-protocol", -] - -[[package]] -name = "x11rb-protocol" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec107c4503ea0b4a98ef47356329af139c0a4f7750e621cf2973cd3385ebcb3d" - [[package]] name = "yaml-rust" version = "0.4.5" diff --git a/codex-rs/Cargo.toml b/codex-rs/Cargo.toml index 0ed8852228..8a48ef8187 100644 --- a/codex-rs/Cargo.toml +++ b/codex-rs/Cargo.toml @@ -15,6 +15,8 @@ members = [ "mcp-server", "mcp-types", "ollama", + "protocol", + "protocol-ts", "tui", ] resolver = "2" diff --git a/codex-rs/apply-patch/src/lib.rs b/codex-rs/apply-patch/src/lib.rs index 61b1b68f9e..87df66dd23 100644 --- a/codex-rs/apply-patch/src/lib.rs +++ b/codex-rs/apply-patch/src/lib.rs @@ -82,8 +82,9 @@ pub struct ApplyPatchArgs { } pub fn maybe_parse_apply_patch(argv: &[String]) -> MaybeApplyPatch { + const APPLY_PATCH_COMMANDS: [&str; 2] = ["apply_patch", "applypatch"]; match argv { - [cmd, body] if cmd == "apply_patch" => match parse_patch(body) { + [cmd, body] if APPLY_PATCH_COMMANDS.contains(&cmd.as_str()) => match parse_patch(body) { Ok(source) => MaybeApplyPatch::Body(source), Err(e) => MaybeApplyPatch::PatchParseError(e), }, @@ -165,7 +166,7 @@ impl ApplyPatchAction { panic!("path must be absolute"); } - #[allow(clippy::expect_used)] + #[expect(clippy::expect_used)] let filename = path .file_name() .expect("path should not be empty") @@ -178,7 +179,7 @@ impl ApplyPatchAction { *** End Patch"#, ); let changes = HashMap::from([(path.to_path_buf(), ApplyPatchFileChange::Add { content })]); - #[allow(clippy::expect_used)] + #[expect(clippy::expect_used)] Self { changes, cwd: path @@ -681,8 +682,6 @@ pub fn print_summary( #[cfg(test)] mod tests { - #![allow(clippy::unwrap_used)] - use super::*; use pretty_assertions::assert_eq; use std::fs; @@ -722,6 +721,31 @@ mod tests { } } + #[test] + fn test_literal_applypatch() { + let args = strs_to_strings(&[ + "applypatch", + r#"*** Begin Patch +*** Add File: foo ++hi +*** End Patch +"#, + ]); + + match maybe_parse_apply_patch(&args) { + MaybeApplyPatch::Body(ApplyPatchArgs { hunks, patch: _ }) => { + assert_eq!( + hunks, + vec![Hunk::AddFile { + path: PathBuf::from("foo"), + contents: "hi\n".to_string() + }] + ); + } + result => panic!("expected MaybeApplyPatch::Body got {result:?}"), + } + } + #[test] fn test_heredoc() { let args = strs_to_strings(&[ diff --git a/codex-rs/apply-patch/src/parser.rs b/codex-rs/apply-patch/src/parser.rs index 44c5b14619..ff9dfd6f8f 100644 --- a/codex-rs/apply-patch/src/parser.rs +++ b/codex-rs/apply-patch/src/parser.rs @@ -427,7 +427,6 @@ fn parse_update_file_chunk( } #[test] -#[allow(clippy::unwrap_used)] fn test_parse_patch() { assert_eq!( parse_patch_text("bad", ParseMode::Strict), diff --git a/codex-rs/arg0/src/lib.rs b/codex-rs/arg0/src/lib.rs index c097ebc11c..b7766fe70a 100644 --- a/codex-rs/arg0/src/lib.rs +++ b/codex-rs/arg0/src/lib.rs @@ -82,10 +82,34 @@ where }) } +const ILLEGAL_ENV_VAR_PREFIX: &str = "CODEX_"; + /// Load env vars from ~/.codex/.env and `$(pwd)/.env`. +/// +/// Security: Do not allow `.env` files to create or modify any variables +/// with names starting with `CODEX_`. fn load_dotenv() { if let Ok(codex_home) = codex_core::config::find_codex_home() { - dotenvy::from_path(codex_home.join(".env")).ok(); + if let Ok(iter) = dotenvy::from_path_iter(codex_home.join(".env")) { + set_filtered(iter); + } + } + + if let Ok(iter) = dotenvy::dotenv_iter() { + set_filtered(iter); + } +} + +/// Helper to set vars from a dotenvy iterator while filtering out `CODEX_` keys. +fn set_filtered(iter: I) +where + I: IntoIterator>, +{ + for (key, value) in iter.into_iter().flatten() { + if !key.to_ascii_uppercase().starts_with(ILLEGAL_ENV_VAR_PREFIX) { + // It is safe to call set_var() because our process is + // single-threaded at this point in its execution. + unsafe { std::env::set_var(&key, &value) }; + } } - dotenvy::dotenv().ok(); } diff --git a/codex-rs/chatgpt/src/chatgpt_client.rs b/codex-rs/chatgpt/src/chatgpt_client.rs index 907783bb81..db75632191 100644 --- a/codex-rs/chatgpt/src/chatgpt_client.rs +++ b/codex-rs/chatgpt/src/chatgpt_client.rs @@ -1,4 +1,5 @@ use codex_core::config::Config; +use codex_core::user_agent::get_codex_user_agent; use crate::chatgpt_token::get_chatgpt_token_data; use crate::chatgpt_token::init_chatgpt_token_from_auth; @@ -30,7 +31,7 @@ pub(crate) async fn chatgpt_get_request( .bearer_auth(&token.access_token) .header("chatgpt-account-id", account_id?) .header("Content-Type", "application/json") - .header("User-Agent", "codex-cli") + .header("User-Agent", get_codex_user_agent(None)) .send() .await .context("Failed to send request")?; diff --git a/codex-rs/chatgpt/src/chatgpt_token.rs b/codex-rs/chatgpt/src/chatgpt_token.rs index c674afbc57..f003c4392b 100644 --- a/codex-rs/chatgpt/src/chatgpt_token.rs +++ b/codex-rs/chatgpt/src/chatgpt_token.rs @@ -1,3 +1,4 @@ +use codex_login::AuthMode; use codex_login::CodexAuth; use std::path::Path; use std::sync::LazyLock; @@ -19,7 +20,7 @@ pub fn set_chatgpt_token_data(value: TokenData) { /// Initialize the ChatGPT token from auth.json file pub async fn init_chatgpt_token_from_auth(codex_home: &Path) -> std::io::Result<()> { - let auth = CodexAuth::from_codex_home(codex_home)?; + let auth = CodexAuth::from_codex_home(codex_home, AuthMode::ChatGPT)?; if let Some(auth) = auth { let token_data = auth.get_token_data().await?; set_chatgpt_token_data(token_data); diff --git a/codex-rs/chatgpt/tests/apply_command_e2e.rs b/codex-rs/chatgpt/tests/apply_command_e2e.rs index f1a35e1521..2aa8b809bb 100644 --- a/codex-rs/chatgpt/tests/apply_command_e2e.rs +++ b/codex-rs/chatgpt/tests/apply_command_e2e.rs @@ -1,5 +1,3 @@ -#![expect(clippy::expect_used)] - use codex_chatgpt::apply_command::apply_diff_from_task; use codex_chatgpt::get_task::GetTaskResponse; use std::path::Path; diff --git a/codex-rs/cli/Cargo.toml b/codex-rs/cli/Cargo.toml index 0f370691cf..f7af3349e0 100644 --- a/codex-rs/cli/Cargo.toml +++ b/codex-rs/cli/Cargo.toml @@ -25,6 +25,7 @@ codex-core = { path = "../core" } codex-exec = { path = "../exec" } codex-login = { path = "../login" } codex-mcp-server = { path = "../mcp-server" } +codex-protocol = { path = "../protocol" } codex-tui = { path = "../tui" } serde_json = "1" tokio = { version = "1", features = [ @@ -36,3 +37,4 @@ tokio = { version = "1", features = [ ] } tracing = "0.1.41" tracing-subscriber = "0.3.19" +codex-protocol-ts = { path = "../protocol-ts" } diff --git a/codex-rs/cli/src/debug_sandbox.rs b/codex-rs/cli/src/debug_sandbox.rs index 7f0983cbc6..6fe7f003c7 100644 --- a/codex-rs/cli/src/debug_sandbox.rs +++ b/codex-rs/cli/src/debug_sandbox.rs @@ -3,11 +3,11 @@ use std::path::PathBuf; use codex_common::CliConfigOverrides; use codex_core::config::Config; use codex_core::config::ConfigOverrides; -use codex_core::config_types::SandboxMode; -use codex_core::exec::spawn_command_under_linux_sandbox; use codex_core::exec_env::create_env; +use codex_core::landlock::spawn_command_under_linux_sandbox; use codex_core::seatbelt::spawn_command_under_seatbelt; use codex_core::spawn::StdioPolicy; +use codex_protocol::config_types::SandboxMode; use crate::LandlockCommand; use crate::SeatbeltCommand; diff --git a/codex-rs/cli/src/login.rs b/codex-rs/cli/src/login.rs index 1a70bd27b6..72eb7b4faf 100644 --- a/codex-rs/cli/src/login.rs +++ b/codex-rs/cli/src/login.rs @@ -1,20 +1,33 @@ -use std::env; - use codex_common::CliConfigOverrides; use codex_core::config::Config; use codex_core::config::ConfigOverrides; use codex_login::AuthMode; +use codex_login::CLIENT_ID; use codex_login::CodexAuth; use codex_login::OPENAI_API_KEY_ENV_VAR; +use codex_login::ServerOptions; use codex_login::login_with_api_key; -use codex_login::login_with_chatgpt; use codex_login::logout; +use codex_login::run_login_server; +use std::env; +use std::path::PathBuf; + +pub async fn login_with_chatgpt(codex_home: PathBuf) -> std::io::Result<()> { + let opts = ServerOptions::new(codex_home, CLIENT_ID.to_string()); + let server = run_login_server(opts)?; + + eprintln!( + "Starting local login server on http://localhost:{}.\nIf your browser did not open, navigate to this URL to authenticate:\n\n{}", + server.actual_port, server.auth_url, + ); + + server.block_until_done().await +} pub async fn run_login_with_chatgpt(cli_config_overrides: CliConfigOverrides) -> ! { let config = load_config_or_exit(cli_config_overrides); - let capture_output = false; - match login_with_chatgpt(&config.codex_home, capture_output).await { + match login_with_chatgpt(config.codex_home).await { Ok(_) => { eprintln!("Successfully logged in"); std::process::exit(0); @@ -47,7 +60,7 @@ pub async fn run_login_with_api_key( pub async fn run_login_status(cli_config_overrides: CliConfigOverrides) -> ! { let config = load_config_or_exit(cli_config_overrides); - match CodexAuth::from_codex_home(&config.codex_home) { + match CodexAuth::from_codex_home(&config.codex_home, config.preferred_auth_method) { Ok(Some(auth)) => match auth.mode { AuthMode::ApiKey => match auth.get_token().await { Ok(api_key) => { diff --git a/codex-rs/cli/src/main.rs b/codex-rs/cli/src/main.rs index 9aef22c09f..d237fe6729 100644 --- a/codex-rs/cli/src/main.rs +++ b/codex-rs/cli/src/main.rs @@ -27,7 +27,11 @@ use crate::proto::ProtoCli; author, version, // If a sub‑command is given, ignore requirements of the default args. - subcommand_negates_reqs = true + subcommand_negates_reqs = true, + // The executable is sometimes invoked via a platform‑specific name like + // `codex-x86_64-unknown-linux-musl`, but the help output should always use + // the generic `codex` command name that users run. + bin_name = "codex" )] struct MultitoolCli { #[clap(flatten)] @@ -68,6 +72,10 @@ enum Subcommand { /// Apply the latest diff produced by Codex agent as a `git apply` to your local working tree. #[clap(visible_alias = "a")] Apply(ApplyCommand), + + /// Internal: generate TypeScript protocol bindings. + #[clap(hide = true)] + GenerateTs(GenerateTsCommand), } #[derive(Debug, Parser)] @@ -116,6 +124,17 @@ struct LogoutCommand { config_overrides: CliConfigOverrides, } +#[derive(Debug, Parser)] +struct GenerateTsCommand { + /// Output directory where .ts files will be written + #[arg(short = 'o', long = "out", value_name = "DIR")] + out_dir: PathBuf, + + /// Optional path to the Prettier executable to format generated files + #[arg(short = 'p', long = "prettier", value_name = "PRETTIER_BIN")] + prettier: Option, +} + fn main() -> anyhow::Result<()> { arg0_dispatch_or_else(|codex_linux_sandbox_exe| async move { cli_main(codex_linux_sandbox_exe).await?; @@ -190,6 +209,9 @@ async fn cli_main(codex_linux_sandbox_exe: Option) -> anyhow::Result<() prepend_config_flags(&mut apply_cli.config_overrides, cli.config_overrides); run_apply_command(apply_cli, None).await?; } + Some(Subcommand::GenerateTs(gen_cli)) => { + codex_protocol_ts::generate_ts(&gen_cli.out_dir, gen_cli.prettier.as_deref())?; + } } Ok(()) diff --git a/codex-rs/cli/src/proto.rs b/codex-rs/cli/src/proto.rs index 6c1de7eaa9..3bc4d81618 100644 --- a/codex-rs/cli/src/proto.rs +++ b/codex-rs/cli/src/proto.rs @@ -1,15 +1,14 @@ use std::io::IsTerminal; -use std::sync::Arc; use clap::Parser; use codex_common::CliConfigOverrides; -use codex_core::Codex; -use codex_core::CodexSpawnOk; +use codex_core::ConversationManager; +use codex_core::NewConversation; use codex_core::config::Config; use codex_core::config::ConfigOverrides; +use codex_core::protocol::Event; +use codex_core::protocol::EventMsg; use codex_core::protocol::Submission; -use codex_core::util::notify_on_sigint; -use codex_login::CodexAuth; use tokio::io::AsyncBufReadExt; use tokio::io::BufReader; use tracing::error; @@ -36,22 +35,38 @@ pub async fn run_main(opts: ProtoCli) -> anyhow::Result<()> { .map_err(anyhow::Error::msg)?; let config = Config::load_with_cli_overrides(overrides_vec, ConfigOverrides::default())?; - let auth = CodexAuth::from_codex_home(&config.codex_home)?; - let ctrl_c = notify_on_sigint(); - let CodexSpawnOk { codex, .. } = Codex::spawn(config, auth, ctrl_c.clone()).await?; - let codex = Arc::new(codex); + // Use conversation_manager API to start a conversation + let conversation_manager = ConversationManager::default(); + let NewConversation { + conversation_id: _, + conversation, + session_configured, + } = conversation_manager.new_conversation(config).await?; + + // Simulate streaming the session_configured event. + let synthetic_event = Event { + // Fake id value. + id: "".to_string(), + msg: EventMsg::SessionConfigured(session_configured), + }; + let session_configured_event = match serde_json::to_string(&synthetic_event) { + Ok(s) => s, + Err(e) => { + error!("Failed to serialize session_configured: {e}"); + return Err(anyhow::Error::from(e)); + } + }; + println!("{session_configured_event}"); // Task that reads JSON lines from stdin and forwards to Submission Queue let sq_fut = { - let codex = codex.clone(); - let ctrl_c = ctrl_c.clone(); + let conversation = conversation.clone(); async move { let stdin = BufReader::new(tokio::io::stdin()); let mut lines = stdin.lines(); loop { let result = tokio::select! { - _ = ctrl_c.notified() => { - info!("Interrupted, exiting"); + _ = tokio::signal::ctrl_c() => { break }, res = lines.next_line() => res, @@ -65,7 +80,7 @@ pub async fn run_main(opts: ProtoCli) -> anyhow::Result<()> { } match serde_json::from_str::(line) { Ok(sub) => { - if let Err(e) = codex.submit_with_id(sub).await { + if let Err(e) = conversation.submit_with_id(sub).await { error!("{e:#}"); break; } @@ -88,8 +103,8 @@ pub async fn run_main(opts: ProtoCli) -> anyhow::Result<()> { let eq_fut = async move { loop { let event = tokio::select! { - _ = ctrl_c.notified() => break, - event = codex.next_event() => event, + _ = tokio::signal::ctrl_c() => break, + event = conversation.next_event() => event, }; match event { Ok(event) => { diff --git a/codex-rs/clippy.toml b/codex-rs/clippy.toml new file mode 100644 index 0000000000..5a6ff7f052 --- /dev/null +++ b/codex-rs/clippy.toml @@ -0,0 +1,9 @@ +allow-expect-in-tests = true +allow-unwrap-in-tests = true +disallowed-methods = [ + { path = "ratatui::style::Color::Rgb", reason = "Use ANSI colors, which work better in various terminal themes." }, + { path = "ratatui::style::Color::Indexed", reason = "Use ANSI colors, which work better in various terminal themes." }, + { path = "ratatui::style::Stylize::white", reason = "Avoid hardcoding white; prefer default fg or dim/bold. Exception: Disable this rule if rendering over a hardcoded ANSI background." }, + { path = "ratatui::style::Stylize::black", reason = "Avoid hardcoding black; prefer default fg or dim/bold. Exception: Disable this rule if rendering over a hardcoded ANSI background." }, + { path = "ratatui::style::Stylize::yellow", reason = "Avoid yellow; prefer other colors in `tui/styles.md`." }, +] diff --git a/codex-rs/common/Cargo.toml b/codex-rs/common/Cargo.toml index 1723098b8a..b10600574c 100644 --- a/codex-rs/common/Cargo.toml +++ b/codex-rs/common/Cargo.toml @@ -9,6 +9,7 @@ workspace = true [dependencies] clap = { version = "4", features = ["derive", "wrap_help"], optional = true } codex-core = { path = "../core" } +codex-protocol = { path = "../protocol" } serde = { version = "1", optional = true } toml = { version = "0.9", optional = true } diff --git a/codex-rs/common/src/config_override.rs b/codex-rs/common/src/config_override.rs index c9b18edc7c..6b77099524 100644 --- a/codex-rs/common/src/config_override.rs +++ b/codex-rs/common/src/config_override.rs @@ -142,7 +142,6 @@ fn parse_toml_value(raw: &str) -> Result { } #[cfg(all(test, feature = "cli"))] -#[allow(clippy::expect_used, clippy::unwrap_used)] mod tests { use super::*; diff --git a/codex-rs/common/src/sandbox_mode_cli_arg.rs b/codex-rs/common/src/sandbox_mode_cli_arg.rs index 588637aebb..fa5662ce66 100644 --- a/codex-rs/common/src/sandbox_mode_cli_arg.rs +++ b/codex-rs/common/src/sandbox_mode_cli_arg.rs @@ -7,7 +7,7 @@ //! `config.toml`. use clap::ValueEnum; -use codex_core::config_types::SandboxMode; +use codex_protocol::config_types::SandboxMode; #[derive(Clone, Copy, Debug, ValueEnum)] #[value(rename_all = "kebab-case")] diff --git a/codex-rs/config.md b/codex-rs/config.md index 848e7c0444..0d5df17cc8 100644 --- a/codex-rs/config.md +++ b/codex-rs/config.md @@ -17,7 +17,7 @@ Both the `--config` flag and the `config.toml` file support the following option The model that Codex should use. ```toml -model = "o3" # overrides the default of "codex-mini-latest" +model = "o3" # overrides the default of "gpt-5" ``` ## model_providers @@ -213,7 +213,7 @@ Users can specify config values at multiple levels. Order of precedence is as fo 1. custom command-line argument, e.g., `--model o3` 2. as part of a profile, where the `--profile` is specified via a CLI (or in the config file itself) 3. as an entry in `config.toml`, e.g., `model = "o3"` -4. the default value that comes with Codex CLI (i.e., Codex CLI defaults to `codex-mini-latest`) +4. the default value that comes with Codex CLI (i.e., Codex CLI defaults to `gpt-5`) ## model_reasoning_effort diff --git a/codex-rs/core/Cargo.toml b/codex-rs/core/Cargo.toml index 006a218abf..78942ff69f 100644 --- a/codex-rs/core/Cargo.toml +++ b/codex-rs/core/Cargo.toml @@ -19,19 +19,22 @@ chrono = { version = "0.4", features = ["serde"] } codex-apply-patch = { path = "../apply-patch" } codex-login = { path = "../login" } codex-mcp-client = { path = "../mcp-client" } +codex-protocol = { path = "../protocol" } dirs = "6" env-flags = "0.1.1" eventsource-stream = "0.2.3" fs2 = "0.4.3" futures = "0.3" -libc = "0.2.174" +libc = "0.2.175" mcp-types = { path = "../mcp-types" } mime_guess = "2.0" +os_info = "3.12.0" rand = "0.9" +regex-lite = "0.1.6" reqwest = { version = "0.12", features = ["json", "stream"] } serde = { version = "1", features = ["derive"] } -serde_json = "1" serde_bytes = "0.11" +serde_json = "1" sha1 = "0.10.6" shlex = "1.3.0" similar = "2.7.0" @@ -46,8 +49,8 @@ tokio = { version = "1", features = [ "rt-multi-thread", "signal", ] } -tokio-util = "0.7.14" -toml = "0.9.4" +tokio-util = "0.7.16" +toml = "0.9.5" toml_edit = "0.23.3" tracing = { version = "0.1.41", features = ["log"] } tree-sitter = "0.25.8" diff --git a/codex-rs/core/prompt.md b/codex-rs/core/prompt.md index 4711dd749a..ff5c2acde6 100644 --- a/codex-rs/core/prompt.md +++ b/codex-rs/core/prompt.md @@ -1,6 +1,7 @@ You are a coding agent running in the Codex CLI, a terminal-based coding assistant. Codex CLI is an open source project led by OpenAI. You are expected to be precise, safe, and helpful. Your capabilities: + - Receive user prompts and other context provided by the harness, such as files in the workspace. - Communicate with the user by streaming thinking & responses, and by making & updating plans. - Emit function calls to run terminal commands and apply patches. Depending on how this specific run is configured, you can request that these function calls be escalated to the user for approval before running. More on this in the "Sandbox and approvals" section. @@ -20,11 +21,13 @@ Your default personality and tone is concise, direct, and friendly. You communic Before making tool calls, send a brief preamble to the user explaining what you’re about to do. When sending preamble messages, follow these principles and examples: - **Logically group related actions**: if you’re about to run several related commands, describe them together in one preamble rather than sending a separate note for each. -- **Keep it concise**: be no more than 1-2 sentences (8–12 words for quick updates). +- **Keep it concise**: be no more than 1-2 sentences, focused on immediate, tangible next steps. (8–12 words for quick updates). - **Build on prior context**: if this is not your first tool call, use the preamble message to connect the dots with what’s been done so far and create a sense of momentum and clarity for the user to understand your next actions. - **Keep your tone light, friendly and curious**: add small touches of personality in preambles feel collaborative and engaging. +- **Exception**: Avoid adding a preamble for every trivial read (e.g., `cat` a single file) unless it’s part of a larger grouped action. **Examples:** + - “I’ve explored the repo; now checking the API route definitions.” - “Next, I’ll patch the config and update the related tests.” - “I’m about to scaffold the CLI commands and helper functions.” @@ -34,15 +37,18 @@ Before making tool calls, send a brief preamble to the user explaining what you - “Alright, build pipeline order is interesting. Checking how it reports failures.” - “Spotted a clever caching util; now hunting where it gets used.” -**Avoiding a preamble for every trivial read (e.g., `cat` a single file) unless it’s part of a larger grouped action. -- Jumping straight into tool calls without explaining what’s about to happen. -- Writing overly long or speculative preambles — focus on immediate, tangible next steps. - ## Planning -You have access to an `update_plan` tool which tracks steps and progress and renders them to the user. Using the tool helps demonstrate that you've understood the task and convey how you're approaching it. Plans can help to make complex, ambiguous, or multi-phase work clearer and more collaborative for the user. A good plan should break the task into meaningful, logically ordered steps that are easy to verify as you go. Note that plans are not for padding out simple work with filler steps or stating the obvious. Do not repeat the full contents of the plan after an `update_plan` call — the harness already displays it. Instead, summarize the change made and highlight any important context or next step. +You have access to an `update_plan` tool which tracks steps and progress and renders them to the user. Using the tool helps demonstrate that you've understood the task and convey how you're approaching it. Plans can help to make complex, ambiguous, or multi-phase work clearer and more collaborative for the user. A good plan should break the task into meaningful, logically ordered steps that are easy to verify as you go. + +Note that plans are not for padding out simple work with filler steps or stating the obvious. The content of your plan should not involve doing anything that you aren't capable of doing (i.e. don't try to test things that you can't test). Do not use plans for simple or single-step queries that you can just do or answer immediately. + +Do not repeat the full contents of the plan after an `update_plan` call — the harness already displays it. Instead, summarize the change made and highlight any important context or next step. + +Before running a command, consider whether or not you have completed the previous step, and make sure to mark it as completed before moving on to the next step. It may be the case that you complete all steps in your plan after a single pass of implementation. If this is the case, you can simply mark all the planned steps as completed. Sometimes, you may need to change plans in the middle of a task: call `update_plan` with the updated plan and make sure to provide an `explanation` of the rationale when doing so. Use a plan when: + - The task is non-trivial and will require multiple actions over a long time horizon. - There are logical phases or dependencies where sequencing matters. - The work has ambiguity that benefits from outlining high-level goals. @@ -51,14 +57,6 @@ Use a plan when: - The user has asked you to use the plan tool (aka "TODOs") - You generate additional steps while working, and plan to do them before yielding to the user -Skip a plan when: -- The task is simple and direct. -- Breaking it down would only produce literal or trivial steps. - -Planning steps are called "steps" in the tool, but really they're more like tasks or TODOs. As such they should be very concise descriptions of non-obvious work that an engineer might do like "Write the API spec", then "Update the backend", then "Implement the frontend". On the other hand, it's obvious that you'll usually have to "Explore the codebase" or "Implement the changes", so those are not worth tracking in your plan. - -It may be the case that you complete all steps in your plan after a single pass of implementation. If this is the case, you can simply mark all the planned steps as completed. The content of your plan should not involve doing anything that you aren't capable of doing (i.e. don't try to test things that you can't test). Do not use plans for simple or single-step queries that you can just do or answer immediately. - ### Examples **High-quality plans** @@ -115,10 +113,11 @@ If you need to write a plan, only write high quality plans, not low quality ones You are a coding agent. Please keep going until the query is completely resolved, before ending your turn and yielding back to the user. Only terminate your turn when you are sure that the problem is solved. Autonomously resolve the query to the best of your ability, using the tools available to you, before coming back to the user. Do NOT guess or make up an answer. You MUST adhere to the following criteria when solving queries: + - Working on the repo(s) in the current environment is allowed, even if they are proprietary. - Analyzing code for vulnerabilities is allowed. - Showing user code and tool call details is allowed. -- Use the `apply_patch` tool to edit files (NEVER try `applypatch` or `apply-patch`, only `apply_patch`): {"command":["apply_patch","*** Begin Patch\\n*** Update File: path/to/file.py\\n@@ def example():\\n- pass\\n+ return 123\\n*** End Patch"]} +- Use the `apply_patch` tool to edit files (NEVER try `applypatch` or `apply-patch`, only `apply_patch`): {"command":["apply_patch","*** Begin Patch\\n*** Update File: path/to/file.py\\n@@ def example():\\n- pass\\n+ return 123\\n*** End Patch"]} If completing the user's task requires writing or modifying files, your code and final answer should follow these coding guidelines, though user instructions (i.e. AGENTS.md) may override these guidelines: @@ -148,21 +147,25 @@ For all of testing, running, building, and formatting, do not attempt to fix unr The Codex CLI harness supports several different sandboxing, and approval configurations that the user can choose from. Filesystem sandboxing prevents you from editing files without user approval. The options are: -- *read-only*: You can only read files. -- *workspace-write*: You can read files. You can write to files in your workspace folder, but not outside it. -- *danger-full-access*: No filesystem sandboxing. + +- **read-only**: You can only read files. +- **workspace-write**: You can read files. You can write to files in your workspace folder, but not outside it. +- **danger-full-access**: No filesystem sandboxing. Network sandboxing prevents you from accessing network without approval. Options are -- *ON* -- *OFF* + +- **restricted** +- **enabled** Approvals are your mechanism to get user consent to perform more privileged actions. Although they introduce friction to the user because your work is paused until the user responds, you should leverage them to accomplish your important work. Do not let these settings or the sandbox deter you from attempting to accomplish the user's task. Approval options are -- *untrusted*: The harness will escalate most commands for user approval, apart from a limited allowlist of safe "read" commands. -- *on-failure*: The harness will allow all commands to run in the sandbox (if enabled), and failures will be escalated to the user for approval to run again without the sandbox. -- *on-request*: Commands will be run in the sandbox by default, and you can specify in your tool call if you want to escalate a command to run without sandboxing. (Note that this mode is not always available. If it is, you'll see parameters for it in the `shell` command description.) -- *never*: This is a non-interactive mode where you may NEVER ask the user for approval to run commands. Instead, you must always persist and work around constraints to solve the task for the user. You MUST do your utmost best to finish the task and validate your work before yielding. If this mode is pared with `danger-full-access`, take advantage of it to deliver the best outcome for the user. Further, in this mode, your default testing philosophy is overridden: Even if you don't see local patterns for testing, you may add tests and scripts to validate your work. Just remove them before yielding. + +- **untrusted**: The harness will escalate most commands for user approval, apart from a limited allowlist of safe "read" commands. +- **on-failure**: The harness will allow all commands to run in the sandbox (if enabled), and failures will be escalated to the user for approval to run again without the sandbox. +- **on-request**: Commands will be run in the sandbox by default, and you can specify in your tool call if you want to escalate a command to run without sandboxing. (Note that this mode is not always available. If it is, you'll see parameters for it in the `shell` command description.) +- **never**: This is a non-interactive mode where you may NEVER ask the user for approval to run commands. Instead, you must always persist and work around constraints to solve the task for the user. You MUST do your utmost best to finish the task and validate your work before yielding. If this mode is pared with `danger-full-access`, take advantage of it to deliver the best outcome for the user. Further, in this mode, your default testing philosophy is overridden: Even if you don't see local patterns for testing, you may add tests and scripts to validate your work. Just remove them before yielding. When you are running with approvals `on-request`, and sandboxing enabled, here are scenarios where you'll need to request approval: + - You need to run a command that writes to a directory that requires it (e.g. running tests that write to /tmp) - You need to run a GUI app (e.g., open/xdg-open/osascript) to open browsers or files. - You are running sandboxed and need to run a command that requires network access (e.g. installing packages) @@ -207,6 +210,7 @@ Brevity is very important as a default. You should be very concise (i.e. no more You are producing plain text that will later be styled by the CLI. Follow these rules exactly. Formatting should make results easy to scan, but not feel mechanical. Use judgment to decide how much structure adds value. **Section Headers** + - Use only when they improve clarity — they are not mandatory for every answer. - Choose descriptive names that fit the content - Keep headers short (1–3 words) and in `**Title Case**`. Always start headers with `**` and end with `**` @@ -214,6 +218,7 @@ You are producing plain text that will later be styled by the CLI. Follow these - Section headers should only be used where they genuinely improve scanability; avoid fragmenting the answer. **Bullets** + - Use `-` followed by a space for every bullet. - Bold the keyword, then colon + concise description. - Merge related points when possible; avoid a bullet for every trivial detail. @@ -222,11 +227,13 @@ You are producing plain text that will later be styled by the CLI. Follow these - Use consistent keyword phrasing and formatting across sections. **Monospace** + - Wrap all commands, file paths, env vars, and code identifiers in backticks (`` `...` ``). - Apply to inline examples and to bullet keywords if the keyword itself is a literal file/command. - Never mix monospace and bold markers; choose one based on whether it’s a keyword (`**`) or inline code/path (`` ` ``). **Structure** + - Place related bullets together; don’t mix unrelated concepts in the same section. - Order sections from general → specific → supporting info. - For subsections (e.g., “Binaries” under “Rust Workspace”), introduce with a bolded keyword bullet, then list items under it. @@ -235,6 +242,7 @@ You are producing plain text that will later be styled by the CLI. Follow these - Simple results → minimal headers, possibly just a short list or paragraph. **Tone** + - Keep the voice collaborative and natural, like a coding partner handing off work. - Be concise and factual — no filler or conversational commentary and avoid unnecessary repetition - Use present tense and active voice (e.g., “Runs tests” not “This will run tests”). @@ -242,6 +250,7 @@ You are producing plain text that will later be styled by the CLI. Follow these - Use parallel structure in lists for consistency. **Don’t** + - Don’t use literal words “bold” or “monospace” in the content. - Don’t nest bullets or create deep hierarchies. - Don’t output ANSI escape codes directly — the CLI renderer applies them. @@ -252,7 +261,14 @@ Generally, ensure your final answers adapt their shape and depth to the request. For casual greetings, acknowledgements, or other one-off conversational messages that are not delivering substantive information or structured results, respond naturally without section headers or bullet formatting. -# Tools +# Tool Guidelines + +## Shell commands + +When using the shell, you must adhere to the following guidelines: + +- When searching for text or files, prefer using `rg` or `rg --files` respectively because `rg` is much faster than alternatives like `grep`. (If the `rg` command is not found, then use alternatives.) +- Read files in chunks with a max chunk size of 250 lines. Do not use python scripts to attempt to output larger chunks of a file. Command line output will be truncated after 10 kilobytes or 256 lines of output, regardless of the command used. ## `apply_patch` diff --git a/codex-rs/core/src/apply_patch.rs b/codex-rs/core/src/apply_patch.rs index dc11aed023..4f9292b6d7 100644 --- a/codex-rs/core/src/apply_patch.rs +++ b/codex-rs/core/src/apply_patch.rs @@ -1,4 +1,5 @@ use crate::codex::Session; +use crate::codex::TurnContext; use crate::models::FunctionCallOutputPayload; use crate::models::ResponseInputItem; use crate::protocol::FileChange; @@ -8,7 +9,6 @@ use crate::safety::assess_patch_safety; use codex_apply_patch::ApplyPatchAction; use codex_apply_patch::ApplyPatchFileChange; use std::collections::HashMap; -use std::path::Path; use std::path::PathBuf; pub const CODEX_APPLY_PATCH_ARG1: &str = "--codex-run-as-apply-patch"; @@ -41,21 +41,16 @@ impl From for InternalApplyPatchInvocation { pub(crate) async fn apply_patch( sess: &Session, + turn_context: &TurnContext, sub_id: &str, call_id: &str, action: ApplyPatchAction, ) -> InternalApplyPatchInvocation { - let writable_roots_snapshot = { - #[allow(clippy::unwrap_used)] - let guard = sess.writable_roots.lock().unwrap(); - guard.clone() - }; - match assess_patch_safety( &action, - sess.approval_policy, - &writable_roots_snapshot, - &sess.cwd, + turn_context.approval_policy, + &turn_context.sandbox_policy, + &turn_context.cwd, ) { SafetyCheck::AutoApprove { .. } => { InternalApplyPatchInvocation::DelegateToExec(ApplyPatchExec { @@ -128,30 +123,3 @@ pub(crate) fn convert_apply_patch_to_protocol( } result } - -pub(crate) fn get_writable_roots(cwd: &Path) -> Vec { - let mut writable_roots = Vec::new(); - if cfg!(target_os = "macos") { - // On macOS, $TMPDIR is private to the user. - writable_roots.push(std::env::temp_dir()); - - // Allow pyenv to update its shims directory. Without this, any tool - // that happens to be managed by `pyenv` will fail with an error like: - // - // pyenv: cannot rehash: $HOME/.pyenv/shims isn't writable - // - // which is emitted every time `pyenv` tries to run `rehash` (for - // example, after installing a new Python package that drops an entry - // point). Although the sandbox is intentionally read‑only by default, - // writing to the user's local `pyenv` directory is safe because it - // is already user‑writable and scoped to the current user account. - if let Ok(home_dir) = std::env::var("HOME") { - let pyenv_dir = PathBuf::from(home_dir).join(".pyenv"); - writable_roots.push(pyenv_dir); - } - } - - writable_roots.push(cwd.to_path_buf()); - - writable_roots -} diff --git a/codex-rs/core/src/bash.rs b/codex-rs/core/src/bash.rs index b9cd444356..5b94daf252 100644 --- a/codex-rs/core/src/bash.rs +++ b/codex-rs/core/src/bash.rs @@ -132,7 +132,6 @@ fn parse_plain_command_from_node(cmd: tree_sitter::Node, src: &str) -> Option Option>> { diff --git a/codex-rs/core/src/chat_completions.rs b/codex-rs/core/src/chat_completions.rs index dae140bc02..fc0ceca5ad 100644 --- a/codex-rs/core/src/chat_completions.rs +++ b/codex-rs/core/src/chat_completions.rs @@ -213,7 +213,9 @@ async fn process_chat_sse( let sse = match timeout(idle_timeout, stream.next()).await { Ok(Some(Ok(ev))) => ev, Ok(Some(Err(e))) => { - let _ = tx_event.send(Err(CodexErr::Stream(e.to_string()))).await; + let _ = tx_event + .send(Err(CodexErr::Stream(e.to_string(), None))) + .await; return; } Ok(None) => { @@ -228,7 +230,10 @@ async fn process_chat_sse( } Err(_) => { let _ = tx_event - .send(Err(CodexErr::Stream("idle timeout waiting for SSE".into()))) + .send(Err(CodexErr::Stream( + "idle timeout waiting for SSE".into(), + None, + ))) .await; return; } @@ -588,6 +593,9 @@ where Poll::Ready(Some(Ok(ResponseEvent::ReasoningSummaryDelta(_)))) => { continue; } + Poll::Ready(Some(Ok(ResponseEvent::ReasoningSummaryPartAdded))) => { + continue; + } } } } diff --git a/codex-rs/core/src/client.rs b/codex-rs/core/src/client.rs index 0caf1170a6..86a711e436 100644 --- a/codex-rs/core/src/client.rs +++ b/codex-rs/core/src/client.rs @@ -1,5 +1,6 @@ use std::io::BufRead; use std::path::Path; +use std::sync::OnceLock; use std::time::Duration; use bytes::Bytes; @@ -7,6 +8,7 @@ use codex_login::AuthMode; use codex_login::CodexAuth; use eventsource_stream::Eventsource; use futures::prelude::*; +use regex_lite::Regex; use reqwest::StatusCode; use serde::Deserialize; use serde::Serialize; @@ -27,18 +29,20 @@ use crate::client_common::ResponseStream; use crate::client_common::ResponsesApiRequest; use crate::client_common::create_reasoning_param_for_request; use crate::config::Config; -use crate::config_types::ReasoningEffort as ReasoningEffortConfig; -use crate::config_types::ReasoningSummary as ReasoningSummaryConfig; use crate::error::CodexErr; use crate::error::Result; use crate::error::UsageLimitReachedError; use crate::flags::CODEX_RS_SSE_FIXTURE; +use crate::model_family::ModelFamily; use crate::model_provider_info::ModelProviderInfo; use crate::model_provider_info::WireApi; use crate::models::ResponseItem; use crate::openai_tools::create_tools_json_for_responses_api; use crate::protocol::TokenUsage; +use crate::user_agent::get_codex_user_agent; use crate::util::backoff; +use codex_protocol::config_types::ReasoningEffort as ReasoningEffortConfig; +use codex_protocol::config_types::ReasoningSummary as ReasoningSummaryConfig; use std::sync::Arc; #[derive(Debug, Deserialize)] @@ -48,10 +52,12 @@ struct ErrorResponse { #[derive(Debug, Deserialize)] struct Error { - r#type: String, + r#type: Option, + code: Option, + message: Option, } -#[derive(Clone)] +#[derive(Debug, Clone)] pub struct ModelClient { config: Arc, auth: Option, @@ -169,6 +175,7 @@ impl ModelClient { store, stream: true, include, + prompt_cache_key: Some(self.session_id.to_string()), }; let mut attempt = 0; @@ -207,6 +214,7 @@ impl ModelClient { .as_deref() .unwrap_or("codex_cli_rs"); req_builder = req_builder.header("originator", originator); + req_builder = req_builder.header("User-Agent", get_codex_user_agent(Some(originator))); let res = req_builder.send().await; if let Ok(resp) = &res { @@ -260,14 +268,18 @@ impl ModelClient { if status == StatusCode::TOO_MANY_REQUESTS { let body = res.json::().await.ok(); if let Some(ErrorResponse { - error: Error { r#type, .. }, + error: + Error { + r#type: Some(error_type), + .. + }, }) = body { - if r#type == "usage_limit_reached" { + if error_type == "usage_limit_reached" { return Err(CodexErr::UsageLimitReached(UsageLimitReachedError { plan_type: auth.and_then(|a| a.get_plan_type()), })); - } else if r#type == "usage_not_included" { + } else if error_type == "usage_not_included" { return Err(CodexErr::UsageNotIncluded); } } @@ -300,6 +312,30 @@ impl ModelClient { pub fn get_provider(&self) -> ModelProviderInfo { self.provider.clone() } + + /// Returns the currently configured model slug. + pub fn get_model(&self) -> String { + self.config.model.clone() + } + + /// Returns the currently configured model family. + pub fn get_model_family(&self) -> ModelFamily { + self.config.model_family.clone() + } + + /// Returns the current reasoning effort setting. + pub fn get_reasoning_effort(&self) -> ReasoningEffortConfig { + self.effort + } + + /// Returns the current reasoning summary setting. + pub fn get_reasoning_summary(&self) -> ReasoningSummaryConfig { + self.summary + } + + pub fn get_auth(&self) -> Option { + self.auth.clone() + } } #[derive(Debug, Deserialize, Serialize)] @@ -363,13 +399,14 @@ async fn process_sse( // If the stream stays completely silent for an extended period treat it as disconnected. // The response id returned from the "complete" message. let mut response_completed: Option = None; + let mut response_error: Option = None; loop { let sse = match timeout(idle_timeout, stream.next()).await { Ok(Some(Ok(sse))) => sse, Ok(Some(Err(e))) => { debug!("SSE Error: {e:#}"); - let event = CodexErr::Stream(e.to_string()); + let event = CodexErr::Stream(e.to_string(), None); let _ = tx_event.send(Err(event)).await; return; } @@ -387,9 +424,10 @@ async fn process_sse( } None => { let _ = tx_event - .send(Err(CodexErr::Stream( + .send(Err(response_error.unwrap_or(CodexErr::Stream( "stream closed before response.completed".into(), - ))) + None, + )))) .await; } } @@ -397,12 +435,17 @@ async fn process_sse( } Err(_) => { let _ = tx_event - .send(Err(CodexErr::Stream("idle timeout waiting for SSE".into()))) + .send(Err(CodexErr::Stream( + "idle timeout waiting for SSE".into(), + None, + ))) .await; return; } }; + trace!("SSE event: {}", sse.data); + let event: SseEvent = match serde_json::from_str(&sse.data) { Ok(event) => event, Err(e) => { @@ -411,7 +454,6 @@ async fn process_sse( } }; - trace!(?event, "SSE event"); match event.kind.as_str() { // Individual output item finalised. Forward immediately so the // rest of the agent can stream assistant text/functions *live* @@ -474,15 +516,25 @@ async fn process_sse( } "response.failed" => { if let Some(resp_val) = event.response { - let error = resp_val - .get("error") - .and_then(|v| v.get("message")) - .and_then(|v| v.as_str()) - .unwrap_or("response.failed event received"); + response_error = Some(CodexErr::Stream( + "response.failed event received".to_string(), + None, + )); - let _ = tx_event - .send(Err(CodexErr::Stream(error.to_string()))) - .await; + let error = resp_val.get("error"); + + if let Some(error) = error { + match serde_json::from_value::(error.clone()) { + Ok(error) => { + let delay = try_parse_retry_after(&error); + let message = error.message.unwrap_or_default(); + response_error = Some(CodexErr::Stream(message, delay)); + } + Err(e) => { + debug!("failed to parse ErrorResponse: {e}"); + } + } + } } } // Final response completed – includes array of output items & id @@ -503,12 +555,18 @@ async fn process_sse( | "response.function_call_arguments.delta" | "response.in_progress" | "response.output_item.added" - | "response.output_text.done" - | "response.reasoning_summary_part.added" - | "response.reasoning_summary_text.done" => { - // Currently, we ignore these events, but we handle them + | "response.output_text.done" => { + // Currently, we ignore this event, but we handle it // separately to skip the logging message in the `other` case. } + "response.reasoning_summary_part.added" => { + // Boundary between reasoning summary sections (e.g., titles). + let event = ResponseEvent::ReasoningSummaryPartAdded; + if tx_event.send(Ok(event)).await.is_err() { + return; + } + } + "response.reasoning_summary_text.done" => {} other => debug!(other, "sse event"), } } @@ -540,10 +598,42 @@ async fn stream_from_fixture( Ok(ResponseStream { rx_event }) } +fn rate_limit_regex() -> &'static Regex { + static RE: OnceLock = OnceLock::new(); + + #[expect(clippy::unwrap_used)] + RE.get_or_init(|| Regex::new(r"Please try again in (\d+(?:\.\d+)?)(s|ms)").unwrap()) +} + +fn try_parse_retry_after(err: &Error) -> Option { + if err.code != Some("rate_limit_exceeded".to_string()) { + return None; + } + + // parse the Please try again in 1.898s format using regex + let re = rate_limit_regex(); + if let Some(message) = &err.message + && let Some(captures) = re.captures(message) + { + let seconds = captures.get(1); + let unit = captures.get(2); + + if let (Some(value), Some(unit)) = (seconds, unit) { + let value = value.as_str().parse::().ok()?; + let unit = unit.as_str(); + + if unit == "s" { + return Some(Duration::from_secs_f64(value)); + } else if unit == "ms" { + return Some(Duration::from_millis(value as u64)); + } + } + } + None +} + #[cfg(test)] mod tests { - #![allow(clippy::expect_used, clippy::unwrap_used)] - use super::*; use serde_json::json; use tokio::sync::mpsc; @@ -725,13 +815,49 @@ mod tests { matches!(events[0], Ok(ResponseEvent::OutputItemDone(_))); match &events[1] { - Err(CodexErr::Stream(msg)) => { + Err(CodexErr::Stream(msg, _)) => { assert_eq!(msg, "stream closed before response.completed") } other => panic!("unexpected second event: {other:?}"), } } + #[tokio::test] + async fn error_when_error_event() { + let raw_error = r#"{"type":"response.failed","sequence_number":3,"response":{"id":"resp_689bcf18d7f08194bf3440ba62fe05d803fee0cdac429894","object":"response","created_at":1755041560,"status":"failed","background":false,"error":{"code":"rate_limit_exceeded","message":"Rate limit reached for gpt-5 in organization org-AAA on tokens per min (TPM): Limit 30000, Used 22999, Requested 12528. Please try again in 11.054s. Visit https://platform.openai.com/account/rate-limits to learn more."}, "usage":null,"user":null,"metadata":{}}}"#; + + let sse1 = format!("event: response.failed\ndata: {raw_error}\n\n"); + let provider = ModelProviderInfo { + name: "test".to_string(), + base_url: Some("https://test.com".to_string()), + env_key: Some("TEST_API_KEY".to_string()), + env_key_instructions: None, + wire_api: WireApi::Responses, + query_params: None, + http_headers: None, + env_http_headers: None, + request_max_retries: Some(0), + stream_max_retries: Some(0), + stream_idle_timeout_ms: Some(1000), + requires_openai_auth: false, + }; + + let events = collect_events(&[sse1.as_bytes()], provider).await; + + assert_eq!(events.len(), 1); + + match &events[0] { + Err(CodexErr::Stream(msg, delay)) => { + assert_eq!( + msg, + "Rate limit reached for gpt-5 in organization org-AAA on tokens per min (TPM): Limit 30000, Used 22999, Requested 12528. Please try again in 11.054s. Visit https://platform.openai.com/account/rate-limits to learn more." + ); + assert_eq!(*delay, Some(Duration::from_secs_f64(11.054))); + } + other => panic!("unexpected second event: {other:?}"), + } + } + // ──────────────────────────── // Table-driven test from `main` // ──────────────────────────── @@ -830,4 +956,27 @@ mod tests { ); } } + + #[test] + fn test_try_parse_retry_after() { + let err = Error { + r#type: None, + message: Some("Rate limit reached for gpt-5 in organization org- on tokens per min (TPM): Limit 1, Used 1, Requested 19304. Please try again in 28ms. Visit https://platform.openai.com/account/rate-limits to learn more.".to_string()), + code: Some("rate_limit_exceeded".to_string()), + }; + + let delay = try_parse_retry_after(&err); + assert_eq!(delay, Some(Duration::from_millis(28))); + } + + #[test] + fn test_try_parse_retry_after_no_delay() { + let err = Error { + r#type: None, + message: Some("Rate limit reached for gpt-5 in organization on tokens per min (TPM): Limit 30000, Used 6899, Requested 24050. Please try again in 1.898s. Visit https://platform.openai.com/account/rate-limits to learn more.".to_string()), + code: Some("rate_limit_exceeded".to_string()), + }; + let delay = try_parse_retry_after(&err); + assert_eq!(delay, Some(Duration::from_secs_f64(1.898))); + } } diff --git a/codex-rs/core/src/client_common.rs b/codex-rs/core/src/client_common.rs index b37b1e3f80..15b8ea89c0 100644 --- a/codex-rs/core/src/client_common.rs +++ b/codex-rs/core/src/client_common.rs @@ -1,19 +1,15 @@ -use crate::config_types::ReasoningEffort as ReasoningEffortConfig; -use crate::config_types::ReasoningSummary as ReasoningSummaryConfig; use crate::error::Result; use crate::model_family::ModelFamily; use crate::models::ContentItem; use crate::models::ResponseItem; use crate::openai_tools::OpenAiTool; -use crate::protocol::AskForApproval; -use crate::protocol::SandboxPolicy; use crate::protocol::TokenUsage; use codex_apply_patch::APPLY_PATCH_TOOL_INSTRUCTIONS; +use codex_protocol::config_types::ReasoningEffort as ReasoningEffortConfig; +use codex_protocol::config_types::ReasoningSummary as ReasoningSummaryConfig; use futures::Stream; use serde::Serialize; use std::borrow::Cow; -use std::fmt::Display; -use std::path::PathBuf; use std::pin::Pin; use std::task::Context; use std::task::Poll; @@ -23,62 +19,19 @@ use tokio::sync::mpsc; /// with this content. const BASE_INSTRUCTIONS: &str = include_str!("../prompt.md"); -/// wraps environment context message in a tag for the model to parse more easily. -const ENVIRONMENT_CONTEXT_START: &str = "\n\n"; -const ENVIRONMENT_CONTEXT_END: &str = "\n\n"; - /// wraps user instructions message in a tag for the model to parse more easily. const USER_INSTRUCTIONS_START: &str = "\n\n"; const USER_INSTRUCTIONS_END: &str = "\n\n"; -#[derive(Debug, Clone)] -pub(crate) struct EnvironmentContext { - pub cwd: PathBuf, - pub approval_policy: AskForApproval, - pub sandbox_policy: SandboxPolicy, -} - -impl Display for EnvironmentContext { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - writeln!( - f, - "Current working directory: {}", - self.cwd.to_string_lossy() - )?; - writeln!(f, "Approval policy: {}", self.approval_policy)?; - writeln!(f, "Sandbox policy: {}", self.sandbox_policy)?; - - let network_access = match self.sandbox_policy.clone() { - SandboxPolicy::DangerFullAccess => "enabled", - SandboxPolicy::ReadOnly => "restricted", - SandboxPolicy::WorkspaceWrite { network_access, .. } => { - if network_access { - "enabled" - } else { - "restricted" - } - } - }; - writeln!(f, "Network access: {network_access}")?; - Ok(()) - } -} - -/// API request payload for a single model turn. +/// API request payload for a single model turn #[derive(Default, Debug, Clone)] pub struct Prompt { /// Conversation context input items. pub input: Vec, - /// Optional instructions from the user to amend to the built-in agent - /// instructions. - pub user_instructions: Option, + /// Whether to store response on server side (disable_response_storage = !store). pub store: bool, - /// A list of key-value pairs that will be added as a developer message - /// for the model to use - pub environment_context: Option, - /// Tools available to the model, including additional tools sourced from /// external MCP servers. pub tools: Vec, @@ -100,36 +53,19 @@ impl Prompt { Cow::Owned(sections.join("\n")) } - fn get_formatted_user_instructions(&self) -> Option { - self.user_instructions - .as_ref() - .map(|ui| format!("{USER_INSTRUCTIONS_START}{ui}{USER_INSTRUCTIONS_END}")) - } - - fn get_formatted_environment_context(&self) -> Option { - self.environment_context - .as_ref() - .map(|ec| format!("{ENVIRONMENT_CONTEXT_START}{ec}{ENVIRONMENT_CONTEXT_END}")) - } - pub(crate) fn get_formatted_input(&self) -> Vec { - let mut input_with_instructions = Vec::with_capacity(self.input.len() + 2); - if let Some(ec) = self.get_formatted_environment_context() { - input_with_instructions.push(ResponseItem::Message { - id: None, - role: "user".to_string(), - content: vec![ContentItem::InputText { text: ec }], - }); + self.input.clone() + } + + /// Creates a formatted user instructions message from a string + pub(crate) fn format_user_instructions_message(ui: &str) -> ResponseItem { + ResponseItem::Message { + id: None, + role: "user".to_string(), + content: vec![ContentItem::InputText { + text: format!("{USER_INSTRUCTIONS_START}{ui}{USER_INSTRUCTIONS_END}"), + }], } - if let Some(ui) = self.get_formatted_user_instructions() { - input_with_instructions.push(ResponseItem::Message { - id: None, - role: "user".to_string(), - content: vec![ContentItem::InputText { text: ui }], - }); - } - input_with_instructions.extend(self.input.clone()); - input_with_instructions } } @@ -144,57 +80,13 @@ pub enum ResponseEvent { OutputTextDelta(String), ReasoningSummaryDelta(String), ReasoningContentDelta(String), + ReasoningSummaryPartAdded, } #[derive(Debug, Serialize)] pub(crate) struct Reasoning { - pub(crate) effort: OpenAiReasoningEffort, - #[serde(skip_serializing_if = "Option::is_none")] - pub(crate) summary: Option, -} - -/// See https://platform.openai.com/docs/guides/reasoning?api-mode=responses#get-started-with-reasoning -#[derive(Debug, Serialize, Default, Clone, Copy)] -#[serde(rename_all = "lowercase")] -pub(crate) enum OpenAiReasoningEffort { - Low, - #[default] - Medium, - High, -} - -impl From for Option { - fn from(effort: ReasoningEffortConfig) -> Self { - match effort { - ReasoningEffortConfig::Low => Some(OpenAiReasoningEffort::Low), - ReasoningEffortConfig::Medium => Some(OpenAiReasoningEffort::Medium), - ReasoningEffortConfig::High => Some(OpenAiReasoningEffort::High), - ReasoningEffortConfig::None => None, - } - } -} - -/// A summary of the reasoning performed by the model. This can be useful for -/// debugging and understanding the model's reasoning process. -/// See https://platform.openai.com/docs/guides/reasoning?api-mode=responses#reasoning-summaries -#[derive(Debug, Serialize, Default, Clone, Copy)] -#[serde(rename_all = "lowercase")] -pub(crate) enum OpenAiReasoningSummary { - #[default] - Auto, - Concise, - Detailed, -} - -impl From for Option { - fn from(summary: ReasoningSummaryConfig) -> Self { - match summary { - ReasoningSummaryConfig::Auto => Some(OpenAiReasoningSummary::Auto), - ReasoningSummaryConfig::Concise => Some(OpenAiReasoningSummary::Concise), - ReasoningSummaryConfig::Detailed => Some(OpenAiReasoningSummary::Detailed), - ReasoningSummaryConfig::None => None, - } - } + pub(crate) effort: ReasoningEffortConfig, + pub(crate) summary: ReasoningSummaryConfig, } /// Request object that is serialized as JSON and POST'ed when using the @@ -215,6 +107,8 @@ pub(crate) struct ResponsesApiRequest<'a> { pub(crate) store: bool, pub(crate) stream: bool, pub(crate) include: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) prompt_cache_key: Option, } pub(crate) fn create_reasoning_param_for_request( @@ -223,12 +117,7 @@ pub(crate) fn create_reasoning_param_for_request( summary: ReasoningSummaryConfig, ) -> Option { if model_family.supports_reasoning_summaries { - let effort: Option = effort.into(); - let effort = effort?; - Some(Reasoning { - effort, - summary: summary.into(), - }) + Some(Reasoning { effort, summary }) } else { None } @@ -248,7 +137,6 @@ impl Stream for ResponseStream { #[cfg(test)] mod tests { - #![allow(clippy::expect_used)] use crate::model_family::find_family_for_model; use super::*; @@ -256,7 +144,6 @@ mod tests { #[test] fn get_full_instructions_no_user_content() { let prompt = Prompt { - user_instructions: Some("custom instruction".to_string()), ..Default::default() }; let expected = format!("{BASE_INSTRUCTIONS}\n{APPLY_PATCH_TOOL_INSTRUCTIONS}"); diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index 3bf6288fdf..6423fb44ef 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -1,12 +1,10 @@ -// Poisoned mutex should fail the program -#![allow(clippy::unwrap_used)] - use std::borrow::Cow; use std::collections::HashMap; use std::collections::HashSet; use std::path::PathBuf; use std::sync::Arc; use std::sync::Mutex; +use std::sync::MutexGuard; use std::sync::atomic::AtomicU64; use std::time::Duration; @@ -16,11 +14,12 @@ use codex_apply_patch::ApplyPatchAction; use codex_apply_patch::MaybeApplyPatchVerified; use codex_apply_patch::maybe_parse_apply_patch_verified; use codex_login::CodexAuth; +use codex_protocol::protocol::TurnAbortReason; +use codex_protocol::protocol::TurnAbortedEvent; use futures::prelude::*; use mcp_types::CallToolResult; use serde::Serialize; use serde_json; -use tokio::sync::Notify; use tokio::sync::oneshot; use tokio::task::AbortHandle; use tracing::debug; @@ -30,19 +29,19 @@ use tracing::trace; use tracing::warn; use uuid::Uuid; +use crate::ModelProviderInfo; +use crate::apply_patch; use crate::apply_patch::ApplyPatchExec; use crate::apply_patch::CODEX_APPLY_PATCH_ARG1; use crate::apply_patch::InternalApplyPatchInvocation; use crate::apply_patch::convert_apply_patch_to_protocol; -use crate::apply_patch::get_writable_roots; -use crate::apply_patch::{self}; use crate::client::ModelClient; -use crate::client_common::EnvironmentContext; use crate::client_common::Prompt; use crate::client_common::ResponseEvent; use crate::config::Config; use crate::config_types::ShellEnvironmentPolicy; use crate::conversation_history::ConversationHistory; +use crate::environment_context::EnvironmentContext; use crate::error::CodexErr; use crate::error::Result as CodexResult; use crate::error::SandboxErr; @@ -51,10 +50,12 @@ use crate::exec::ExecParams; use crate::exec::ExecToolCallOutput; use crate::exec::SandboxType; use crate::exec::StdoutStream; +use crate::exec::StreamOutput; use crate::exec::process_exec_tool_call; use crate::exec_env::create_env; use crate::mcp_connection_manager::McpConnectionManager; use crate::mcp_tool_call::handle_mcp_tool_call; +use crate::model_family::find_family_for_model; use crate::models::ContentItem; use crate::models::FunctionCallOutputPayload; use crate::models::LocalShellAction; @@ -63,8 +64,10 @@ use crate::models::ReasoningItemReasoningSummary; use crate::models::ResponseInputItem; use crate::models::ResponseItem; use crate::models::ShellToolCallParams; +use crate::openai_tools::ApplyPatchToolArgs; use crate::openai_tools::ToolsConfig; use crate::openai_tools::get_openai_tools; +use crate::parse_command::parse_command; use crate::plan_tool::handle_update_plan; use crate::project_doc::get_user_instructions; use crate::protocol::AgentMessageDeltaEvent; @@ -73,6 +76,7 @@ use crate::protocol::AgentReasoningDeltaEvent; use crate::protocol::AgentReasoningEvent; use crate::protocol::AgentReasoningRawContentDeltaEvent; use crate::protocol::AgentReasoningRawContentEvent; +use crate::protocol::AgentReasoningSectionBreakEvent; use crate::protocol::ApplyPatchApprovalRequestEvent; use crate::protocol::AskForApproval; use crate::protocol::BackgroundEventEvent; @@ -101,6 +105,23 @@ use crate::shell; use crate::turn_diff_tracker::TurnDiffTracker; use crate::user_notification::UserNotification; use crate::util::backoff; +use codex_protocol::config_types::ReasoningEffort as ReasoningEffortConfig; +use codex_protocol::config_types::ReasoningSummary as ReasoningSummaryConfig; + +// A convenience extension trait for acquiring mutex locks where poisoning is +// unrecoverable and should abort the program. This avoids scattered `.unwrap()` +// calls on `lock()` while still surfacing a clear panic message when a lock is +// poisoned. +trait MutexExt { + fn lock_unchecked(&self) -> MutexGuard<'_, T>; +} + +impl MutexExt for Mutex { + fn lock_unchecked(&self) -> MutexGuard<'_, T> { + #[expect(clippy::expect_used)] + self.lock().expect("poisoned lock") + } +} /// The high-level interface to the Codex system. /// It operates as a queue pair where you send submissions and receive events. @@ -115,26 +136,23 @@ pub struct Codex { /// unique session id. pub struct CodexSpawnOk { pub codex: Codex, - pub init_id: String, pub session_id: Uuid, } +pub(crate) const INITIAL_SUBMIT_ID: &str = ""; + impl Codex { /// Spawn a new [`Codex`] and initialize the session. - pub async fn spawn( - config: Config, - auth: Option, - ctrl_c: Arc, - ) -> CodexResult { - // experimental resume path (undocumented) - let resume_path = config.experimental_resume.clone(); - info!("resume_path: {resume_path:?}"); + pub async fn spawn(config: Config, auth: Option) -> CodexResult { let (tx_sub, rx_sub) = async_channel::bounded(64); let (tx_event, rx_event) = async_channel::unbounded(); let user_instructions = get_user_instructions(&config).await; - let configure_session = Op::ConfigureSession { + let config = Arc::new(config); + let resume_path = config.experimental_resume.clone(); + + let configure_session = ConfigureSession { provider: config.model_provider.clone(), model: config.model.clone(), model_reasoning_effort: config.model_reasoning_effort, @@ -146,28 +164,28 @@ impl Codex { disable_response_storage: config.disable_response_storage, notify: config.notify.clone(), cwd: config.cwd.clone(), - resume_path: resume_path.clone(), + resume_path, }; - let config = Arc::new(config); - // Generate a unique ID for the lifetime of this Codex session. - let session_id = Uuid::new_v4(); - tokio::spawn(submission_loop( - session_id, config, auth, rx_sub, tx_event, ctrl_c, - )); + let (session, turn_context) = + Session::new(configure_session, config.clone(), auth, tx_event.clone()) + .await + .map_err(|e| { + error!("Failed to create session: {e:#}"); + CodexErr::InternalAgentDied + })?; + let session_id = session.session_id; + + // This task will run until Op::Shutdown is received. + tokio::spawn(submission_loop(session, turn_context, config, rx_sub)); let codex = Codex { next_id: AtomicU64::new(0), tx_sub, rx_event, }; - let init_id = codex.submit(configure_session).await?; - Ok(CodexSpawnOk { - codex, - init_id, - session_id, - }) + Ok(CodexSpawnOk { codex, session_id }) } /// Submit the `op` wrapped in a `Submission` with a unique ID. @@ -201,26 +219,22 @@ impl Codex { } } +/// Mutable state of the agent +#[derive(Default)] +struct State { + approved_commands: HashSet>, + current_task: Option, + pending_approvals: HashMap>, + pending_input: Vec, + history: ConversationHistory, +} + /// Context for an initialized model agent /// /// A session has at most 1 running task at a time, and can be interrupted by user input. pub(crate) struct Session { - client: ModelClient, - pub(crate) tx_event: Sender, - ctrl_c: Arc, - - /// The session's current working directory. All relative paths provided by - /// the model as well as sandbox policies are resolved against this path - /// instead of `std::env::current_dir()`. - pub(crate) cwd: PathBuf, - base_instructions: Option, - user_instructions: Option, - pub(crate) approval_policy: AskForApproval, - sandbox_policy: SandboxPolicy, - shell_environment_policy: ShellEnvironmentPolicy, - pub(crate) writable_roots: Mutex>, - disable_response_storage: bool, - tools_config: ToolsConfig, + session_id: Uuid, + tx_event: Sender, /// Manager for external MCP servers/tools. mcp_connection_manager: McpConnectionManager, @@ -238,7 +252,24 @@ pub(crate) struct Session { show_raw_agent_reasoning: bool, } -impl Session { +/// The context needed for a single turn of the conversation. +#[derive(Debug)] +pub(crate) struct TurnContext { + pub(crate) client: ModelClient, + /// The session's current working directory. All relative paths provided by + /// the model as well as sandbox policies are resolved against this path + /// instead of `std::env::current_dir()`. + pub(crate) cwd: PathBuf, + pub(crate) base_instructions: Option, + pub(crate) user_instructions: Option, + pub(crate) approval_policy: AskForApproval, + pub(crate) sandbox_policy: SandboxPolicy, + pub(crate) shell_environment_policy: ShellEnvironmentPolicy, + pub(crate) disable_response_storage: bool, + pub(crate) tools_config: ToolsConfig, +} + +impl TurnContext { fn resolve_path(&self, path: Option) -> PathBuf { path.as_ref() .map(PathBuf::from) @@ -246,27 +277,273 @@ impl Session { } } -/// Mutable state of the agent -#[derive(Default)] -struct State { - approved_commands: HashSet>, - current_task: Option, - pending_approvals: HashMap>, - pending_input: Vec, - history: ConversationHistory, +/// Configure the model session. +struct ConfigureSession { + /// Provider identifier ("openai", "openrouter", ...). + provider: ModelProviderInfo, + + /// If not specified, server will use its default model. + model: String, + + model_reasoning_effort: ReasoningEffortConfig, + model_reasoning_summary: ReasoningSummaryConfig, + + /// Model instructions that are appended to the base instructions. + user_instructions: Option, + + /// Base instructions override. + base_instructions: Option, + + /// When to escalate for approval for execution + approval_policy: AskForApproval, + /// How to sandbox commands executed in the system + sandbox_policy: SandboxPolicy, + /// Disable server-side response storage (send full context each request) + disable_response_storage: bool, + + /// Optional external notifier command tokens. Present only when the + /// client wants the agent to spawn a program after each completed + /// turn. + notify: Option>, + + /// Working directory that should be treated as the *root* of the + /// session. All relative paths supplied by the model as well as the + /// execution sandbox are resolved against this directory **instead** + /// of the process-wide current working directory. CLI front-ends are + /// expected to expand this to an absolute path before sending the + /// `ConfigureSession` operation so that the business-logic layer can + /// operate deterministically. + cwd: PathBuf, + + resume_path: Option, } impl Session { + async fn new( + configure_session: ConfigureSession, + config: Arc, + auth: Option, + tx_event: Sender, + ) -> anyhow::Result<(Arc, TurnContext)> { + let ConfigureSession { + provider, + model, + model_reasoning_effort, + model_reasoning_summary, + user_instructions, + base_instructions, + approval_policy, + sandbox_policy, + disable_response_storage, + notify, + cwd, + resume_path, + } = configure_session; + debug!("Configuring session: model={model}; provider={provider:?}"); + if !cwd.is_absolute() { + return Err(anyhow::anyhow!("cwd is not absolute: {cwd:?}")); + } + + // Error messages to dispatch after SessionConfigured is sent. + let mut post_session_configured_error_events = Vec::::new(); + + // Kick off independent async setup tasks in parallel to reduce startup latency. + // + // - initialize RolloutRecorder with new or resumed session info + // - spin up MCP connection manager + // - perform default shell discovery + // - load history metadata + let rollout_fut = async { + match resume_path.as_ref() { + Some(path) => RolloutRecorder::resume(path, cwd.clone()) + .await + .map(|(rec, saved)| (saved.session_id, Some(saved), rec)), + None => { + let session_id = Uuid::new_v4(); + RolloutRecorder::new(&config, session_id, user_instructions.clone()) + .await + .map(|rec| (session_id, None, rec)) + } + } + }; + + let mcp_fut = McpConnectionManager::new(config.mcp_servers.clone()); + let default_shell_fut = shell::default_user_shell(); + let history_meta_fut = crate::message_history::history_metadata(&config); + + // Join all independent futures. + let (rollout_res, mcp_res, default_shell, (history_log_id, history_entry_count)) = + tokio::join!(rollout_fut, mcp_fut, default_shell_fut, history_meta_fut); + + // Handle rollout result, which determines the session_id. + struct RolloutResult { + session_id: Uuid, + rollout_recorder: Option, + restored_items: Option>, + } + let rollout_result = match rollout_res { + Ok((session_id, maybe_saved, recorder)) => { + let restored_items: Option> = + maybe_saved.and_then(|saved_session| { + if saved_session.items.is_empty() { + None + } else { + Some(saved_session.items) + } + }); + RolloutResult { + session_id, + rollout_recorder: Some(recorder), + restored_items, + } + } + Err(e) => { + if let Some(path) = resume_path.as_ref() { + return Err(anyhow::anyhow!( + "failed to resume rollout from {path:?}: {e}" + )); + } + + let message = format!("failed to initialize rollout recorder: {e}"); + post_session_configured_error_events.push(Event { + id: INITIAL_SUBMIT_ID.to_owned(), + msg: EventMsg::Error(ErrorEvent { + message: message.clone(), + }), + }); + warn!("{message}"); + + RolloutResult { + session_id: Uuid::new_v4(), + rollout_recorder: None, + restored_items: None, + } + } + }; + + let RolloutResult { + session_id, + rollout_recorder, + restored_items, + } = rollout_result; + + // Create the mutable state for the Session. + let mut state = State { + history: ConversationHistory::new(), + ..Default::default() + }; + if let Some(restored_items) = restored_items { + state.history.record_items(&restored_items); + } + + // Handle MCP manager result and record any startup failures. + let (mcp_connection_manager, failed_clients) = match mcp_res { + Ok((mgr, failures)) => (mgr, failures), + Err(e) => { + let message = format!("Failed to create MCP connection manager: {e:#}"); + error!("{message}"); + post_session_configured_error_events.push(Event { + id: INITIAL_SUBMIT_ID.to_owned(), + msg: EventMsg::Error(ErrorEvent { message }), + }); + (McpConnectionManager::default(), Default::default()) + } + }; + + // Surface individual client start-up failures to the user. + if !failed_clients.is_empty() { + for (server_name, err) in failed_clients { + let message = format!("MCP client for `{server_name}` failed to start: {err:#}"); + error!("{message}"); + post_session_configured_error_events.push(Event { + id: INITIAL_SUBMIT_ID.to_owned(), + msg: EventMsg::Error(ErrorEvent { message }), + }); + } + } + + // Now that `session_id` is final (may have been updated by resume), + // construct the model client. + let client = ModelClient::new( + config.clone(), + auth.clone(), + provider.clone(), + model_reasoning_effort, + model_reasoning_summary, + session_id, + ); + let turn_context = TurnContext { + client, + tools_config: ToolsConfig::new( + &config.model_family, + approval_policy, + sandbox_policy.clone(), + config.include_plan_tool, + config.include_apply_patch_tool, + ), + user_instructions, + base_instructions, + approval_policy, + sandbox_policy, + shell_environment_policy: config.shell_environment_policy.clone(), + cwd, + disable_response_storage, + }; + let sess = Arc::new(Session { + session_id, + tx_event: tx_event.clone(), + mcp_connection_manager, + notify, + state: Mutex::new(state), + rollout: Mutex::new(rollout_recorder), + codex_linux_sandbox_exe: config.codex_linux_sandbox_exe.clone(), + user_shell: default_shell, + show_raw_agent_reasoning: config.show_raw_agent_reasoning, + }); + + // record the initial user instructions and environment context, + // regardless of whether we restored items. + let mut conversation_items = Vec::::with_capacity(2); + if let Some(user_instructions) = turn_context.user_instructions.as_deref() { + conversation_items.push(Prompt::format_user_instructions_message(user_instructions)); + } + conversation_items.push(ResponseItem::from(EnvironmentContext::new( + turn_context.cwd.to_path_buf(), + turn_context.approval_policy, + turn_context.sandbox_policy.clone(), + ))); + sess.record_conversation_items(&conversation_items).await; + + // Dispatch the SessionConfiguredEvent first and then report any errors. + let events = std::iter::once(Event { + id: INITIAL_SUBMIT_ID.to_owned(), + msg: EventMsg::SessionConfigured(SessionConfiguredEvent { + session_id, + model, + history_log_id, + history_entry_count, + }), + }) + .chain(post_session_configured_error_events.into_iter()); + for event in events { + if let Err(e) = tx_event.send(event).await { + error!("failed to send event: {e:?}"); + } + } + + Ok((sess, turn_context)) + } + pub fn set_task(&self, task: AgentTask) { - let mut state = self.state.lock().unwrap(); + let mut state = self.state.lock_unchecked(); if let Some(current_task) = state.current_task.take() { - current_task.abort(); + current_task.abort(TurnAbortReason::Replaced); } state.current_task = Some(task); } pub fn remove_task(&self, sub_id: &str) { - let mut state = self.state.lock().unwrap(); + let mut state = self.state.lock_unchecked(); if let Some(task) = &state.current_task { if task.sub_id == sub_id { state.current_task.take(); @@ -302,7 +579,7 @@ impl Session { }; let _ = self.tx_event.send(event).await; { - let mut state = self.state.lock().unwrap(); + let mut state = self.state.lock_unchecked(); state.pending_approvals.insert(sub_id, tx_approve); } rx_approve @@ -328,21 +605,21 @@ impl Session { }; let _ = self.tx_event.send(event).await; { - let mut state = self.state.lock().unwrap(); + let mut state = self.state.lock_unchecked(); state.pending_approvals.insert(sub_id, tx_approve); } rx_approve } pub fn notify_approval(&self, sub_id: &str, decision: ReviewDecision) { - let mut state = self.state.lock().unwrap(); + let mut state = self.state.lock_unchecked(); if let Some(tx_approve) = state.pending_approvals.remove(sub_id) { tx_approve.send(decision).ok(); } } pub fn add_approved_command(&self, cmd: Vec) { - let mut state = self.state.lock().unwrap(); + let mut state = self.state.lock_unchecked(); state.approved_commands.insert(cmd); } @@ -352,14 +629,14 @@ impl Session { debug!("Recording items for conversation: {items:?}"); self.record_state_snapshot(items).await; - self.state.lock().unwrap().history.record_items(items); + self.state.lock_unchecked().history.record_items(items); } async fn record_state_snapshot(&self, items: &[ResponseItem]) { let snapshot = { crate::rollout::SessionStateSnapshot {} }; let recorder = { - let guard = self.rollout.lock().unwrap(); + let guard = self.rollout.lock_unchecked(); guard.as_ref().cloned() }; @@ -402,6 +679,10 @@ impl Session { call_id, command: command_for_display.clone(), cwd, + parsed_cmd: parse_command(&command_for_display) + .into_iter() + .map(Into::into) + .collect(), }), }; let event = Event { @@ -429,8 +710,8 @@ impl Session { // Because stdout and stderr could each be up to 100 KiB, we send // truncated versions. const MAX_STREAM_OUTPUT: usize = 5 * 1024; // 5KiB - let stdout = stdout.chars().take(MAX_STREAM_OUTPUT).collect(); - let stderr = stderr.chars().take(MAX_STREAM_OUTPUT).collect(); + let stdout = stdout.text.chars().take(MAX_STREAM_OUTPUT).collect(); + let stderr = stderr.text.chars().take(MAX_STREAM_OUTPUT).collect(); let msg = if is_apply_patch { EventMsg::PatchApplyEnd(PatchApplyEndEvent { @@ -489,7 +770,6 @@ impl Session { let result = process_exec_tool_call( exec_args.params, exec_args.sandbox_type, - exec_args.ctrl_c, exec_args.sandbox_policy, exec_args.codex_linux_sandbox_exe, exec_args.stdout_stream, @@ -502,8 +782,8 @@ impl Session { Err(e) => { output_stderr = ExecToolCallOutput { exit_code: -1, - stdout: String::new(), - stderr: get_error_message_ui(e), + stdout: StreamOutput::new(String::new()), + stderr: StreamOutput::new(get_error_message_ui(e)), duration: Duration::default(), }; &output_stderr @@ -537,12 +817,12 @@ impl Session { /// Build the full turn input by concatenating the current conversation /// history with additional items for this turn. pub fn turn_input_with_history(&self, extra: Vec) -> Vec { - [self.state.lock().unwrap().history.contents(), extra].concat() + [self.state.lock_unchecked().history.contents(), extra].concat() } /// Returns the input if there was no task running to inject into pub fn inject_input(&self, input: Vec) -> Result<(), Vec> { - let mut state = self.state.lock().unwrap(); + let mut state = self.state.lock_unchecked(); if state.current_task.is_some() { state.pending_input.push(input.into()); Ok(()) @@ -552,7 +832,7 @@ impl Session { } pub fn get_pending_input(&self) -> Vec { - let mut state = self.state.lock().unwrap(); + let mut state = self.state.lock_unchecked(); if state.pending_input.is_empty() { Vec::with_capacity(0) } else { @@ -574,13 +854,13 @@ impl Session { .await } - pub fn abort(&self) { - info!("Aborting existing session"); - let mut state = self.state.lock().unwrap(); + fn interrupt_task(&self) { + info!("interrupt received: abort current task, if any"); + let mut state = self.state.lock_unchecked(); state.pending_approvals.clear(); state.pending_input.clear(); if let Some(task) = state.current_task.take() { - task.abort(); + task.abort(TurnAbortReason::Interrupted); } } @@ -616,17 +896,7 @@ impl Session { impl Drop for Session { fn drop(&mut self) { - self.abort(); - } -} - -impl State { - pub fn partial_clone(&self) -> Self { - Self { - approved_commands: self.approved_commands.clone(), - history: self.history.clone(), - ..Default::default() - } + self.interrupt_task(); } } @@ -653,28 +923,19 @@ pub(crate) struct AgentTask { } impl AgentTask { - fn spawn(sess: Arc, sub_id: String, input: Vec) -> Self { - let handle = - tokio::spawn(run_task(Arc::clone(&sess), sub_id.clone(), input)).abort_handle(); - Self { - sess, - sub_id, - handle, - } - } - fn compact( + fn spawn( sess: Arc, + turn_context: Arc, sub_id: String, input: Vec, - compact_instructions: String, ) -> Self { - let handle = tokio::spawn(run_compact_task( - Arc::clone(&sess), - sub_id.clone(), - input, - compact_instructions, - )) - .abort_handle(); + let handle = { + let sess = sess.clone(); + let sub_id = sub_id.clone(); + let tc = Arc::clone(&turn_context); + tokio::spawn(async move { run_task(sess, tc.as_ref(), sub_id, input).await }) + .abort_handle() + }; Self { sess, sub_id, @@ -682,14 +943,36 @@ impl AgentTask { } } - fn abort(self) { + fn compact( + sess: Arc, + turn_context: Arc, + sub_id: String, + input: Vec, + compact_instructions: String, + ) -> Self { + let handle = { + let sess = sess.clone(); + let sub_id = sub_id.clone(); + let tc = Arc::clone(&turn_context); + tokio::spawn(async move { + run_compact_task(sess, tc.as_ref(), sub_id, input, compact_instructions).await + }) + .abort_handle() + }; + Self { + sess, + sub_id, + handle, + } + } + + fn abort(self, reason: TurnAbortReason) { + // TOCTOU? if !self.handle.is_finished() { self.handle.abort(); let event = Event { id: self.sub_id, - msg: EventMsg::Error(ErrorEvent { - message: " Turn interrupted".to_string(), - }), + msg: EventMsg::TurnAborted(TurnAbortedEvent { reason }), }; let tx_event = self.sess.tx_event.clone(); tokio::spawn(async move { @@ -700,276 +983,178 @@ impl AgentTask { } async fn submission_loop( - mut session_id: Uuid, + sess: Arc, + turn_context: TurnContext, config: Arc, - auth: Option, rx_sub: Receiver, - tx_event: Sender, - ctrl_c: Arc, ) { - let mut sess: Option> = None; - // shorthand - send an event when there is no active session - let send_no_session_event = |sub_id: String| async { - let event = Event { - id: sub_id, - msg: EventMsg::Error(ErrorEvent { - message: "No session initialized, expected 'ConfigureSession' as first Op" - .to_string(), - }), - }; - tx_event.send(event).await.ok(); - }; - - loop { - let interrupted = ctrl_c.notified(); - let sub = tokio::select! { - res = rx_sub.recv() => match res { - Ok(sub) => sub, - Err(_) => break, - }, - _ = interrupted => { - if let Some(sess) = sess.as_ref(){ - sess.abort(); - } - continue; - }, - }; - + // Wrap once to avoid cloning TurnContext for each task. + let mut turn_context = Arc::new(turn_context); + // To break out of this loop, send Op::Shutdown. + while let Ok(sub) = rx_sub.recv().await { debug!(?sub, "Submission"); match sub.op { Op::Interrupt => { - let sess = match sess.as_ref() { - Some(sess) => sess, - None => { - send_no_session_event(sub.id).await; - continue; - } - }; - sess.abort(); + sess.interrupt_task(); } - Op::ConfigureSession { - provider, - model, - model_reasoning_effort, - model_reasoning_summary, - user_instructions, - base_instructions, + Op::OverrideTurnContext { + cwd, approval_policy, sandbox_policy, - disable_response_storage, - notify, - cwd, - resume_path, + model, + effort, + summary, } => { - debug!( - "Configuring session: model={model}; provider={provider:?}; resume={resume_path:?}" - ); - if !cwd.is_absolute() { - let message = format!("cwd is not absolute: {cwd:?}"); - error!(message); - let event = Event { - id: sub.id, - msg: EventMsg::Error(ErrorEvent { message }), - }; - if let Err(e) = tx_event.send(event).await { - error!("failed to send error message: {e:?}"); - } - return; - } - // Optionally resume an existing rollout. - let mut restored_items: Option> = None; - let rollout_recorder: Option = - if let Some(path) = resume_path.as_ref() { - match RolloutRecorder::resume(path, cwd.clone()).await { - Ok((rec, saved)) => { - session_id = saved.session_id; - if !saved.items.is_empty() { - restored_items = Some(saved.items); - } - Some(rec) - } - Err(e) => { - warn!("failed to resume rollout from {path:?}: {e}"); - None - } - } - } else { - None - }; + // Recalculate the persistent turn context with provided overrides. + let prev = Arc::clone(&turn_context); + let provider = prev.client.get_provider(); - let rollout_recorder = match rollout_recorder { - Some(rec) => Some(rec), - None => { - match RolloutRecorder::new(&config, session_id, user_instructions.clone()) - .await - { - Ok(r) => Some(r), - Err(e) => { - warn!("failed to initialise rollout recorder: {e}"); - None - } - } - } + // Effective model + family + let (effective_model, effective_family) = if let Some(m) = model { + let fam = + find_family_for_model(&m).unwrap_or_else(|| config.model_family.clone()); + (m, fam) + } else { + (prev.client.get_model(), prev.client.get_model_family()) }; + // Effective reasoning settings + let effective_effort = effort.unwrap_or(prev.client.get_reasoning_effort()); + let effective_summary = summary.unwrap_or(prev.client.get_reasoning_summary()); + + let auth = prev.client.get_auth(); + // Build updated config for the client + let mut updated_config = (*config).clone(); + updated_config.model = effective_model.clone(); + updated_config.model_family = effective_family.clone(); + let client = ModelClient::new( - config.clone(), - auth.clone(), - provider.clone(), - model_reasoning_effort, - model_reasoning_summary, - session_id, + Arc::new(updated_config), + auth, + provider, + effective_effort, + effective_summary, + sess.session_id, ); - // abort any current running session and clone its state - let state = match sess.take() { - Some(sess) => { - sess.abort(); - sess.state.lock().unwrap().partial_clone() - } - None => State { - history: ConversationHistory::new(), - ..Default::default() - }, + let new_approval_policy = approval_policy.unwrap_or(prev.approval_policy); + let new_sandbox_policy = sandbox_policy + .clone() + .unwrap_or(prev.sandbox_policy.clone()); + let new_cwd = cwd.clone().unwrap_or_else(|| prev.cwd.clone()); + + let tools_config = ToolsConfig::new( + &effective_family, + new_approval_policy, + new_sandbox_policy.clone(), + config.include_plan_tool, + config.include_apply_patch_tool, + ); + + let new_turn_context = TurnContext { + client, + tools_config, + user_instructions: prev.user_instructions.clone(), + base_instructions: prev.base_instructions.clone(), + approval_policy: new_approval_policy, + sandbox_policy: new_sandbox_policy.clone(), + shell_environment_policy: prev.shell_environment_policy.clone(), + cwd: new_cwd.clone(), + disable_response_storage: prev.disable_response_storage, }; - let writable_roots = Mutex::new(get_writable_roots(&cwd)); - - // Error messages to dispatch after SessionConfigured is sent. - let mut mcp_connection_errors = Vec::::new(); - let (mcp_connection_manager, failed_clients) = - match McpConnectionManager::new(config.mcp_servers.clone()).await { - Ok((mgr, failures)) => (mgr, failures), - Err(e) => { - let message = format!("Failed to create MCP connection manager: {e:#}"); - error!("{message}"); - mcp_connection_errors.push(Event { - id: sub.id.clone(), - msg: EventMsg::Error(ErrorEvent { message }), - }); - (McpConnectionManager::default(), Default::default()) - } - }; - - // Surface individual client start-up failures to the user. - if !failed_clients.is_empty() { - for (server_name, err) in failed_clients { - let message = - format!("MCP client for `{server_name}` failed to start: {err:#}"); - error!("{message}"); - mcp_connection_errors.push(Event { - id: sub.id.clone(), - msg: EventMsg::Error(ErrorEvent { message }), - }); - } - } - let default_shell = shell::default_user_shell().await; - sess = Some(Arc::new(Session { - client, - tools_config: ToolsConfig::new( - &config.model_family, - approval_policy, - sandbox_policy.clone(), - config.include_plan_tool, - ), - tx_event: tx_event.clone(), - ctrl_c: Arc::clone(&ctrl_c), - user_instructions, - base_instructions, - approval_policy, - sandbox_policy, - shell_environment_policy: config.shell_environment_policy.clone(), - cwd, - writable_roots, - mcp_connection_manager, - notify, - state: Mutex::new(state), - rollout: Mutex::new(rollout_recorder), - codex_linux_sandbox_exe: config.codex_linux_sandbox_exe.clone(), - disable_response_storage, - user_shell: default_shell, - show_raw_agent_reasoning: config.show_raw_agent_reasoning, - })); - - // Patch restored state into the newly created session. - if let Some(sess_arc) = &sess { - if restored_items.is_some() { - let mut st = sess_arc.state.lock().unwrap(); - st.history.record_items(restored_items.unwrap().iter()); - } - } - - // Gather history metadata for SessionConfiguredEvent. - let (history_log_id, history_entry_count) = - crate::message_history::history_metadata(&config).await; - - // ack - let events = std::iter::once(Event { - id: sub.id.clone(), - msg: EventMsg::SessionConfigured(SessionConfiguredEvent { - session_id, - model, - history_log_id, - history_entry_count, - }), - }) - .chain(mcp_connection_errors.into_iter()); - for event in events { - if let Err(e) = tx_event.send(event).await { - error!("failed to send event: {e:?}"); - } + // Install the new persistent context for subsequent tasks/turns. + turn_context = Arc::new(new_turn_context); + if cwd.is_some() || approval_policy.is_some() || sandbox_policy.is_some() { + sess.record_conversation_items(&[ResponseItem::from(EnvironmentContext::new( + new_cwd, + new_approval_policy, + new_sandbox_policy, + ))]) + .await; } } Op::UserInput { items } => { - let sess = match sess.as_ref() { - Some(sess) => sess, - None => { - send_no_session_event(sub.id).await; - continue; - } - }; - // attempt to inject input into current task if let Err(items) = sess.inject_input(items) { // no current task, spawn a new one - let task = AgentTask::spawn(Arc::clone(sess), sub.id, items); + let task = + AgentTask::spawn(sess.clone(), Arc::clone(&turn_context), sub.id, items); sess.set_task(task); } } - Op::ExecApproval { id, decision } => { - let sess = match sess.as_ref() { - Some(sess) => sess, - None => { - send_no_session_event(sub.id).await; - continue; - } - }; - match decision { - ReviewDecision::Abort => { - sess.abort(); - } - other => sess.notify_approval(&id, other), + Op::UserTurn { + items, + cwd, + approval_policy, + sandbox_policy, + model, + effort, + summary, + } => { + // attempt to inject input into current task + if let Err(items) = sess.inject_input(items) { + // Derive a fresh TurnContext for this turn using the provided overrides. + let provider = turn_context.client.get_provider(); + + // Derive a model family for the requested model; fall back to the session's. + let model_family = find_family_for_model(&model) + .unwrap_or_else(|| config.model_family.clone()); + + // Create a per‑turn Config clone with the requested model/family. + let mut per_turn_config = (*config).clone(); + per_turn_config.model = model.clone(); + per_turn_config.model_family = model_family.clone(); + + // Build a new client with per‑turn reasoning settings. + // Reuse the same provider and session id; auth defaults to env/API key. + let client = ModelClient::new( + Arc::new(per_turn_config), + None, + provider, + effort, + summary, + sess.session_id, + ); + + let fresh_turn_context = TurnContext { + client, + tools_config: ToolsConfig::new( + &model_family, + approval_policy, + sandbox_policy.clone(), + config.include_plan_tool, + config.include_apply_patch_tool, + ), + user_instructions: turn_context.user_instructions.clone(), + base_instructions: turn_context.base_instructions.clone(), + approval_policy, + sandbox_policy, + shell_environment_policy: turn_context.shell_environment_policy.clone(), + cwd, + disable_response_storage: turn_context.disable_response_storage, + }; + // TODO: record the new environment context in the conversation history + // no current task, spawn a new one with the per‑turn context + let task = + AgentTask::spawn(sess.clone(), Arc::new(fresh_turn_context), sub.id, items); + sess.set_task(task); } } - Op::PatchApproval { id, decision } => { - let sess = match sess.as_ref() { - Some(sess) => sess, - None => { - send_no_session_event(sub.id).await; - continue; - } - }; - match decision { - ReviewDecision::Abort => { - sess.abort(); - } - other => sess.notify_approval(&id, other), + Op::ExecApproval { id, decision } => match decision { + ReviewDecision::Abort => { + sess.interrupt_task(); } - } + other => sess.notify_approval(&id, other), + }, + Op::PatchApproval { id, decision } => match decision { + ReviewDecision::Abort => { + sess.interrupt_task(); + } + other => sess.notify_approval(&id, other), + }, Op::AddToHistory { text } => { - // TODO: What should we do if we got AddToHistory before ConfigureSession? - // currently, if ConfigureSession has resume path, this history will be ignored - let id = session_id; + let id = sess.session_id; let config = config.clone(); tokio::spawn(async move { if let Err(e) = crate::message_history::append_entry(&text, &id, &config).await @@ -981,7 +1166,7 @@ async fn submission_loop( Op::GetHistoryEntryRequest { offset, log_id } => { let config = config.clone(); - let tx_event = tx_event.clone(); + let tx_event = sess.tx_event.clone(); let sub_id = sub.id.clone(); tokio::spawn(async move { @@ -998,7 +1183,13 @@ async fn submission_loop( crate::protocol::GetHistoryEntryResponseEvent { offset, log_id, - entry: entry_opt, + entry: entry_opt.map(|e| { + codex_protocol::message_history::HistoryEntry { + session_id: e.session_id, + ts: e.ts, + text: e.text, + } + }), }, ), }; @@ -1008,15 +1199,23 @@ async fn submission_loop( } }); } - Op::Compact => { - let sess = match sess.as_ref() { - Some(sess) => sess, - None => { - send_no_session_event(sub.id).await; - continue; - } - }; + Op::ListMcpTools => { + let tx_event = sess.tx_event.clone(); + let sub_id = sub.id.clone(); + // This is a cheap lookup from the connection manager's cache. + let tools = sess.mcp_connection_manager.list_all_tools(); + let event = Event { + id: sub_id, + msg: EventMsg::McpListToolsResponse( + crate::protocol::McpListToolsResponseEvent { tools }, + ), + }; + if let Err(e) = tx_event.send(event).await { + warn!("failed to send McpListToolsResponse event: {e}"); + } + } + Op::Compact => { // Create a summarization request as user input const SUMMARIZATION_PROMPT: &str = include_str!("prompt_for_compact_command.md"); @@ -1026,6 +1225,7 @@ async fn submission_loop( }]) { let task = AgentTask::compact( sess.clone(), + Arc::clone(&turn_context), sub.id, items, SUMMARIZATION_PROMPT.to_string(), @@ -1038,32 +1238,34 @@ async fn submission_loop( // Gracefully flush and shutdown rollout recorder on session end so tests // that inspect the rollout file do not race with the background writer. - if let Some(sess_arc) = sess { - let recorder_opt = sess_arc.rollout.lock().unwrap().take(); - if let Some(rec) = recorder_opt { - if let Err(e) = rec.shutdown().await { - warn!("failed to shutdown rollout recorder: {e}"); - let event = Event { - id: sub.id.clone(), - msg: EventMsg::Error(ErrorEvent { - message: "Failed to shutdown rollout recorder".to_string(), - }), - }; - if let Err(e) = tx_event.send(event).await { - warn!("failed to send error message: {e:?}"); - } + let recorder_opt = sess.rollout.lock_unchecked().take(); + if let Some(rec) = recorder_opt { + if let Err(e) = rec.shutdown().await { + warn!("failed to shutdown rollout recorder: {e}"); + let event = Event { + id: sub.id.clone(), + msg: EventMsg::Error(ErrorEvent { + message: "Failed to shutdown rollout recorder".to_string(), + }), + }; + if let Err(e) = sess.tx_event.send(event).await { + warn!("failed to send error message: {e:?}"); } } } + let event = Event { id: sub.id.clone(), msg: EventMsg::ShutdownComplete, }; - if let Err(e) = tx_event.send(event).await { + if let Err(e) = sess.tx_event.send(event).await { warn!("failed to send Shutdown event: {e}"); } break; } + _ => { + // Ignore unknown ops; enum is non_exhaustive to allow extensions. + } } } debug!("Agent loop exited"); @@ -1082,7 +1284,12 @@ async fn submission_loop( /// back to the model in the next turn. /// - If the model sends only an assistant message, we record it in the /// conversation history and consider the task complete. -async fn run_task(sess: Arc, sub_id: String, input: Vec) { +async fn run_task( + sess: Arc, + turn_context: &TurnContext, + sub_id: String, + input: Vec, +) { if input.is_empty() { return; } @@ -1134,7 +1341,15 @@ async fn run_task(sess: Arc, sub_id: String, input: Vec) { }) }) .collect(); - match run_turn(&sess, &mut turn_diff_tracker, sub_id.clone(), turn_input).await { + match run_turn( + &sess, + turn_context, + &mut turn_diff_tracker, + sub_id.clone(), + turn_input, + ) + .await + { Ok(turn_output) => { let mut items_to_record_in_conversation_history = Vec::::new(); let mut responses = Vec::::new(); @@ -1263,31 +1478,26 @@ async fn run_task(sess: Arc, sub_id: String, input: Vec) { async fn run_turn( sess: &Session, + turn_context: &TurnContext, turn_diff_tracker: &mut TurnDiffTracker, sub_id: String, input: Vec, ) -> CodexResult> { let tools = get_openai_tools( - &sess.tools_config, + &turn_context.tools_config, Some(sess.mcp_connection_manager.list_all_tools()), ); let prompt = Prompt { input, - user_instructions: sess.user_instructions.clone(), - store: !sess.disable_response_storage, + store: !turn_context.disable_response_storage, tools, - base_instructions_override: sess.base_instructions.clone(), - environment_context: Some(EnvironmentContext { - cwd: sess.cwd.clone(), - approval_policy: sess.approval_policy, - sandbox_policy: sess.sandbox_policy.clone(), - }), + base_instructions_override: turn_context.base_instructions.clone(), }; let mut retries = 0; loop { - match try_run_turn(sess, turn_diff_tracker, &sub_id, &prompt).await { + match try_run_turn(sess, turn_context, turn_diff_tracker, &sub_id, &prompt).await { Ok(output) => return Ok(output), Err(CodexErr::Interrupted) => return Err(CodexErr::Interrupted), Err(CodexErr::EnvVar(var)) => return Err(CodexErr::EnvVar(var)), @@ -1296,10 +1506,13 @@ async fn run_turn( } Err(e) => { // Use the configured provider-specific stream retry budget. - let max_retries = sess.client.get_provider().stream_max_retries(); + let max_retries = turn_context.client.get_provider().stream_max_retries(); if retries < max_retries { retries += 1; - let delay = backoff(retries); + let delay = match e { + CodexErr::Stream(_, Some(delay)) => delay, + _ => backoff(retries), + }; warn!( "stream disconnected - retrying turn ({retries}/{max_retries} in {delay:?})...", ); @@ -1336,6 +1549,7 @@ struct ProcessedResponseItem { async fn try_run_turn( sess: &Session, + turn_context: &TurnContext, turn_diff_tracker: &mut TurnDiffTracker, sub_id: &str, prompt: &Prompt, @@ -1396,7 +1610,7 @@ async fn try_run_turn( }) }; - let mut stream = sess.client.clone().stream(&prompt).await?; + let mut stream = turn_context.client.clone().stream(&prompt).await?; let mut output = Vec::new(); loop { @@ -1409,6 +1623,7 @@ async fn try_run_turn( // Treat as a disconnected stream so the caller can retry. return Err(CodexErr::Stream( "stream closed before response.completed".into(), + None, )); }; @@ -1424,9 +1639,14 @@ async fn try_run_turn( match event { ResponseEvent::Created => {} ResponseEvent::OutputItemDone(item) => { - let response = - handle_response_item(sess, turn_diff_tracker, sub_id, item.clone()).await?; - + let response = handle_response_item( + sess, + turn_context, + turn_diff_tracker, + sub_id, + item.clone(), + ) + .await?; output.push(ProcessedResponseItem { item, response }); } ResponseEvent::Completed { @@ -1457,7 +1677,7 @@ async fn try_run_turn( } ResponseEvent::OutputTextDelta(delta) => { { - let mut st = sess.state.lock().unwrap(); + let mut st = sess.state.lock_unchecked(); st.history.append_assistant_text(&delta); } @@ -1474,6 +1694,13 @@ async fn try_run_turn( }; sess.tx_event.send(event).await.ok(); } + ResponseEvent::ReasoningSummaryPartAdded => { + let event = Event { + id: sub_id.to_string(), + msg: EventMsg::AgentReasoningSectionBreak(AgentReasoningSectionBreakEvent {}), + }; + sess.tx_event.send(event).await.ok(); + } ResponseEvent::ReasoningContentDelta(delta) => { if sess.show_raw_agent_reasoning { let event = Event { @@ -1491,6 +1718,7 @@ async fn try_run_turn( async fn run_compact_task( sess: Arc, + turn_context: &TurnContext, sub_id: String, input: Vec, compact_instructions: String, @@ -1509,18 +1737,16 @@ async fn run_compact_task( let prompt = Prompt { input: turn_input, - user_instructions: None, - store: !sess.disable_response_storage, - environment_context: None, + store: !turn_context.disable_response_storage, tools: Vec::new(), base_instructions_override: Some(compact_instructions.clone()), }; - let max_retries = sess.client.get_provider().stream_max_retries(); + let max_retries = turn_context.client.get_provider().stream_max_retries(); let mut retries = 0; loop { - let attempt_result = drain_to_completed(&sess, &sub_id, &prompt).await; + let attempt_result = drain_to_completed(&sess, turn_context, &sub_id, &prompt).await; match attempt_result { Ok(()) => break, @@ -1568,12 +1794,13 @@ async fn run_compact_task( }; sess.send_event(event).await; - let mut state = sess.state.lock().unwrap(); + let mut state = sess.state.lock_unchecked(); state.history.keep_last_messages(1); } async fn handle_response_item( sess: &Session, + turn_context: &TurnContext, turn_diff_tracker: &mut TurnDiffTracker, sub_id: &str, item: ResponseItem, @@ -1608,11 +1835,13 @@ async fn handle_response_item( }; sess.tx_event.send(event).await.ok(); } - if sess.show_raw_agent_reasoning && content.is_some() { - let content = content.unwrap(); + if sess.show_raw_agent_reasoning + && let Some(content) = content + { for item in content { let text = match item { ReasoningItemContent::ReasoningText { text } => text, + ReasoningItemContent::Text { text } => text, }; let event = Event { id: sub_id.to_string(), @@ -1635,6 +1864,7 @@ async fn handle_response_item( Some( handle_function_call( sess, + turn_context, turn_diff_tracker, sub_id.to_string(), name, @@ -1674,11 +1904,12 @@ async fn handle_response_item( } }; - let exec_params = to_exec_params(params, sess); + let exec_params = to_exec_params(params, turn_context); Some( handle_container_exec_with_params( exec_params, sess, + turn_context, turn_diff_tracker, sub_id.to_string(), effective_call_id, @@ -1697,6 +1928,7 @@ async fn handle_response_item( async fn handle_function_call( sess: &Session, + turn_context: &TurnContext, turn_diff_tracker: &mut TurnDiffTracker, sub_id: String, name: String, @@ -1705,14 +1937,52 @@ async fn handle_function_call( ) -> ResponseInputItem { match name.as_str() { "container.exec" | "shell" => { - let params = match parse_container_exec_arguments(arguments, sess, &call_id) { + let params = match parse_container_exec_arguments(arguments, turn_context, &call_id) { Ok(params) => params, Err(output) => { return *output; } }; - handle_container_exec_with_params(params, sess, turn_diff_tracker, sub_id, call_id) - .await + handle_container_exec_with_params( + params, + sess, + turn_context, + turn_diff_tracker, + sub_id, + call_id, + ) + .await + } + "apply_patch" => { + let args = match serde_json::from_str::(&arguments) { + Ok(a) => a, + Err(e) => { + return ResponseInputItem::FunctionCallOutput { + call_id, + output: FunctionCallOutputPayload { + content: format!("failed to parse function arguments: {e}"), + success: None, + }, + }; + } + }; + let exec_params = ExecParams { + command: vec!["apply_patch".to_string(), args.input.clone()], + cwd: turn_context.cwd.clone(), + timeout_ms: None, + env: HashMap::new(), + with_escalated_permissions: None, + justification: None, + }; + handle_container_exec_with_params( + exec_params, + sess, + turn_context, + turn_diff_tracker, + sub_id, + call_id, + ) + .await } "update_plan" => handle_update_plan(sess, arguments, sub_id, call_id).await, _ => { @@ -1740,12 +2010,12 @@ async fn handle_function_call( } } -fn to_exec_params(params: ShellToolCallParams, sess: &Session) -> ExecParams { +fn to_exec_params(params: ShellToolCallParams, turn_context: &TurnContext) -> ExecParams { ExecParams { command: params.command, - cwd: sess.resolve_path(params.workdir.clone()), + cwd: turn_context.resolve_path(params.workdir.clone()), timeout_ms: params.timeout_ms, - env: create_env(&sess.shell_environment_policy), + env: create_env(&turn_context.shell_environment_policy), with_escalated_permissions: params.with_escalated_permissions, justification: params.justification, } @@ -1753,12 +2023,12 @@ fn to_exec_params(params: ShellToolCallParams, sess: &Session) -> ExecParams { fn parse_container_exec_arguments( arguments: String, - sess: &Session, + turn_context: &TurnContext, call_id: &str, ) -> Result> { // parse command match serde_json::from_str::(&arguments) { - Ok(shell_tool_call_params) => Ok(to_exec_params(shell_tool_call_params, sess)), + Ok(shell_tool_call_params) => Ok(to_exec_params(shell_tool_call_params, turn_context)), Err(e) => { // allow model to re-sample let output = ResponseInputItem::FunctionCallOutput { @@ -1776,14 +2046,17 @@ fn parse_container_exec_arguments( pub struct ExecInvokeArgs<'a> { pub params: ExecParams, pub sandbox_type: SandboxType, - pub ctrl_c: Arc, pub sandbox_policy: &'a SandboxPolicy, pub codex_linux_sandbox_exe: &'a Option, pub stdout_stream: Option, } -fn maybe_run_with_user_profile(params: ExecParams, sess: &Session) -> ExecParams { - if sess.shell_environment_policy.use_profile { +fn maybe_run_with_user_profile( + params: ExecParams, + sess: &Session, + turn_context: &TurnContext, +) -> ExecParams { + if turn_context.shell_environment_policy.use_profile { let command = sess .user_shell .format_default_shell_invocation(params.command.clone()); @@ -1797,6 +2070,7 @@ fn maybe_run_with_user_profile(params: ExecParams, sess: &Session) -> ExecParams async fn handle_container_exec_with_params( params: ExecParams, sess: &Session, + turn_context: &TurnContext, turn_diff_tracker: &mut TurnDiffTracker, sub_id: String, call_id: String, @@ -1804,7 +2078,7 @@ async fn handle_container_exec_with_params( // check if this was a patch, and apply it if so let apply_patch_exec = match maybe_parse_apply_patch_verified(¶ms.command, ¶ms.cwd) { MaybeApplyPatchVerified::Body(changes) => { - match apply_patch::apply_patch(sess, &sub_id, &call_id, changes).await { + match apply_patch::apply_patch(sess, turn_context, &sub_id, &call_id, changes).await { InternalApplyPatchInvocation::Output(item) => return item, InternalApplyPatchInvocation::DelegateToExec(apply_patch_exec) => { Some(apply_patch_exec) @@ -1866,8 +2140,8 @@ async fn handle_container_exec_with_params( } } else { assess_safety_for_untrusted_command( - sess.approval_policy, - &sess.sandbox_policy, + turn_context.approval_policy, + &turn_context.sandbox_policy, params.with_escalated_permissions.unwrap_or(false), ) }; @@ -1879,11 +2153,11 @@ async fn handle_container_exec_with_params( } None => { let safety = { - let state = sess.state.lock().unwrap(); + let state = sess.state.lock_unchecked(); assess_command_safety( ¶ms.command, - sess.approval_policy, - &sess.sandbox_policy, + turn_context.approval_policy, + &turn_context.sandbox_policy, &state.approved_commands, params.with_escalated_permissions.unwrap_or(false), ) @@ -1953,7 +2227,7 @@ async fn handle_container_exec_with_params( ), }; - let params = maybe_run_with_user_profile(params, sess); + let params = maybe_run_with_user_profile(params, sess, turn_context); let output_result = sess .run_exec_with_events( turn_diff_tracker, @@ -1961,8 +2235,7 @@ async fn handle_container_exec_with_params( ExecInvokeArgs { params: params.clone(), sandbox_type, - ctrl_c: sess.ctrl_c.clone(), - sandbox_policy: &sess.sandbox_policy, + sandbox_policy: &turn_context.sandbox_policy, codex_linux_sandbox_exe: &sess.codex_linux_sandbox_exe, stdout_stream: Some(StdoutStream { sub_id: sub_id.clone(), @@ -1975,19 +2248,10 @@ async fn handle_container_exec_with_params( match output_result { Ok(output) => { - let ExecToolCallOutput { - exit_code, - stdout, - stderr, - duration, - } = &output; + let ExecToolCallOutput { exit_code, .. } = &output; let is_success = *exit_code == 0; - let content = format_exec_output( - if is_success { stdout } else { stderr }, - *exit_code, - *duration, - ); + let content = format_exec_output(output); ResponseInputItem::FunctionCallOutput { call_id: call_id.clone(), output: FunctionCallOutputPayload { @@ -2004,6 +2268,7 @@ async fn handle_container_exec_with_params( error, sandbox_type, sess, + turn_context, ) .await } @@ -2024,6 +2289,7 @@ async fn handle_sandbox_error( error: SandboxErr, sandbox_type: SandboxType, sess: &Session, + turn_context: &TurnContext, ) -> ResponseInputItem { let call_id = exec_command_context.call_id.clone(); let sub_id = exec_command_context.sub_id.clone(); @@ -2031,7 +2297,7 @@ async fn handle_sandbox_error( // Early out if either the user never wants to be asked for approval, or // we're letting the model manage escalation requests. Otherwise, continue - match sess.approval_policy { + match turn_context.approval_policy { AskForApproval::Never | AskForApproval::OnRequest => { return ResponseInputItem::FunctionCallOutput { call_id, @@ -2102,8 +2368,7 @@ async fn handle_sandbox_error( ExecInvokeArgs { params, sandbox_type: SandboxType::None, - ctrl_c: sess.ctrl_c.clone(), - sandbox_policy: &sess.sandbox_policy, + sandbox_policy: &turn_context.sandbox_policy, codex_linux_sandbox_exe: &sess.codex_linux_sandbox_exe, stdout_stream: Some(StdoutStream { sub_id: sub_id.clone(), @@ -2116,19 +2381,10 @@ async fn handle_sandbox_error( match retry_output_result { Ok(retry_output) => { - let ExecToolCallOutput { - exit_code, - stdout, - stderr, - duration, - } = &retry_output; + let ExecToolCallOutput { exit_code, .. } = &retry_output; let is_success = *exit_code == 0; - let content = format_exec_output( - if is_success { stdout } else { stderr }, - *exit_code, - *duration, - ); + let content = format_exec_output(retry_output); ResponseInputItem::FunctionCallOutput { call_id: call_id.clone(), @@ -2161,7 +2417,14 @@ async fn handle_sandbox_error( } /// Exec output is a pre-serialized JSON payload -fn format_exec_output(output: &str, exit_code: i32, duration: Duration) -> String { +fn format_exec_output(exec_output: ExecToolCallOutput) -> String { + let ExecToolCallOutput { + exit_code, + stdout, + stderr, + duration, + } = exec_output; + #[derive(Serialize)] struct ExecMetadata { exit_code: i32, @@ -2177,8 +2440,18 @@ fn format_exec_output(output: &str, exit_code: i32, duration: Duration) -> Strin // round to 1 decimal place let duration_seconds = ((duration.as_secs_f32()) * 10.0).round() / 10.0; + let is_success = exit_code == 0; + let output = if is_success { stdout } else { stderr }; + + let mut formatted_output = output.text; + if let Some(truncated_after_lines) = output.truncated_after_lines { + formatted_output.push_str(&format!( + "\n\n[Output truncated after {truncated_after_lines} lines: too many lines or bytes.]", + )); + } + let payload = ExecOutput { - output, + output: &formatted_output, metadata: ExecMetadata { exit_code, duration_seconds, @@ -2209,19 +2482,25 @@ fn get_last_assistant_message_from_turn(responses: &[ResponseItem]) -> Option CodexResult<()> { - let mut stream = sess.client.clone().stream(prompt).await?; +async fn drain_to_completed( + sess: &Session, + turn_context: &TurnContext, + sub_id: &str, + prompt: &Prompt, +) -> CodexResult<()> { + let mut stream = turn_context.client.clone().stream(prompt).await?; loop { let maybe_event = stream.next().await; let Some(event) = maybe_event else { return Err(CodexErr::Stream( "stream closed before response.completed".into(), + None, )); }; match event { Ok(ResponseEvent::OutputItemDone(item)) => { // Record only to in-memory conversation history; avoid state snapshot. - let mut state = sess.state.lock().unwrap(); + let mut state = sess.state.lock_unchecked(); state.history.record_items(std::slice::from_ref(&item)); } Ok(ResponseEvent::Completed { @@ -2233,6 +2512,7 @@ async fn drain_to_completed(sess: &Session, sub_id: &str, prompt: &Prompt) -> Co None => { return Err(CodexErr::Stream( "token_usage was None in ResponseEvent::Completed".into(), + None, )); } }; diff --git a/codex-rs/core/src/codex_conversation.rs b/codex-rs/core/src/codex_conversation.rs new file mode 100644 index 0000000000..d3b00046fc --- /dev/null +++ b/codex-rs/core/src/codex_conversation.rs @@ -0,0 +1,30 @@ +use crate::codex::Codex; +use crate::error::Result as CodexResult; +use crate::protocol::Event; +use crate::protocol::Op; +use crate::protocol::Submission; + +pub struct CodexConversation { + codex: Codex, +} + +/// Conduit for the bidirectional stream of messages that compose a conversation +/// in Codex. +impl CodexConversation { + pub(crate) fn new(codex: Codex) -> Self { + Self { codex } + } + + pub async fn submit(&self, op: Op) -> CodexResult { + self.codex.submit(op).await + } + + /// Use sparingly: this is intended to be removed soon. + pub async fn submit_with_id(&self, sub: Submission) -> CodexResult<()> { + self.codex.submit_with_id(sub).await + } + + pub async fn next_event(&self) -> CodexResult { + self.codex.next_event().await + } +} diff --git a/codex-rs/core/src/codex_wrapper.rs b/codex-rs/core/src/codex_wrapper.rs deleted file mode 100644 index dc10ec8d84..0000000000 --- a/codex-rs/core/src/codex_wrapper.rs +++ /dev/null @@ -1,59 +0,0 @@ -use std::sync::Arc; - -use crate::Codex; -use crate::CodexSpawnOk; -use crate::config::Config; -use crate::protocol::Event; -use crate::protocol::EventMsg; -use crate::util::notify_on_sigint; -use codex_login::CodexAuth; -use tokio::sync::Notify; -use uuid::Uuid; - -/// Represents an active Codex conversation, including the first event -/// (which is [`EventMsg::SessionConfigured`]). -pub struct CodexConversation { - pub codex: Codex, - pub session_id: Uuid, - pub session_configured: Event, - pub ctrl_c: Arc, -} - -/// Spawn a new [`Codex`] and initialize the session. -/// -/// Returns the wrapped [`Codex`] **and** the `SessionInitialized` event that -/// is received as a response to the initial `ConfigureSession` submission so -/// that callers can surface the information to the UI. -pub async fn init_codex(config: Config) -> anyhow::Result { - let ctrl_c = notify_on_sigint(); - let auth = CodexAuth::from_codex_home(&config.codex_home)?; - let CodexSpawnOk { - codex, - init_id, - session_id, - } = Codex::spawn(config, auth, ctrl_c.clone()).await?; - - // The first event must be `SessionInitialized`. Validate and forward it to - // the caller so that they can display it in the conversation history. - let event = codex.next_event().await?; - if event.id != init_id - || !matches!( - &event, - Event { - id: _id, - msg: EventMsg::SessionConfigured(_), - } - ) - { - return Err(anyhow::anyhow!( - "expected SessionInitialized but got {event:?}" - )); - } - - Ok(CodexConversation { - codex, - session_id, - session_configured: event, - ctrl_c, - }) -} diff --git a/codex-rs/core/src/config.rs b/codex-rs/core/src/config.rs index f9c15b9eed..f20912b22e 100644 --- a/codex-rs/core/src/config.rs +++ b/codex-rs/core/src/config.rs @@ -1,9 +1,6 @@ use crate::config_profile::ConfigProfile; use crate::config_types::History; use crate::config_types::McpServerConfig; -use crate::config_types::ReasoningEffort; -use crate::config_types::ReasoningSummary; -use crate::config_types::SandboxMode; use crate::config_types::SandboxWorkspaceWrite; use crate::config_types::ShellEnvironmentPolicy; use crate::config_types::ShellEnvironmentPolicyToml; @@ -16,6 +13,10 @@ use crate::model_provider_info::built_in_model_providers; use crate::openai_model_info::get_model_info; use crate::protocol::AskForApproval; use crate::protocol::SandboxPolicy; +use codex_login::AuthMode; +use codex_protocol::config_types::ReasoningEffort; +use codex_protocol::config_types::ReasoningSummary; +use codex_protocol::config_types::SandboxMode; use dirs::home_dir; use serde::Deserialize; use std::collections::HashMap; @@ -156,8 +157,16 @@ pub struct Config { /// Include an experimental plan tool that the model can use to update its current plan and status of each step. pub include_plan_tool: bool, + /// Include the `apply_patch` tool for models that benefit from invoking + /// file edits as a structured tool call. When unset, this falls back to the + /// model family's default preference. + pub include_apply_patch_tool: bool, + /// The value for the `originator` header included with Responses API requests. pub internal_originator: Option, + + /// If set to `true`, the API key will be signed with the `originator` header. + pub preferred_auth_method: AuthMode, } impl Config { @@ -404,6 +413,9 @@ pub struct ConfigToml { pub internal_originator: Option, pub projects: Option>, + + /// If set to `true`, the API key will be signed with the `originator` header. + pub preferred_auth_method: Option, } #[derive(Deserialize, Debug, Clone, PartialEq, Eq)] @@ -480,6 +492,7 @@ pub struct ConfigOverrides { pub codex_linux_sandbox_exe: Option, pub base_instructions: Option, pub include_plan_tool: Option, + pub include_apply_patch_tool: Option, pub disable_response_storage: Option, pub show_raw_agent_reasoning: Option, } @@ -505,6 +518,7 @@ impl Config { codex_linux_sandbox_exe, base_instructions, include_plan_tool, + include_apply_patch_tool, disable_response_storage, show_raw_agent_reasoning, } = overrides; @@ -581,6 +595,7 @@ impl Config { needs_special_apply_patch_instructions: false, supports_reasoning_summaries, uses_local_shell_tool: false, + uses_apply_patch_tool: false, } }); @@ -607,6 +622,9 @@ impl Config { Self::get_base_instructions(experimental_instructions_path, &resolved_cwd)?; let base_instructions = base_instructions.or(file_base_instructions); + let include_apply_patch_tool_val = + include_apply_patch_tool.unwrap_or(model_family.uses_apply_patch_tool); + let config = Self { model, model_family, @@ -659,7 +677,9 @@ impl Config { experimental_resume, include_plan_tool: include_plan_tool.unwrap_or(false), + include_apply_patch_tool: include_apply_patch_tool_val, internal_originator: cfg.internal_originator, + preferred_auth_method: cfg.preferred_auth_method.unwrap_or(AuthMode::ChatGPT), }; Ok(config) } @@ -765,7 +785,6 @@ pub fn log_dir(cfg: &Config) -> std::io::Result { #[cfg(test)] mod tests { - #![allow(clippy::expect_used, clippy::unwrap_used)] use crate::config_types::HistoryPersistence; use super::*; @@ -1023,7 +1042,9 @@ disable_response_storage = true experimental_resume: None, base_instructions: None, include_plan_tool: false, + include_apply_patch_tool: false, internal_originator: None, + preferred_auth_method: AuthMode::ChatGPT, }, o3_profile_config ); @@ -1074,7 +1095,9 @@ disable_response_storage = true experimental_resume: None, base_instructions: None, include_plan_tool: false, + include_apply_patch_tool: false, internal_originator: None, + preferred_auth_method: AuthMode::ChatGPT, }; assert_eq!(expected_gpt3_profile_config, gpt3_profile_config); @@ -1140,7 +1163,9 @@ disable_response_storage = true experimental_resume: None, base_instructions: None, include_plan_tool: false, + include_apply_patch_tool: false, internal_originator: None, + preferred_auth_method: AuthMode::ChatGPT, }; assert_eq!(expected_zdr_profile_config, zdr_profile_config); diff --git a/codex-rs/core/src/config_profile.rs b/codex-rs/core/src/config_profile.rs index d945d1df7a..8bb0738788 100644 --- a/codex-rs/core/src/config_profile.rs +++ b/codex-rs/core/src/config_profile.rs @@ -1,9 +1,9 @@ use serde::Deserialize; use std::path::PathBuf; -use crate::config_types::ReasoningEffort; -use crate::config_types::ReasoningSummary; use crate::protocol::AskForApproval; +use codex_protocol::config_types::ReasoningEffort; +use codex_protocol::config_types::ReasoningSummary; /// Collection of common configuration options that a user can define as a unit /// in `config.toml`. diff --git a/codex-rs/core/src/config_types.rs b/codex-rs/core/src/config_types.rs index 291dcb6422..d6661699fe 100644 --- a/codex-rs/core/src/config_types.rs +++ b/codex-rs/core/src/config_types.rs @@ -5,11 +5,9 @@ use std::collections::HashMap; use std::path::PathBuf; -use strum_macros::Display; use wildmatch::WildMatchPattern; use serde::Deserialize; -use serde::Serialize; #[derive(Deserialize, Debug, Clone, PartialEq)] pub struct McpServerConfig { @@ -78,20 +76,6 @@ pub enum HistoryPersistence { #[derive(Deserialize, Debug, Clone, PartialEq, Default)] pub struct Tui {} -#[derive(Deserialize, Debug, Clone, Copy, PartialEq, Default, Serialize)] -#[serde(rename_all = "kebab-case")] -pub enum SandboxMode { - #[serde(rename = "read-only")] - #[default] - ReadOnly, - - #[serde(rename = "workspace-write")] - WorkspaceWrite, - - #[serde(rename = "danger-full-access")] - DangerFullAccess, -} - #[derive(Deserialize, Debug, Clone, PartialEq, Default)] pub struct SandboxWorkspaceWrite { #[serde(default)] @@ -199,31 +183,3 @@ impl From for ShellEnvironmentPolicy { } } } - -/// See https://platform.openai.com/docs/guides/reasoning?api-mode=responses#get-started-with-reasoning -#[derive(Debug, Serialize, Deserialize, Default, Clone, Copy, PartialEq, Eq, Display)] -#[serde(rename_all = "lowercase")] -#[strum(serialize_all = "lowercase")] -pub enum ReasoningEffort { - Low, - #[default] - Medium, - High, - /// Option to disable reasoning. - None, -} - -/// A summary of the reasoning performed by the model. This can be useful for -/// debugging and understanding the model's reasoning process. -/// See https://platform.openai.com/docs/guides/reasoning?api-mode=responses#reasoning-summaries -#[derive(Debug, Serialize, Deserialize, Default, Clone, Copy, PartialEq, Eq, Display)] -#[serde(rename_all = "lowercase")] -#[strum(serialize_all = "lowercase")] -pub enum ReasoningSummary { - #[default] - Auto, - Concise, - Detailed, - /// Option to disable reasoning summaries. - None, -} diff --git a/codex-rs/core/src/conversation_manager.rs b/codex-rs/core/src/conversation_manager.rs new file mode 100644 index 0000000000..2dc69be45a --- /dev/null +++ b/codex-rs/core/src/conversation_manager.rs @@ -0,0 +1,96 @@ +use std::collections::HashMap; +use std::sync::Arc; + +use codex_login::CodexAuth; +use tokio::sync::RwLock; +use uuid::Uuid; + +use crate::codex::Codex; +use crate::codex::CodexSpawnOk; +use crate::codex::INITIAL_SUBMIT_ID; +use crate::codex_conversation::CodexConversation; +use crate::config::Config; +use crate::error::CodexErr; +use crate::error::Result as CodexResult; +use crate::protocol::Event; +use crate::protocol::EventMsg; +use crate::protocol::SessionConfiguredEvent; + +/// Represents a newly created Codex conversation, including the first event +/// (which is [`EventMsg::SessionConfigured`]). +pub struct NewConversation { + pub conversation_id: Uuid, + pub conversation: Arc, + pub session_configured: SessionConfiguredEvent, +} + +/// [`ConversationManager`] is responsible for creating conversations and +/// maintaining them in memory. +pub struct ConversationManager { + conversations: Arc>>>, +} + +impl Default for ConversationManager { + fn default() -> Self { + Self { + conversations: Arc::new(RwLock::new(HashMap::new())), + } + } +} + +impl ConversationManager { + pub async fn new_conversation(&self, config: Config) -> CodexResult { + let auth = CodexAuth::from_codex_home(&config.codex_home, config.preferred_auth_method)?; + self.new_conversation_with_auth(config, auth).await + } + + /// Used for integration tests: should not be used by ordinary business + /// logic. + pub async fn new_conversation_with_auth( + &self, + config: Config, + auth: Option, + ) -> CodexResult { + let CodexSpawnOk { + codex, + session_id: conversation_id, + } = Codex::spawn(config, auth).await?; + + // The first event must be `SessionInitialized`. Validate and forward it + // to the caller so that they can display it in the conversation + // history. + let event = codex.next_event().await?; + let session_configured = match event { + Event { + id, + msg: EventMsg::SessionConfigured(session_configured), + } if id == INITIAL_SUBMIT_ID => session_configured, + _ => { + return Err(CodexErr::SessionConfiguredNotFirstEvent); + } + }; + + let conversation = Arc::new(CodexConversation::new(codex)); + self.conversations + .write() + .await + .insert(conversation_id, conversation.clone()); + + Ok(NewConversation { + conversation_id, + conversation, + session_configured, + }) + } + + pub async fn get_conversation( + &self, + conversation_id: Uuid, + ) -> CodexResult> { + let conversations = self.conversations.read().await; + conversations + .get(&conversation_id) + .cloned() + .ok_or_else(|| CodexErr::ConversationNotFound(conversation_id)) + } +} diff --git a/codex-rs/core/src/environment_context.rs b/codex-rs/core/src/environment_context.rs new file mode 100644 index 0000000000..b37645a32b --- /dev/null +++ b/codex-rs/core/src/environment_context.rs @@ -0,0 +1,86 @@ +use serde::Deserialize; +use serde::Serialize; +use strum_macros::Display as DeriveDisplay; + +use crate::models::ContentItem; +use crate::models::ResponseItem; +use crate::protocol::AskForApproval; +use crate::protocol::SandboxPolicy; +use codex_protocol::config_types::SandboxMode; +use std::fmt::Display; +use std::path::PathBuf; + +/// wraps environment context message in a tag for the model to parse more easily. +pub(crate) const ENVIRONMENT_CONTEXT_START: &str = "\n"; +pub(crate) const ENVIRONMENT_CONTEXT_END: &str = ""; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, DeriveDisplay)] +#[serde(rename_all = "kebab-case")] +#[strum(serialize_all = "kebab-case")] +pub enum NetworkAccess { + Restricted, + Enabled, +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(rename = "environment_context", rename_all = "snake_case")] +pub(crate) struct EnvironmentContext { + pub cwd: PathBuf, + pub approval_policy: AskForApproval, + pub sandbox_mode: SandboxMode, + pub network_access: NetworkAccess, +} + +impl EnvironmentContext { + pub fn new( + cwd: PathBuf, + approval_policy: AskForApproval, + sandbox_policy: SandboxPolicy, + ) -> Self { + Self { + cwd, + approval_policy, + sandbox_mode: match sandbox_policy { + SandboxPolicy::DangerFullAccess => SandboxMode::DangerFullAccess, + SandboxPolicy::ReadOnly => SandboxMode::ReadOnly, + SandboxPolicy::WorkspaceWrite { .. } => SandboxMode::WorkspaceWrite, + }, + network_access: match sandbox_policy { + SandboxPolicy::DangerFullAccess => NetworkAccess::Enabled, + SandboxPolicy::ReadOnly => NetworkAccess::Restricted, + SandboxPolicy::WorkspaceWrite { network_access, .. } => { + if network_access { + NetworkAccess::Enabled + } else { + NetworkAccess::Restricted + } + } + }, + } + } +} + +impl Display for EnvironmentContext { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + writeln!( + f, + "Current working directory: {}", + self.cwd.to_string_lossy() + )?; + writeln!(f, "Approval policy: {}", self.approval_policy)?; + writeln!(f, "Sandbox mode: {}", self.sandbox_mode)?; + writeln!(f, "Network access: {}", self.network_access)?; + Ok(()) + } +} + +impl From for ResponseItem { + fn from(ec: EnvironmentContext) -> Self { + ResponseItem::Message { + id: None, + role: "user".to_string(), + content: vec![ContentItem::InputText { + text: format!("{ENVIRONMENT_CONTEXT_START}{ec}{ENVIRONMENT_CONTEXT_END}"), + }], + } + } +} diff --git a/codex-rs/core/src/error.rs b/codex-rs/core/src/error.rs index 2931d30636..356c23011c 100644 --- a/codex-rs/core/src/error.rs +++ b/codex-rs/core/src/error.rs @@ -1,8 +1,10 @@ use reqwest::StatusCode; use serde_json; use std::io; +use std::time::Duration; use thiserror::Error; use tokio::task::JoinError; +use uuid::Uuid; pub type Result = std::result::Result; @@ -41,8 +43,16 @@ pub enum CodexErr { /// handshake has succeeded but **before** it finished emitting `response.completed`. /// /// The Session loop treats this as a transient error and will automatically retry the turn. + /// + /// Optionally includes the requested delay before retrying the turn. #[error("stream disconnected before completion: {0}")] - Stream(String), + Stream(String, Option), + + #[error("no conversation with id: {0}")] + ConversationNotFound(Uuid), + + #[error("session configured event was not the first event in the stream")] + SessionConfiguredNotFirstEvent, /// Returned by run_command_stream when the spawned child process timed out (10s). #[error("timeout waiting for child process to exit")] diff --git a/codex-rs/core/src/exec.rs b/codex-rs/core/src/exec.rs index 10606b6821..9430433c11 100644 --- a/codex-rs/core/src/exec.rs +++ b/codex-rs/core/src/exec.rs @@ -3,10 +3,8 @@ use std::os::unix::process::ExitStatusExt; use std::collections::HashMap; use std::io; -use std::path::Path; use std::path::PathBuf; use std::process::ExitStatus; -use std::sync::Arc; use std::time::Duration; use std::time::Instant; @@ -15,11 +13,11 @@ use tokio::io::AsyncRead; use tokio::io::AsyncReadExt; use tokio::io::BufReader; use tokio::process::Child; -use tokio::sync::Notify; use crate::error::CodexErr; use crate::error::Result; use crate::error::SandboxErr; +use crate::landlock::spawn_command_under_linux_sandbox; use crate::protocol::Event; use crate::protocol::EventMsg; use crate::protocol::ExecCommandOutputDeltaEvent; @@ -80,7 +78,6 @@ pub struct StdoutStream { pub async fn process_exec_tool_call( params: ExecParams, sandbox_type: SandboxType, - ctrl_c: Arc, sandbox_policy: &SandboxPolicy, codex_linux_sandbox_exe: &Option, stdout_stream: Option, @@ -89,7 +86,7 @@ pub async fn process_exec_tool_call( let raw_output_result: std::result::Result = match sandbox_type { - SandboxType::None => exec(params, sandbox_policy, ctrl_c, stdout_stream.clone()).await, + SandboxType::None => exec(params, sandbox_policy, stdout_stream.clone()).await, SandboxType::MacosSeatbelt => { let timeout = params.timeout_duration(); let ExecParams { @@ -103,7 +100,7 @@ pub async fn process_exec_tool_call( env, ) .await?; - consume_truncated_output(child, ctrl_c, timeout, stdout_stream.clone()).await + consume_truncated_output(child, timeout, stdout_stream.clone()).await } SandboxType::LinuxSeccomp => { let timeout = params.timeout_duration(); @@ -124,14 +121,14 @@ pub async fn process_exec_tool_call( ) .await?; - consume_truncated_output(child, ctrl_c, timeout, stdout_stream).await + consume_truncated_output(child, timeout, stdout_stream).await } }; let duration = start.elapsed(); match raw_output_result { Ok(raw_output) => { - let stdout = String::from_utf8_lossy(&raw_output.stdout).to_string(); - let stderr = String::from_utf8_lossy(&raw_output.stderr).to_string(); + let stdout = raw_output.stdout.from_utf8_lossy(); + let stderr = raw_output.stderr.from_utf8_lossy(); #[cfg(target_family = "unix")] match raw_output.exit_status.signal() { @@ -146,7 +143,9 @@ pub async fn process_exec_tool_call( if exit_code != 0 && is_likely_sandbox_denied(sandbox_type, exit_code) { return Err(CodexErr::Sandbox(SandboxErr::Denied( - exit_code, stdout, stderr, + exit_code, + stdout.text, + stderr.text, ))); } @@ -164,65 +163,6 @@ pub async fn process_exec_tool_call( } } -/// Spawn a shell tool command under the Linux Landlock+seccomp sandbox helper -/// (codex-linux-sandbox). -/// -/// Unlike macOS Seatbelt where we directly embed the policy text, the Linux -/// helper accepts a list of `--sandbox-permission`/`-s` flags mirroring the -/// public CLI. We convert the internal [`SandboxPolicy`] representation into -/// the equivalent CLI options. -pub async fn spawn_command_under_linux_sandbox

( - codex_linux_sandbox_exe: P, - command: Vec, - sandbox_policy: &SandboxPolicy, - cwd: PathBuf, - stdio_policy: StdioPolicy, - env: HashMap, -) -> std::io::Result -where - P: AsRef, -{ - let args = create_linux_sandbox_command_args(command, sandbox_policy, &cwd); - let arg0 = Some("codex-linux-sandbox"); - spawn_child_async( - codex_linux_sandbox_exe.as_ref().to_path_buf(), - args, - arg0, - cwd, - sandbox_policy, - stdio_policy, - env, - ) - .await -} - -/// Converts the sandbox policy into the CLI invocation for `codex-linux-sandbox`. -fn create_linux_sandbox_command_args( - command: Vec, - sandbox_policy: &SandboxPolicy, - cwd: &Path, -) -> Vec { - #[expect(clippy::expect_used)] - let sandbox_policy_cwd = cwd.to_str().expect("cwd must be valid UTF-8").to_string(); - - #[expect(clippy::expect_used)] - let sandbox_policy_json = - serde_json::to_string(sandbox_policy).expect("Failed to serialize SandboxPolicy to JSON"); - - let mut linux_cmd: Vec = vec![ - sandbox_policy_cwd, - sandbox_policy_json, - // Separator so that command arguments starting with `-` are not parsed as - // options of the helper itself. - "--".to_string(), - ]; - - // Append the original tool command. - linux_cmd.extend(command); - - linux_cmd -} - /// We don't have a fully deterministic way to tell if our command failed /// because of the sandbox - a command in the user's zshrc file might hit an /// error, but the command itself might fail or succeed for other reasons. @@ -243,25 +183,47 @@ fn is_likely_sandbox_denied(sandbox_type: SandboxType, exit_code: i32) -> bool { true } +#[derive(Debug)] +pub struct StreamOutput { + pub text: T, + pub truncated_after_lines: Option, +} #[derive(Debug)] pub struct RawExecToolCallOutput { pub exit_status: ExitStatus, - pub stdout: Vec, - pub stderr: Vec, + pub stdout: StreamOutput>, + pub stderr: StreamOutput>, +} + +impl StreamOutput { + pub fn new(text: String) -> Self { + Self { + text, + truncated_after_lines: None, + } + } +} + +impl StreamOutput> { + pub fn from_utf8_lossy(&self) -> StreamOutput { + StreamOutput { + text: String::from_utf8_lossy(&self.text).to_string(), + truncated_after_lines: self.truncated_after_lines, + } + } } #[derive(Debug)] pub struct ExecToolCallOutput { pub exit_code: i32, - pub stdout: String, - pub stderr: String, + pub stdout: StreamOutput, + pub stderr: StreamOutput, pub duration: Duration, } async fn exec( params: ExecParams, sandbox_policy: &SandboxPolicy, - ctrl_c: Arc, stdout_stream: Option, ) -> Result { let timeout = params.timeout_duration(); @@ -286,14 +248,13 @@ async fn exec( env, ) .await?; - consume_truncated_output(child, ctrl_c, timeout, stdout_stream).await + consume_truncated_output(child, timeout, stdout_stream).await } /// Consumes the output of a child process, truncating it so it is suitable for /// use as the output of a `shell` tool call. Also enforces specified timeout. pub(crate) async fn consume_truncated_output( mut child: Child, - ctrl_c: Arc, timeout: Duration, stdout_stream: Option, ) -> Result { @@ -327,7 +288,6 @@ pub(crate) async fn consume_truncated_output( true, )); - let interrupted = ctrl_c.notified(); let exit_status = tokio::select! { result = tokio::time::timeout(timeout, child.wait()) => { match result { @@ -341,7 +301,7 @@ pub(crate) async fn consume_truncated_output( } } } - _ = interrupted => { + _ = tokio::signal::ctrl_c() => { child.start_kill()?; synthetic_exit_status(128 + SIGKILL_CODE) } @@ -363,7 +323,7 @@ async fn read_capped( max_lines: usize, stream: Option, is_stderr: bool, -) -> io::Result> { +) -> io::Result>> { let mut buf = Vec::with_capacity(max_output.min(8 * 1024)); let mut tmp = [0u8; 8192]; @@ -413,7 +373,16 @@ async fn read_capped( // Continue reading to EOF to avoid back-pressure, but discard once caps are hit. } - Ok(buf) + let truncated = remaining_lines == 0 || remaining_bytes == 0; + + Ok(StreamOutput { + text: buf, + truncated_after_lines: if truncated { + Some((max_lines - remaining_lines) as u32) + } else { + None + }, + }) } #[cfg(unix)] diff --git a/codex-rs/core/src/exec_env.rs b/codex-rs/core/src/exec_env.rs index 2957f3da15..88246b063c 100644 --- a/codex-rs/core/src/exec_env.rs +++ b/codex-rs/core/src/exec_env.rs @@ -70,8 +70,6 @@ where #[cfg(test)] mod tests { - #![allow(clippy::unwrap_used, clippy::expect_used)] - use super::*; use crate::config_types::ShellEnvironmentPolicyInherit; use maplit::hashmap; diff --git a/codex-rs/core/src/git_info.rs b/codex-rs/core/src/git_info.rs index 52d029f669..09a59b7b8c 100644 --- a/codex-rs/core/src/git_info.rs +++ b/codex-rs/core/src/git_info.rs @@ -99,9 +99,6 @@ async fn run_git_command_with_timeout(args: &[&str], cwd: &Path) -> Option) -> bool { #[cfg(test)] mod tests { - #![allow(clippy::unwrap_used)] use super::*; fn vec_str(args: &[&str]) -> Vec { diff --git a/codex-rs/core/src/landlock.rs b/codex-rs/core/src/landlock.rs new file mode 100644 index 0000000000..8c8840c269 --- /dev/null +++ b/codex-rs/core/src/landlock.rs @@ -0,0 +1,66 @@ +use crate::protocol::SandboxPolicy; +use crate::spawn::StdioPolicy; +use crate::spawn::spawn_child_async; +use std::collections::HashMap; +use std::path::Path; +use std::path::PathBuf; +use tokio::process::Child; + +/// Spawn a shell tool command under the Linux Landlock+seccomp sandbox helper +/// (codex-linux-sandbox). +/// +/// Unlike macOS Seatbelt where we directly embed the policy text, the Linux +/// helper accepts a list of `--sandbox-permission`/`-s` flags mirroring the +/// public CLI. We convert the internal [`SandboxPolicy`] representation into +/// the equivalent CLI options. +pub async fn spawn_command_under_linux_sandbox

( + codex_linux_sandbox_exe: P, + command: Vec, + sandbox_policy: &SandboxPolicy, + cwd: PathBuf, + stdio_policy: StdioPolicy, + env: HashMap, +) -> std::io::Result +where + P: AsRef, +{ + let args = create_linux_sandbox_command_args(command, sandbox_policy, &cwd); + let arg0 = Some("codex-linux-sandbox"); + spawn_child_async( + codex_linux_sandbox_exe.as_ref().to_path_buf(), + args, + arg0, + cwd, + sandbox_policy, + stdio_policy, + env, + ) + .await +} + +/// Converts the sandbox policy into the CLI invocation for `codex-linux-sandbox`. +fn create_linux_sandbox_command_args( + command: Vec, + sandbox_policy: &SandboxPolicy, + cwd: &Path, +) -> Vec { + #[expect(clippy::expect_used)] + let sandbox_policy_cwd = cwd.to_str().expect("cwd must be valid UTF-8").to_string(); + + #[expect(clippy::expect_used)] + let sandbox_policy_json = + serde_json::to_string(sandbox_policy).expect("Failed to serialize SandboxPolicy to JSON"); + + let mut linux_cmd: Vec = vec![ + sandbox_policy_cwd, + sandbox_policy_json, + // Separator so that command arguments starting with `-` are not parsed as + // options of the helper itself. + "--".to_string(), + ]; + + // Append the original tool command. + linux_cmd.extend(command); + + linux_cmd +} diff --git a/codex-rs/core/src/lib.rs b/codex-rs/core/src/lib.rs index c728bd3125..28d35f5376 100644 --- a/codex-rs/core/src/lib.rs +++ b/codex-rs/core/src/lib.rs @@ -11,42 +11,53 @@ mod chat_completions; mod client; mod client_common; pub mod codex; -pub use codex::Codex; -pub use codex::CodexSpawnOk; -pub mod codex_wrapper; +mod codex_conversation; +pub use codex_conversation::CodexConversation; pub mod config; pub mod config_profile; pub mod config_types; mod conversation_history; +mod environment_context; pub mod error; pub mod exec; pub mod exec_env; mod flags; pub mod git_info; mod is_safe_command; +pub mod landlock; mod mcp_connection_manager; mod mcp_tool_call; mod message_history; mod model_provider_info; +pub mod parse_command; pub use model_provider_info::BUILT_IN_OSS_MODEL_PROVIDER_ID; pub use model_provider_info::ModelProviderInfo; pub use model_provider_info::WireApi; pub use model_provider_info::built_in_model_providers; pub use model_provider_info::create_oss_provider_with_base_url; +mod conversation_manager; +pub use conversation_manager::ConversationManager; +pub use conversation_manager::NewConversation; pub mod model_family; mod models; mod openai_model_info; mod openai_tools; pub mod plan_tool; mod project_doc; -pub mod protocol; mod rollout; pub(crate) mod safety; pub mod seatbelt; pub mod shell; pub mod spawn; pub mod turn_diff_tracker; +pub mod user_agent; mod user_notification; pub mod util; pub use apply_patch::CODEX_APPLY_PATCH_ARG1; pub use safety::get_platform_sandbox; +// Re-export the protocol types from the standalone `codex-protocol` crate so existing +// `codex_core::protocol::...` references continue to work across the workspace. +pub use codex_protocol::protocol; +// Re-export protocol config enums to ensure call sites can use the same types +// as those in the protocol crate when constructing protocol messages. +pub use codex_protocol::config_types as protocol_config_types; diff --git a/codex-rs/core/src/mcp_connection_manager.rs b/codex-rs/core/src/mcp_connection_manager.rs index 2e33c8754b..b5813a0462 100644 --- a/codex-rs/core/src/mcp_connection_manager.rs +++ b/codex-rs/core/src/mcp_connection_manager.rs @@ -281,7 +281,6 @@ fn is_valid_mcp_server_name(server_name: &str) -> bool { } #[cfg(test)] -#[allow(clippy::unwrap_used)] mod tests { use super::*; use mcp_types::ToolInputSchema; diff --git a/codex-rs/core/src/model_family.rs b/codex-rs/core/src/model_family.rs index 1245a030c6..6d1c2efcc1 100644 --- a/codex-rs/core/src/model_family.rs +++ b/codex-rs/core/src/model_family.rs @@ -23,6 +23,10 @@ pub struct ModelFamily { // the model such that its description can be omitted. // See https://platform.openai.com/docs/guides/tools-local-shell pub uses_local_shell_tool: bool, + + /// True if the model performs better when `apply_patch` is provided as + /// a tool call instead of just a bash command. + pub uses_apply_patch_tool: bool, } macro_rules! model_family { @@ -36,6 +40,7 @@ macro_rules! model_family { needs_special_apply_patch_instructions: false, supports_reasoning_summaries: false, uses_local_shell_tool: false, + uses_apply_patch_tool: false, }; // apply overrides $( @@ -55,6 +60,7 @@ macro_rules! simple_model_family { needs_special_apply_patch_instructions: false, supports_reasoning_summaries: false, uses_local_shell_tool: false, + uses_apply_patch_tool: false, }) }}; } @@ -78,15 +84,20 @@ pub fn find_family_for_model(slug: &str) -> Option { supports_reasoning_summaries: true, uses_local_shell_tool: true, ) + } else if slug.starts_with("codex-") { + model_family!( + slug, slug, + supports_reasoning_summaries: true, + ) } else if slug.starts_with("gpt-4.1") { model_family!( slug, "gpt-4.1", needs_special_apply_patch_instructions: true, ) + } else if slug.starts_with("gpt-oss") { + model_family!(slug, "gpt-oss", uses_apply_patch_tool: true) } else if slug.starts_with("gpt-4o") { simple_model_family!(slug, "gpt-4o") - } else if slug.starts_with("gpt-oss") { - simple_model_family!(slug, "gpt-oss") } else if slug.starts_with("gpt-3.5") { simple_model_family!(slug, "gpt-3.5") } else if slug.starts_with("gpt-5") { diff --git a/codex-rs/core/src/model_provider_info.rs b/codex-rs/core/src/model_provider_info.rs index 98f07deb1e..00eabc539c 100644 --- a/codex-rs/core/src/model_provider_info.rs +++ b/codex-rs/core/src/model_provider_info.rs @@ -322,7 +322,6 @@ pub fn create_oss_provider_with_base_url(base_url: &str) -> ModelProviderInfo { #[cfg(test)] mod tests { - #![allow(clippy::unwrap_used)] use super::*; use pretty_assertions::assert_eq; diff --git a/codex-rs/core/src/models.rs b/codex-rs/core/src/models.rs index e052bc43a4..f6323e2724 100644 --- a/codex-rs/core/src/models.rs +++ b/codex-rs/core/src/models.rs @@ -45,7 +45,7 @@ pub enum ResponseItem { Reasoning { id: String, summary: Vec, - #[serde(default, skip_serializing_if = "Option::is_none")] + #[serde(default, skip_serializing_if = "should_serialize_reasoning_content")] content: Option>, encrypted_content: Option, }, @@ -81,6 +81,15 @@ pub enum ResponseItem { Other, } +fn should_serialize_reasoning_content(content: &Option>) -> bool { + match content { + Some(content) => !content + .iter() + .any(|c| matches!(c, ReasoningItemContent::ReasoningText { .. })), + None => false, + } +} + impl From for ResponseItem { fn from(item: ResponseInputItem) -> Self { match item { @@ -142,6 +151,7 @@ pub enum ReasoningItemReasoningSummary { #[serde(tag = "type", rename_all = "snake_case")] pub enum ReasoningItemContent { ReasoningText { text: String }, + Text { text: String }, } impl From> for ResponseInputItem { @@ -173,6 +183,7 @@ impl From> for ResponseInputItem { None } }, + _ => None, }) .collect::>(), } @@ -256,7 +267,6 @@ impl std::ops::Deref for FunctionCallOutputPayload { #[cfg(test)] mod tests { - #![allow(clippy::unwrap_used)] use super::*; #[test] diff --git a/codex-rs/core/src/openai_model_info.rs b/codex-rs/core/src/openai_model_info.rs index a072d409c6..66f3c626ea 100644 --- a/codex-rs/core/src/openai_model_info.rs +++ b/codex-rs/core/src/openai_model_info.rs @@ -15,7 +15,8 @@ pub(crate) struct ModelInfo { } pub(crate) fn get_model_info(model_family: &ModelFamily) -> Option { - match model_family.slug.as_str() { + let slug = model_family.slug.as_str(); + match slug { // OSS models have a 128k shared token pool. // Arbitrarily splitting it: 3/4 input context, 1/4 output. // https://openai.com/index/gpt-oss-model-card/ @@ -82,6 +83,11 @@ pub(crate) fn get_model_info(model_family: &ModelFamily) -> Option { max_output_tokens: 100_000, }), + _ if slug.starts_with("codex-") => Some(ModelInfo { + context_window: 200_000, + max_output_tokens: 100_000, + }), + _ => None, } } diff --git a/codex-rs/core/src/openai_tools.rs b/codex-rs/core/src/openai_tools.rs index 1c92c07c12..32ead20e7d 100644 --- a/codex-rs/core/src/openai_tools.rs +++ b/codex-rs/core/src/openai_tools.rs @@ -1,5 +1,6 @@ use serde::Deserialize; use serde::Serialize; +use serde_json::Value as JsonValue; use serde_json::json; use std::collections::BTreeMap; use std::collections::HashMap; @@ -42,6 +43,7 @@ pub enum ConfigShellToolType { pub struct ToolsConfig { pub shell_type: ConfigShellToolType, pub plan_tool: bool, + pub apply_patch_tool: bool, } impl ToolsConfig { @@ -50,6 +52,7 @@ impl ToolsConfig { approval_policy: AskForApproval, sandbox_policy: SandboxPolicy, include_plan_tool: bool, + include_apply_patch_tool: bool, ) -> Self { let mut shell_type = if model_family.uses_local_shell_tool { ConfigShellToolType::LocalShell @@ -65,6 +68,7 @@ impl ToolsConfig { Self { shell_type, plan_tool: include_plan_tool, + apply_patch_tool: include_apply_patch_tool || model_family.uses_apply_patch_tool, } } } @@ -81,6 +85,8 @@ pub(crate) enum JsonSchema { #[serde(skip_serializing_if = "Option::is_none")] description: Option, }, + /// MCP schema allows "number" | "integer" for Number + #[serde(alias = "integer")] Number { #[serde(skip_serializing_if = "Option::is_none")] description: Option, @@ -232,6 +238,87 @@ The shell tool is used to execute shell commands. }) } +#[derive(Serialize, Deserialize)] +pub(crate) struct ApplyPatchToolArgs { + pub(crate) input: String, +} + +fn create_apply_patch_tool() -> OpenAiTool { + // Minimal schema: one required string argument containing the patch body + let mut properties = BTreeMap::new(); + properties.insert( + "input".to_string(), + JsonSchema::String { + description: Some(r#"The entire contents of the apply_patch command"#.to_string()), + }, + ); + + OpenAiTool::Function(ResponsesApiTool { + name: "apply_patch".to_string(), + description: r#"Use this tool to edit files. +Your patch language is a stripped‑down, file‑oriented diff format designed to be easy to parse and safe to apply. You can think of it as a high‑level envelope: + +**_ Begin Patch +[ one or more file sections ] +_** End Patch + +Within that envelope, you get a sequence of file operations. +You MUST include a header to specify the action you are taking. +Each operation starts with one of three headers: + +**_ Add File: - create a new file. Every following line is a + line (the initial contents). +_** Delete File: - remove an existing file. Nothing follows. +\*\*\* Update File: - patch an existing file in place (optionally with a rename). + +May be immediately followed by \*\*\* Move to: if you want to rename the file. +Then one or more “hunks”, each introduced by @@ (optionally followed by a hunk header). +Within a hunk each line starts with: + +- for inserted text, + +* for removed text, or + space ( ) for context. + At the end of a truncated hunk you can emit \*\*\* End of File. + +Patch := Begin { FileOp } End +Begin := "**_ Begin Patch" NEWLINE +End := "_** End Patch" NEWLINE +FileOp := AddFile | DeleteFile | UpdateFile +AddFile := "**_ Add File: " path NEWLINE { "+" line NEWLINE } +DeleteFile := "_** Delete File: " path NEWLINE +UpdateFile := "**_ Update File: " path NEWLINE [ MoveTo ] { Hunk } +MoveTo := "_** Move to: " newPath NEWLINE +Hunk := "@@" [ header ] NEWLINE { HunkLine } [ "*** End of File" NEWLINE ] +HunkLine := (" " | "-" | "+") text NEWLINE + +A full patch can combine several operations: + +**_ Begin Patch +_** Add File: hello.txt ++Hello world +**_ Update File: src/app.py +_** Move to: src/main.py +@@ def greet(): +-print("Hi") ++print("Hello, world!") +**_ Delete File: obsolete.txt +_** End Patch + +It is important to remember: + +- You must include a header with your intended action (Add/Delete/Update) +- You must prefix new lines with `+` even when creating a new file +"# + .to_string(), + strict: false, + parameters: JsonSchema::Object { + properties, + required: Some(vec!["input".to_string()]), + additional_properties: Some(false), + }, + }) +} + /// Returns JSON values that are compatible with Function Calling in the /// Responses API: /// https://platform.openai.com/docs/guides/function-calling?api-mode=responses @@ -296,7 +383,13 @@ pub(crate) fn mcp_tool_to_openai_tool( input_schema.properties = Some(serde_json::Value::Object(serde_json::Map::new())); } - let serialized_input_schema = serde_json::to_value(input_schema)?; + // Serialize to a raw JSON value so we can sanitize schemas coming from MCP + // servers. Some servers omit the top-level or nested `type` in JSON + // Schemas (e.g. using enum/anyOf), or use unsupported variants like + // `integer`. Our internal JsonSchema is a small subset and requires + // `type`, so we coerce/sanitize here for compatibility. + let mut serialized_input_schema = serde_json::to_value(input_schema)?; + sanitize_json_schema(&mut serialized_input_schema); let input_schema = serde_json::from_value::(serialized_input_schema)?; Ok(ResponsesApiTool { @@ -307,6 +400,120 @@ pub(crate) fn mcp_tool_to_openai_tool( }) } +/// Sanitize a JSON Schema (as serde_json::Value) so it can fit our limited +/// JsonSchema enum. This function: +/// - Ensures every schema object has a "type". If missing, infers it from +/// common keywords (properties => object, items => array, enum/const/format => string) +/// and otherwise defaults to "string". +/// - Fills required child fields (e.g. array items, object properties) with +/// permissive defaults when absent. +fn sanitize_json_schema(value: &mut JsonValue) { + match value { + JsonValue::Bool(_) => { + // JSON Schema boolean form: true/false. Coerce to an accept-all string. + *value = json!({ "type": "string" }); + } + JsonValue::Array(arr) => { + for v in arr.iter_mut() { + sanitize_json_schema(v); + } + } + JsonValue::Object(map) => { + // First, recursively sanitize known nested schema holders + if let Some(props) = map.get_mut("properties") { + if let Some(props_map) = props.as_object_mut() { + for (_k, v) in props_map.iter_mut() { + sanitize_json_schema(v); + } + } + } + if let Some(items) = map.get_mut("items") { + sanitize_json_schema(items); + } + // Some schemas use oneOf/anyOf/allOf - sanitize their entries + for combiner in ["oneOf", "anyOf", "allOf", "prefixItems"] { + if let Some(v) = map.get_mut(combiner) { + sanitize_json_schema(v); + } + } + + // Normalize/ensure type + let mut ty = map + .get("type") + .and_then(|v| v.as_str()) + .map(|s| s.to_string()); + + // If type is an array (union), pick first supported; else leave to inference + if ty.is_none() { + if let Some(JsonValue::Array(types)) = map.get("type") { + for t in types { + if let Some(tt) = t.as_str() { + if matches!( + tt, + "object" | "array" | "string" | "number" | "integer" | "boolean" + ) { + ty = Some(tt.to_string()); + break; + } + } + } + } + } + + // Infer type if still missing + if ty.is_none() { + if map.contains_key("properties") + || map.contains_key("required") + || map.contains_key("additionalProperties") + { + ty = Some("object".to_string()); + } else if map.contains_key("items") || map.contains_key("prefixItems") { + ty = Some("array".to_string()); + } else if map.contains_key("enum") + || map.contains_key("const") + || map.contains_key("format") + { + ty = Some("string".to_string()); + } else if map.contains_key("minimum") + || map.contains_key("maximum") + || map.contains_key("exclusiveMinimum") + || map.contains_key("exclusiveMaximum") + || map.contains_key("multipleOf") + { + ty = Some("number".to_string()); + } + } + // If we still couldn't infer, default to string + let ty = ty.unwrap_or_else(|| "string".to_string()); + map.insert("type".to_string(), JsonValue::String(ty.to_string())); + + // Ensure object schemas have properties map + if ty == "object" { + if !map.contains_key("properties") { + map.insert( + "properties".to_string(), + JsonValue::Object(serde_json::Map::new()), + ); + } + // If additionalProperties is an object schema, sanitize it too. + // Leave booleans as-is, since JSON Schema allows boolean here. + if let Some(ap) = map.get_mut("additionalProperties") { + let is_bool = matches!(ap, JsonValue::Bool(_)); + if !is_bool { + sanitize_json_schema(ap); + } + } + } + + // Ensure array schemas have items + if ty == "array" && !map.contains_key("items") { + map.insert("items".to_string(), json!({ "type": "string" })); + } + } + _ => {} + } +} + /// Returns a list of OpenAiTools based on the provided config and MCP tools. /// Note that the keys of mcp_tools should be fully qualified names. See /// [`McpConnectionManager`] for more details. @@ -332,6 +539,10 @@ pub(crate) fn get_openai_tools( tools.push(PLAN_TOOL.clone()); } + if config.apply_patch_tool { + tools.push(create_apply_patch_tool()); + } + if let Some(mcp_tools) = mcp_tools { for (name, tool) in mcp_tools { match mcp_tool_to_openai_tool(name.clone(), tool.clone()) { @@ -347,10 +558,10 @@ pub(crate) fn get_openai_tools( } #[cfg(test)] -#[allow(clippy::expect_used)] mod tests { use crate::model_family::find_family_for_model; use mcp_types::ToolInputSchema; + use pretty_assertions::assert_eq; use super::*; @@ -385,6 +596,7 @@ mod tests { AskForApproval::Never, SandboxPolicy::ReadOnly, true, + model_family.uses_apply_patch_tool, ); let tools = get_openai_tools(&config, Some(HashMap::new())); @@ -399,6 +611,7 @@ mod tests { AskForApproval::Never, SandboxPolicy::ReadOnly, true, + model_family.uses_apply_patch_tool, ); let tools = get_openai_tools(&config, Some(HashMap::new())); @@ -413,6 +626,7 @@ mod tests { AskForApproval::Never, SandboxPolicy::ReadOnly, false, + model_family.uses_apply_patch_tool, ); let tools = get_openai_tools( &config, @@ -497,4 +711,216 @@ mod tests { }) ); } + + #[test] + fn test_mcp_tool_property_missing_type_defaults_to_string() { + let model_family = find_family_for_model("o3").expect("o3 should be a valid model family"); + let config = ToolsConfig::new( + &model_family, + AskForApproval::Never, + SandboxPolicy::ReadOnly, + false, + model_family.uses_apply_patch_tool, + ); + + let tools = get_openai_tools( + &config, + Some(HashMap::from([( + "dash/search".to_string(), + mcp_types::Tool { + name: "search".to_string(), + input_schema: ToolInputSchema { + properties: Some(serde_json::json!({ + "query": { + "description": "search query" + } + })), + required: None, + r#type: "object".to_string(), + }, + output_schema: None, + title: None, + annotations: None, + description: Some("Search docs".to_string()), + }, + )])), + ); + + assert_eq_tool_names(&tools, &["shell", "dash/search"]); + + assert_eq!( + tools[1], + OpenAiTool::Function(ResponsesApiTool { + name: "dash/search".to_string(), + parameters: JsonSchema::Object { + properties: BTreeMap::from([( + "query".to_string(), + JsonSchema::String { + description: Some("search query".to_string()) + } + )]), + required: None, + additional_properties: None, + }, + description: "Search docs".to_string(), + strict: false, + }) + ); + } + + #[test] + fn test_mcp_tool_integer_normalized_to_number() { + let model_family = find_family_for_model("o3").expect("o3 should be a valid model family"); + let config = ToolsConfig::new( + &model_family, + AskForApproval::Never, + SandboxPolicy::ReadOnly, + false, + model_family.uses_apply_patch_tool, + ); + + let tools = get_openai_tools( + &config, + Some(HashMap::from([( + "dash/paginate".to_string(), + mcp_types::Tool { + name: "paginate".to_string(), + input_schema: ToolInputSchema { + properties: Some(serde_json::json!({ + "page": { "type": "integer" } + })), + required: None, + r#type: "object".to_string(), + }, + output_schema: None, + title: None, + annotations: None, + description: Some("Pagination".to_string()), + }, + )])), + ); + + assert_eq_tool_names(&tools, &["shell", "dash/paginate"]); + assert_eq!( + tools[1], + OpenAiTool::Function(ResponsesApiTool { + name: "dash/paginate".to_string(), + parameters: JsonSchema::Object { + properties: BTreeMap::from([( + "page".to_string(), + JsonSchema::Number { description: None } + )]), + required: None, + additional_properties: None, + }, + description: "Pagination".to_string(), + strict: false, + }) + ); + } + + #[test] + fn test_mcp_tool_array_without_items_gets_default_string_items() { + let model_family = find_family_for_model("o3").expect("o3 should be a valid model family"); + let config = ToolsConfig::new( + &model_family, + AskForApproval::Never, + SandboxPolicy::ReadOnly, + false, + model_family.uses_apply_patch_tool, + ); + + let tools = get_openai_tools( + &config, + Some(HashMap::from([( + "dash/tags".to_string(), + mcp_types::Tool { + name: "tags".to_string(), + input_schema: ToolInputSchema { + properties: Some(serde_json::json!({ + "tags": { "type": "array" } + })), + required: None, + r#type: "object".to_string(), + }, + output_schema: None, + title: None, + annotations: None, + description: Some("Tags".to_string()), + }, + )])), + ); + + assert_eq_tool_names(&tools, &["shell", "dash/tags"]); + assert_eq!( + tools[1], + OpenAiTool::Function(ResponsesApiTool { + name: "dash/tags".to_string(), + parameters: JsonSchema::Object { + properties: BTreeMap::from([( + "tags".to_string(), + JsonSchema::Array { + items: Box::new(JsonSchema::String { description: None }), + description: None + } + )]), + required: None, + additional_properties: None, + }, + description: "Tags".to_string(), + strict: false, + }) + ); + } + + #[test] + fn test_mcp_tool_anyof_defaults_to_string() { + let model_family = find_family_for_model("o3").expect("o3 should be a valid model family"); + let config = ToolsConfig::new( + &model_family, + AskForApproval::Never, + SandboxPolicy::ReadOnly, + false, + model_family.uses_apply_patch_tool, + ); + + let tools = get_openai_tools( + &config, + Some(HashMap::from([( + "dash/value".to_string(), + mcp_types::Tool { + name: "value".to_string(), + input_schema: ToolInputSchema { + properties: Some(serde_json::json!({ + "value": { "anyOf": [ { "type": "string" }, { "type": "number" } ] } + })), + required: None, + r#type: "object".to_string(), + }, + output_schema: None, + title: None, + annotations: None, + description: Some("AnyOf Value".to_string()), + }, + )])), + ); + + assert_eq_tool_names(&tools, &["shell", "dash/value"]); + assert_eq!( + tools[1], + OpenAiTool::Function(ResponsesApiTool { + name: "dash/value".to_string(), + parameters: JsonSchema::Object { + properties: BTreeMap::from([( + "value".to_string(), + JsonSchema::String { description: None } + )]), + required: None, + additional_properties: None, + }, + description: "AnyOf Value".to_string(), + strict: false, + }) + ); + } } diff --git a/codex-rs/core/src/parse_command.rs b/codex-rs/core/src/parse_command.rs new file mode 100644 index 0000000000..6ea0626814 --- /dev/null +++ b/codex-rs/core/src/parse_command.rs @@ -0,0 +1,2068 @@ +use crate::bash::try_parse_bash; +use crate::bash::try_parse_word_only_commands_sequence; +use serde::Deserialize; +use serde::Serialize; +use shlex::split as shlex_split; +use shlex::try_join as shlex_try_join; + +#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)] +pub enum ParsedCommand { + Read { + cmd: String, + name: String, + }, + ListFiles { + cmd: String, + path: Option, + }, + Search { + cmd: String, + query: Option, + path: Option, + }, + Format { + cmd: String, + tool: Option, + targets: Option>, + }, + Test { + cmd: String, + }, + Lint { + cmd: String, + tool: Option, + targets: Option>, + }, + Noop { + cmd: String, + }, + Unknown { + cmd: String, + }, +} + +// Convert core's parsed command enum into the protocol's simplified type so +// events can carry the canonical representation across process boundaries. +impl From for codex_protocol::parse_command::ParsedCommand { + fn from(v: ParsedCommand) -> Self { + use codex_protocol::parse_command::ParsedCommand as P; + match v { + ParsedCommand::Read { cmd, name } => P::Read { cmd, name }, + ParsedCommand::ListFiles { cmd, path } => P::ListFiles { cmd, path }, + ParsedCommand::Search { cmd, query, path } => P::Search { cmd, query, path }, + ParsedCommand::Format { cmd, tool, targets } => P::Format { cmd, tool, targets }, + ParsedCommand::Test { cmd } => P::Test { cmd }, + ParsedCommand::Lint { cmd, tool, targets } => P::Lint { cmd, tool, targets }, + ParsedCommand::Noop { cmd } => P::Noop { cmd }, + ParsedCommand::Unknown { cmd } => P::Unknown { cmd }, + } + } +} + +fn shlex_join(tokens: &[String]) -> String { + shlex_try_join(tokens.iter().map(|s| s.as_str())) + .unwrap_or_else(|_| "".to_string()) +} + +/// DO NOT REVIEW THIS CODE BY HAND +/// This parsing code is quite complex and not easy to hand-modify. +/// The easiest way to iterate is to add unit tests and have Codex fix the implementation. +/// To encourage this, the tests have been put directly below this function rather than at the bottom of the +/// +/// Parses metadata out of an arbitrary command. +/// These commands are model driven and could include just about anything. +/// The parsing is slightly lossy due to the ~infinite expressiveness of an arbitrary command. +/// The goal of the parsed metadata is to be able to provide the user with a human readable gis +/// of what it is doing. +pub fn parse_command(command: &[String]) -> Vec { + // Parse and then collapse consecutive duplicate commands to avoid redundant summaries. + let parsed = parse_command_impl(command); + let mut deduped: Vec = Vec::with_capacity(parsed.len()); + for cmd in parsed.into_iter() { + if deduped.last().is_some_and(|prev| prev == &cmd) { + continue; + } + deduped.push(cmd); + } + deduped +} + +#[cfg(test)] +#[allow(clippy::items_after_test_module)] +/// Tests are at the top to encourage using TDD + Codex to fix the implementation. +mod tests { + use super::*; + + fn shlex_split_safe(s: &str) -> Vec { + shlex_split(s).unwrap_or_else(|| s.split_whitespace().map(|s| s.to_string()).collect()) + } + + fn vec_str(args: &[&str]) -> Vec { + args.iter().map(|s| s.to_string()).collect() + } + + fn assert_parsed(args: &[String], expected: Vec) { + let out = parse_command(args); + assert_eq!(out, expected); + } + + #[test] + fn git_status_is_unknown() { + assert_parsed( + &vec_str(&["git", "status"]), + vec![ParsedCommand::Unknown { + cmd: "git status".to_string(), + }], + ); + } + + #[test] + fn handles_git_pipe_wc() { + let inner = "git status | wc -l"; + assert_parsed( + &vec_str(&["bash", "-lc", inner]), + vec![ParsedCommand::Unknown { + cmd: "git status | wc -l".to_string(), + }], + ); + } + + #[test] + fn bash_lc_redirect_not_quoted() { + let inner = "echo foo > bar"; + assert_parsed( + &vec_str(&["bash", "-lc", inner]), + vec![ParsedCommand::Unknown { + cmd: "echo foo > bar".to_string(), + }], + ); + } + + #[test] + fn handles_complex_bash_command_head() { + let inner = + "rg --version && node -v && pnpm -v && rg --files | wc -l && rg --files | head -n 40"; + assert_parsed( + &vec_str(&["bash", "-lc", inner]), + vec![ + // Expect commands in left-to-right execution order + ParsedCommand::Search { + cmd: "rg --version".to_string(), + query: None, + path: None, + }, + ParsedCommand::Unknown { + cmd: "node -v".to_string(), + }, + ParsedCommand::Unknown { + cmd: "pnpm -v".to_string(), + }, + ParsedCommand::Search { + cmd: "rg --files".to_string(), + query: None, + path: None, + }, + ParsedCommand::Unknown { + cmd: "head -n 40".to_string(), + }, + ], + ); + } + + #[test] + fn supports_searching_for_navigate_to_route() -> anyhow::Result<()> { + let inner = "rg -n \"navigate-to-route\" -S"; + assert_parsed( + &vec_str(&["bash", "-lc", inner]), + vec![ParsedCommand::Search { + cmd: "rg -n navigate-to-route -S".to_string(), + query: Some("navigate-to-route".to_string()), + path: None, + }], + ); + Ok(()) + } + + #[test] + fn handles_complex_bash_command() { + let inner = "rg -n \"BUG|FIXME|TODO|XXX|HACK\" -S | head -n 200"; + assert_parsed( + &vec_str(&["bash", "-lc", inner]), + vec![ + ParsedCommand::Search { + cmd: "rg -n 'BUG|FIXME|TODO|XXX|HACK' -S".to_string(), + query: Some("BUG|FIXME|TODO|XXX|HACK".to_string()), + path: None, + }, + ParsedCommand::Unknown { + cmd: "head -n 200".to_string(), + }, + ], + ); + } + + #[test] + fn supports_rg_files_with_path_and_pipe() { + let inner = "rg --files webview/src | sed -n"; + assert_parsed( + &vec_str(&["bash", "-lc", inner]), + vec![ParsedCommand::Search { + cmd: "rg --files webview/src".to_string(), + query: None, + path: Some("webview".to_string()), + }], + ); + } + + #[test] + fn supports_rg_files_then_head() { + let inner = "rg --files | head -n 50"; + assert_parsed( + &vec_str(&["bash", "-lc", inner]), + vec![ + ParsedCommand::Search { + cmd: "rg --files".to_string(), + query: None, + path: None, + }, + ParsedCommand::Unknown { + cmd: "head -n 50".to_string(), + }, + ], + ); + } + + #[test] + fn supports_cat() { + let inner = "cat webview/README.md"; + assert_parsed( + &vec_str(&["bash", "-lc", inner]), + vec![ParsedCommand::Read { + cmd: inner.to_string(), + name: "README.md".to_string(), + }], + ); + } + + #[test] + fn supports_ls_with_pipe() { + let inner = "ls -la | sed -n '1,120p'"; + assert_parsed( + &vec_str(&["bash", "-lc", inner]), + vec![ParsedCommand::ListFiles { + cmd: "ls -la".to_string(), + path: None, + }], + ); + } + + #[test] + fn supports_head_n() { + let inner = "head -n 50 Cargo.toml"; + assert_parsed( + &vec_str(&["bash", "-lc", inner]), + vec![ParsedCommand::Read { + cmd: inner.to_string(), + name: "Cargo.toml".to_string(), + }], + ); + } + + #[test] + fn supports_cat_sed_n() { + let inner = "cat tui/Cargo.toml | sed -n '1,200p'"; + assert_parsed( + &vec_str(&["bash", "-lc", inner]), + vec![ParsedCommand::Read { + cmd: inner.to_string(), + name: "Cargo.toml".to_string(), + }], + ); + } + + #[test] + fn supports_tail_n_plus() { + let inner = "tail -n +522 README.md"; + assert_parsed( + &vec_str(&["bash", "-lc", inner]), + vec![ParsedCommand::Read { + cmd: inner.to_string(), + name: "README.md".to_string(), + }], + ); + } + + #[test] + fn supports_tail_n_last_lines() { + let inner = "tail -n 30 README.md"; + let out = parse_command(&vec_str(&["bash", "-lc", inner])); + assert_eq!( + out, + vec![ParsedCommand::Read { + cmd: inner.to_string(), + name: "README.md".to_string(), + }] + ); + } + + #[test] + fn supports_npm_run_build_is_unknown() { + assert_parsed( + &vec_str(&["npm", "run", "build"]), + vec![ParsedCommand::Unknown { + cmd: "npm run build".to_string(), + }], + ); + } + + #[test] + fn supports_npm_run_with_forwarded_args() { + assert_parsed( + &vec_str(&[ + "npm", + "run", + "lint", + "--", + "--max-warnings", + "0", + "--format", + "json", + ]), + vec![ParsedCommand::Lint { + cmd: "npm run lint -- --max-warnings 0 --format json".to_string(), + tool: Some("npm-script:lint".to_string()), + targets: None, + }], + ); + } + + #[test] + fn supports_grep_recursive_current_dir() { + assert_parsed( + &vec_str(&["grep", "-R", "CODEX_SANDBOX_ENV_VAR", "-n", "."]), + vec![ParsedCommand::Search { + cmd: "grep -R CODEX_SANDBOX_ENV_VAR -n .".to_string(), + query: Some("CODEX_SANDBOX_ENV_VAR".to_string()), + path: Some(".".to_string()), + }], + ); + } + + #[test] + fn supports_grep_recursive_specific_file() { + assert_parsed( + &vec_str(&[ + "grep", + "-R", + "CODEX_SANDBOX_ENV_VAR", + "-n", + "core/src/spawn.rs", + ]), + vec![ParsedCommand::Search { + cmd: "grep -R CODEX_SANDBOX_ENV_VAR -n core/src/spawn.rs".to_string(), + query: Some("CODEX_SANDBOX_ENV_VAR".to_string()), + path: Some("spawn.rs".to_string()), + }], + ); + } + + #[test] + fn supports_grep_query_with_slashes_not_shortened() { + // Query strings may contain slashes and should not be shortened to the basename. + // Previously, grep queries were passed through short_display_path, which is incorrect. + assert_parsed( + &shlex_split_safe("grep -R src/main.rs -n ."), + vec![ParsedCommand::Search { + cmd: "grep -R src/main.rs -n .".to_string(), + query: Some("src/main.rs".to_string()), + path: Some(".".to_string()), + }], + ); + } + + #[test] + fn supports_grep_weird_backtick_in_query() { + assert_parsed( + &shlex_split_safe("grep -R COD`EX_SANDBOX -n"), + vec![ParsedCommand::Search { + cmd: "grep -R 'COD`EX_SANDBOX' -n".to_string(), + query: Some("COD`EX_SANDBOX".to_string()), + path: None, + }], + ); + } + + #[test] + fn supports_cd_and_rg_files() { + assert_parsed( + &shlex_split_safe("cd codex-rs && rg --files"), + vec![ + ParsedCommand::Unknown { + cmd: "cd codex-rs".to_string(), + }, + ParsedCommand::Search { + cmd: "rg --files".to_string(), + query: None, + path: None, + }, + ], + ); + } + + #[test] + fn echo_then_cargo_test_sequence() { + assert_parsed( + &shlex_split_safe("echo Running tests... && cargo test --all-features --quiet"), + vec![ParsedCommand::Test { + cmd: "cargo test --all-features --quiet".to_string(), + }], + ); + } + + #[test] + fn supports_cargo_fmt_and_test_with_config() { + assert_parsed( + &shlex_split_safe( + "cargo fmt -- --config imports_granularity=Item && cargo test -p core --all-features", + ), + vec![ + ParsedCommand::Format { + cmd: "cargo fmt -- --config 'imports_granularity=Item'".to_string(), + tool: Some("cargo fmt".to_string()), + targets: None, + }, + ParsedCommand::Test { + cmd: "cargo test -p core --all-features".to_string(), + }, + ], + ); + } + + #[test] + fn recognizes_rustfmt_and_clippy() { + assert_parsed( + &shlex_split_safe("rustfmt src/main.rs"), + vec![ParsedCommand::Format { + cmd: "rustfmt src/main.rs".to_string(), + tool: Some("rustfmt".to_string()), + targets: Some(vec!["src/main.rs".to_string()]), + }], + ); + + assert_parsed( + &shlex_split_safe("cargo clippy -p core --all-features -- -D warnings"), + vec![ParsedCommand::Lint { + cmd: "cargo clippy -p core --all-features -- -D warnings".to_string(), + tool: Some("cargo clippy".to_string()), + targets: None, + }], + ); + } + + #[test] + fn recognizes_pytest_go_and_tools() { + assert_parsed( + &shlex_split_safe( + "pytest -k 'Login and not slow' tests/test_login.py::TestLogin::test_ok", + ), + vec![ParsedCommand::Test { + cmd: "pytest -k 'Login and not slow' tests/test_login.py::TestLogin::test_ok" + .to_string(), + }], + ); + + assert_parsed( + &shlex_split_safe("go fmt ./..."), + vec![ParsedCommand::Format { + cmd: "go fmt ./...".to_string(), + tool: Some("go fmt".to_string()), + targets: Some(vec!["./...".to_string()]), + }], + ); + + assert_parsed( + &shlex_split_safe("go test ./pkg -run TestThing"), + vec![ParsedCommand::Test { + cmd: "go test ./pkg -run TestThing".to_string(), + }], + ); + + assert_parsed( + &shlex_split_safe("eslint . --max-warnings 0"), + vec![ParsedCommand::Lint { + cmd: "eslint . --max-warnings 0".to_string(), + tool: Some("eslint".to_string()), + targets: Some(vec![".".to_string()]), + }], + ); + + assert_parsed( + &shlex_split_safe("prettier -w ."), + vec![ParsedCommand::Format { + cmd: "prettier -w .".to_string(), + tool: Some("prettier".to_string()), + targets: Some(vec![".".to_string()]), + }], + ); + } + + #[test] + fn recognizes_jest_and_vitest_filters() { + assert_parsed( + &shlex_split_safe("jest -t 'should work' src/foo.test.ts"), + vec![ParsedCommand::Test { + cmd: "jest -t 'should work' src/foo.test.ts".to_string(), + }], + ); + + assert_parsed( + &shlex_split_safe("vitest -t 'runs' src/foo.test.tsx"), + vec![ParsedCommand::Test { + cmd: "vitest -t runs src/foo.test.tsx".to_string(), + }], + ); + } + + #[test] + fn recognizes_npx_and_scripts() { + assert_parsed( + &shlex_split_safe("npx eslint src"), + vec![ParsedCommand::Lint { + cmd: "npx eslint src".to_string(), + tool: Some("eslint".to_string()), + targets: Some(vec!["src".to_string()]), + }], + ); + + assert_parsed( + &shlex_split_safe("npx prettier -c ."), + vec![ParsedCommand::Format { + cmd: "npx prettier -c .".to_string(), + tool: Some("prettier".to_string()), + targets: Some(vec![".".to_string()]), + }], + ); + + assert_parsed( + &shlex_split_safe("pnpm run lint -- --max-warnings 0"), + vec![ParsedCommand::Lint { + cmd: "pnpm run lint -- --max-warnings 0".to_string(), + tool: Some("pnpm-script:lint".to_string()), + targets: None, + }], + ); + + assert_parsed( + &shlex_split_safe("npm test"), + vec![ParsedCommand::Test { + cmd: "npm test".to_string(), + }], + ); + + assert_parsed( + &shlex_split_safe("yarn test"), + vec![ParsedCommand::Test { + cmd: "yarn test".to_string(), + }], + ); + } + + // ---- is_small_formatting_command unit tests ---- + #[test] + fn small_formatting_always_true_commands() { + for cmd in [ + "wc", "tr", "cut", "sort", "uniq", "xargs", "tee", "column", "awk", + ] { + assert!(is_small_formatting_command(&shlex_split_safe(cmd))); + assert!(is_small_formatting_command(&shlex_split_safe(&format!( + "{cmd} -x" + )))); + } + } + + #[test] + fn head_behavior() { + // No args -> small formatting + assert!(is_small_formatting_command(&vec_str(&["head"]))); + // Numeric count only -> not considered small formatting by implementation + assert!(!is_small_formatting_command(&shlex_split_safe( + "head -n 40" + ))); + // With explicit file -> not small formatting + assert!(!is_small_formatting_command(&shlex_split_safe( + "head -n 40 file.txt" + ))); + // File only (no count) -> treated as small formatting by implementation + assert!(is_small_formatting_command(&vec_str(&["head", "file.txt"]))); + } + + #[test] + fn tail_behavior() { + // No args -> small formatting + assert!(is_small_formatting_command(&vec_str(&["tail"]))); + // Numeric with plus offset -> not small formatting + assert!(!is_small_formatting_command(&shlex_split_safe( + "tail -n +10" + ))); + assert!(!is_small_formatting_command(&shlex_split_safe( + "tail -n +10 file.txt" + ))); + // Numeric count + assert!(!is_small_formatting_command(&shlex_split_safe( + "tail -n 30" + ))); + assert!(!is_small_formatting_command(&shlex_split_safe( + "tail -n 30 file.txt" + ))); + // File only -> small formatting by implementation + assert!(is_small_formatting_command(&vec_str(&["tail", "file.txt"]))); + } + + #[test] + fn sed_behavior() { + // Plain sed -> small formatting + assert!(is_small_formatting_command(&vec_str(&["sed"]))); + // sed -n (no file) -> still small formatting + assert!(is_small_formatting_command(&vec_str(&["sed", "-n", "10p"]))); + // Valid range with file -> not small formatting + assert!(!is_small_formatting_command(&shlex_split_safe( + "sed -n 10p file.txt" + ))); + assert!(!is_small_formatting_command(&shlex_split_safe( + "sed -n 1,200p file.txt" + ))); + // Invalid ranges with file -> small formatting + assert!(is_small_formatting_command(&shlex_split_safe( + "sed -n p file.txt" + ))); + assert!(is_small_formatting_command(&shlex_split_safe( + "sed -n +10p file.txt" + ))); + } + + #[test] + fn empty_tokens_is_not_small() { + let empty: Vec = Vec::new(); + assert!(!is_small_formatting_command(&empty)); + } + + #[test] + fn supports_nl_then_sed_reading() { + let inner = "nl -ba core/src/parse_command.rs | sed -n '1200,1720p'"; + assert_parsed( + &vec_str(&["bash", "-lc", inner]), + vec![ParsedCommand::Read { + cmd: inner.to_string(), + name: "parse_command.rs".to_string(), + }], + ); + } + + #[test] + fn supports_sed_n() { + let inner = "sed -n '2000,2200p' tui/src/history_cell.rs"; + assert_parsed( + &vec_str(&["bash", "-lc", inner]), + vec![ParsedCommand::Read { + cmd: inner.to_string(), + name: "history_cell.rs".to_string(), + }], + ); + } + + #[test] + fn filters_out_printf() { + let inner = + r#"printf "\n===== ansi-escape/Cargo.toml =====\n"; cat -- ansi-escape/Cargo.toml"#; + assert_parsed( + &vec_str(&["bash", "-lc", inner]), + vec![ParsedCommand::Read { + cmd: "cat -- ansi-escape/Cargo.toml".to_string(), + name: "Cargo.toml".to_string(), + }], + ); + } + + #[test] + fn drops_yes_in_pipelines() { + // Inside bash -lc, `yes | rg --files` should focus on the primary command. + let inner = "yes | rg --files"; + assert_parsed( + &vec_str(&["bash", "-lc", inner]), + vec![ParsedCommand::Search { + cmd: "rg --files".to_string(), + query: None, + path: None, + }], + ); + } + + #[test] + fn supports_sed_n_then_nl_as_search() { + // Ensure `sed -n '' | nl -ba` is summarized as a search for that file. + let args = shlex_split_safe( + "sed -n '260,640p' exec/src/event_processor_with_human_output.rs | nl -ba", + ); + assert_parsed( + &args, + vec![ParsedCommand::Read { + cmd: "sed -n '260,640p' exec/src/event_processor_with_human_output.rs".to_string(), + name: "event_processor_with_human_output.rs".to_string(), + }], + ); + } + + #[test] + fn preserves_rg_with_spaces() { + assert_parsed( + &shlex_split_safe("yes | rg -n 'foo bar' -S"), + vec![ParsedCommand::Search { + cmd: "rg -n 'foo bar' -S".to_string(), + query: Some("foo bar".to_string()), + path: None, + }], + ); + } + + #[test] + fn ls_with_glob() { + assert_parsed( + &shlex_split_safe("ls -I '*.test.js'"), + vec![ParsedCommand::ListFiles { + cmd: "ls -I '*.test.js'".to_string(), + path: None, + }], + ); + } + + #[test] + fn trim_on_semicolon() { + assert_parsed( + &shlex_split_safe("rg foo ; echo done"), + vec![ + ParsedCommand::Search { + cmd: "rg foo".to_string(), + query: Some("foo".to_string()), + path: None, + }, + ParsedCommand::Unknown { + cmd: "echo done".to_string(), + }, + ], + ); + } + + #[test] + fn split_on_or_connector() { + // Ensure we split commands on the logical OR operator as well. + assert_parsed( + &shlex_split_safe("rg foo || echo done"), + vec![ + ParsedCommand::Search { + cmd: "rg foo".to_string(), + query: Some("foo".to_string()), + path: None, + }, + ParsedCommand::Unknown { + cmd: "echo done".to_string(), + }, + ], + ); + } + + #[test] + fn strips_true_in_sequence() { + // `true` should be dropped from parsed sequences + assert_parsed( + &shlex_split_safe("true && rg --files"), + vec![ParsedCommand::Search { + cmd: "rg --files".to_string(), + query: None, + path: None, + }], + ); + + assert_parsed( + &shlex_split_safe("rg --files && true"), + vec![ParsedCommand::Search { + cmd: "rg --files".to_string(), + query: None, + path: None, + }], + ); + } + + #[test] + fn strips_true_inside_bash_lc() { + let inner = "true && rg --files"; + assert_parsed( + &vec_str(&["bash", "-lc", inner]), + vec![ParsedCommand::Search { + cmd: "rg --files".to_string(), + query: None, + path: None, + }], + ); + + let inner2 = "rg --files || true"; + assert_parsed( + &vec_str(&["bash", "-lc", inner2]), + vec![ParsedCommand::Search { + cmd: "rg --files".to_string(), + query: None, + path: None, + }], + ); + } + + #[test] + fn shorten_path_on_windows() { + assert_parsed( + &shlex_split_safe(r#"cat "pkg\src\main.rs""#), + vec![ParsedCommand::Read { + cmd: r#"cat "pkg\\src\\main.rs""#.to_string(), + name: "main.rs".to_string(), + }], + ); + } + + #[test] + fn head_with_no_space() { + assert_parsed( + &shlex_split_safe("bash -lc 'head -n50 Cargo.toml'"), + vec![ParsedCommand::Read { + cmd: "head -n50 Cargo.toml".to_string(), + name: "Cargo.toml".to_string(), + }], + ); + } + + #[test] + fn bash_dash_c_pipeline_parsing() { + // Ensure -c is handled similarly to -lc by normalization + let inner = "rg --files | head -n 1"; + assert_parsed( + &shlex_split_safe(inner), + vec![ + ParsedCommand::Search { + cmd: "rg --files".to_string(), + query: None, + path: None, + }, + ParsedCommand::Unknown { + cmd: "head -n 1".to_string(), + }, + ], + ); + } + + #[test] + fn tail_with_no_space() { + assert_parsed( + &shlex_split_safe("bash -lc 'tail -n+10 README.md'"), + vec![ParsedCommand::Read { + cmd: "tail -n+10 README.md".to_string(), + name: "README.md".to_string(), + }], + ); + } + + #[test] + fn pnpm_test_is_parsed_as_test() { + assert_parsed( + &shlex_split_safe("pnpm test"), + vec![ParsedCommand::Test { + cmd: "pnpm test".to_string(), + }], + ); + } + + #[test] + fn pnpm_exec_vitest_is_unknown() { + // From commands_combined: cd codex-cli && pnpm exec vitest run tests/... --threads=false --passWithNoTests + let inner = "cd codex-cli && pnpm exec vitest run tests/file-tag-utils.test.ts --threads=false --passWithNoTests"; + assert_parsed( + &shlex_split_safe(inner), + vec![ + ParsedCommand::Unknown { + cmd: "cd codex-cli".to_string(), + }, + ParsedCommand::Unknown { + cmd: "pnpm exec vitest run tests/file-tag-utils.test.ts '--threads=false' --passWithNoTests".to_string(), + }, + ], + ); + } + + #[test] + fn cargo_test_with_crate() { + assert_parsed( + &shlex_split_safe("cargo test -p codex-core parse_command::"), + vec![ParsedCommand::Test { + cmd: "cargo test -p codex-core parse_command::".to_string(), + }], + ); + } + + #[test] + fn cargo_test_with_crate_2() { + assert_parsed( + &shlex_split_safe( + "cd core && cargo test -q parse_command::tests::bash_dash_c_pipeline_parsing parse_command::tests::fd_file_finder_variants", + ), + vec![ParsedCommand::Test { + cmd: "cargo test -q parse_command::tests::bash_dash_c_pipeline_parsing parse_command::tests::fd_file_finder_variants".to_string(), + }], + ); + } + + #[test] + fn cargo_test_with_crate_3() { + assert_parsed( + &shlex_split_safe("cd core && cargo test -q parse_command::tests"), + vec![ParsedCommand::Test { + cmd: "cargo test -q parse_command::tests".to_string(), + }], + ); + } + + #[test] + fn cargo_test_with_crate_4() { + assert_parsed( + &shlex_split_safe("cd core && cargo test --all-features parse_command -- --nocapture"), + vec![ParsedCommand::Test { + cmd: "cargo test --all-features parse_command -- --nocapture".to_string(), + }], + ); + } + + // Additional coverage for other common tools/frameworks + #[test] + fn recognizes_black_and_ruff() { + // black formats Python code + assert_parsed( + &shlex_split_safe("black src"), + vec![ParsedCommand::Format { + cmd: "black src".to_string(), + tool: Some("black".to_string()), + targets: Some(vec!["src".to_string()]), + }], + ); + + // ruff check is a linter; ensure we collect targets + assert_parsed( + &shlex_split_safe("ruff check ."), + vec![ParsedCommand::Lint { + cmd: "ruff check .".to_string(), + tool: Some("ruff".to_string()), + targets: Some(vec![".".to_string()]), + }], + ); + + // ruff format is a formatter + assert_parsed( + &shlex_split_safe("ruff format pkg/"), + vec![ParsedCommand::Format { + cmd: "ruff format pkg/".to_string(), + tool: Some("ruff".to_string()), + targets: Some(vec!["pkg/".to_string()]), + }], + ); + } + + #[test] + fn recognizes_pnpm_monorepo_test_and_npm_format_script() { + // pnpm -r test in a monorepo should still parse as a test action + assert_parsed( + &shlex_split_safe("pnpm -r test"), + vec![ParsedCommand::Test { + cmd: "pnpm -r test".to_string(), + }], + ); + + // npm run format should be recognized as a format action + assert_parsed( + &shlex_split_safe("npm run format -- -w ."), + vec![ParsedCommand::Format { + cmd: "npm run format -- -w .".to_string(), + tool: Some("npm-script:format".to_string()), + targets: None, + }], + ); + } + + #[test] + fn yarn_test_is_parsed_as_test() { + assert_parsed( + &shlex_split_safe("yarn test"), + vec![ParsedCommand::Test { + cmd: "yarn test".to_string(), + }], + ); + } + + #[test] + fn pytest_file_only_and_go_run_regex() { + // pytest invoked with a file path should be captured as a filter + assert_parsed( + &shlex_split_safe("pytest tests/test_example.py"), + vec![ParsedCommand::Test { + cmd: "pytest tests/test_example.py".to_string(), + }], + ); + + // go test with -run regex should capture the filter + assert_parsed( + &shlex_split_safe("go test ./... -run '^TestFoo$'"), + vec![ParsedCommand::Test { + cmd: "go test ./... -run '^TestFoo$'".to_string(), + }], + ); + } + + #[test] + fn grep_with_query_and_path() { + assert_parsed( + &shlex_split_safe("grep -R TODO src"), + vec![ParsedCommand::Search { + cmd: "grep -R TODO src".to_string(), + query: Some("TODO".to_string()), + path: Some("src".to_string()), + }], + ); + } + + #[test] + fn rg_with_equals_style_flags() { + assert_parsed( + &shlex_split_safe("rg --colors=never -n foo src"), + vec![ParsedCommand::Search { + cmd: "rg '--colors=never' -n foo src".to_string(), + query: Some("foo".to_string()), + path: Some("src".to_string()), + }], + ); + } + + #[test] + fn cat_with_double_dash_and_sed_ranges() { + // cat -- should be treated as a read of that file + assert_parsed( + &shlex_split_safe("cat -- ./-strange-file-name"), + vec![ParsedCommand::Read { + cmd: "cat -- ./-strange-file-name".to_string(), + name: "-strange-file-name".to_string(), + }], + ); + + // sed -n should be treated as a read of + assert_parsed( + &shlex_split_safe("sed -n '12,20p' Cargo.toml"), + vec![ParsedCommand::Read { + cmd: "sed -n '12,20p' Cargo.toml".to_string(), + name: "Cargo.toml".to_string(), + }], + ); + } + + #[test] + fn drop_trailing_nl_in_pipeline() { + // When an `nl` stage has only flags, it should be dropped from the summary + assert_parsed( + &shlex_split_safe("rg --files | nl -ba"), + vec![ParsedCommand::Search { + cmd: "rg --files".to_string(), + query: None, + path: None, + }], + ); + } + + #[test] + fn ls_with_time_style_and_path() { + assert_parsed( + &shlex_split_safe("ls --time-style=long-iso ./dist"), + vec![ParsedCommand::ListFiles { + cmd: "ls '--time-style=long-iso' ./dist".to_string(), + // short_display_path drops "dist" and shows "." as the last useful segment + path: Some(".".to_string()), + }], + ); + } + + #[test] + fn eslint_with_config_path_and_target() { + assert_parsed( + &shlex_split_safe("eslint -c .eslintrc.json src"), + vec![ParsedCommand::Lint { + cmd: "eslint -c .eslintrc.json src".to_string(), + tool: Some("eslint".to_string()), + targets: Some(vec!["src".to_string()]), + }], + ); + } + + #[test] + fn npx_eslint_with_config_path_and_target() { + assert_parsed( + &shlex_split_safe("npx eslint -c .eslintrc src"), + vec![ParsedCommand::Lint { + cmd: "npx eslint -c .eslintrc src".to_string(), + tool: Some("eslint".to_string()), + targets: Some(vec!["src".to_string()]), + }], + ); + } + + #[test] + fn fd_file_finder_variants() { + assert_parsed( + &shlex_split_safe("fd -t f src/"), + vec![ParsedCommand::Search { + cmd: "fd -t f src/".to_string(), + query: None, + path: Some("src".to_string()), + }], + ); + + // fd with query and path should capture both + assert_parsed( + &shlex_split_safe("fd main src"), + vec![ParsedCommand::Search { + cmd: "fd main src".to_string(), + query: Some("main".to_string()), + path: Some("src".to_string()), + }], + ); + } + + #[test] + fn find_basic_name_filter() { + assert_parsed( + &shlex_split_safe("find . -name '*.rs'"), + vec![ParsedCommand::Search { + cmd: "find . -name '*.rs'".to_string(), + query: Some("*.rs".to_string()), + path: Some(".".to_string()), + }], + ); + } + + #[test] + fn find_type_only_path() { + assert_parsed( + &shlex_split_safe("find src -type f"), + vec![ParsedCommand::Search { + cmd: "find src -type f".to_string(), + query: None, + path: Some("src".to_string()), + }], + ); + } +} + +pub fn parse_command_impl(command: &[String]) -> Vec { + if let Some(commands) = parse_bash_lc_commands(command) { + return commands; + } + + let normalized = normalize_tokens(command); + + let parts = if contains_connectors(&normalized) { + split_on_connectors(&normalized) + } else { + vec![normalized.clone()] + }; + + // Preserve left-to-right execution order for all commands, including bash -c/-lc + // so summaries reflect the order they will run. + + // Map each pipeline segment to its parsed summary. + let mut commands: Vec = parts + .iter() + .map(|tokens| summarize_main_tokens(tokens)) + .collect(); + + while let Some(next) = simplify_once(&commands) { + commands = next; + } + + commands +} + +fn simplify_once(commands: &[ParsedCommand]) -> Option> { + if commands.len() <= 1 { + return None; + } + + // echo ... && ...rest => ...rest + if let ParsedCommand::Unknown { cmd } = &commands[0] { + if shlex_split(cmd).is_some_and(|t| t.first().map(|s| s.as_str()) == Some("echo")) { + return Some(commands[1..].to_vec()); + } + } + + // cd foo && [any Test command] => [any Test command] + if let Some(idx) = commands.iter().position(|pc| match pc { + ParsedCommand::Unknown { cmd } => { + shlex_split(cmd).is_some_and(|t| t.first().map(|s| s.as_str()) == Some("cd")) + } + _ => false, + }) { + if commands + .iter() + .skip(idx + 1) + .any(|pc| matches!(pc, ParsedCommand::Test { .. })) + { + let mut out = Vec::with_capacity(commands.len() - 1); + out.extend_from_slice(&commands[..idx]); + out.extend_from_slice(&commands[idx + 1..]); + return Some(out); + } + } + + // cmd || true => cmd + if let Some(idx) = commands.iter().position(|pc| match pc { + ParsedCommand::Noop { cmd } => cmd == "true", + _ => false, + }) { + let mut out = Vec::with_capacity(commands.len() - 1); + out.extend_from_slice(&commands[..idx]); + out.extend_from_slice(&commands[idx + 1..]); + return Some(out); + } + + // nl -[any_flags] && ...rest => ...rest + if let Some(idx) = commands.iter().position(|pc| match pc { + ParsedCommand::Unknown { cmd } => { + if let Some(tokens) = shlex_split(cmd) { + tokens.first().is_some_and(|s| s.as_str() == "nl") + && tokens.iter().skip(1).all(|t| t.starts_with('-')) + } else { + false + } + } + _ => false, + }) { + let mut out = Vec::with_capacity(commands.len() - 1); + out.extend_from_slice(&commands[..idx]); + out.extend_from_slice(&commands[idx + 1..]); + return Some(out); + } + + None +} + +/// Validates that this is a `sed -n 123,123p` command. +fn is_valid_sed_n_arg(arg: Option<&str>) -> bool { + let s = match arg { + Some(s) => s, + None => return false, + }; + let core = match s.strip_suffix('p') { + Some(rest) => rest, + None => return false, + }; + let parts: Vec<&str> = core.split(',').collect(); + match parts.as_slice() { + [num] => !num.is_empty() && num.chars().all(|c| c.is_ascii_digit()), + [a, b] => { + !a.is_empty() + && !b.is_empty() + && a.chars().all(|c| c.is_ascii_digit()) + && b.chars().all(|c| c.is_ascii_digit()) + } + _ => false, + } +} + +/// Normalize a command by: +/// - Removing `yes`/`no`/`bash -c`/`bash -lc` prefixes. +/// - Splitting on `|` and `&&`/`||`/`; +fn normalize_tokens(cmd: &[String]) -> Vec { + match cmd { + [first, pipe, rest @ ..] if (first == "yes" || first == "y") && pipe == "|" => { + // Do not re-shlex already-tokenized input; just drop the prefix. + rest.to_vec() + } + [first, pipe, rest @ ..] if (first == "no" || first == "n") && pipe == "|" => { + // Do not re-shlex already-tokenized input; just drop the prefix. + rest.to_vec() + } + [bash, flag, script] if bash == "bash" && (flag == "-c" || flag == "-lc") => { + shlex_split(script) + .unwrap_or_else(|| vec!["bash".to_string(), flag.clone(), script.clone()]) + } + _ => cmd.to_vec(), + } +} + +fn contains_connectors(tokens: &[String]) -> bool { + tokens + .iter() + .any(|t| t == "&&" || t == "||" || t == "|" || t == ";") +} + +fn split_on_connectors(tokens: &[String]) -> Vec> { + let mut out: Vec> = Vec::new(); + let mut cur: Vec = Vec::new(); + for t in tokens { + if t == "&&" || t == "||" || t == "|" || t == ";" { + if !cur.is_empty() { + out.push(std::mem::take(&mut cur)); + } + } else { + cur.push(t.clone()); + } + } + if !cur.is_empty() { + out.push(cur); + } + out +} + +fn trim_at_connector(tokens: &[String]) -> Vec { + let idx = tokens + .iter() + .position(|t| t == "|" || t == "&&" || t == "||" || t == ";") + .unwrap_or(tokens.len()); + tokens[..idx].to_vec() +} + +/// Shorten a path to the last component, excluding `build`/`dist`/`node_modules`/`src`. +/// It also pulls out a useful path from a directory such as: +/// - webview/src -> webview +/// - foo/src/ -> foo +/// - packages/app/node_modules/ -> app +fn short_display_path(path: &str) -> String { + // Normalize separators and drop any trailing slash for display. + let normalized = path.replace('\\', "/"); + let trimmed = normalized.trim_end_matches('/'); + let mut parts = trimmed.split('/').rev().filter(|p| { + !p.is_empty() && *p != "build" && *p != "dist" && *p != "node_modules" && *p != "src" + }); + parts + .next() + .map(|s| s.to_string()) + .unwrap_or_else(|| trimmed.to_string()) +} + +// Skip values consumed by specific flags and ignore --flag=value style arguments. +fn skip_flag_values<'a>(args: &'a [String], flags_with_vals: &[&str]) -> Vec<&'a String> { + let mut out: Vec<&'a String> = Vec::new(); + let mut skip_next = false; + for (i, a) in args.iter().enumerate() { + if skip_next { + skip_next = false; + continue; + } + if a == "--" { + // From here on, everything is positional operands; push the rest and break. + for rest in &args[i + 1..] { + out.push(rest); + } + break; + } + if a.starts_with("--") && a.contains('=') { + // --flag=value form: treat as a flag taking a value; skip entirely. + continue; + } + if flags_with_vals.contains(&a.as_str()) { + // This flag consumes the next argument as its value. + if i + 1 < args.len() { + skip_next = true; + } + continue; + } + out.push(a); + } + out +} + +/// Common flags for ESLint that take a following value and should not be +/// considered positional targets. +const ESLINT_FLAGS_WITH_VALUES: &[&str] = &[ + "-c", + "--config", + "--parser", + "--parser-options", + "--rulesdir", + "--plugin", + "--max-warnings", + "--format", +]; + +fn collect_non_flag_targets(args: &[String]) -> Option> { + let mut targets = Vec::new(); + let mut skip_next = false; + for (i, a) in args.iter().enumerate() { + if a == "--" { + break; + } + if skip_next { + skip_next = false; + continue; + } + if a == "-p" + || a == "--package" + || a == "--features" + || a == "-C" + || a == "--config" + || a == "--config-path" + || a == "--out-dir" + || a == "-o" + || a == "--run" + || a == "--max-warnings" + || a == "--format" + { + if i + 1 < args.len() { + skip_next = true; + } + continue; + } + if a.starts_with('-') { + continue; + } + targets.push(a.clone()); + } + if targets.is_empty() { + None + } else { + Some(targets) + } +} + +fn collect_non_flag_targets_with_flags( + args: &[String], + flags_with_vals: &[&str], +) -> Option> { + let targets: Vec = skip_flag_values(args, flags_with_vals) + .into_iter() + .filter(|a| !a.starts_with('-')) + .cloned() + .collect(); + if targets.is_empty() { + None + } else { + Some(targets) + } +} + +fn is_pathish(s: &str) -> bool { + s == "." + || s == ".." + || s.starts_with("./") + || s.starts_with("../") + || s.contains('/') + || s.contains('\\') +} + +fn parse_fd_query_and_path(tail: &[String]) -> (Option, Option) { + let args_no_connector = trim_at_connector(tail); + // fd has several flags that take values (e.g., -t/--type, -e/--extension). + // Skip those values when extracting positional operands. + let candidates = skip_flag_values( + &args_no_connector, + &[ + "-t", + "--type", + "-e", + "--extension", + "-E", + "--exclude", + "--search-path", + ], + ); + let non_flags: Vec<&String> = candidates + .into_iter() + .filter(|p| !p.starts_with('-')) + .collect(); + match non_flags.as_slice() { + [one] => { + if is_pathish(one) { + (None, Some(short_display_path(one))) + } else { + (Some((*one).clone()), None) + } + } + [q, p, ..] => (Some((*q).clone()), Some(short_display_path(p))), + _ => (None, None), + } +} + +fn parse_find_query_and_path(tail: &[String]) -> (Option, Option) { + let args_no_connector = trim_at_connector(tail); + // First positional argument (excluding common unary operators) is the root path + let mut path: Option = None; + for a in &args_no_connector { + if !a.starts_with('-') && *a != "!" && *a != "(" && *a != ")" { + path = Some(short_display_path(a)); + break; + } + } + // Extract a common name/path/regex pattern if present + let mut query: Option = None; + let mut i = 0; + while i < args_no_connector.len() { + let a = &args_no_connector[i]; + if a == "-name" || a == "-iname" || a == "-path" || a == "-regex" { + if i + 1 < args_no_connector.len() { + query = Some(args_no_connector[i + 1].clone()); + } + break; + } + i += 1; + } + (query, path) +} + +fn classify_npm_like(tool: &str, tail: &[String], full_cmd: &[String]) -> Option { + let mut r = tail; + if tool == "pnpm" && r.first().map(|s| s.as_str()) == Some("-r") { + r = &r[1..]; + } + let mut script_name: Option = None; + if r.first().map(|s| s.as_str()) == Some("run") { + script_name = r.get(1).cloned(); + } else { + let is_test_cmd = (tool == "npm" && r.first().map(|s| s.as_str()) == Some("t")) + || ((tool == "npm" || tool == "pnpm" || tool == "yarn") + && r.first().map(|s| s.as_str()) == Some("test")); + if is_test_cmd { + script_name = Some("test".to_string()); + } + } + if let Some(name) = script_name { + let lname = name.to_lowercase(); + if lname == "test" || lname == "unit" || lname == "jest" || lname == "vitest" { + return Some(ParsedCommand::Test { + cmd: shlex_join(full_cmd), + }); + } + if lname == "lint" || lname == "eslint" { + return Some(ParsedCommand::Lint { + cmd: shlex_join(full_cmd), + tool: Some(format!("{tool}-script:{name}")), + targets: None, + }); + } + if lname == "format" || lname == "fmt" || lname == "prettier" { + return Some(ParsedCommand::Format { + cmd: shlex_join(full_cmd), + tool: Some(format!("{tool}-script:{name}")), + targets: None, + }); + } + } + None +} + +fn parse_bash_lc_commands(original: &[String]) -> Option> { + let [bash, flag, script] = original else { + return None; + }; + if bash != "bash" || flag != "-lc" { + return None; + } + if let Some(tree) = try_parse_bash(script) { + if let Some(all_commands) = try_parse_word_only_commands_sequence(&tree, script) { + if !all_commands.is_empty() { + let script_tokens = shlex_split(script) + .unwrap_or_else(|| vec!["bash".to_string(), flag.clone(), script.clone()]); + // Strip small formatting helpers (e.g., head/tail/awk/wc/etc) so we + // bias toward the primary command when pipelines are present. + // First, drop obvious small formatting helpers (e.g., wc/awk/etc). + let had_multiple_commands = all_commands.len() > 1; + // The bash AST walker yields commands in right-to-left order for + // connector/pipeline sequences. Reverse to reflect actual execution order. + let mut filtered_commands = drop_small_formatting_commands(all_commands); + filtered_commands.reverse(); + if filtered_commands.is_empty() { + return Some(vec![ParsedCommand::Unknown { + cmd: script.clone(), + }]); + } + let mut commands: Vec = filtered_commands + .into_iter() + .map(|tokens| summarize_main_tokens(&tokens)) + .collect(); + if commands.len() > 1 { + commands.retain(|pc| !matches!(pc, ParsedCommand::Noop { .. })); + } + if commands.len() == 1 { + // If we reduced to a single command, attribute the full original script + // for clearer UX in file-reading and listing scenarios, or when there were + // no connectors in the original script. For search commands that came from + // a pipeline (e.g. `rg --files | sed -n`), keep only the primary command. + let had_connectors = had_multiple_commands + || script_tokens + .iter() + .any(|t| t == "|" || t == "&&" || t == "||" || t == ";"); + commands = commands + .into_iter() + .map(|pc| match pc { + ParsedCommand::Read { name, cmd, .. } => { + if had_connectors { + let has_pipe = script_tokens.iter().any(|t| t == "|"); + let has_sed_n = script_tokens.windows(2).any(|w| { + w.first().map(|s| s.as_str()) == Some("sed") + && w.get(1).map(|s| s.as_str()) == Some("-n") + }); + if has_pipe && has_sed_n { + ParsedCommand::Read { + cmd: script.clone(), + name, + } + } else { + ParsedCommand::Read { + cmd: cmd.clone(), + name, + } + } + } else { + ParsedCommand::Read { + cmd: shlex_join(&script_tokens), + name, + } + } + } + ParsedCommand::ListFiles { path, cmd, .. } => { + if had_connectors { + ParsedCommand::ListFiles { + cmd: cmd.clone(), + path, + } + } else { + ParsedCommand::ListFiles { + cmd: shlex_join(&script_tokens), + path, + } + } + } + ParsedCommand::Search { + query, path, cmd, .. + } => { + if had_connectors { + ParsedCommand::Search { + cmd: cmd.clone(), + query, + path, + } + } else { + ParsedCommand::Search { + cmd: shlex_join(&script_tokens), + query, + path, + } + } + } + ParsedCommand::Format { + tool, targets, cmd, .. + } => ParsedCommand::Format { + cmd: cmd.clone(), + tool, + targets, + }, + ParsedCommand::Test { cmd, .. } => { + ParsedCommand::Test { cmd: cmd.clone() } + } + ParsedCommand::Lint { + tool, targets, cmd, .. + } => ParsedCommand::Lint { + cmd: cmd.clone(), + tool, + targets, + }, + ParsedCommand::Unknown { .. } => ParsedCommand::Unknown { + cmd: script.clone(), + }, + ParsedCommand::Noop { .. } => ParsedCommand::Noop { + cmd: script.clone(), + }, + }) + .collect(); + } + return Some(commands); + } + } + } + Some(vec![ParsedCommand::Unknown { + cmd: script.clone(), + }]) +} + +/// Return true if this looks like a small formatting helper in a pipeline. +/// Examples: `head -n 40`, `tail -n +10`, `wc -l`, `awk ...`, `cut ...`, `tr ...`. +/// We try to keep variants that clearly include a file path (e.g. `tail -n 30 file`). +fn is_small_formatting_command(tokens: &[String]) -> bool { + if tokens.is_empty() { + return false; + } + let cmd = tokens[0].as_str(); + match cmd { + // Always formatting; typically used in pipes. + // `nl` is special-cased below to allow `nl ` to be treated as a read command. + "wc" | "tr" | "cut" | "sort" | "uniq" | "xargs" | "tee" | "column" | "awk" | "yes" + | "printf" => true, + "head" => { + // Treat as formatting when no explicit file operand is present. + // Common forms: `head -n 40`, `head -c 100`. + // Keep cases like `head -n 40 file`. + tokens.len() < 3 + } + "tail" => { + // Treat as formatting when no explicit file operand is present. + // Common forms: `tail -n +10`, `tail -n 30`. + // Keep cases like `tail -n 30 file`. + tokens.len() < 3 + } + "sed" => { + // Keep `sed -n file` (treated as a file read elsewhere); + // otherwise consider it a formatting helper in a pipeline. + tokens.len() < 4 + || !(tokens[1] == "-n" && is_valid_sed_n_arg(tokens.get(2).map(|s| s.as_str()))) + } + _ => false, + } +} + +fn drop_small_formatting_commands(mut commands: Vec>) -> Vec> { + commands.retain(|tokens| !is_small_formatting_command(tokens)); + commands +} + +fn summarize_main_tokens(main_cmd: &[String]) -> ParsedCommand { + match main_cmd.split_first() { + Some((head, tail)) if head == "true" && tail.is_empty() => ParsedCommand::Noop { + cmd: shlex_join(main_cmd), + }, + // (sed-specific logic handled below in dedicated arm returning Read) + Some((head, tail)) + if head == "cargo" && tail.first().map(|s| s.as_str()) == Some("fmt") => + { + ParsedCommand::Format { + cmd: shlex_join(main_cmd), + tool: Some("cargo fmt".to_string()), + targets: collect_non_flag_targets(&tail[1..]), + } + } + Some((head, tail)) + if head == "cargo" && tail.first().map(|s| s.as_str()) == Some("clippy") => + { + ParsedCommand::Lint { + cmd: shlex_join(main_cmd), + tool: Some("cargo clippy".to_string()), + targets: collect_non_flag_targets(&tail[1..]), + } + } + Some((head, tail)) + if head == "cargo" && tail.first().map(|s| s.as_str()) == Some("test") => + { + ParsedCommand::Test { + cmd: shlex_join(main_cmd), + } + } + Some((head, tail)) if head == "rustfmt" => ParsedCommand::Format { + cmd: shlex_join(main_cmd), + tool: Some("rustfmt".to_string()), + targets: collect_non_flag_targets(tail), + }, + Some((head, tail)) if head == "go" && tail.first().map(|s| s.as_str()) == Some("fmt") => { + ParsedCommand::Format { + cmd: shlex_join(main_cmd), + tool: Some("go fmt".to_string()), + targets: collect_non_flag_targets(&tail[1..]), + } + } + Some((head, tail)) if head == "go" && tail.first().map(|s| s.as_str()) == Some("test") => { + ParsedCommand::Test { + cmd: shlex_join(main_cmd), + } + } + Some((head, _)) if head == "pytest" => ParsedCommand::Test { + cmd: shlex_join(main_cmd), + }, + Some((head, tail)) if head == "eslint" => { + // Treat configuration flags with values (e.g. `-c .eslintrc`) as non-targets. + let targets = collect_non_flag_targets_with_flags(tail, ESLINT_FLAGS_WITH_VALUES); + ParsedCommand::Lint { + cmd: shlex_join(main_cmd), + tool: Some("eslint".to_string()), + targets, + } + } + Some((head, tail)) if head == "prettier" => ParsedCommand::Format { + cmd: shlex_join(main_cmd), + tool: Some("prettier".to_string()), + targets: collect_non_flag_targets(tail), + }, + Some((head, tail)) if head == "black" => ParsedCommand::Format { + cmd: shlex_join(main_cmd), + tool: Some("black".to_string()), + targets: collect_non_flag_targets(tail), + }, + Some((head, tail)) + if head == "ruff" && tail.first().map(|s| s.as_str()) == Some("check") => + { + ParsedCommand::Lint { + cmd: shlex_join(main_cmd), + tool: Some("ruff".to_string()), + targets: collect_non_flag_targets(&tail[1..]), + } + } + Some((head, tail)) + if head == "ruff" && tail.first().map(|s| s.as_str()) == Some("format") => + { + ParsedCommand::Format { + cmd: shlex_join(main_cmd), + tool: Some("ruff".to_string()), + targets: collect_non_flag_targets(&tail[1..]), + } + } + Some((head, _)) if (head == "jest" || head == "vitest") => ParsedCommand::Test { + cmd: shlex_join(main_cmd), + }, + Some((head, tail)) + if head == "npx" && tail.first().map(|s| s.as_str()) == Some("eslint") => + { + let targets = collect_non_flag_targets_with_flags(&tail[1..], ESLINT_FLAGS_WITH_VALUES); + ParsedCommand::Lint { + cmd: shlex_join(main_cmd), + tool: Some("eslint".to_string()), + targets, + } + } + Some((head, tail)) + if head == "npx" && tail.first().map(|s| s.as_str()) == Some("prettier") => + { + ParsedCommand::Format { + cmd: shlex_join(main_cmd), + tool: Some("prettier".to_string()), + targets: collect_non_flag_targets(&tail[1..]), + } + } + // NPM-like scripts including yarn + Some((tool, tail)) if (tool == "pnpm" || tool == "npm" || tool == "yarn") => { + if let Some(cmd) = classify_npm_like(tool, tail, main_cmd) { + cmd + } else { + ParsedCommand::Unknown { + cmd: shlex_join(main_cmd), + } + } + } + Some((head, tail)) if head == "ls" => { + // Avoid treating option values as paths (e.g., ls -I "*.test.js"). + let candidates = skip_flag_values( + tail, + &[ + "-I", + "-w", + "--block-size", + "--format", + "--time-style", + "--color", + "--quoting-style", + ], + ); + let path = candidates + .into_iter() + .find(|p| !p.starts_with('-')) + .map(|p| short_display_path(p)); + ParsedCommand::ListFiles { + cmd: shlex_join(main_cmd), + path, + } + } + Some((head, tail)) if head == "rg" => { + let args_no_connector = trim_at_connector(tail); + let has_files_flag = args_no_connector.iter().any(|a| a == "--files"); + let non_flags: Vec<&String> = args_no_connector + .iter() + .filter(|p| !p.starts_with('-')) + .collect(); + let (query, path) = if has_files_flag { + (None, non_flags.first().map(|s| short_display_path(s))) + } else { + ( + non_flags.first().cloned().map(|s| s.to_string()), + non_flags.get(1).map(|s| short_display_path(s)), + ) + }; + ParsedCommand::Search { + cmd: shlex_join(main_cmd), + query, + path, + } + } + Some((head, tail)) if head == "fd" => { + let (query, path) = parse_fd_query_and_path(tail); + ParsedCommand::Search { + cmd: shlex_join(main_cmd), + query, + path, + } + } + Some((head, tail)) if head == "find" => { + // Basic find support: capture path and common name filter + let (query, path) = parse_find_query_and_path(tail); + ParsedCommand::Search { + cmd: shlex_join(main_cmd), + query, + path, + } + } + Some((head, tail)) if head == "grep" => { + let args_no_connector = trim_at_connector(tail); + let non_flags: Vec<&String> = args_no_connector + .iter() + .filter(|p| !p.starts_with('-')) + .collect(); + // Do not shorten the query: grep patterns may legitimately contain slashes + // and should be preserved verbatim. Only paths should be shortened. + let query = non_flags.first().cloned().map(|s| s.to_string()); + let path = non_flags.get(1).map(|s| short_display_path(s)); + ParsedCommand::Search { + cmd: shlex_join(main_cmd), + query, + path, + } + } + Some((head, tail)) if head == "cat" => { + // Support both `cat ` and `cat -- ` forms. + let effective_tail: &[String] = if tail.first().map(|s| s.as_str()) == Some("--") { + &tail[1..] + } else { + tail + }; + if effective_tail.len() == 1 { + let name = short_display_path(&effective_tail[0]); + ParsedCommand::Read { + cmd: shlex_join(main_cmd), + name, + } + } else { + ParsedCommand::Unknown { + cmd: shlex_join(main_cmd), + } + } + } + Some((head, tail)) if head == "head" => { + // Support `head -n 50 file` and `head -n50 file` forms. + let has_valid_n = match tail.split_first() { + Some((first, rest)) if first == "-n" => rest + .first() + .is_some_and(|n| n.chars().all(|c| c.is_ascii_digit())), + Some((first, _)) if first.starts_with("-n") => { + first[2..].chars().all(|c| c.is_ascii_digit()) + } + _ => false, + }; + if has_valid_n { + // Build candidates skipping the numeric value consumed by `-n` when separated. + let mut candidates: Vec<&String> = Vec::new(); + let mut i = 0; + while i < tail.len() { + if i == 0 && tail[i] == "-n" && i + 1 < tail.len() { + let n = &tail[i + 1]; + if n.chars().all(|c| c.is_ascii_digit()) { + i += 2; + continue; + } + } + candidates.push(&tail[i]); + i += 1; + } + if let Some(p) = candidates.into_iter().find(|p| !p.starts_with('-')) { + let name = short_display_path(p); + return ParsedCommand::Read { + cmd: shlex_join(main_cmd), + name, + }; + } + } + ParsedCommand::Unknown { + cmd: shlex_join(main_cmd), + } + } + Some((head, tail)) if head == "tail" => { + // Support `tail -n +10 file` and `tail -n+10 file` forms. + let has_valid_n = match tail.split_first() { + Some((first, rest)) if first == "-n" => rest.first().is_some_and(|n| { + let s = n.strip_prefix('+').unwrap_or(n); + !s.is_empty() && s.chars().all(|c| c.is_ascii_digit()) + }), + Some((first, _)) if first.starts_with("-n") => { + let v = &first[2..]; + let s = v.strip_prefix('+').unwrap_or(v); + !s.is_empty() && s.chars().all(|c| c.is_ascii_digit()) + } + _ => false, + }; + if has_valid_n { + // Build candidates skipping the numeric value consumed by `-n` when separated. + let mut candidates: Vec<&String> = Vec::new(); + let mut i = 0; + while i < tail.len() { + if i == 0 && tail[i] == "-n" && i + 1 < tail.len() { + let n = &tail[i + 1]; + let s = n.strip_prefix('+').unwrap_or(n); + if !s.is_empty() && s.chars().all(|c| c.is_ascii_digit()) { + i += 2; + continue; + } + } + candidates.push(&tail[i]); + i += 1; + } + if let Some(p) = candidates.into_iter().find(|p| !p.starts_with('-')) { + let name = short_display_path(p); + return ParsedCommand::Read { + cmd: shlex_join(main_cmd), + name, + }; + } + } + ParsedCommand::Unknown { + cmd: shlex_join(main_cmd), + } + } + Some((head, tail)) if head == "nl" => { + // Avoid treating option values as paths (e.g., nl -s " "). + let candidates = skip_flag_values(tail, &["-s", "-w", "-v", "-i", "-b"]); + if let Some(p) = candidates.into_iter().find(|p| !p.starts_with('-')) { + let name = short_display_path(p); + ParsedCommand::Read { + cmd: shlex_join(main_cmd), + name, + } + } else { + ParsedCommand::Unknown { + cmd: shlex_join(main_cmd), + } + } + } + Some((head, tail)) + if head == "sed" + && tail.len() >= 3 + && tail[0] == "-n" + && is_valid_sed_n_arg(tail.get(1).map(|s| s.as_str())) => + { + if let Some(path) = tail.get(2) { + let name = short_display_path(path); + ParsedCommand::Read { + cmd: shlex_join(main_cmd), + name, + } + } else { + ParsedCommand::Unknown { + cmd: shlex_join(main_cmd), + } + } + } + // Other commands + _ => ParsedCommand::Unknown { + cmd: shlex_join(main_cmd), + }, + } +} diff --git a/codex-rs/core/src/plan_tool.rs b/codex-rs/core/src/plan_tool.rs index bba5363266..bc39e4f6ce 100644 --- a/codex-rs/core/src/plan_tool.rs +++ b/codex-rs/core/src/plan_tool.rs @@ -1,9 +1,6 @@ use std::collections::BTreeMap; use std::sync::LazyLock; -use serde::Deserialize; -use serde::Serialize; - use crate::codex::Session; use crate::models::FunctionCallOutputPayload; use crate::models::ResponseInputItem; @@ -13,36 +10,22 @@ use crate::openai_tools::ResponsesApiTool; use crate::protocol::Event; use crate::protocol::EventMsg; +// Use the canonical plan tool types from the protocol crate to ensure +// type-identity matches events transported via `codex_protocol`. +pub use codex_protocol::plan_tool::PlanItemArg; +pub use codex_protocol::plan_tool::StepStatus; +pub use codex_protocol::plan_tool::UpdatePlanArgs; + // Types for the TODO tool arguments matching codex-vscode/todo-mcp/src/main.rs -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum StepStatus { - Pending, - InProgress, - Completed, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct PlanItemArg { - pub step: String, - pub status: StepStatus, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct UpdatePlanArgs { - #[serde(default)] - pub explanation: Option, - pub plan: Vec, -} pub(crate) static PLAN_TOOL: LazyLock = LazyLock::new(|| { let mut plan_item_props = BTreeMap::new(); plan_item_props.insert("step".to_string(), JsonSchema::String { description: None }); plan_item_props.insert( "status".to_string(), - JsonSchema::String { description: None }, + JsonSchema::String { + description: Some("One of: pending, in_progress, completed".to_string()), + }, ); let plan_items_schema = JsonSchema::Array { @@ -63,17 +46,11 @@ pub(crate) static PLAN_TOOL: LazyLock = LazyLock::new(|| { OpenAiTool::Function(ResponsesApiTool { name: "update_plan".to_string(), - description: r#"Use the update_plan tool to keep the user updated on the current plan for the task. -After understanding the user's task, call the update_plan tool with an initial plan. An example of a plan: -1. Explore the codebase to find relevant files (status: in_progress) -2. Implement the feature in the XYZ component (status: pending) -3. Commit changes and make a pull request (status: pending) -Each step should be a short, 1-sentence description. -Until all the steps are finished, there should always be exactly one in_progress step in the plan. -Call the update_plan tool whenever you finish a step, marking the completed step as `completed` and marking the next step as `in_progress`. -Before running a command, consider whether or not you have completed the previous step, and make sure to mark it as completed before moving on to the next step. -Sometimes, you may need to change plans in the middle of a task: call `update_plan` with the updated plan and make sure to provide an `explanation` of the rationale when doing so. -When all steps are completed, call update_plan one last time with all steps marked as `completed`."#.to_string(), + description: r#"Updates the task plan. +Provide an optional explanation and a list of plan items, each with a step and status. +At most one step can be in_progress at a time. +"# + .to_string(), strict: false, parameters: JsonSchema::Object { properties, diff --git a/codex-rs/core/src/project_doc.rs b/codex-rs/core/src/project_doc.rs index 9f46159d1d..3591b5b1d8 100644 --- a/codex-rs/core/src/project_doc.rs +++ b/codex-rs/core/src/project_doc.rs @@ -134,8 +134,6 @@ async fn load_first_candidate( #[cfg(test)] mod tests { - #![allow(clippy::expect_used, clippy::unwrap_used)] - use super::*; use crate::config::ConfigOverrides; use crate::config::ConfigToml; diff --git a/codex-rs/core/src/safety.rs b/codex-rs/core/src/safety.rs index 860a728def..c878a71110 100644 --- a/codex-rs/core/src/safety.rs +++ b/codex-rs/core/src/safety.rs @@ -21,7 +21,7 @@ pub enum SafetyCheck { pub fn assess_patch_safety( action: &ApplyPatchAction, policy: AskForApproval, - writable_roots: &[PathBuf], + sandbox_policy: &SandboxPolicy, cwd: &Path, ) -> SafetyCheck { if action.is_empty() { @@ -45,7 +45,7 @@ pub fn assess_patch_safety( // is possible that paths in the patch are hard links to files outside the // writable roots, so we should still run `apply_patch` in a sandbox in that // case. - if is_write_patch_constrained_to_writable_paths(action, writable_roots, cwd) + if is_write_patch_constrained_to_writable_paths(action, sandbox_policy, cwd) || policy == AskForApproval::OnFailure { // Only auto‑approve when we can actually enforce a sandbox. Otherwise @@ -171,13 +171,19 @@ pub fn get_platform_sandbox() -> Option { fn is_write_patch_constrained_to_writable_paths( action: &ApplyPatchAction, - writable_roots: &[PathBuf], + sandbox_policy: &SandboxPolicy, cwd: &Path, ) -> bool { // Early‑exit if there are no declared writable roots. - if writable_roots.is_empty() { - return false; - } + let writable_roots = match sandbox_policy { + SandboxPolicy::ReadOnly => { + return false; + } + SandboxPolicy::DangerFullAccess => { + return true; + } + SandboxPolicy::WorkspaceWrite { .. } => sandbox_policy.get_writable_roots_with_cwd(cwd), + }; // Normalize a path by removing `.` and resolving `..` without touching the // filesystem (works even if the file does not exist). @@ -209,15 +215,9 @@ fn is_write_patch_constrained_to_writable_paths( None => return false, }; - writable_roots.iter().any(|root| { - let root_abs = if root.is_absolute() { - root.clone() - } else { - normalize(&cwd.join(root)).unwrap_or_else(|| cwd.join(root)) - }; - - abs.starts_with(&root_abs) - }) + writable_roots + .iter() + .any(|writable_root| writable_root.is_path_writable(&abs)) }; for (path, change) in action.changes() { @@ -245,40 +245,57 @@ fn is_write_patch_constrained_to_writable_paths( #[cfg(test)] mod tests { - #![allow(clippy::unwrap_used)] use super::*; + use tempfile::TempDir; #[test] fn test_writable_roots_constraint() { - let cwd = std::env::current_dir().unwrap(); + // Use a temporary directory as our workspace to avoid touching + // the real current working directory. + let tmp = TempDir::new().unwrap(); + let cwd = tmp.path().to_path_buf(); let parent = cwd.parent().unwrap().to_path_buf(); - // Helper to build a single‑entry map representing a patch that adds a - // file at `p`. + // Helper to build a single‑entry patch that adds a file at `p`. let make_add_change = |p: PathBuf| ApplyPatchAction::new_add_for_test(&p, "".to_string()); let add_inside = make_add_change(cwd.join("inner.txt")); let add_outside = make_add_change(parent.join("outside.txt")); + // Policy limited to the workspace only; exclude system temp roots so + // only `cwd` is writable by default. + let policy_workspace_only = SandboxPolicy::WorkspaceWrite { + writable_roots: vec![], + network_access: false, + exclude_tmpdir_env_var: true, + exclude_slash_tmp: true, + }; + assert!(is_write_patch_constrained_to_writable_paths( &add_inside, - &[PathBuf::from(".")], + &policy_workspace_only, &cwd, )); - let add_outside_2 = make_add_change(parent.join("outside.txt")); assert!(!is_write_patch_constrained_to_writable_paths( - &add_outside_2, - &[PathBuf::from(".")], + &add_outside, + &policy_workspace_only, &cwd, )); - // With parent dir added as writable root, it should pass. + // With the parent dir explicitly added as a writable root, the + // outside write should be permitted. + let policy_with_parent = SandboxPolicy::WorkspaceWrite { + writable_roots: vec![parent.clone()], + network_access: false, + exclude_tmpdir_env_var: true, + exclude_slash_tmp: true, + }; assert!(is_write_patch_constrained_to_writable_paths( &add_outside, - &[PathBuf::from("..")], + &policy_with_parent, &cwd, - )) + )); } #[test] diff --git a/codex-rs/core/src/seatbelt.rs b/codex-rs/core/src/seatbelt.rs index ff9dbaa7f9..31aa770d2e 100644 --- a/codex-rs/core/src/seatbelt.rs +++ b/codex-rs/core/src/seatbelt.rs @@ -122,7 +122,6 @@ fn create_seatbelt_command_args( #[cfg(test)] mod tests { - #![expect(clippy::expect_used)] use super::MACOS_SEATBELT_BASE_POLICY; use super::create_seatbelt_command_args; use crate::protocol::SandboxPolicy; diff --git a/codex-rs/core/src/shell.rs b/codex-rs/core/src/shell.rs index de0764f75e..0cde3b4a78 100644 --- a/codex-rs/core/src/shell.rs +++ b/codex-rs/core/src/shell.rs @@ -98,7 +98,6 @@ mod tests { use std::process::Command; #[tokio::test] - #[expect(clippy::unwrap_used)] async fn test_current_shell_detects_zsh() { let shell = Command::new("sh") .arg("-c") @@ -129,7 +128,6 @@ mod tests { assert_eq!(actual_cmd, None); } - #[expect(clippy::unwrap_used)] #[tokio::test] async fn test_run_with_profile_escaping_and_execution() { let shell_path = "/bin/zsh"; @@ -167,9 +165,6 @@ mod tests { for (input, expected_cmd, expected_output) in cases { use std::collections::HashMap; use std::path::PathBuf; - use std::sync::Arc; - - use tokio::sync::Notify; use crate::exec::ExecParams; use crate::exec::SandboxType; @@ -219,7 +214,6 @@ mod tests { justification: None, }, SandboxType::None, - Arc::new(Notify::new()), &SandboxPolicy::DangerFullAccess, &None, None, @@ -230,7 +224,7 @@ mod tests { assert_eq!(output.exit_code, 0, "input: {input:?} output: {output:?}"); if let Some(expected) = expected_output { assert_eq!( - output.stdout, expected, + output.stdout.text, expected, "input: {input:?} output: {output:?}" ); } diff --git a/codex-rs/core/src/turn_diff_tracker.rs b/codex-rs/core/src/turn_diff_tracker.rs index 7026d7bb32..5e73fa0650 100644 --- a/codex-rs/core/src/turn_diff_tracker.rs +++ b/codex-rs/core/src/turn_diff_tracker.rs @@ -466,7 +466,6 @@ fn is_windows_drive_or_unc_root(p: &std::path::Path) -> bool { #[cfg(test)] mod tests { - #![allow(clippy::unwrap_used)] use super::*; use pretty_assertions::assert_eq; use tempfile::tempdir; diff --git a/codex-rs/core/src/user_agent.rs b/codex-rs/core/src/user_agent.rs new file mode 100644 index 0000000000..ddcfd4b7f1 --- /dev/null +++ b/codex-rs/core/src/user_agent.rs @@ -0,0 +1,35 @@ +const DEFAULT_ORIGINATOR: &str = "codex_cli_rs"; + +pub fn get_codex_user_agent(originator: Option<&str>) -> String { + let build_version = env!("CARGO_PKG_VERSION"); + let os_info = os_info::get(); + format!( + "{}/{build_version} ({} {}; {})", + originator.unwrap_or(DEFAULT_ORIGINATOR), + os_info.os_type(), + os_info.version(), + os_info.architecture().unwrap_or("unknown"), + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_get_codex_user_agent() { + let user_agent = get_codex_user_agent(None); + assert!(user_agent.starts_with("codex_cli_rs/")); + } + + #[test] + #[cfg(target_os = "macos")] + fn test_macos() { + use regex_lite::Regex; + let user_agent = get_codex_user_agent(None); + let re = + Regex::new(r"^codex_cli_rs/\d+\.\d+\.\d+ \(Mac OS \d+\.\d+\.\d+; (x86_64|arm64)\)$") + .unwrap(); + assert!(re.is_match(&user_agent)); + } +} diff --git a/codex-rs/core/src/user_notification.rs b/codex-rs/core/src/user_notification.rs index e7479f89cd..0a3cb49e78 100644 --- a/codex-rs/core/src/user_notification.rs +++ b/codex-rs/core/src/user_notification.rs @@ -20,7 +20,6 @@ pub(crate) enum UserNotification { #[cfg(test)] mod tests { - #![allow(clippy::unwrap_used)] use super::*; #[test] diff --git a/codex-rs/core/src/util.rs b/codex-rs/core/src/util.rs index fb5f45de6f..e1248a4867 100644 --- a/codex-rs/core/src/util.rs +++ b/codex-rs/core/src/util.rs @@ -1,32 +1,11 @@ use std::path::Path; -use std::sync::Arc; use std::time::Duration; use rand::Rng; -use tokio::sync::Notify; -use tracing::debug; const INITIAL_DELAY_MS: u64 = 200; const BACKOFF_FACTOR: f64 = 2.0; -/// Make a CancellationToken that is fulfilled when SIGINT occurs. -pub fn notify_on_sigint() -> Arc { - let notify = Arc::new(Notify::new()); - - tokio::spawn({ - let notify = Arc::clone(¬ify); - async move { - loop { - tokio::signal::ctrl_c().await.ok(); - debug!("Keyboard interrupt"); - notify.notify_waiters(); - } - } - }); - - notify -} - pub(crate) fn backoff(attempt: u64) -> Duration { let exp = BACKOFF_FACTOR.powi(attempt.saturating_sub(1) as i32); let base = (INITIAL_DELAY_MS as f64 * exp) as u64; diff --git a/codex-rs/core/tests/cli_stream.rs b/codex-rs/core/tests/cli_stream.rs index c8570577ce..219e99787d 100644 --- a/codex-rs/core/tests/cli_stream.rs +++ b/codex-rs/core/tests/cli_stream.rs @@ -1,5 +1,3 @@ -#![expect(clippy::unwrap_used)] - use assert_cmd::Command as AssertCommand; use codex_core::spawn::CODEX_SANDBOX_NETWORK_DISABLED_ENV_VAR; use std::time::Duration; diff --git a/codex-rs/core/tests/client.rs b/codex-rs/core/tests/client.rs index bd7ad2feef..075c496a6b 100644 --- a/codex-rs/core/tests/client.rs +++ b/codex-rs/core/tests/client.rs @@ -1,15 +1,13 @@ -#![allow(clippy::expect_used, clippy::unwrap_used)] - -use codex_core::Codex; -use codex_core::CodexSpawnOk; +use codex_core::ConversationManager; use codex_core::ModelProviderInfo; +use codex_core::NewConversation; use codex_core::WireApi; use codex_core::built_in_model_providers; use codex_core::protocol::EventMsg; use codex_core::protocol::InputItem; use codex_core::protocol::Op; -use codex_core::protocol::SessionConfiguredEvent; use codex_core::spawn::CODEX_SANDBOX_NETWORK_DISABLED_ENV_VAR; +use codex_login::AuthMode; use codex_login::CodexAuth; use core_test_support::load_default_config_for_test; use core_test_support::load_sse_fixture_with_id; @@ -28,10 +26,12 @@ fn sse_completed(id: &str) -> String { load_sse_fixture_with_id("tests/fixtures/completed_template.json", id) } +#[expect(clippy::unwrap_used)] fn assert_message_role(request_body: &serde_json::Value, role: &str) { assert_eq!(request_body["role"].as_str().unwrap(), role); } +#[expect(clippy::expect_used)] fn assert_message_starts_with(request_body: &serde_json::Value, text: &str) { let content = request_body["content"][0]["text"] .as_str() @@ -43,6 +43,7 @@ fn assert_message_starts_with(request_body: &serde_json::Value, text: &str) { ); } +#[expect(clippy::expect_used)] fn assert_message_ends_with(request_body: &serde_json::Value, text: &str) { let content = request_body["content"][0]["text"] .as_str() @@ -54,10 +55,61 @@ fn assert_message_ends_with(request_body: &serde_json::Value, text: &str) { ); } +/// Writes an `auth.json` into the provided `codex_home` with the specified parameters. +/// Returns the fake JWT string written to `tokens.id_token`. +#[expect(clippy::unwrap_used)] +fn write_auth_json( + codex_home: &TempDir, + openai_api_key: Option<&str>, + chatgpt_plan_type: &str, + access_token: &str, + account_id: Option<&str>, +) -> String { + use base64::Engine as _; + use serde_json::json; + + let header = json!({ "alg": "none", "typ": "JWT" }); + let payload = json!({ + "email": "user@example.com", + "https://api.openai.com/auth": { + "chatgpt_plan_type": chatgpt_plan_type, + "chatgpt_account_id": account_id.unwrap_or("acc-123") + } + }); + + let b64 = |b: &[u8]| base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(b); + let header_b64 = b64(&serde_json::to_vec(&header).unwrap()); + let payload_b64 = b64(&serde_json::to_vec(&payload).unwrap()); + let signature_b64 = b64(b"sig"); + let fake_jwt = format!("{header_b64}.{payload_b64}.{signature_b64}"); + + let mut tokens = json!({ + "id_token": fake_jwt, + "access_token": access_token, + "refresh_token": "refresh-test", + }); + if let Some(acc) = account_id { + tokens["account_id"] = json!(acc); + } + + let auth_json = json!({ + "OPENAI_API_KEY": openai_api_key, + "tokens": tokens, + // RFC3339 datetime; value doesn't matter for these tests + "last_refresh": "2025-08-06T20:41:36.232376Z", + }); + + std::fs::write( + codex_home.path().join("auth.json"), + serde_json::to_string_pretty(&auth_json).unwrap(), + ) + .unwrap(); + + fake_jwt +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn includes_session_id_and_model_headers_in_request() { - #![allow(clippy::unwrap_used)] - if std::env::var(CODEX_SANDBOX_NETWORK_DISABLED_ENV_VAR).is_ok() { println!( "Skipping test because it cannot execute when network is disabled in a Codex sandbox." @@ -90,14 +142,15 @@ async fn includes_session_id_and_model_headers_in_request() { let mut config = load_default_config_for_test(&codex_home); config.model_provider = model_provider; - let ctrl_c = std::sync::Arc::new(tokio::sync::Notify::new()); - let CodexSpawnOk { codex, .. } = Codex::spawn( - config, - Some(CodexAuth::from_api_key("Test API Key")), - ctrl_c.clone(), - ) - .await - .unwrap(); + let conversation_manager = ConversationManager::default(); + let NewConversation { + conversation: codex, + conversation_id, + session_configured: _, + } = conversation_manager + .new_conversation_with_auth(config, Some(CodexAuth::from_api_key("Test API Key"))) + .await + .expect("create new conversation"); codex .submit(Op::UserInput { @@ -108,13 +161,6 @@ async fn includes_session_id_and_model_headers_in_request() { .await .unwrap(); - let EventMsg::SessionConfigured(SessionConfiguredEvent { session_id, .. }) = - wait_for_event(&codex, |ev| matches!(ev, EventMsg::SessionConfigured(_))).await - else { - unreachable!() - }; - - let current_session_id = Some(session_id.to_string()); wait_for_event(&codex, |ev| matches!(ev, EventMsg::TaskComplete(_))).await; // get request from the server @@ -123,10 +169,9 @@ async fn includes_session_id_and_model_headers_in_request() { let request_authorization = request.headers.get("authorization").unwrap(); let request_originator = request.headers.get("originator").unwrap(); - assert!(current_session_id.is_some()); assert_eq!( request_session_id.to_str().unwrap(), - current_session_id.as_ref().unwrap() + conversation_id.to_string() ); assert_eq!(request_originator.to_str().unwrap(), "codex_cli_rs"); assert_eq!( @@ -137,8 +182,6 @@ async fn includes_session_id_and_model_headers_in_request() { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn includes_base_instructions_override_in_request() { - #![allow(clippy::unwrap_used)] - // Mock server let server = MockServer::start().await; @@ -164,14 +207,12 @@ async fn includes_base_instructions_override_in_request() { config.base_instructions = Some("test instructions".to_string()); config.model_provider = model_provider; - let ctrl_c = std::sync::Arc::new(tokio::sync::Notify::new()); - let CodexSpawnOk { codex, .. } = Codex::spawn( - config, - Some(CodexAuth::from_api_key("Test API Key")), - ctrl_c.clone(), - ) - .await - .unwrap(); + let conversation_manager = ConversationManager::default(); + let codex = conversation_manager + .new_conversation_with_auth(config, Some(CodexAuth::from_api_key("Test API Key"))) + .await + .expect("create new conversation") + .conversation; codex .submit(Op::UserInput { @@ -197,8 +238,6 @@ async fn includes_base_instructions_override_in_request() { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn originator_config_override_is_used() { - #![allow(clippy::unwrap_used)] - // Mock server let server = MockServer::start().await; @@ -223,14 +262,12 @@ async fn originator_config_override_is_used() { config.model_provider = model_provider; config.internal_originator = Some("my_override".to_string()); - let ctrl_c = std::sync::Arc::new(tokio::sync::Notify::new()); - let CodexSpawnOk { codex, .. } = Codex::spawn( - config, - Some(CodexAuth::from_api_key("Test API Key")), - ctrl_c.clone(), - ) - .await - .unwrap(); + let conversation_manager = ConversationManager::default(); + let codex = conversation_manager + .new_conversation_with_auth(config, Some(CodexAuth::from_api_key("Test API Key"))) + .await + .expect("create new conversation") + .conversation; codex .submit(Op::UserInput { @@ -250,8 +287,6 @@ async fn originator_config_override_is_used() { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn chatgpt_auth_sends_correct_request() { - #![allow(clippy::unwrap_used)] - if std::env::var(CODEX_SANDBOX_NETWORK_DISABLED_ENV_VAR).is_ok() { println!( "Skipping test because it cannot execute when network is disabled in a Codex sandbox." @@ -283,11 +318,15 @@ async fn chatgpt_auth_sends_correct_request() { let codex_home = TempDir::new().unwrap(); let mut config = load_default_config_for_test(&codex_home); config.model_provider = model_provider; - let ctrl_c = std::sync::Arc::new(tokio::sync::Notify::new()); - let CodexSpawnOk { codex, .. } = - Codex::spawn(config, Some(create_dummy_codex_auth()), ctrl_c.clone()) - .await - .unwrap(); + let conversation_manager = ConversationManager::default(); + let NewConversation { + conversation: codex, + conversation_id, + session_configured: _, + } = conversation_manager + .new_conversation_with_auth(config, Some(create_dummy_codex_auth())) + .await + .expect("create new conversation"); codex .submit(Op::UserInput { @@ -298,13 +337,6 @@ async fn chatgpt_auth_sends_correct_request() { .await .unwrap(); - let EventMsg::SessionConfigured(SessionConfiguredEvent { session_id, .. }) = - wait_for_event(&codex, |ev| matches!(ev, EventMsg::SessionConfigured(_))).await - else { - unreachable!() - }; - - let current_session_id = Some(session_id.to_string()); wait_for_event(&codex, |ev| matches!(ev, EventMsg::TaskComplete(_))).await; // get request from the server @@ -315,10 +347,9 @@ async fn chatgpt_auth_sends_correct_request() { let request_chatgpt_account_id = request.headers.get("chatgpt-account-id").unwrap(); let request_body = request.body_json::().unwrap(); - assert!(current_session_id.is_some()); assert_eq!( request_session_id.to_str().unwrap(), - current_session_id.as_ref().unwrap() + conversation_id.to_string() ); assert_eq!(request_originator.to_str().unwrap(), "codex_cli_rs"); assert_eq!( @@ -335,9 +366,157 @@ async fn chatgpt_auth_sends_correct_request() { } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn includes_user_instructions_message_in_request() { - #![allow(clippy::unwrap_used)] +async fn prefers_chatgpt_token_when_config_prefers_chatgpt() { + if std::env::var(CODEX_SANDBOX_NETWORK_DISABLED_ENV_VAR).is_ok() { + println!( + "Skipping test because it cannot execute when network is disabled in a Codex sandbox." + ); + return; + } + // Mock server + let server = MockServer::start().await; + + let first = ResponseTemplate::new(200) + .insert_header("content-type", "text/event-stream") + .set_body_raw(sse_completed("resp1"), "text/event-stream"); + + // Expect ChatGPT base path and correct headers + Mock::given(method("POST")) + .and(path("/v1/responses")) + .and(header_regex("Authorization", r"Bearer Access-123")) + .and(header_regex("chatgpt-account-id", r"acc-123")) + .respond_with(first) + .expect(1) + .mount(&server) + .await; + + let model_provider = ModelProviderInfo { + base_url: Some(format!("{}/v1", server.uri())), + ..built_in_model_providers()["openai"].clone() + }; + + // Init session + let codex_home = TempDir::new().unwrap(); + // Write auth.json that contains both API key and ChatGPT tokens for a plan that should prefer ChatGPT. + let _jwt = write_auth_json( + &codex_home, + Some("sk-test-key"), + "pro", + "Access-123", + Some("acc-123"), + ); + + let mut config = load_default_config_for_test(&codex_home); + config.model_provider = model_provider; + config.preferred_auth_method = AuthMode::ChatGPT; + + let conversation_manager = ConversationManager::default(); + let NewConversation { + conversation: codex, + .. + } = conversation_manager + .new_conversation(config) + .await + .expect("create new conversation"); + + codex + .submit(Op::UserInput { + items: vec![InputItem::Text { + text: "hello".into(), + }], + }) + .await + .unwrap(); + + wait_for_event(&codex, |ev| matches!(ev, EventMsg::TaskComplete(_))).await; + + // verify request body flags + let request = &server.received_requests().await.unwrap()[0]; + let request_body = request.body_json::().unwrap(); + assert!( + !request_body["store"].as_bool().unwrap(), + "store should be false for ChatGPT auth" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn prefers_apikey_when_config_prefers_apikey_even_with_chatgpt_tokens() { + if std::env::var(CODEX_SANDBOX_NETWORK_DISABLED_ENV_VAR).is_ok() { + println!( + "Skipping test because it cannot execute when network is disabled in a Codex sandbox." + ); + return; + } + + // Mock server + let server = MockServer::start().await; + + let first = ResponseTemplate::new(200) + .insert_header("content-type", "text/event-stream") + .set_body_raw(sse_completed("resp1"), "text/event-stream"); + + // Expect API key header, no ChatGPT account header required. + Mock::given(method("POST")) + .and(path("/v1/responses")) + .and(header_regex("Authorization", r"Bearer sk-test-key")) + .respond_with(first) + .expect(1) + .mount(&server) + .await; + + let model_provider = ModelProviderInfo { + base_url: Some(format!("{}/v1", server.uri())), + ..built_in_model_providers()["openai"].clone() + }; + + // Init session + let codex_home = TempDir::new().unwrap(); + // Write auth.json that contains both API key and ChatGPT tokens for a plan that should prefer ChatGPT, + // but config will force API key preference. + let _jwt = write_auth_json( + &codex_home, + Some("sk-test-key"), + "pro", + "Access-123", + Some("acc-123"), + ); + + let mut config = load_default_config_for_test(&codex_home); + config.model_provider = model_provider; + config.preferred_auth_method = AuthMode::ApiKey; + + let conversation_manager = ConversationManager::default(); + let NewConversation { + conversation: codex, + .. + } = conversation_manager + .new_conversation(config) + .await + .expect("create new conversation"); + + codex + .submit(Op::UserInput { + items: vec![InputItem::Text { + text: "hello".into(), + }], + }) + .await + .unwrap(); + + wait_for_event(&codex, |ev| matches!(ev, EventMsg::TaskComplete(_))).await; + + // verify request body flags + let request = &server.received_requests().await.unwrap()[0]; + let request_body = request.body_json::().unwrap(); + assert!( + request_body["store"].as_bool().unwrap(), + "store should be true for API key auth" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn includes_user_instructions_message_in_request() { let server = MockServer::start().await; let first = ResponseTemplate::new(200) @@ -361,14 +540,12 @@ async fn includes_user_instructions_message_in_request() { config.model_provider = model_provider; config.user_instructions = Some("be nice".to_string()); - let ctrl_c = std::sync::Arc::new(tokio::sync::Notify::new()); - let CodexSpawnOk { codex, .. } = Codex::spawn( - config, - Some(CodexAuth::from_api_key("Test API Key")), - ctrl_c.clone(), - ) - .await - .unwrap(); + let conversation_manager = ConversationManager::default(); + let codex = conversation_manager + .new_conversation_with_auth(config, Some(CodexAuth::from_api_key("Test API Key"))) + .await + .expect("create new conversation") + .conversation; codex .submit(Op::UserInput { @@ -391,17 +568,15 @@ async fn includes_user_instructions_message_in_request() { .contains("be nice") ); assert_message_role(&request_body["input"][0], "user"); - assert_message_starts_with(&request_body["input"][0], "\n\n"); - assert_message_ends_with(&request_body["input"][0], ""); + assert_message_starts_with(&request_body["input"][0], ""); + assert_message_ends_with(&request_body["input"][0], ""); assert_message_role(&request_body["input"][1], "user"); - assert_message_starts_with(&request_body["input"][1], "\n\n"); - assert_message_ends_with(&request_body["input"][1], ""); + assert_message_starts_with(&request_body["input"][1], ""); + assert_message_ends_with(&request_body["input"][1], ""); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn azure_overrides_assign_properties_used_for_responses_url() { - #![allow(clippy::unwrap_used)] - let existing_env_var_with_random_value = if cfg!(windows) { "USERNAME" } else { "USER" }; // Mock server @@ -457,8 +632,12 @@ async fn azure_overrides_assign_properties_used_for_responses_url() { let mut config = load_default_config_for_test(&codex_home); config.model_provider = provider; - let ctrl_c = std::sync::Arc::new(tokio::sync::Notify::new()); - let CodexSpawnOk { codex, .. } = Codex::spawn(config, None, ctrl_c.clone()).await.unwrap(); + let conversation_manager = ConversationManager::default(); + let codex = conversation_manager + .new_conversation_with_auth(config, None) + .await + .expect("create new conversation") + .conversation; codex .submit(Op::UserInput { @@ -474,8 +653,6 @@ async fn azure_overrides_assign_properties_used_for_responses_url() { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn env_var_overrides_loaded_auth() { - #![allow(clippy::unwrap_used)] - let existing_env_var_with_random_value = if cfg!(windows) { "USERNAME" } else { "USER" }; // Mock server @@ -531,11 +708,12 @@ async fn env_var_overrides_loaded_auth() { let mut config = load_default_config_for_test(&codex_home); config.model_provider = provider; - let ctrl_c = std::sync::Arc::new(tokio::sync::Notify::new()); - let CodexSpawnOk { codex, .. } = - Codex::spawn(config, Some(create_dummy_codex_auth()), ctrl_c.clone()) - .await - .unwrap(); + let conversation_manager = ConversationManager::default(); + let codex = conversation_manager + .new_conversation_with_auth(config, Some(create_dummy_codex_auth())) + .await + .expect("create new conversation") + .conversation; codex .submit(Op::UserInput { diff --git a/codex-rs/core/tests/common/lib.rs b/codex-rs/core/tests/common/lib.rs index 834ec38298..244d093e7d 100644 --- a/codex-rs/core/tests/common/lib.rs +++ b/codex-rs/core/tests/common/lib.rs @@ -1,7 +1,8 @@ -#![allow(clippy::expect_used)] +#![expect(clippy::expect_used)] use tempfile::TempDir; +use codex_core::CodexConversation; use codex_core::config::Config; use codex_core::config::ConfigOverrides; use codex_core::config::ConfigToml; @@ -46,6 +47,26 @@ pub fn load_sse_fixture(path: impl AsRef) -> String { .collect() } +pub fn load_sse_fixture_with_id_from_str(raw: &str, id: &str) -> String { + let replaced = raw.replace("__ID__", id); + let events: Vec = + serde_json::from_str(&replaced).expect("parse JSON fixture"); + events + .into_iter() + .map(|e| { + let kind = e + .get("type") + .and_then(|v| v.as_str()) + .expect("fixture event missing type"); + if e.as_object().map(|o| o.len() == 1).unwrap_or(false) { + format!("event: {kind}\n\n") + } else { + format!("event: {kind}\ndata: {e}\n\n") + } + }) + .collect() +} + /// Same as [`load_sse_fixture`], but replaces the placeholder `__ID__` in the /// fixture template with the supplied identifier before parsing. This lets a /// single JSON template be reused by multiple tests that each need a unique @@ -72,7 +93,7 @@ pub fn load_sse_fixture_with_id(path: impl AsRef, id: &str) -> } pub async fn wait_for_event( - codex: &codex_core::Codex, + codex: &CodexConversation, predicate: F, ) -> codex_core::protocol::EventMsg where @@ -83,16 +104,18 @@ where } pub async fn wait_for_event_with_timeout( - codex: &codex_core::Codex, + codex: &CodexConversation, mut predicate: F, wait_time: tokio::time::Duration, ) -> codex_core::protocol::EventMsg where F: FnMut(&codex_core::protocol::EventMsg) -> bool, { + use tokio::time::Duration; use tokio::time::timeout; loop { - let ev = timeout(wait_time, codex.next_event()) + // Allow a bit more time to accommodate async startup work (e.g. config IO, tool discovery) + let ev = timeout(wait_time.max(Duration::from_secs(5)), codex.next_event()) .await .expect("timeout waiting for event") .expect("stream ended unexpectedly"); diff --git a/codex-rs/core/tests/compact.rs b/codex-rs/core/tests/compact.rs index fa5c81d883..28b1ca8d74 100644 --- a/codex-rs/core/tests/compact.rs +++ b/codex-rs/core/tests/compact.rs @@ -1,7 +1,6 @@ #![expect(clippy::unwrap_used)] -use codex_core::Codex; -use codex_core::CodexSpawnOk; +use codex_core::ConversationManager; use codex_core::ModelProviderInfo; use codex_core::built_in_model_providers; use codex_core::protocol::EventMsg; @@ -142,14 +141,12 @@ async fn summarize_context_three_requests_and_instructions() { let home = TempDir::new().unwrap(); let mut config = load_default_config_for_test(&home); config.model_provider = model_provider; - let ctrl_c = std::sync::Arc::new(tokio::sync::Notify::new()); - let CodexSpawnOk { codex, .. } = Codex::spawn( - config, - Some(CodexAuth::from_api_key("dummy")), - ctrl_c.clone(), - ) - .await - .unwrap(); + let conversation_manager = ConversationManager::default(); + let codex = conversation_manager + .new_conversation_with_auth(config, Some(CodexAuth::from_api_key("dummy"))) + .await + .unwrap() + .conversation; // 1) Normal user input – should hit server once. codex diff --git a/codex-rs/core/tests/exec.rs b/codex-rs/core/tests/exec.rs index f1b9e78e67..f011aa7a0a 100644 --- a/codex-rs/core/tests/exec.rs +++ b/codex-rs/core/tests/exec.rs @@ -1,25 +1,30 @@ #![cfg(target_os = "macos")] -#![expect(clippy::expect_used)] use std::collections::HashMap; -use std::sync::Arc; use codex_core::exec::ExecParams; +use codex_core::exec::ExecToolCallOutput; use codex_core::exec::SandboxType; use codex_core::exec::process_exec_tool_call; use codex_core::protocol::SandboxPolicy; use codex_core::spawn::CODEX_SANDBOX_ENV_VAR; use tempfile::TempDir; -use tokio::sync::Notify; + +use codex_core::error::Result; use codex_core::get_platform_sandbox; -async fn run_test_cmd(tmp: TempDir, cmd: Vec<&str>, should_be_ok: bool) { +fn skip_test() -> bool { if std::env::var(CODEX_SANDBOX_ENV_VAR) == Ok("seatbelt".to_string()) { eprintln!("{CODEX_SANDBOX_ENV_VAR} is set to 'seatbelt', skipping test."); - return; + return true; } + false +} + +#[expect(clippy::expect_used)] +async fn run_test_cmd(tmp: TempDir, cmd: Vec<&str>) -> Result { let sandbox_type = get_platform_sandbox().expect("should be able to get sandbox type"); assert_eq!(sandbox_type, SandboxType::MacosSeatbelt); @@ -32,34 +37,83 @@ async fn run_test_cmd(tmp: TempDir, cmd: Vec<&str>, should_be_ok: bool) { justification: None, }; - let ctrl_c = Arc::new(Notify::new()); let policy = SandboxPolicy::new_read_only_policy(); - let result = process_exec_tool_call(params, sandbox_type, ctrl_c, &policy, &None, None).await; - - assert!(result.is_ok() == should_be_ok); + process_exec_tool_call(params, sandbox_type, &policy, &None, None).await } /// Command succeeds with exit code 0 normally #[tokio::test] async fn exit_code_0_succeeds() { + if skip_test() { + return; + } + let tmp = TempDir::new().expect("should be able to create temp dir"); let cmd = vec!["echo", "hello"]; - run_test_cmd(tmp, cmd, true).await + let output = run_test_cmd(tmp, cmd).await.unwrap(); + assert_eq!(output.stdout.text, "hello\n"); + assert_eq!(output.stderr.text, ""); + assert_eq!(output.stdout.truncated_after_lines, None); +} + +/// Command succeeds with exit code 0 normally +#[tokio::test] +async fn truncates_output_lines() { + if skip_test() { + return; + } + + let tmp = TempDir::new().expect("should be able to create temp dir"); + let cmd = vec!["seq", "300"]; + + let output = run_test_cmd(tmp, cmd).await.unwrap(); + + let expected_output = (1..=256) + .map(|i| format!("{i}\n")) + .collect::>() + .join(""); + assert_eq!(output.stdout.text, expected_output); + assert_eq!(output.stdout.truncated_after_lines, Some(256)); +} + +/// Command succeeds with exit code 0 normally +#[tokio::test] +async fn truncates_output_bytes() { + if skip_test() { + return; + } + + let tmp = TempDir::new().expect("should be able to create temp dir"); + // each line is 1000 bytes + let cmd = vec!["bash", "-lc", "seq 15 | awk '{printf \"%-1000s\\n\", $0}'"]; + + let output = run_test_cmd(tmp, cmd).await.unwrap(); + + assert_eq!(output.stdout.text.len(), 10240); + assert_eq!(output.stdout.truncated_after_lines, Some(10)); } /// Command not found returns exit code 127, this is not considered a sandbox error #[tokio::test] async fn exit_command_not_found_is_ok() { + if skip_test() { + return; + } + let tmp = TempDir::new().expect("should be able to create temp dir"); let cmd = vec!["/bin/bash", "-c", "nonexistent_command_12345"]; - run_test_cmd(tmp, cmd, true).await + run_test_cmd(tmp, cmd).await.unwrap(); } /// Writing a file fails and should be considered a sandbox error #[tokio::test] async fn write_file_fails_as_sandbox_error() { + if skip_test() { + return; + } + let tmp = TempDir::new().expect("should be able to create temp dir"); let path = tmp.path().join("test.txt"); let cmd = vec![ @@ -67,5 +121,5 @@ async fn write_file_fails_as_sandbox_error() { path.to_str().expect("should be able to get path"), ]; - run_test_cmd(tmp, cmd, false).await; + assert!(run_test_cmd(tmp, cmd).await.is_err()); } diff --git a/codex-rs/core/tests/exec_stream_events.rs b/codex-rs/core/tests/exec_stream_events.rs index 534b25513a..85d4eb3701 100644 --- a/codex-rs/core/tests/exec_stream_events.rs +++ b/codex-rs/core/tests/exec_stream_events.rs @@ -2,7 +2,6 @@ use std::collections::HashMap; use std::path::PathBuf; -use std::sync::Arc; use async_channel::Receiver; use codex_core::exec::ExecParams; @@ -14,7 +13,6 @@ use codex_core::protocol::EventMsg; use codex_core::protocol::ExecCommandOutputDeltaEvent; use codex_core::protocol::ExecOutputStream; use codex_core::protocol::SandboxPolicy; -use tokio::sync::Notify; fn collect_stdout_events(rx: Receiver) -> Vec { let mut out = Vec::new(); @@ -57,13 +55,11 @@ async fn test_exec_stdout_stream_events_echo() { justification: None, }; - let ctrl_c = Arc::new(Notify::new()); let policy = SandboxPolicy::new_read_only_policy(); let result = process_exec_tool_call( params, SandboxType::None, - ctrl_c, &policy, &None, Some(stdout_stream), @@ -76,7 +72,7 @@ async fn test_exec_stdout_stream_events_echo() { }; assert_eq!(result.exit_code, 0); - assert_eq!(result.stdout, "hello-world\n"); + assert_eq!(result.stdout.text, "hello-world\n"); let streamed = collect_stdout_events(rx); // We should have received at least the same contents (possibly in one chunk) @@ -109,13 +105,11 @@ async fn test_exec_stderr_stream_events_echo() { justification: None, }; - let ctrl_c = Arc::new(Notify::new()); let policy = SandboxPolicy::new_read_only_policy(); let result = process_exec_tool_call( params, SandboxType::None, - ctrl_c, &policy, &None, Some(stdout_stream), @@ -128,8 +122,8 @@ async fn test_exec_stderr_stream_events_echo() { }; assert_eq!(result.exit_code, 0); - assert_eq!(result.stdout, ""); - assert_eq!(result.stderr, "oops\n"); + assert_eq!(result.stdout.text, ""); + assert_eq!(result.stderr.text, "oops\n"); // Collect only stderr delta events let mut err = Vec::new(); diff --git a/codex-rs/core/tests/live_agent.rs b/codex-rs/core/tests/live_agent.rs deleted file mode 100644 index 81b3bb2a12..0000000000 --- a/codex-rs/core/tests/live_agent.rs +++ /dev/null @@ -1,209 +0,0 @@ -#![expect(clippy::unwrap_used, clippy::expect_used)] - -//! Live integration tests that exercise the full [`Agent`] stack **against the real -//! OpenAI `/v1/responses` API**. These tests complement the lightweight mock‑based -//! unit tests by verifying that the agent can drive an end‑to‑end conversation, -//! stream incremental events, execute function‑call tool invocations and safely -//! chain multiple turns inside a single session – the exact scenarios that have -//! historically been brittle. -//! -//! The live tests are **ignored by default** so CI remains deterministic and free -//! of external dependencies. Developers can opt‑in locally with e.g. -//! -//! ```bash -//! OPENAI_API_KEY=sk‑... cargo test --test live_agent -- --ignored --nocapture -//! ``` -//! -//! Make sure your key has access to the experimental *Responses* API and that -//! any billable usage is acceptable. - -use std::time::Duration; - -use codex_core::Codex; -use codex_core::CodexSpawnOk; -use codex_core::error::CodexErr; -use codex_core::protocol::AgentMessageEvent; -use codex_core::protocol::ErrorEvent; -use codex_core::protocol::EventMsg; -use codex_core::protocol::InputItem; -use codex_core::protocol::Op; -use core_test_support::load_default_config_for_test; -use tempfile::TempDir; -use tokio::sync::Notify; -use tokio::time::timeout; - -fn api_key_available() -> bool { - std::env::var("OPENAI_API_KEY").is_ok() -} - -/// Helper that spawns a fresh Agent and sends the mandatory *ConfigureSession* -/// submission. The caller receives the constructed [`Agent`] plus the unique -/// submission id used for the initialization message. -async fn spawn_codex() -> Result { - assert!( - api_key_available(), - "OPENAI_API_KEY must be set for live tests" - ); - - let codex_home = TempDir::new().unwrap(); - let mut config = load_default_config_for_test(&codex_home); - config.model_provider.request_max_retries = Some(2); - config.model_provider.stream_max_retries = Some(2); - let CodexSpawnOk { codex: agent, .. } = - Codex::spawn(config, None, std::sync::Arc::new(Notify::new())).await?; - - Ok(agent) -} - -/// Verifies that the agent streams incremental *AgentMessage* events **before** -/// emitting `TaskComplete` and that a second task inside the same session does -/// not get tripped up by a stale `previous_response_id`. -#[ignore] -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn live_streaming_and_prev_id_reset() { - if !api_key_available() { - eprintln!("skipping live_streaming_and_prev_id_reset – OPENAI_API_KEY not set"); - return; - } - - let codex = spawn_codex().await.unwrap(); - - // ---------- Task 1 ---------- - codex - .submit(Op::UserInput { - items: vec![InputItem::Text { - text: "Say the words 'stream test'".into(), - }], - }) - .await - .unwrap(); - - let mut saw_message_before_complete = false; - loop { - let ev = timeout(Duration::from_secs(60), codex.next_event()) - .await - .expect("timeout waiting for task1 events") - .expect("agent closed"); - - match ev.msg { - EventMsg::AgentMessage(_) => saw_message_before_complete = true, - EventMsg::TaskComplete(_) => break, - EventMsg::Error(ErrorEvent { message }) => { - panic!("agent reported error in task1: {message}") - } - _ => { - // Ignore other events. - } - } - } - - assert!( - saw_message_before_complete, - "Agent did not stream any AgentMessage before TaskComplete" - ); - - // ---------- Task 2 (same session) ---------- - codex - .submit(Op::UserInput { - items: vec![InputItem::Text { - text: "Respond with exactly: second turn succeeded".into(), - }], - }) - .await - .unwrap(); - - let mut got_expected = false; - loop { - let ev = timeout(Duration::from_secs(60), codex.next_event()) - .await - .expect("timeout waiting for task2 events") - .expect("agent closed"); - - match &ev.msg { - EventMsg::AgentMessage(AgentMessageEvent { message }) - if message.contains("second turn succeeded") => - { - got_expected = true; - } - EventMsg::TaskComplete(_) => break, - EventMsg::Error(ErrorEvent { message }) => { - panic!("agent reported error in task2: {message}") - } - _ => { - // Ignore other events. - } - } - } - - assert!(got_expected, "second task did not receive expected answer"); -} - -/// Exercises a *function‑call → shell execution* round‑trip by instructing the -/// model to run a harmless `echo` command. The test asserts that: -/// 1. the function call is executed (we see `ExecCommandBegin`/`End` events) -/// 2. the captured stdout reaches the client unchanged. -#[ignore] -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn live_shell_function_call() { - if !api_key_available() { - eprintln!("skipping live_shell_function_call – OPENAI_API_KEY not set"); - return; - } - - let codex = spawn_codex().await.unwrap(); - - const MARKER: &str = "codex_live_echo_ok"; - - codex - .submit(Op::UserInput { - items: vec![InputItem::Text { - text: format!( - "Use the shell function to run the command `echo {MARKER}` and no other commands." - ), - }], - }) - .await - .unwrap(); - - let mut saw_begin = false; - let mut saw_end_with_output = false; - - loop { - let ev = timeout(Duration::from_secs(60), codex.next_event()) - .await - .expect("timeout waiting for function‑call events") - .expect("agent closed"); - - match ev.msg { - EventMsg::ExecCommandBegin(codex_core::protocol::ExecCommandBeginEvent { - command, - .. - }) => { - assert_eq!(command, vec!["echo", MARKER]); - saw_begin = true; - } - EventMsg::ExecCommandEnd(codex_core::protocol::ExecCommandEndEvent { - stdout, - exit_code, - .. - }) => { - assert_eq!(exit_code, 0, "echo returned non‑zero exit code"); - assert!(stdout.contains(MARKER)); - saw_end_with_output = true; - } - EventMsg::TaskComplete(_) => break, - EventMsg::Error(codex_core::protocol::ErrorEvent { message }) => { - panic!("agent error during shell test: {message}") - } - _ => { - // Ignore other events. - } - } - } - - assert!(saw_begin, "ExecCommandBegin event missing"); - assert!( - saw_end_with_output, - "ExecCommandEnd with expected output missing" - ); -} diff --git a/codex-rs/core/tests/live_cli.rs b/codex-rs/core/tests/live_cli.rs index d79e242c4d..e77fe6998e 100644 --- a/codex-rs/core/tests/live_cli.rs +++ b/codex-rs/core/tests/live_cli.rs @@ -17,7 +17,7 @@ fn require_api_key() -> String { /// Helper that spawns the binary inside a TempDir with minimal flags. Returns (Assert, TempDir). fn run_live(prompt: &str) -> (assert_cmd::assert::Assert, TempDir) { - #![allow(clippy::unwrap_used)] + #![expect(clippy::unwrap_used)] use std::io::Read; use std::io::Write; use std::thread; @@ -113,7 +113,6 @@ fn run_live(prompt: &str) -> (assert_cmd::assert::Assert, TempDir) { #[ignore] #[test] fn live_create_file_hello_txt() { - #![allow(clippy::unwrap_used)] if std::env::var("OPENAI_API_KEY").is_err() { eprintln!("skipping live_create_file_hello_txt – OPENAI_API_KEY not set"); return; diff --git a/codex-rs/core/tests/prompt_caching.rs b/codex-rs/core/tests/prompt_caching.rs new file mode 100644 index 0000000000..9f5829e113 --- /dev/null +++ b/codex-rs/core/tests/prompt_caching.rs @@ -0,0 +1,362 @@ +use codex_core::ConversationManager; +use codex_core::ModelProviderInfo; +use codex_core::built_in_model_providers; +use codex_core::protocol::AskForApproval; +use codex_core::protocol::EventMsg; +use codex_core::protocol::InputItem; +use codex_core::protocol::Op; +use codex_core::protocol::SandboxPolicy; +use codex_core::protocol_config_types::ReasoningEffort; +use codex_core::protocol_config_types::ReasoningSummary; +use codex_login::CodexAuth; +use core_test_support::load_default_config_for_test; +use core_test_support::load_sse_fixture_with_id; +use core_test_support::wait_for_event; +use tempfile::TempDir; +use wiremock::Mock; +use wiremock::MockServer; +use wiremock::ResponseTemplate; +use wiremock::matchers::method; +use wiremock::matchers::path; + +/// Build minimal SSE stream with completed marker using the JSON fixture. +fn sse_completed(id: &str) -> String { + load_sse_fixture_with_id("tests/fixtures/completed_template.json", id) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn prefixes_context_and_instructions_once_and_consistently_across_requests() { + use pretty_assertions::assert_eq; + + let server = MockServer::start().await; + + let sse = sse_completed("resp"); + let template = ResponseTemplate::new(200) + .insert_header("content-type", "text/event-stream") + .set_body_raw(sse, "text/event-stream"); + + // Expect two POSTs to /v1/responses + Mock::given(method("POST")) + .and(path("/v1/responses")) + .respond_with(template) + .expect(2) + .mount(&server) + .await; + + let model_provider = ModelProviderInfo { + base_url: Some(format!("{}/v1", server.uri())), + ..built_in_model_providers()["openai"].clone() + }; + + let cwd = TempDir::new().unwrap(); + let codex_home = TempDir::new().unwrap(); + let mut config = load_default_config_for_test(&codex_home); + config.cwd = cwd.path().to_path_buf(); + config.model_provider = model_provider; + config.user_instructions = Some("be consistent and helpful".to_string()); + + let conversation_manager = ConversationManager::default(); + let codex = conversation_manager + .new_conversation_with_auth(config, Some(CodexAuth::from_api_key("Test API Key"))) + .await + .expect("create new conversation") + .conversation; + + codex + .submit(Op::UserInput { + items: vec![InputItem::Text { + text: "hello 1".into(), + }], + }) + .await + .unwrap(); + wait_for_event(&codex, |ev| matches!(ev, EventMsg::TaskComplete(_))).await; + + codex + .submit(Op::UserInput { + items: vec![InputItem::Text { + text: "hello 2".into(), + }], + }) + .await + .unwrap(); + wait_for_event(&codex, |ev| matches!(ev, EventMsg::TaskComplete(_))).await; + + let requests = server.received_requests().await.unwrap(); + assert_eq!(requests.len(), 2, "expected two POST requests"); + + let expected_env_text = format!( + "\nCurrent working directory: {}\nApproval policy: on-request\nSandbox mode: read-only\nNetwork access: restricted\n", + cwd.path().to_string_lossy() + ); + let expected_ui_text = + "\n\nbe consistent and helpful\n\n"; + + let expected_env_msg = serde_json::json!({ + "type": "message", + "id": serde_json::Value::Null, + "role": "user", + "content": [ { "type": "input_text", "text": expected_env_text } ] + }); + let expected_ui_msg = serde_json::json!({ + "type": "message", + "id": serde_json::Value::Null, + "role": "user", + "content": [ { "type": "input_text", "text": expected_ui_text } ] + }); + + let expected_user_message_1 = serde_json::json!({ + "type": "message", + "id": serde_json::Value::Null, + "role": "user", + "content": [ { "type": "input_text", "text": "hello 1" } ] + }); + let body1 = requests[0].body_json::().unwrap(); + assert_eq!( + body1["input"], + serde_json::json!([expected_ui_msg, expected_env_msg, expected_user_message_1]) + ); + + let expected_user_message_2 = serde_json::json!({ + "type": "message", + "id": serde_json::Value::Null, + "role": "user", + "content": [ { "type": "input_text", "text": "hello 2" } ] + }); + let body2 = requests[1].body_json::().unwrap(); + let expected_body2 = serde_json::json!( + [ + body1["input"].as_array().unwrap().as_slice(), + [expected_user_message_2].as_slice(), + ] + .concat() + ); + assert_eq!(body2["input"], expected_body2); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn overrides_turn_context_but_keeps_cached_prefix_and_key_constant() { + use pretty_assertions::assert_eq; + + let server = MockServer::start().await; + + let sse = sse_completed("resp"); + let template = ResponseTemplate::new(200) + .insert_header("content-type", "text/event-stream") + .set_body_raw(sse, "text/event-stream"); + + // Expect two POSTs to /v1/responses + Mock::given(method("POST")) + .and(path("/v1/responses")) + .respond_with(template) + .expect(2) + .mount(&server) + .await; + + let model_provider = ModelProviderInfo { + base_url: Some(format!("{}/v1", server.uri())), + ..built_in_model_providers()["openai"].clone() + }; + + let cwd = TempDir::new().unwrap(); + let codex_home = TempDir::new().unwrap(); + let mut config = load_default_config_for_test(&codex_home); + config.cwd = cwd.path().to_path_buf(); + config.model_provider = model_provider; + config.user_instructions = Some("be consistent and helpful".to_string()); + + let conversation_manager = ConversationManager::default(); + let codex = conversation_manager + .new_conversation_with_auth(config, Some(CodexAuth::from_api_key("Test API Key"))) + .await + .expect("create new conversation") + .conversation; + + // First turn + codex + .submit(Op::UserInput { + items: vec![InputItem::Text { + text: "hello 1".into(), + }], + }) + .await + .unwrap(); + wait_for_event(&codex, |ev| matches!(ev, EventMsg::TaskComplete(_))).await; + + // Change everything about the turn context. + let new_cwd = TempDir::new().unwrap(); + let writable = TempDir::new().unwrap(); + codex + .submit(Op::OverrideTurnContext { + cwd: Some(new_cwd.path().to_path_buf()), + approval_policy: Some(AskForApproval::Never), + sandbox_policy: Some(SandboxPolicy::WorkspaceWrite { + writable_roots: vec![writable.path().to_path_buf()], + network_access: true, + exclude_tmpdir_env_var: true, + exclude_slash_tmp: true, + }), + model: Some("o3".to_string()), + effort: Some(ReasoningEffort::High), + summary: Some(ReasoningSummary::Detailed), + }) + .await + .unwrap(); + + // Second turn after overrides + codex + .submit(Op::UserInput { + items: vec![InputItem::Text { + text: "hello 2".into(), + }], + }) + .await + .unwrap(); + wait_for_event(&codex, |ev| matches!(ev, EventMsg::TaskComplete(_))).await; + + // Verify we issued exactly two requests, and the cached prefix stayed identical. + let requests = server.received_requests().await.unwrap(); + assert_eq!(requests.len(), 2, "expected two POST requests"); + + let body1 = requests[0].body_json::().unwrap(); + let body2 = requests[1].body_json::().unwrap(); + + // prompt_cache_key should remain constant across overrides + assert_eq!( + body1["prompt_cache_key"], body2["prompt_cache_key"], + "prompt_cache_key should not change across overrides" + ); + + // The entire prefix from the first request should be identical and reused + // as the prefix of the second request, ensuring cache hit potential. + let expected_user_message_2 = serde_json::json!({ + "type": "message", + "id": serde_json::Value::Null, + "role": "user", + "content": [ { "type": "input_text", "text": "hello 2" } ] + }); + // After overriding the turn context, the environment context should be emitted again + // reflecting the new cwd, approval policy and sandbox settings. + let expected_env_text_2 = format!( + "\nCurrent working directory: {}\nApproval policy: never\nSandbox mode: workspace-write\nNetwork access: enabled\n", + new_cwd.path().to_string_lossy() + ); + let expected_env_msg_2 = serde_json::json!({ + "type": "message", + "id": serde_json::Value::Null, + "role": "user", + "content": [ { "type": "input_text", "text": expected_env_text_2 } ] + }); + let expected_body2 = serde_json::json!( + [ + body1["input"].as_array().unwrap().as_slice(), + [expected_env_msg_2, expected_user_message_2].as_slice(), + ] + .concat() + ); + assert_eq!(body2["input"], expected_body2); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn per_turn_overrides_keep_cached_prefix_and_key_constant() { + use pretty_assertions::assert_eq; + + let server = MockServer::start().await; + + let sse = sse_completed("resp"); + let template = ResponseTemplate::new(200) + .insert_header("content-type", "text/event-stream") + .set_body_raw(sse, "text/event-stream"); + + // Expect two POSTs to /v1/responses + Mock::given(method("POST")) + .and(path("/v1/responses")) + .respond_with(template) + .expect(2) + .mount(&server) + .await; + + let model_provider = ModelProviderInfo { + base_url: Some(format!("{}/v1", server.uri())), + ..built_in_model_providers()["openai"].clone() + }; + + let cwd = TempDir::new().unwrap(); + let codex_home = TempDir::new().unwrap(); + let mut config = load_default_config_for_test(&codex_home); + config.cwd = cwd.path().to_path_buf(); + config.model_provider = model_provider; + config.user_instructions = Some("be consistent and helpful".to_string()); + + let conversation_manager = ConversationManager::default(); + let codex = conversation_manager + .new_conversation_with_auth(config, Some(CodexAuth::from_api_key("Test API Key"))) + .await + .expect("create new conversation") + .conversation; + + // First turn + codex + .submit(Op::UserInput { + items: vec![InputItem::Text { + text: "hello 1".into(), + }], + }) + .await + .unwrap(); + wait_for_event(&codex, |ev| matches!(ev, EventMsg::TaskComplete(_))).await; + + // Second turn using per-turn overrides via UserTurn + let new_cwd = TempDir::new().unwrap(); + let writable = TempDir::new().unwrap(); + codex + .submit(Op::UserTurn { + items: vec![InputItem::Text { + text: "hello 2".into(), + }], + cwd: new_cwd.path().to_path_buf(), + approval_policy: AskForApproval::Never, + sandbox_policy: SandboxPolicy::WorkspaceWrite { + writable_roots: vec![writable.path().to_path_buf()], + network_access: true, + exclude_tmpdir_env_var: true, + exclude_slash_tmp: true, + }, + model: "o3".to_string(), + effort: ReasoningEffort::High, + summary: ReasoningSummary::Detailed, + }) + .await + .unwrap(); + wait_for_event(&codex, |ev| matches!(ev, EventMsg::TaskComplete(_))).await; + + // Verify we issued exactly two requests, and the cached prefix stayed identical. + let requests = server.received_requests().await.unwrap(); + assert_eq!(requests.len(), 2, "expected two POST requests"); + + let body1 = requests[0].body_json::().unwrap(); + let body2 = requests[1].body_json::().unwrap(); + + // prompt_cache_key should remain constant across per-turn overrides + assert_eq!( + body1["prompt_cache_key"], body2["prompt_cache_key"], + "prompt_cache_key should not change across per-turn overrides" + ); + + // The entire prefix from the first request should be identical and reused + // as the prefix of the second request. + let expected_user_message_2 = serde_json::json!({ + "type": "message", + "id": serde_json::Value::Null, + "role": "user", + "content": [ { "type": "input_text", "text": "hello 2" } ] + }); + let expected_body2 = serde_json::json!( + [ + body1["input"].as_array().unwrap().as_slice(), + [expected_user_message_2].as_slice(), + ] + .concat() + ); + assert_eq!(body2["input"], expected_body2); +} diff --git a/codex-rs/core/tests/sandbox.rs b/codex-rs/core/tests/seatbelt.rs similarity index 96% rename from codex-rs/core/tests/sandbox.rs rename to codex-rs/core/tests/seatbelt.rs index ae5bdc44a6..638ac48bb8 100644 --- a/codex-rs/core/tests/sandbox.rs +++ b/codex-rs/core/tests/seatbelt.rs @@ -1,5 +1,7 @@ #![cfg(target_os = "macos")] -#![expect(clippy::expect_used)] + +//! Tests for the macOS sandboxing that are specific to Seatbelt. +//! Tests that apply to both Mac and Linux sandboxing should go in sandbox.rs. use std::collections::HashMap; use std::path::Path; @@ -157,6 +159,7 @@ async fn read_only_forbids_all_writes() { .await; } +#[expect(clippy::expect_used)] fn create_test_scenario(tmp: &TempDir) -> TestScenario { let repo_parent = tmp.path().to_path_buf(); let repo_root = repo_parent.join("repo"); @@ -174,6 +177,7 @@ fn create_test_scenario(tmp: &TempDir) -> TestScenario { } } +#[expect(clippy::expect_used)] /// Note that `path` must be absolute. async fn touch(path: &Path, policy: &SandboxPolicy) -> bool { assert!(path.is_absolute(), "Path must be absolute: {path:?}"); diff --git a/codex-rs/core/tests/stream_error_allows_next_turn.rs b/codex-rs/core/tests/stream_error_allows_next_turn.rs index 1500c789e9..415e75a426 100644 --- a/codex-rs/core/tests/stream_error_allows_next_turn.rs +++ b/codex-rs/core/tests/stream_error_allows_next_turn.rs @@ -1,7 +1,6 @@ use std::time::Duration; -use codex_core::Codex; -use codex_core::CodexSpawnOk; +use codex_core::ConversationManager; use codex_core::ModelProviderInfo; use codex_core::WireApi; use codex_core::protocol::EventMsg; @@ -26,7 +25,6 @@ fn sse_completed(id: &str) -> String { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn continue_after_stream_error() { - #![allow(clippy::unwrap_used)] if std::env::var(CODEX_SANDBOX_NETWORK_DISABLED_ENV_VAR).is_ok() { println!( "Skipping test because it cannot execute when network is disabled in a Codex sandbox." @@ -90,13 +88,12 @@ async fn continue_after_stream_error() { config.base_instructions = Some("You are a helpful assistant".to_string()); config.model_provider = provider; - let CodexSpawnOk { codex, .. } = Codex::spawn( - config, - Some(CodexAuth::from_api_key("Test API Key")), - std::sync::Arc::new(tokio::sync::Notify::new()), - ) - .await - .unwrap(); + let conversation_manager = ConversationManager::default(); + let codex = conversation_manager + .new_conversation_with_auth(config, Some(CodexAuth::from_api_key("Test API Key"))) + .await + .unwrap() + .conversation; codex .submit(Op::UserInput { diff --git a/codex-rs/core/tests/stream_no_completed.rs b/codex-rs/core/tests/stream_no_completed.rs index 0ded3337ab..3fb3f642d7 100644 --- a/codex-rs/core/tests/stream_no_completed.rs +++ b/codex-rs/core/tests/stream_no_completed.rs @@ -3,8 +3,7 @@ use std::time::Duration; -use codex_core::Codex; -use codex_core::CodexSpawnOk; +use codex_core::ConversationManager; use codex_core::ModelProviderInfo; use codex_core::protocol::EventMsg; use codex_core::protocol::InputItem; @@ -34,8 +33,6 @@ fn sse_completed(id: &str) -> String { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn retries_on_early_close() { - #![allow(clippy::unwrap_used)] - if std::env::var(CODEX_SANDBOX_NETWORK_DISABLED_ENV_VAR).is_ok() { println!( "Skipping test because it cannot execute when network is disabled in a Codex sandbox." @@ -93,17 +90,15 @@ async fn retries_on_early_close() { requires_openai_auth: false, }; - let ctrl_c = std::sync::Arc::new(tokio::sync::Notify::new()); let codex_home = TempDir::new().unwrap(); let mut config = load_default_config_for_test(&codex_home); config.model_provider = model_provider; - let CodexSpawnOk { codex, .. } = Codex::spawn( - config, - Some(CodexAuth::from_api_key("Test API Key")), - ctrl_c, - ) - .await - .unwrap(); + let conversation_manager = ConversationManager::default(); + let codex = conversation_manager + .new_conversation_with_auth(config, Some(CodexAuth::from_api_key("Test API Key"))) + .await + .unwrap() + .conversation; codex .submit(Op::UserInput { diff --git a/codex-rs/exec/Cargo.toml b/codex-rs/exec/Cargo.toml index aee480d7b4..89dc3951e7 100644 --- a/codex-rs/exec/Cargo.toml +++ b/codex-rs/exec/Cargo.toml @@ -26,6 +26,7 @@ codex-common = { path = "../common", features = [ ] } codex-core = { path = "../core" } codex-ollama = { path = "../ollama" } +codex-protocol = { path = "../protocol" } owo-colors = "4.2.0" serde_json = "1" shlex = "1.3.0" @@ -41,5 +42,8 @@ tracing-subscriber = { version = "0.3.19", features = ["env-filter"] } [dev-dependencies] assert_cmd = "2" +core_test_support = { path = "../core/tests/common" } +libc = "0.2" predicates = "3" tempfile = "3.13.0" +wiremock = "0.6" diff --git a/codex-rs/exec/src/event_processor_with_human_output.rs b/codex-rs/exec/src/event_processor_with_human_output.rs index a2ae813183..498b1d29a4 100644 --- a/codex-rs/exec/src/event_processor_with_human_output.rs +++ b/codex-rs/exec/src/event_processor_with_human_output.rs @@ -21,6 +21,7 @@ use codex_core::protocol::PatchApplyBeginEvent; use codex_core::protocol::PatchApplyEndEvent; use codex_core::protocol::SessionConfiguredEvent; use codex_core::protocol::TaskCompleteEvent; +use codex_core::protocol::TurnAbortReason; use codex_core::protocol::TurnDiffEvent; use owo_colors::OwoColorize; use owo_colors::Style; @@ -191,7 +192,7 @@ impl EventProcessor for EventProcessorWithHumanOutput { self.answer_started = true; } print!("{delta}"); - #[allow(clippy::expect_used)] + #[expect(clippy::expect_used)] std::io::stdout().flush().expect("could not flush stdout"); } EventMsg::AgentReasoningDelta(AgentReasoningDeltaEvent { delta }) => { @@ -207,7 +208,15 @@ impl EventProcessor for EventProcessorWithHumanOutput { self.reasoning_started = true; } print!("{delta}"); - #[allow(clippy::expect_used)] + #[expect(clippy::expect_used)] + std::io::stdout().flush().expect("could not flush stdout"); + } + EventMsg::AgentReasoningSectionBreak(_) => { + if !self.show_agent_reasoning { + return CodexStatus::Running; + } + println!(); + #[expect(clippy::expect_used)] std::io::stdout().flush().expect("could not flush stdout"); } EventMsg::AgentReasoningRawContent(AgentReasoningRawContentEvent { text }) => { @@ -216,7 +225,7 @@ impl EventProcessor for EventProcessorWithHumanOutput { } if !self.raw_reasoning_started { print!("{text}"); - #[allow(clippy::expect_used)] + #[expect(clippy::expect_used)] std::io::stdout().flush().expect("could not flush stdout"); } else { println!(); @@ -233,7 +242,7 @@ impl EventProcessor for EventProcessorWithHumanOutput { self.raw_reasoning_started = true; } print!("{delta}"); - #[allow(clippy::expect_used)] + #[expect(clippy::expect_used)] std::io::stdout().flush().expect("could not flush stdout"); } EventMsg::AgentMessage(AgentMessageEvent { message }) => { @@ -255,6 +264,7 @@ impl EventProcessor for EventProcessorWithHumanOutput { call_id, command, cwd, + parsed_cmd: _, }) => { self.call_id_to_command.insert( call_id.clone(), @@ -513,6 +523,17 @@ impl EventProcessor for EventProcessorWithHumanOutput { EventMsg::GetHistoryEntryResponse(_) => { // Currently ignored in exec output. } + EventMsg::McpListToolsResponse(_) => { + // Currently ignored in exec output. + } + EventMsg::TurnAborted(abort_reason) => match abort_reason.reason { + TurnAbortReason::Interrupted => { + ts_println!(self, "task interrupted"); + } + TurnAbortReason::Replaced => { + ts_println!(self, "task aborted: replaced by a new task"); + } + }, EventMsg::ShutdownComplete => return CodexStatus::Shutdown, } CodexStatus::Running diff --git a/codex-rs/exec/src/event_processor_with_json_output.rs b/codex-rs/exec/src/event_processor_with_json_output.rs index 76985518e6..11e9732e99 100644 --- a/codex-rs/exec/src/event_processor_with_json_output.rs +++ b/codex-rs/exec/src/event_processor_with_json_output.rs @@ -28,7 +28,7 @@ impl EventProcessor for EventProcessorWithJsonOutput { .into_iter() .map(|(key, value)| (key.to_string(), value)) .collect::>(); - #[allow(clippy::expect_used)] + #[expect(clippy::expect_used)] let config_json = serde_json::to_string(&entries).expect("Failed to serialize config summary to JSON"); println!("{config_json}"); diff --git a/codex-rs/exec/src/lib.rs b/codex-rs/exec/src/lib.rs index 6ed57898b2..e18314fbc7 100644 --- a/codex-rs/exec/src/lib.rs +++ b/codex-rs/exec/src/lib.rs @@ -6,15 +6,13 @@ mod event_processor_with_json_output; use std::io::IsTerminal; use std::io::Read; use std::path::PathBuf; -use std::sync::Arc; pub use cli::Cli; use codex_core::BUILT_IN_OSS_MODEL_PROVIDER_ID; -use codex_core::codex_wrapper::CodexConversation; -use codex_core::codex_wrapper::{self}; +use codex_core::ConversationManager; +use codex_core::NewConversation; use codex_core::config::Config; use codex_core::config::ConfigOverrides; -use codex_core::config_types::SandboxMode; use codex_core::protocol::AskForApproval; use codex_core::protocol::Event; use codex_core::protocol::EventMsg; @@ -23,6 +21,7 @@ use codex_core::protocol::Op; use codex_core::protocol::TaskCompleteEvent; use codex_core::util::is_inside_git_repo; use codex_ollama::DEFAULT_OSS_MODEL; +use codex_protocol::config_types::SandboxMode; use event_processor_with_human_output::EventProcessorWithHumanOutput; use event_processor_with_json_output::EventProcessorWithJsonOutput; use tracing::debug; @@ -147,6 +146,7 @@ pub async fn run_main(cli: Cli, codex_linux_sandbox_exe: Option) -> any codex_linux_sandbox_exe, base_instructions: None, include_plan_tool: None, + include_apply_patch_tool: None, disable_response_storage: oss.then_some(true), show_raw_agent_reasoning: oss.then_some(true), }; @@ -185,35 +185,30 @@ pub async fn run_main(cli: Cli, codex_linux_sandbox_exe: Option) -> any std::process::exit(1); } - let CodexConversation { - codex: codex_wrapper, + let conversation_manager = ConversationManager::default(); + let NewConversation { + conversation_id: _, + conversation, session_configured, - ctrl_c, - .. - } = codex_wrapper::init_codex(config).await?; - let codex = Arc::new(codex_wrapper); + } = conversation_manager.new_conversation(config).await?; info!("Codex initialized with event: {session_configured:?}"); let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel::(); { - let codex = codex.clone(); + let conversation = conversation.clone(); tokio::spawn(async move { loop { - let interrupted = ctrl_c.notified(); tokio::select! { - _ = interrupted => { - // Forward an interrupt to the codex so it can abort any in‑flight task. - let _ = codex - .submit( - Op::Interrupt, - ) - .await; + _ = tokio::signal::ctrl_c() => { + tracing::debug!("Keyboard interrupt"); + // Immediately notify Codex to abort any in‑flight task. + conversation.submit(Op::Interrupt).await.ok(); - // Exit the inner loop and return to the main input prompt. The codex + // Exit the inner loop and return to the main input prompt. The codex // will emit a `TurnInterrupted` (Error) event which is drained later. break; } - res = codex.next_event() => match res { + res = conversation.next_event() => match res { Ok(event) => { debug!("Received event: {event:?}"); @@ -243,9 +238,9 @@ pub async fn run_main(cli: Cli, codex_linux_sandbox_exe: Option) -> any .into_iter() .map(|path| InputItem::LocalImage { path }) .collect(); - let initial_images_event_id = codex.submit(Op::UserInput { items }).await?; + let initial_images_event_id = conversation.submit(Op::UserInput { items }).await?; info!("Sent images with event ID: {initial_images_event_id}"); - while let Ok(event) = codex.next_event().await { + while let Ok(event) = conversation.next_event().await { if event.id == initial_images_event_id && matches!( event.msg, @@ -261,7 +256,7 @@ pub async fn run_main(cli: Cli, codex_linux_sandbox_exe: Option) -> any // Send the prompt. let items: Vec = vec![InputItem::Text { text: prompt }]; - let initial_prompt_task_id = codex.submit(Op::UserInput { items }).await?; + let initial_prompt_task_id = conversation.submit(Op::UserInput { items }).await?; info!("Sent prompt with event ID: {initial_prompt_task_id}"); // Run the loop until the task is complete. @@ -270,7 +265,7 @@ pub async fn run_main(cli: Cli, codex_linux_sandbox_exe: Option) -> any match shutdown { CodexStatus::Running => continue, CodexStatus::InitiateShutdown => { - codex.submit(Op::Shutdown).await?; + conversation.submit(Op::Shutdown).await?; } CodexStatus::Shutdown => { break; diff --git a/codex-rs/exec/tests/apply_patch.rs b/codex-rs/exec/tests/apply_patch.rs index f65d32e1c8..ecce43d732 100644 --- a/codex-rs/exec/tests/apply_patch.rs +++ b/codex-rs/exec/tests/apply_patch.rs @@ -1,3 +1,5 @@ +#![allow(clippy::expect_used, clippy::unwrap_used)] + use anyhow::Context; use assert_cmd::prelude::*; use codex_core::CODEX_APPLY_PATCH_ARG1; @@ -37,3 +39,152 @@ fn test_standalone_exec_cli_can_use_apply_patch() -> anyhow::Result<()> { ); Ok(()) } + +#[cfg(not(target_os = "windows"))] +#[tokio::test] +async fn test_apply_patch_tool() -> anyhow::Result<()> { + use core_test_support::load_sse_fixture_with_id_from_str; + use tempfile::TempDir; + use wiremock::Mock; + use wiremock::MockServer; + use wiremock::ResponseTemplate; + use wiremock::matchers::method; + use wiremock::matchers::path; + + const SSE_TOOL_CALL_ADD: &str = r#"[ + { + "type": "response.output_item.done", + "item": { + "type": "function_call", + "name": "apply_patch", + "arguments": "{\n \"input\": \"*** Begin Patch\\n*** Add File: test.md\\n+Hello world\\n*** End Patch\"\n}", + "call_id": "__ID__" + } + }, + { + "type": "response.completed", + "response": { + "id": "__ID__", + "usage": { + "input_tokens": 0, + "input_tokens_details": null, + "output_tokens": 0, + "output_tokens_details": null, + "total_tokens": 0 + }, + "output": [] + } + } +]"#; + + const SSE_TOOL_CALL_UPDATE: &str = r#"[ + { + "type": "response.output_item.done", + "item": { + "type": "function_call", + "name": "apply_patch", + "arguments": "{\n \"input\": \"*** Begin Patch\\n*** Update File: test.md\\n@@\\n-Hello world\\n+Final text\\n*** End Patch\"\n}", + "call_id": "__ID__" + } + }, + { + "type": "response.completed", + "response": { + "id": "__ID__", + "usage": { + "input_tokens": 0, + "input_tokens_details": null, + "output_tokens": 0, + "output_tokens_details": null, + "total_tokens": 0 + }, + "output": [] + } + } +]"#; + + const SSE_TOOL_CALL_COMPLETED: &str = r#"[ + { + "type": "response.completed", + "response": { + "id": "__ID__", + "usage": { + "input_tokens": 0, + "input_tokens_details": null, + "output_tokens": 0, + "output_tokens_details": null, + "total_tokens": 0 + }, + "output": [] + } + } +]"#; + + // Start a mock model server + let server = MockServer::start().await; + + // First response: model calls apply_patch to create test.md + let first = ResponseTemplate::new(200) + .insert_header("content-type", "text/event-stream") + .set_body_raw( + load_sse_fixture_with_id_from_str(SSE_TOOL_CALL_ADD, "call1"), + "text/event-stream", + ); + + Mock::given(method("POST")) + // .and(path("/v1/responses")) + .respond_with(first) + .up_to_n_times(1) + .mount(&server) + .await; + + // Second response: model calls apply_patch to update test.md + let second = ResponseTemplate::new(200) + .insert_header("content-type", "text/event-stream") + .set_body_raw( + load_sse_fixture_with_id_from_str(SSE_TOOL_CALL_UPDATE, "call2"), + "text/event-stream", + ); + + Mock::given(method("POST")) + .and(path("/v1/responses")) + .respond_with(second) + .up_to_n_times(1) + .mount(&server) + .await; + + let final_completed = ResponseTemplate::new(200) + .insert_header("content-type", "text/event-stream") + .set_body_raw( + load_sse_fixture_with_id_from_str(SSE_TOOL_CALL_COMPLETED, "resp3"), + "text/event-stream", + ); + + Mock::given(method("POST")) + // .and(path("/v1/responses")) + .respond_with(final_completed) + .expect(1) + .mount(&server) + .await; + + let tmp_cwd = TempDir::new().unwrap(); + Command::cargo_bin("codex-exec") + .context("should find binary for codex-exec")? + .current_dir(tmp_cwd.path()) + .env("CODEX_HOME", tmp_cwd.path()) + .env("OPENAI_API_KEY", "dummy") + .env("OPENAI_BASE_URL", format!("{}/v1", server.uri())) + .arg("--skip-git-repo-check") + .arg("-s") + .arg("workspace-write") + .arg("foo") + .assert() + .success(); + + // Verify final file contents + let final_path = tmp_cwd.path().join("test.md"); + let contents = std::fs::read_to_string(&final_path) + .unwrap_or_else(|e| panic!("failed reading {}: {e}", final_path.display())); + assert_eq!(contents, "Final text\n"); + Ok(()) +} diff --git a/codex-rs/exec/tests/sandbox.rs b/codex-rs/exec/tests/sandbox.rs new file mode 100644 index 0000000000..a98e0a9760 --- /dev/null +++ b/codex-rs/exec/tests/sandbox.rs @@ -0,0 +1,219 @@ +#![cfg(unix)] +use codex_core::protocol::SandboxPolicy; +use codex_core::spawn::StdioPolicy; +use std::collections::HashMap; +use std::future::Future; +use std::io; +use std::path::PathBuf; +use std::process::ExitStatus; +use tokio::process::Child; + +#[cfg(target_os = "macos")] +async fn spawn_command_under_sandbox( + command: Vec, + sandbox_policy: &SandboxPolicy, + cwd: PathBuf, + stdio_policy: StdioPolicy, + env: HashMap, +) -> std::io::Result { + use codex_core::seatbelt::spawn_command_under_seatbelt; + spawn_command_under_seatbelt(command, sandbox_policy, cwd, stdio_policy, env).await +} + +#[cfg(target_os = "linux")] +async fn spawn_command_under_sandbox( + command: Vec, + sandbox_policy: &SandboxPolicy, + cwd: PathBuf, + stdio_policy: StdioPolicy, + env: HashMap, +) -> std::io::Result { + use codex_core::landlock::spawn_command_under_linux_sandbox; + let codex_linux_sandbox_exe = assert_cmd::cargo::cargo_bin("codex-exec"); + spawn_command_under_linux_sandbox( + codex_linux_sandbox_exe, + command, + sandbox_policy, + cwd, + stdio_policy, + env, + ) + .await +} + +#[tokio::test] +async fn python_multiprocessing_lock_works_under_sandbox() { + #[cfg(target_os = "macos")] + let writable_roots = Vec::::new(); + + // From https://man7.org/linux/man-pages/man7/sem_overview.7.html + // + // > On Linux, named semaphores are created in a virtual filesystem, + // > normally mounted under /dev/shm. + #[cfg(target_os = "linux")] + let writable_roots = vec![PathBuf::from("/dev/shm")]; + + let policy = SandboxPolicy::WorkspaceWrite { + writable_roots, + network_access: false, + exclude_tmpdir_env_var: false, + exclude_slash_tmp: false, + }; + + let python_code = r#"import multiprocessing +from multiprocessing import Lock, Process + +def f(lock): + with lock: + print("Lock acquired in child process") + +if __name__ == '__main__': + lock = Lock() + p = Process(target=f, args=(lock,)) + p.start() + p.join() +"#; + + let mut child = spawn_command_under_sandbox( + vec![ + "python3".to_string(), + "-c".to_string(), + python_code.to_string(), + ], + &policy, + std::env::current_dir().expect("should be able to get current dir"), + StdioPolicy::Inherit, + HashMap::new(), + ) + .await + .expect("should be able to spawn python under sandbox"); + + let status = child.wait().await.expect("should wait for child process"); + assert!(status.success(), "python exited with {status:?}"); +} + +fn unix_sock_body() { + unsafe { + let mut fds = [0i32; 2]; + let r = libc::socketpair(libc::AF_UNIX, libc::SOCK_DGRAM, 0, fds.as_mut_ptr()); + assert_eq!( + r, + 0, + "socketpair(AF_UNIX, SOCK_DGRAM) failed: {}", + io::Error::last_os_error() + ); + + let msg = b"hello_unix"; + // write() from one end (generic write is allowed) + let sent = libc::write(fds[0], msg.as_ptr() as *const libc::c_void, msg.len()); + assert!(sent >= 0, "write() failed: {}", io::Error::last_os_error()); + + // recvfrom() on the other end. We don’t need the address for socketpair, + // so we pass null pointers for src address. + let mut buf = [0u8; 64]; + let recvd = libc::recvfrom( + fds[1], + buf.as_mut_ptr() as *mut libc::c_void, + buf.len(), + 0, + std::ptr::null_mut(), + std::ptr::null_mut(), + ); + assert!( + recvd >= 0, + "recvfrom() failed: {}", + io::Error::last_os_error() + ); + + let recvd_slice = &buf[..(recvd as usize)]; + assert_eq!( + recvd_slice, + &msg[..], + "payload mismatch: sent {} bytes, got {} bytes", + msg.len(), + recvd + ); + + // Also exercise AF_UNIX stream socketpair quickly to ensure AF_UNIX in general works. + let mut sfds = [0i32; 2]; + let sr = libc::socketpair(libc::AF_UNIX, libc::SOCK_STREAM, 0, sfds.as_mut_ptr()); + assert_eq!( + sr, + 0, + "socketpair(AF_UNIX, SOCK_STREAM) failed: {}", + io::Error::last_os_error() + ); + let snt2 = libc::write(sfds[0], msg.as_ptr() as *const libc::c_void, msg.len()); + assert!( + snt2 >= 0, + "write(stream) failed: {}", + io::Error::last_os_error() + ); + let mut b2 = [0u8; 64]; + let rcv2 = libc::recv(sfds[1], b2.as_mut_ptr() as *mut libc::c_void, b2.len(), 0); + assert!( + rcv2 >= 0, + "recv(stream) failed: {}", + io::Error::last_os_error() + ); + + // Clean up + let _ = libc::close(sfds[0]); + let _ = libc::close(sfds[1]); + let _ = libc::close(fds[0]); + let _ = libc::close(fds[1]); + } +} + +#[tokio::test] +async fn allow_unix_socketpair_recvfrom() { + run_code_under_sandbox( + "allow_unix_socketpair_recvfrom", + &SandboxPolicy::ReadOnly, + || async { unix_sock_body() }, + ) + .await + .expect("should be able to reexec"); +} + +const IN_SANDBOX_ENV_VAR: &str = "IN_SANDBOX"; + +#[expect(clippy::expect_used)] +pub async fn run_code_under_sandbox( + test_selector: &str, + policy: &SandboxPolicy, + child_body: F, +) -> io::Result> +where + F: FnOnce() -> Fut + Send + 'static, + Fut: Future + Send + 'static, +{ + if std::env::var(IN_SANDBOX_ENV_VAR).is_err() { + let exe = std::env::current_exe()?; + let mut cmds = vec![exe.to_string_lossy().into_owned(), "--exact".into()]; + let mut stdio_policy = StdioPolicy::RedirectForShellTool; + // Allow for us to pass forward --nocapture / use the right stdio policy. + if std::env::args().any(|a| a == "--nocapture") { + cmds.push("--nocapture".into()); + stdio_policy = StdioPolicy::Inherit; + } + cmds.push(test_selector.into()); + + // Your existing launcher: + let mut child = spawn_command_under_sandbox( + cmds, + policy, + std::env::current_dir().expect("should be able to get current dir"), + stdio_policy, + HashMap::from([("IN_SANDBOX".into(), "1".into())]), + ) + .await?; + + let status = child.wait().await?; + Ok(Some(status)) + } else { + // Child branch: run the provided body. + child_body().await; + Ok(None) + } +} diff --git a/codex-rs/execpolicy/src/execv_checker.rs b/codex-rs/execpolicy/src/execv_checker.rs index 3c9084e825..242ea6d177 100644 --- a/codex-rs/execpolicy/src/execv_checker.rs +++ b/codex-rs/execpolicy/src/execv_checker.rs @@ -140,7 +140,6 @@ fn is_executable_file(path: &str) -> bool { #[cfg(test)] mod tests { - #![allow(clippy::unwrap_used)] use tempfile::TempDir; use super::*; diff --git a/codex-rs/execpolicy/tests/bad.rs b/codex-rs/execpolicy/tests/bad.rs index 5c2999a0cb..8b6e195fb0 100644 --- a/codex-rs/execpolicy/tests/bad.rs +++ b/codex-rs/execpolicy/tests/bad.rs @@ -1,4 +1,3 @@ -#![expect(clippy::expect_used)] use codex_execpolicy::NegativeExamplePassedCheck; use codex_execpolicy::get_default_policy; diff --git a/codex-rs/execpolicy/tests/cp.rs b/codex-rs/execpolicy/tests/cp.rs index 14fd24410f..aa19f0b5d5 100644 --- a/codex-rs/execpolicy/tests/cp.rs +++ b/codex-rs/execpolicy/tests/cp.rs @@ -1,4 +1,3 @@ -#![expect(clippy::expect_used)] extern crate codex_execpolicy; use codex_execpolicy::ArgMatcher; @@ -12,6 +11,7 @@ use codex_execpolicy::Result; use codex_execpolicy::ValidExec; use codex_execpolicy::get_default_policy; +#[expect(clippy::expect_used)] fn setup() -> Policy { get_default_policy().expect("failed to load default policy") } diff --git a/codex-rs/execpolicy/tests/good.rs b/codex-rs/execpolicy/tests/good.rs index 645728afbd..3b7313a335 100644 --- a/codex-rs/execpolicy/tests/good.rs +++ b/codex-rs/execpolicy/tests/good.rs @@ -1,4 +1,3 @@ -#![expect(clippy::expect_used)] use codex_execpolicy::PositiveExampleFailedCheck; use codex_execpolicy::get_default_policy; diff --git a/codex-rs/execpolicy/tests/head.rs b/codex-rs/execpolicy/tests/head.rs index 8d7a165cca..3c32ccfbfe 100644 --- a/codex-rs/execpolicy/tests/head.rs +++ b/codex-rs/execpolicy/tests/head.rs @@ -1,4 +1,3 @@ -#![expect(clippy::expect_used)] use codex_execpolicy::ArgMatcher; use codex_execpolicy::ArgType; use codex_execpolicy::Error; @@ -13,6 +12,7 @@ use codex_execpolicy::get_default_policy; extern crate codex_execpolicy; +#[expect(clippy::expect_used)] fn setup() -> Policy { get_default_policy().expect("failed to load default policy") } diff --git a/codex-rs/execpolicy/tests/literal.rs b/codex-rs/execpolicy/tests/literal.rs index 629aeb9cfe..d849371e3b 100644 --- a/codex-rs/execpolicy/tests/literal.rs +++ b/codex-rs/execpolicy/tests/literal.rs @@ -1,4 +1,3 @@ -#![expect(clippy::expect_used)] use codex_execpolicy::ArgType; use codex_execpolicy::Error; use codex_execpolicy::ExecCall; diff --git a/codex-rs/execpolicy/tests/ls.rs b/codex-rs/execpolicy/tests/ls.rs index 854ec3bf58..e52316c06e 100644 --- a/codex-rs/execpolicy/tests/ls.rs +++ b/codex-rs/execpolicy/tests/ls.rs @@ -1,4 +1,3 @@ -#![expect(clippy::expect_used)] extern crate codex_execpolicy; use codex_execpolicy::ArgType; @@ -12,6 +11,7 @@ use codex_execpolicy::Result; use codex_execpolicy::ValidExec; use codex_execpolicy::get_default_policy; +#[expect(clippy::expect_used)] fn setup() -> Policy { get_default_policy().expect("failed to load default policy") } diff --git a/codex-rs/execpolicy/tests/pwd.rs b/codex-rs/execpolicy/tests/pwd.rs index 339908e928..fdf5a4f1a5 100644 --- a/codex-rs/execpolicy/tests/pwd.rs +++ b/codex-rs/execpolicy/tests/pwd.rs @@ -1,4 +1,3 @@ -#![expect(clippy::expect_used)] extern crate codex_execpolicy; use std::vec; @@ -12,6 +11,7 @@ use codex_execpolicy::PositionalArg; use codex_execpolicy::ValidExec; use codex_execpolicy::get_default_policy; +#[expect(clippy::expect_used)] fn setup() -> Policy { get_default_policy().expect("failed to load default policy") } diff --git a/codex-rs/execpolicy/tests/sed.rs b/codex-rs/execpolicy/tests/sed.rs index 064e539305..bf35bf6d46 100644 --- a/codex-rs/execpolicy/tests/sed.rs +++ b/codex-rs/execpolicy/tests/sed.rs @@ -1,4 +1,3 @@ -#![expect(clippy::expect_used)] extern crate codex_execpolicy; use codex_execpolicy::ArgType; @@ -13,6 +12,7 @@ use codex_execpolicy::Result; use codex_execpolicy::ValidExec; use codex_execpolicy::get_default_policy; +#[expect(clippy::expect_used)] fn setup() -> Policy { get_default_policy().expect("failed to load default policy") } diff --git a/codex-rs/linux-sandbox/Cargo.toml b/codex-rs/linux-sandbox/Cargo.toml index ea7052c409..d769fae2c6 100644 --- a/codex-rs/linux-sandbox/Cargo.toml +++ b/codex-rs/linux-sandbox/Cargo.toml @@ -20,7 +20,7 @@ clap = { version = "4", features = ["derive"] } codex-common = { path = "../common", features = ["cli"] } codex-core = { path = "../core" } landlock = "0.4.1" -libc = "0.2.172" +libc = "0.2.175" seccompiler = "0.5.0" [target.'cfg(target_os = "linux")'.dev-dependencies] diff --git a/codex-rs/linux-sandbox/src/landlock.rs b/codex-rs/linux-sandbox/src/landlock.rs index e13e3c8b75..5bc96130dd 100644 --- a/codex-rs/linux-sandbox/src/landlock.rs +++ b/codex-rs/linux-sandbox/src/landlock.rs @@ -104,7 +104,9 @@ fn install_network_seccomp_filter_on_current_thread() -> std::result::Result<(), deny_syscall(libc::SYS_sendto); deny_syscall(libc::SYS_sendmsg); deny_syscall(libc::SYS_sendmmsg); - deny_syscall(libc::SYS_recvfrom); + // NOTE: allowing recvfrom allows some tools like: `cargo clippy` to run + // with their socketpair + child processes for sub-proc management + // deny_syscall(libc::SYS_recvfrom); deny_syscall(libc::SYS_recvmsg); deny_syscall(libc::SYS_recvmmsg); deny_syscall(libc::SYS_getsockopt); @@ -115,12 +117,12 @@ fn install_network_seccomp_filter_on_current_thread() -> std::result::Result<(), let unix_only_rule = SeccompRule::new(vec![SeccompCondition::new( 0, // first argument (domain) SeccompCmpArgLen::Dword, - SeccompCmpOp::Eq, + SeccompCmpOp::Ne, libc::AF_UNIX as u64, )?])?; - rules.insert(libc::SYS_socket, vec![unix_only_rule]); - rules.insert(libc::SYS_socketpair, vec![]); // always deny (Unix can use socketpair but fine, keep open?) + rules.insert(libc::SYS_socket, vec![unix_only_rule.clone()]); + rules.insert(libc::SYS_socketpair, vec![unix_only_rule]); // always deny (Unix can use socketpair but fine, keep open?) let filter = SeccompFilter::new( rules, diff --git a/codex-rs/linux-sandbox/tests/landlock.rs b/codex-rs/linux-sandbox/tests/landlock.rs index 96298c6563..8faa02d158 100644 --- a/codex-rs/linux-sandbox/tests/landlock.rs +++ b/codex-rs/linux-sandbox/tests/landlock.rs @@ -1,6 +1,4 @@ #![cfg(target_os = "linux")] -#![expect(clippy::unwrap_used, clippy::expect_used)] - use codex_core::config_types::ShellEnvironmentPolicy; use codex_core::error::CodexErr; use codex_core::error::SandboxErr; @@ -11,9 +9,7 @@ use codex_core::exec_env::create_env; use codex_core::protocol::SandboxPolicy; use std::collections::HashMap; use std::path::PathBuf; -use std::sync::Arc; use tempfile::NamedTempFile; -use tokio::sync::Notify; // At least on GitHub CI, the arm64 tests appear to need longer timeouts. @@ -37,7 +33,7 @@ fn create_env_from_core_vars() -> HashMap { create_env(&policy) } -#[allow(clippy::print_stdout)] +#[expect(clippy::print_stdout, clippy::expect_used, clippy::unwrap_used)] async fn run_cmd(cmd: &[&str], writable_roots: &[PathBuf], timeout_ms: u64) { let params = ExecParams { command: cmd.iter().map(|elm| elm.to_string()).collect(), @@ -59,11 +55,9 @@ async fn run_cmd(cmd: &[&str], writable_roots: &[PathBuf], timeout_ms: u64) { }; let sandbox_program = env!("CARGO_BIN_EXE_codex-linux-sandbox"); let codex_linux_sandbox_exe = Some(PathBuf::from(sandbox_program)); - let ctrl_c = Arc::new(Notify::new()); let res = process_exec_tool_call( params, SandboxType::LinuxSeccomp, - ctrl_c, &sandbox_policy, &codex_linux_sandbox_exe, None, @@ -72,8 +66,8 @@ async fn run_cmd(cmd: &[&str], writable_roots: &[PathBuf], timeout_ms: u64) { .unwrap(); if res.exit_code != 0 { - println!("stdout:\n{}", res.stdout); - println!("stderr:\n{}", res.stderr); + println!("stdout:\n{}", res.stdout.text); + println!("stderr:\n{}", res.stderr.text); panic!("exit code: {}", res.exit_code); } } @@ -136,6 +130,7 @@ async fn test_timeout() { /// does NOT succeed (i.e. returns a non‑zero exit code) **unless** the binary /// is missing in which case we silently treat it as an accepted skip so the /// suite remains green on leaner CI images. +#[expect(clippy::expect_used)] async fn assert_network_blocked(cmd: &[&str]) { let cwd = std::env::current_dir().expect("cwd should exist"); let params = ExecParams { @@ -150,13 +145,11 @@ async fn assert_network_blocked(cmd: &[&str]) { }; let sandbox_policy = SandboxPolicy::new_read_only_policy(); - let ctrl_c = Arc::new(Notify::new()); let sandbox_program = env!("CARGO_BIN_EXE_codex-linux-sandbox"); let codex_linux_sandbox_exe: Option = Some(PathBuf::from(sandbox_program)); let result = process_exec_tool_call( params, SandboxType::LinuxSeccomp, - ctrl_c, &sandbox_policy, &codex_linux_sandbox_exe, None, @@ -164,7 +157,7 @@ async fn assert_network_blocked(cmd: &[&str]) { .await; let (exit_code, stdout, stderr) = match result { - Ok(output) => (output.exit_code, output.stdout, output.stderr), + Ok(output) => (output.exit_code, output.stdout.text, output.stderr.text), Err(CodexErr::Sandbox(SandboxErr::Denied(exit_code, stdout, stderr))) => { (exit_code, stdout, stderr) } diff --git a/codex-rs/login/Cargo.toml b/codex-rs/login/Cargo.toml index 85c11505ec..c1e21ca627 100644 --- a/codex-rs/login/Cargo.toml +++ b/codex-rs/login/Cargo.toml @@ -9,11 +9,14 @@ workspace = true [dependencies] base64 = "0.22" chrono = { version = "0.4", features = ["serde"] } -reqwest = { version = "0.12", features = ["json"] } +rand = "0.8" +reqwest = { version = "0.12", features = ["json", "blocking"] } serde = { version = "1", features = ["derive"] } serde_json = "1" +sha2 = "0.10" tempfile = "3" thiserror = "2.0.12" +tiny_http = "0.12" tokio = { version = "1", features = [ "io-std", "macros", @@ -21,6 +24,9 @@ tokio = { version = "1", features = [ "rt-multi-thread", "signal", ] } +url = "2" +urlencoding = "2.1" +webbrowser = "1.0" [dev-dependencies] pretty_assertions = "1.4.1" diff --git a/codex-rs/login/src/assets/success.html b/codex-rs/login/src/assets/success.html new file mode 100644 index 0000000000..eb2a0ee719 --- /dev/null +++ b/codex-rs/login/src/assets/success.html @@ -0,0 +1,198 @@ + + + + + Sign into Codex CLI + + + + +

+
+
+ +
Signed in to Codex CLI
+
+ + +
+
+ + + + \ No newline at end of file diff --git a/codex-rs/login/src/lib.rs b/codex-rs/login/src/lib.rs index a1dad79e9d..327daa7e9b 100644 --- a/codex-rs/login/src/lib.rs +++ b/codex-rs/login/src/lib.rs @@ -1,5 +1,4 @@ use chrono::DateTime; - use chrono::Utc; use serde::Deserialize; use serde::Serialize; @@ -9,30 +8,30 @@ use std::fs::OpenOptions; use std::fs::remove_file; use std::io::Read; use std::io::Write; -use std::io::{self}; #[cfg(unix)] use std::os::unix::fs::OpenOptionsExt; use std::path::Path; use std::path::PathBuf; -use std::process::Child; -use std::process::Stdio; use std::sync::Arc; use std::sync::Mutex; use std::time::Duration; -use tempfile::NamedTempFile; -use tokio::process::Command; +pub use crate::server::LoginServer; +pub use crate::server::ServerOptions; +pub use crate::server::ShutdownHandle; +pub use crate::server::run_login_server; pub use crate::token_data::TokenData; use crate::token_data::parse_id_token; +mod pkce; +mod server; mod token_data; -const SOURCE_FOR_PYTHON_SERVER: &str = include_str!("./login_with_chatgpt.py"); - -const CLIENT_ID: &str = "app_EMoamEEZ73f0CkXaXp7hrann"; +pub const CLIENT_ID: &str = "app_EMoamEEZ73f0CkXaXp7hrann"; pub const OPENAI_API_KEY_ENV_VAR: &str = "OPENAI_API_KEY"; -#[derive(Clone, Debug, PartialEq, Copy)] +#[derive(Clone, Debug, PartialEq, Copy, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] pub enum AuthMode { ApiKey, ChatGPT, @@ -65,8 +64,11 @@ impl CodexAuth { /// Loads the available auth information from the auth.json or /// OPENAI_API_KEY environment variable. - pub fn from_codex_home(codex_home: &Path) -> std::io::Result> { - load_auth(codex_home, true) + pub fn from_codex_home( + codex_home: &Path, + preferred_auth_method: AuthMode, + ) -> std::io::Result> { + load_auth(codex_home, true, preferred_auth_method) } pub async fn get_token_data(&self) -> Result { @@ -167,7 +169,11 @@ impl CodexAuth { } } -fn load_auth(codex_home: &Path, include_env_var: bool) -> std::io::Result> { +fn load_auth( + codex_home: &Path, + include_env_var: bool, + preferred_auth_method: AuthMode, +) -> std::io::Result> { // First, check to see if there is a valid auth.json file. If not, we fall // back to AuthMode::ApiKey using the OPENAI_API_KEY environment variable // (if it is set). @@ -203,7 +209,7 @@ fn load_auth(codex_home: &Path, include_env_var: bool) -> std::io::Result