diff --git a/.github/workflows/cargo-deny.yml b/.github/workflows/cargo-deny.yml new file mode 100644 index 0000000000..e365420cca --- /dev/null +++ b/.github/workflows/cargo-deny.yml @@ -0,0 +1,26 @@ +name: cargo-deny + +on: + pull_request: + push: + branches: + - main + +jobs: + cargo-deny: + runs-on: ubuntu-latest + defaults: + run: + working-directory: ./codex-rs + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + + - name: Run cargo-deny + uses: EmbarkStudios/cargo-deny-action@v1 + with: + rust-version: stable + manifest-path: ./codex-rs/Cargo.toml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 38773bb9f1..724995e9c9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,7 +12,7 @@ jobs: NODE_OPTIONS: --max-old-space-size=4096 steps: - name: Checkout repository - uses: actions/checkout@v5 + uses: actions/checkout@v6 - name: Setup pnpm uses: pnpm/action-setup@v4 diff --git a/.github/workflows/cla.yml b/.github/workflows/cla.yml index 17d54f214a..bab34d0365 100644 --- a/.github/workflows/cla.yml +++ b/.github/workflows/cla.yml @@ -46,6 +46,4 @@ jobs: path-to-document: https://github.com/openai/codex/blob/main/docs/CLA.md path-to-signatures: signatures/cla.json branch: cla-signatures - allowlist: | - codex - dependabot[bot] + allowlist: codex,dependabot,dependabot[bot],github-actions[bot] diff --git a/.github/workflows/codespell.yml b/.github/workflows/codespell.yml index c036581325..bbbb06d06f 100644 --- a/.github/workflows/codespell.yml +++ b/.github/workflows/codespell.yml @@ -18,7 +18,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v5 + uses: actions/checkout@v6 - name: Annotate locations with typos uses: codespell-project/codespell-problem-matcher@b80729f885d32f78a716c2f107b4db1025001c42 # v1 - name: Codespell diff --git a/.github/workflows/issue-deduplicator.yml b/.github/workflows/issue-deduplicator.yml index 579b6a3685..c78b1f3161 100644 --- a/.github/workflows/issue-deduplicator.yml +++ b/.github/workflows/issue-deduplicator.yml @@ -16,7 +16,7 @@ jobs: outputs: codex_output: ${{ steps.codex.outputs.final-message }} steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - name: Prepare Codex inputs env: @@ -46,7 +46,6 @@ jobs: with: openai-api-key: ${{ secrets.CODEX_OPENAI_API_KEY }} allow-users: "*" - model: gpt-5.1 prompt: | You are an assistant that triages new GitHub issues by identifying potential duplicates. diff --git a/.github/workflows/issue-labeler.yml b/.github/workflows/issue-labeler.yml index 39f9d47f14..424c7e7263 100644 --- a/.github/workflows/issue-labeler.yml +++ b/.github/workflows/issue-labeler.yml @@ -16,7 +16,7 @@ jobs: outputs: codex_output: ${{ steps.codex.outputs.final-message }} steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - id: codex uses: openai/codex-action@main diff --git a/.github/workflows/rust-ci.yml b/.github/workflows/rust-ci.yml index 0bd91ca53b..08c39db69d 100644 --- a/.github/workflows/rust-ci.yml +++ b/.github/workflows/rust-ci.yml @@ -17,7 +17,7 @@ jobs: codex: ${{ steps.detect.outputs.codex }} workflows: ${{ steps.detect.outputs.workflows }} steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 with: fetch-depth: 0 - name: Detect changed paths (no external action) @@ -56,7 +56,7 @@ jobs: run: working-directory: codex-rs steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - uses: dtolnay/rust-toolchain@1.90 with: components: rustfmt @@ -74,7 +74,7 @@ jobs: run: working-directory: codex-rs steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - uses: dtolnay/rust-toolchain@1.90 - uses: taiki-e/install-action@44c6d64aa62cd779e873306675c7a58e86d6d532 # v2 with: @@ -147,12 +147,21 @@ jobs: profile: release steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - uses: dtolnay/rust-toolchain@1.90 with: targets: ${{ matrix.target }} components: clippy + - name: Compute lockfile hash + id: lockhash + working-directory: codex-rs + shell: bash + run: | + set -euo pipefail + echo "hash=$(sha256sum Cargo.lock | cut -d' ' -f1)" >> "$GITHUB_OUTPUT" + echo "toolchain_hash=$(sha256sum rust-toolchain.toml | cut -d' ' -f1)" >> "$GITHUB_OUTPUT" + # Explicit cache restore: split cargo home vs target, so we can # avoid caching the large target dir on the gnu-dev job. - name: Restore cargo home cache @@ -164,7 +173,7 @@ jobs: ~/.cargo/registry/index/ ~/.cargo/registry/cache/ ~/.cargo/git/db/ - key: cargo-home-${{ matrix.runner }}-${{ matrix.target }}-${{ matrix.profile }}-${{ hashFiles('**/Cargo.lock') }}-${{ hashFiles('codex-rs/rust-toolchain.toml') }} + key: cargo-home-${{ matrix.runner }}-${{ matrix.target }}-${{ matrix.profile }}-${{ steps.lockhash.outputs.hash }}-${{ steps.lockhash.outputs.toolchain_hash }} restore-keys: | cargo-home-${{ matrix.runner }}-${{ matrix.target }}-${{ matrix.profile }}- @@ -201,9 +210,9 @@ jobs: uses: actions/cache/restore@v4 with: path: ${{ github.workspace }}/.sccache/ - key: sccache-${{ matrix.runner }}-${{ matrix.target }}-${{ matrix.profile }}-${{ hashFiles('**/Cargo.lock') }}-${{ github.run_id }} + key: sccache-${{ matrix.runner }}-${{ matrix.target }}-${{ matrix.profile }}-${{ steps.lockhash.outputs.hash }}-${{ github.run_id }} restore-keys: | - sccache-${{ matrix.runner }}-${{ matrix.target }}-${{ matrix.profile }}-${{ hashFiles('**/Cargo.lock') }}- + sccache-${{ matrix.runner }}-${{ matrix.target }}-${{ matrix.profile }}-${{ steps.lockhash.outputs.hash }}- sccache-${{ matrix.runner }}-${{ matrix.target }}-${{ matrix.profile }}- - if: ${{ matrix.target == 'x86_64-unknown-linux-musl' || matrix.target == 'aarch64-unknown-linux-musl'}} @@ -278,7 +287,7 @@ jobs: ~/.cargo/registry/index/ ~/.cargo/registry/cache/ ~/.cargo/git/db/ - key: cargo-home-${{ matrix.runner }}-${{ matrix.target }}-${{ matrix.profile }}-${{ hashFiles('**/Cargo.lock') }}-${{ hashFiles('codex-rs/rust-toolchain.toml') }} + key: cargo-home-${{ matrix.runner }}-${{ matrix.target }}-${{ matrix.profile }}-${{ steps.lockhash.outputs.hash }}-${{ steps.lockhash.outputs.toolchain_hash }} - name: Save sccache cache (fallback) if: always() && !cancelled() && env.USE_SCCACHE == 'true' && env.SCCACHE_GHA_ENABLED != 'true' @@ -286,7 +295,7 @@ jobs: uses: actions/cache/save@v4 with: path: ${{ github.workspace }}/.sccache/ - key: sccache-${{ matrix.runner }}-${{ matrix.target }}-${{ matrix.profile }}-${{ hashFiles('**/Cargo.lock') }}-${{ github.run_id }} + key: sccache-${{ matrix.runner }}-${{ matrix.target }}-${{ matrix.profile }}-${{ steps.lockhash.outputs.hash }}-${{ github.run_id }} - name: sccache stats if: always() && env.USE_SCCACHE == 'true' @@ -359,11 +368,20 @@ jobs: profile: dev steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - uses: dtolnay/rust-toolchain@1.90 with: targets: ${{ matrix.target }} + - name: Compute lockfile hash + id: lockhash + working-directory: codex-rs + shell: bash + run: | + set -euo pipefail + echo "hash=$(sha256sum Cargo.lock | cut -d' ' -f1)" >> "$GITHUB_OUTPUT" + echo "toolchain_hash=$(sha256sum rust-toolchain.toml | cut -d' ' -f1)" >> "$GITHUB_OUTPUT" + - name: Restore cargo home cache id: cache_cargo_home_restore uses: actions/cache/restore@v4 @@ -373,7 +391,7 @@ jobs: ~/.cargo/registry/index/ ~/.cargo/registry/cache/ ~/.cargo/git/db/ - key: cargo-home-${{ matrix.runner }}-${{ matrix.target }}-${{ matrix.profile }}-${{ hashFiles('**/Cargo.lock') }}-${{ hashFiles('codex-rs/rust-toolchain.toml') }} + key: cargo-home-${{ matrix.runner }}-${{ matrix.target }}-${{ matrix.profile }}-${{ steps.lockhash.outputs.hash }}-${{ steps.lockhash.outputs.toolchain_hash }} restore-keys: | cargo-home-${{ matrix.runner }}-${{ matrix.target }}-${{ matrix.profile }}- @@ -409,9 +427,9 @@ jobs: uses: actions/cache/restore@v4 with: path: ${{ github.workspace }}/.sccache/ - key: sccache-${{ matrix.runner }}-${{ matrix.target }}-${{ matrix.profile }}-${{ hashFiles('**/Cargo.lock') }}-${{ github.run_id }} + key: sccache-${{ matrix.runner }}-${{ matrix.target }}-${{ matrix.profile }}-${{ steps.lockhash.outputs.hash }}-${{ github.run_id }} restore-keys: | - sccache-${{ matrix.runner }}-${{ matrix.target }}-${{ matrix.profile }}-${{ hashFiles('**/Cargo.lock') }}- + sccache-${{ matrix.runner }}-${{ matrix.target }}-${{ matrix.profile }}-${{ steps.lockhash.outputs.hash }}- sccache-${{ matrix.runner }}-${{ matrix.target }}-${{ matrix.profile }}- - uses: taiki-e/install-action@44c6d64aa62cd779e873306675c7a58e86d6d532 # v2 @@ -436,7 +454,7 @@ jobs: ~/.cargo/registry/index/ ~/.cargo/registry/cache/ ~/.cargo/git/db/ - key: cargo-home-${{ matrix.runner }}-${{ matrix.target }}-${{ matrix.profile }}-${{ hashFiles('**/Cargo.lock') }}-${{ hashFiles('codex-rs/rust-toolchain.toml') }} + key: cargo-home-${{ matrix.runner }}-${{ matrix.target }}-${{ matrix.profile }}-${{ steps.lockhash.outputs.hash }}-${{ steps.lockhash.outputs.toolchain_hash }} - name: Save sccache cache (fallback) if: always() && !cancelled() && env.USE_SCCACHE == 'true' && env.SCCACHE_GHA_ENABLED != 'true' @@ -444,7 +462,7 @@ jobs: uses: actions/cache/save@v4 with: path: ${{ github.workspace }}/.sccache/ - key: sccache-${{ matrix.runner }}-${{ matrix.target }}-${{ matrix.profile }}-${{ hashFiles('**/Cargo.lock') }}-${{ github.run_id }} + key: sccache-${{ matrix.runner }}-${{ matrix.target }}-${{ matrix.profile }}-${{ steps.lockhash.outputs.hash }}-${{ github.run_id }} - name: sccache stats if: always() && env.USE_SCCACHE == 'true' diff --git a/.github/workflows/rust-release.yml b/.github/workflows/rust-release.yml index 6f27fbf543..14f8aa0327 100644 --- a/.github/workflows/rust-release.yml +++ b/.github/workflows/rust-release.yml @@ -19,7 +19,7 @@ jobs: tag-check: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - name: Validate tag matches Cargo.toml version shell: bash @@ -76,7 +76,7 @@ jobs: target: aarch64-pc-windows-msvc steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - uses: dtolnay/rust-toolchain@1.90 with: targets: ${{ matrix.target }} @@ -371,8 +371,19 @@ jobs: path: | codex-rs/dist/${{ matrix.target }}/* + shell-tool-mcp: + name: shell-tool-mcp + needs: tag-check + uses: ./.github/workflows/shell-tool-mcp.yml + with: + release-tag: ${{ github.ref_name }} + publish: true + secrets: inherit + release: - needs: build + needs: + - build + - shell-tool-mcp name: release runs-on: ubuntu-latest permissions: @@ -386,7 +397,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v5 + uses: actions/checkout@v6 - uses: actions/download-artifact@v4 with: @@ -395,6 +406,14 @@ jobs: - name: List run: ls -R dist/ + # This is a temporary fix: we should modify shell-tool-mcp.yml so these + # files do not end up in dist/ in the first place. + - name: Delete entries from dist/ that should not go in the release + run: | + rm -rf dist/shell-tool-mcp* + + ls -R dist/ + - name: Define release name id: release_name run: | diff --git a/.github/workflows/sdk.yml b/.github/workflows/sdk.yml index 0f3a7a194b..d0c9f6eb95 100644 --- a/.github/workflows/sdk.yml +++ b/.github/workflows/sdk.yml @@ -11,7 +11,7 @@ jobs: timeout-minutes: 10 steps: - name: Checkout repository - uses: actions/checkout@v5 + uses: actions/checkout@v6 - name: Setup pnpm uses: pnpm/action-setup@v4 diff --git a/.github/workflows/shell-tool-mcp-ci.yml b/.github/workflows/shell-tool-mcp-ci.yml new file mode 100644 index 0000000000..5e09459640 --- /dev/null +++ b/.github/workflows/shell-tool-mcp-ci.yml @@ -0,0 +1,48 @@ +name: shell-tool-mcp CI + +on: + push: + paths: + - "shell-tool-mcp/**" + - ".github/workflows/shell-tool-mcp-ci.yml" + - "pnpm-lock.yaml" + - "pnpm-workspace.yaml" + pull_request: + paths: + - "shell-tool-mcp/**" + - ".github/workflows/shell-tool-mcp-ci.yml" + - "pnpm-lock.yaml" + - "pnpm-workspace.yaml" + +env: + NODE_VERSION: 22 + +jobs: + test: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Setup pnpm + uses: pnpm/action-setup@v4 + with: + run_install: false + + - name: Setup Node.js + uses: actions/setup-node@v5 + with: + node-version: ${{ env.NODE_VERSION }} + cache: "pnpm" + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + - name: Format check + run: pnpm --filter @openai/codex-shell-tool-mcp run format + + - name: Run tests + run: pnpm --filter @openai/codex-shell-tool-mcp test + + - name: Build + run: pnpm --filter @openai/codex-shell-tool-mcp run build diff --git a/.github/workflows/shell-tool-mcp.yml b/.github/workflows/shell-tool-mcp.yml new file mode 100644 index 0000000000..63ce308d10 --- /dev/null +++ b/.github/workflows/shell-tool-mcp.yml @@ -0,0 +1,405 @@ +name: shell-tool-mcp + +on: + workflow_call: + inputs: + release-version: + description: Version to publish (x.y.z or x.y.z-alpha.N). Defaults to GITHUB_REF_NAME when it starts with rust-v. + required: false + type: string + release-tag: + description: Tag name to use when downloading release artifacts (defaults to rust-v). + required: false + type: string + publish: + description: Whether to publish to npm when the version is releasable. + required: false + default: true + type: boolean + +env: + NODE_VERSION: 22 + +jobs: + metadata: + runs-on: ubuntu-latest + outputs: + version: ${{ steps.compute.outputs.version }} + release_tag: ${{ steps.compute.outputs.release_tag }} + should_publish: ${{ steps.compute.outputs.should_publish }} + npm_tag: ${{ steps.compute.outputs.npm_tag }} + steps: + - name: Compute version and tags + id: compute + run: | + set -euo pipefail + + version="${{ inputs.release-version }}" + release_tag="${{ inputs.release-tag }}" + + if [[ -z "$version" ]]; then + if [[ -n "$release_tag" && "$release_tag" =~ ^rust-v.+ ]]; then + version="${release_tag#rust-v}" + elif [[ "${GITHUB_REF_NAME:-}" =~ ^rust-v.+ ]]; then + version="${GITHUB_REF_NAME#rust-v}" + release_tag="${GITHUB_REF_NAME}" + else + echo "release-version is required when GITHUB_REF_NAME is not a rust-v tag." + exit 1 + fi + fi + + if [[ -z "$release_tag" ]]; then + release_tag="rust-v${version}" + fi + + npm_tag="" + should_publish="false" + if [[ "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + should_publish="true" + elif [[ "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+-alpha\.[0-9]+$ ]]; then + should_publish="true" + npm_tag="alpha" + fi + + echo "version=${version}" >> "$GITHUB_OUTPUT" + echo "release_tag=${release_tag}" >> "$GITHUB_OUTPUT" + echo "npm_tag=${npm_tag}" >> "$GITHUB_OUTPUT" + echo "should_publish=${should_publish}" >> "$GITHUB_OUTPUT" + + rust-binaries: + name: Build Rust - ${{ matrix.target }} + needs: metadata + runs-on: ${{ matrix.runner }} + timeout-minutes: 30 + defaults: + run: + working-directory: codex-rs + strategy: + fail-fast: false + matrix: + include: + - runner: macos-15-xlarge + target: aarch64-apple-darwin + - runner: macos-15-xlarge + target: x86_64-apple-darwin + - runner: ubuntu-24.04 + target: x86_64-unknown-linux-musl + install_musl: true + - runner: ubuntu-24.04-arm + target: aarch64-unknown-linux-musl + install_musl: true + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - uses: dtolnay/rust-toolchain@1.90 + with: + targets: ${{ matrix.target }} + + - if: ${{ matrix.install_musl }} + name: Install musl build dependencies + run: | + sudo apt-get update + sudo apt-get install -y musl-tools pkg-config + + - name: Build exec server binaries + run: cargo build --release --target ${{ matrix.target }} --bin codex-exec-mcp-server --bin codex-execve-wrapper + + - name: Stage exec server binaries + run: | + dest="${GITHUB_WORKSPACE}/artifacts/vendor/${{ matrix.target }}" + mkdir -p "$dest" + cp "target/${{ matrix.target }}/release/codex-exec-mcp-server" "$dest/" + cp "target/${{ matrix.target }}/release/codex-execve-wrapper" "$dest/" + + - uses: actions/upload-artifact@v5 + with: + name: shell-tool-mcp-rust-${{ matrix.target }} + path: artifacts/** + if-no-files-found: error + + bash-linux: + name: Build Bash (Linux) - ${{ matrix.variant }} - ${{ matrix.target }} + needs: metadata + runs-on: ${{ matrix.runner }} + timeout-minutes: 30 + container: + image: ${{ matrix.image }} + strategy: + fail-fast: false + matrix: + include: + - runner: ubuntu-24.04 + target: x86_64-unknown-linux-musl + variant: ubuntu-24.04 + image: ubuntu:24.04 + - runner: ubuntu-24.04 + target: x86_64-unknown-linux-musl + variant: ubuntu-22.04 + image: ubuntu:22.04 + - runner: ubuntu-24.04 + target: x86_64-unknown-linux-musl + variant: debian-12 + image: debian:12 + - runner: ubuntu-24.04 + target: x86_64-unknown-linux-musl + variant: debian-11 + image: debian:11 + - runner: ubuntu-24.04 + target: x86_64-unknown-linux-musl + variant: centos-9 + image: quay.io/centos/centos:stream9 + - runner: ubuntu-24.04-arm + target: aarch64-unknown-linux-musl + variant: ubuntu-24.04 + image: arm64v8/ubuntu:24.04 + - runner: ubuntu-24.04-arm + target: aarch64-unknown-linux-musl + variant: ubuntu-22.04 + image: arm64v8/ubuntu:22.04 + - runner: ubuntu-24.04-arm + target: aarch64-unknown-linux-musl + variant: ubuntu-20.04 + image: arm64v8/ubuntu:20.04 + - runner: ubuntu-24.04-arm + target: aarch64-unknown-linux-musl + variant: debian-12 + image: arm64v8/debian:12 + - runner: ubuntu-24.04-arm + target: aarch64-unknown-linux-musl + variant: debian-11 + image: arm64v8/debian:11 + - runner: ubuntu-24.04-arm + target: aarch64-unknown-linux-musl + variant: centos-9 + image: quay.io/centos/centos:stream9 + steps: + - name: Install build prerequisites + shell: bash + run: | + set -euo pipefail + if command -v apt-get >/dev/null 2>&1; then + apt-get update + DEBIAN_FRONTEND=noninteractive apt-get install -y git build-essential bison autoconf gettext + elif command -v dnf >/dev/null 2>&1; then + dnf install -y git gcc gcc-c++ make bison autoconf gettext + elif command -v yum >/dev/null 2>&1; then + yum install -y git gcc gcc-c++ make bison autoconf gettext + else + echo "Unsupported package manager in container" + exit 1 + fi + + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Build patched Bash + shell: bash + run: | + set -euo pipefail + git clone --depth 1 https://github.com/bminor/bash /tmp/bash + cd /tmp/bash + git fetch --depth 1 origin a8a1c2fac029404d3f42cd39f5a20f24b6e4fe4b + git checkout a8a1c2fac029404d3f42cd39f5a20f24b6e4fe4b + git apply "${GITHUB_WORKSPACE}/shell-tool-mcp/patches/bash-exec-wrapper.patch" + ./configure --without-bash-malloc + cores="$(command -v nproc >/dev/null 2>&1 && nproc || getconf _NPROCESSORS_ONLN)" + make -j"${cores}" + + dest="${GITHUB_WORKSPACE}/artifacts/vendor/${{ matrix.target }}/bash/${{ matrix.variant }}" + mkdir -p "$dest" + cp bash "$dest/bash" + + - uses: actions/upload-artifact@v5 + with: + name: shell-tool-mcp-bash-${{ matrix.target }}-${{ matrix.variant }} + path: artifacts/** + if-no-files-found: error + + bash-darwin: + name: Build Bash (macOS) - ${{ matrix.variant }} - ${{ matrix.target }} + needs: metadata + runs-on: ${{ matrix.runner }} + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + include: + - runner: macos-15-xlarge + target: aarch64-apple-darwin + variant: macos-15 + - runner: macos-14 + target: aarch64-apple-darwin + variant: macos-14 + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Build patched Bash + shell: bash + run: | + set -euo pipefail + git clone --depth 1 https://github.com/bminor/bash /tmp/bash + cd /tmp/bash + git fetch --depth 1 origin a8a1c2fac029404d3f42cd39f5a20f24b6e4fe4b + git checkout a8a1c2fac029404d3f42cd39f5a20f24b6e4fe4b + git apply "${GITHUB_WORKSPACE}/shell-tool-mcp/patches/bash-exec-wrapper.patch" + ./configure --without-bash-malloc + cores="$(getconf _NPROCESSORS_ONLN)" + make -j"${cores}" + + dest="${GITHUB_WORKSPACE}/artifacts/vendor/${{ matrix.target }}/bash/${{ matrix.variant }}" + mkdir -p "$dest" + cp bash "$dest/bash" + + - uses: actions/upload-artifact@v5 + with: + name: shell-tool-mcp-bash-${{ matrix.target }}-${{ matrix.variant }} + path: artifacts/** + if-no-files-found: error + + package: + name: Package npm module + needs: + - metadata + - rust-binaries + - bash-linux + - bash-darwin + runs-on: ubuntu-latest + env: + PACKAGE_VERSION: ${{ needs.metadata.outputs.version }} + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Setup pnpm + uses: pnpm/action-setup@v4 + with: + version: 10.8.1 + run_install: false + + - name: Setup Node.js + uses: actions/setup-node@v5 + with: + node-version: ${{ env.NODE_VERSION }} + + - name: Install JavaScript dependencies + run: pnpm install --frozen-lockfile + + - name: Build (shell-tool-mcp) + run: pnpm --filter @openai/codex-shell-tool-mcp run build + + - name: Download build artifacts + uses: actions/download-artifact@v4 + with: + path: artifacts + + - name: Assemble staging directory + id: staging + shell: bash + run: | + set -euo pipefail + staging="${STAGING_DIR}" + mkdir -p "$staging" "$staging/vendor" + cp shell-tool-mcp/README.md "$staging/" + cp shell-tool-mcp/package.json "$staging/" + cp -R shell-tool-mcp/bin "$staging/" + + found_vendor="false" + shopt -s nullglob + for vendor_dir in artifacts/*/vendor; do + rsync -av "$vendor_dir/" "$staging/vendor/" + found_vendor="true" + done + if [[ "$found_vendor" == "false" ]]; then + echo "No vendor payloads were downloaded." + exit 1 + fi + + node - <<'NODE' + import fs from "node:fs"; + import path from "node:path"; + + const stagingDir = process.env.STAGING_DIR; + const version = process.env.PACKAGE_VERSION; + const pkgPath = path.join(stagingDir, "package.json"); + const pkg = JSON.parse(fs.readFileSync(pkgPath, "utf8")); + pkg.version = version; + fs.writeFileSync(pkgPath, JSON.stringify(pkg, null, 2) + "\n"); + NODE + + echo "dir=$staging" >> "$GITHUB_OUTPUT" + env: + STAGING_DIR: ${{ runner.temp }}/shell-tool-mcp + + - name: Ensure binaries are executable + run: | + set -euo pipefail + staging="${{ steps.staging.outputs.dir }}" + chmod +x \ + "$staging"/vendor/*/codex-exec-mcp-server \ + "$staging"/vendor/*/codex-execve-wrapper \ + "$staging"/vendor/*/bash/*/bash + + - name: Create npm tarball + shell: bash + run: | + set -euo pipefail + mkdir -p dist/npm + staging="${{ steps.staging.outputs.dir }}" + pack_info=$(cd "$staging" && npm pack --ignore-scripts --json --pack-destination "${GITHUB_WORKSPACE}/dist/npm") + filename=$(PACK_INFO="$pack_info" node -e 'const data = JSON.parse(process.env.PACK_INFO); console.log(data[0].filename);') + mv "dist/npm/${filename}" "dist/npm/codex-shell-tool-mcp-npm-${PACKAGE_VERSION}.tgz" + + - uses: actions/upload-artifact@v5 + with: + name: codex-shell-tool-mcp-npm + path: dist/npm/codex-shell-tool-mcp-npm-${{ env.PACKAGE_VERSION }}.tgz + if-no-files-found: error + + publish: + name: Publish npm package + needs: + - metadata + - package + if: ${{ inputs.publish && needs.metadata.outputs.should_publish == 'true' }} + runs-on: ubuntu-latest + permissions: + id-token: write + contents: read + steps: + - name: Setup pnpm + uses: pnpm/action-setup@v4 + with: + version: 10.8.1 + run_install: false + + - name: Setup Node.js + uses: actions/setup-node@v5 + with: + node-version: ${{ env.NODE_VERSION }} + registry-url: https://registry.npmjs.org + scope: "@openai" + + - name: Update npm + run: npm install -g npm@latest + + - name: Download npm tarball + uses: actions/download-artifact@v4 + with: + name: codex-shell-tool-mcp-npm + path: dist/npm + + - name: Publish to npm + env: + NPM_TAG: ${{ needs.metadata.outputs.npm_tag }} + VERSION: ${{ needs.metadata.outputs.version }} + shell: bash + run: | + set -euo pipefail + tag_args=() + if [[ -n "${NPM_TAG}" ]]; then + tag_args+=(--tag "${NPM_TAG}") + fi + npm publish "dist/npm/codex-shell-tool-mcp-npm-${VERSION}.tgz" "${tag_args[@]}" diff --git a/README.md b/README.md index 4fed276405..78eaf9eb35 100644 --- a/README.md +++ b/README.md @@ -69,38 +69,9 @@ Codex can access MCP servers. To configure them, refer to the [config docs](./do Codex CLI supports a rich set of configuration options, with preferences stored in `~/.codex/config.toml`. For full configuration options, see [Configuration](./docs/config.md). -### Execpolicy Quickstart +### Execpolicy -Codex can enforce your own rules-based execution policy before it runs shell commands. - -1. Create a policy directory: `mkdir -p ~/.codex/policy`. -2. Create one or more `.codexpolicy` files in that folder. Codex automatically loads every `.codexpolicy` file in there on startup. -3. Write `prefix_rule` entries to describe the commands you want to allow, prompt, or block: - -```starlark -prefix_rule( - pattern = ["git", ["push", "fetch"]], - decision = "prompt", # allow | prompt | forbidden - match = [["git", "push", "origin", "main"]], # examples that must match - not_match = [["git", "status"]], # examples that must not match -) -``` - -- `pattern` is a list of shell tokens, evaluated from left to right; wrap tokens in a nested list to express alternatives (e.g., match both `push` and `fetch`). -- `decision` sets the severity; Codex picks the strictest decision when multiple rules match (forbidden > prompt > allow). -- `match` and `not_match` act as (optional) unit tests. Codex validates them when it loads your policy, so you get feedback if an example has unexpected behavior. - -In this example rule, if Codex wants to run commands with the prefix `git push` or `git fetch`, it will first ask for user approval. - -Use [`execpolicy2` CLI](./codex-rs/execpolicy2/README.md) to preview decisions for policy files: - -```shell -cargo run -p codex-execpolicy2 -- check --policy ~/.codex/policy/default.codexpolicy git push origin main -``` - -Pass multiple `--policy` flags to test how several files combine. See the [`codex-rs/execpolicy2` README](./codex-rs/execpolicy2/README.md) for a more detailed walkthrough of the available syntax. - ---- +See the [Execpolicy quickstart](./docs/execpolicy.md) to set up rules that govern what commands Codex can execute. ### Docs & FAQ @@ -114,6 +85,7 @@ Pass multiple `--policy` flags to test how several files combine. See the [`code - [**Configuration**](./docs/config.md) - [Example config](./docs/example-config.md) - [**Sandbox & approvals**](./docs/sandbox.md) +- [**Execpolicy quickstart**](./docs/execpolicy.md) - [**Authentication**](./docs/authentication.md) - [Auth methods](./docs/authentication.md#forcing-a-specific-auth-method-advanced) - [Login on a "Headless" machine](./docs/authentication.md#connecting-on-a-headless-machine) diff --git a/codex-rs/.cargo/audit.toml b/codex-rs/.cargo/audit.toml new file mode 100644 index 0000000000..143e64163a --- /dev/null +++ b/codex-rs/.cargo/audit.toml @@ -0,0 +1,6 @@ +[advisories] +ignore = [ + "RUSTSEC-2024-0388", # derivative 2.2.0 via starlark; upstream crate is unmaintained + "RUSTSEC-2025-0057", # fxhash 0.2.1 via starlark_map; upstream crate is unmaintained + "RUSTSEC-2024-0436", # paste 1.0.15 via starlark/ratatui; upstream crate is unmaintained +] diff --git a/codex-rs/.config/nextest.toml b/codex-rs/.config/nextest.toml index 3ca7cfe503..f432af88ea 100644 --- a/codex-rs/.config/nextest.toml +++ b/codex-rs/.config/nextest.toml @@ -7,3 +7,7 @@ slow-timeout = { period = "15s", terminate-after = 2 } # Do not add new tests here filter = 'test(rmcp_client) | test(humanlike_typing_1000_chars_appears_live_no_placeholder)' slow-timeout = { period = "1m", terminate-after = 4 } + +[[profile.default.overrides]] +filter = 'test(approval_matrix_covers_all_modes)' +slow-timeout = { period = "30s", terminate-after = 2 } diff --git a/codex-rs/.github/workflows/cargo-audit.yml b/codex-rs/.github/workflows/cargo-audit.yml new file mode 100644 index 0000000000..e75c841ab4 --- /dev/null +++ b/codex-rs/.github/workflows/cargo-audit.yml @@ -0,0 +1,26 @@ +name: Cargo audit + +on: + pull_request: + push: + branches: + - main + +permissions: + contents: read + +jobs: + audit: + runs-on: ubuntu-latest + defaults: + run: + working-directory: codex-rs + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - name: Install cargo-audit + uses: taiki-e/install-action@v2 + with: + tool: cargo-audit + - name: Run cargo audit + run: cargo audit --deny warnings diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index b8175ee36b..b9fcc969b3 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -187,8 +187,10 @@ dependencies = [ "codex-app-server-protocol", "codex-core", "codex-protocol", + "core_test_support", "serde", "serde_json", + "shlex", "tokio", "uuid", "wiremock", @@ -196,9 +198,9 @@ dependencies = [ [[package]] name = "arboard" -version = "3.6.0" +version = "3.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55f533f8e0af236ffe5eb979b99381df3258853f00ba2e44b6e1955292c75227" +checksum = "0348a1c054491f4bfe6ab86a7b6ab1e44e45d899005de92f58b3df180b36ddaf" dependencies = [ "clipboard-win", "image", @@ -210,7 +212,7 @@ dependencies = [ "objc2-foundation", "parking_lot", "percent-encoding", - "windows-sys 0.59.0", + "windows-sys 0.60.2", "wl-clipboard-rs", "x11rb", ] @@ -260,7 +262,7 @@ dependencies = [ "memchr", "proc-macro2", "quote", - "rustc-hash 2.1.1", + "rustc-hash", "serde", "serde_derive", "syn 2.0.104", @@ -726,6 +728,17 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" +[[package]] +name = "chardetng" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14b8f0b65b7b08ae3c8187e8d77174de20cb6777864c6b832d8ad365999cf1ea" +dependencies = [ + "cfg-if", + "encoding_rs", + "memchr", +] + [[package]] name = "chrono" version = "0.4.42" @@ -830,6 +843,30 @@ dependencies = [ "tracing", ] +[[package]] +name = "codex-api" +version = "0.0.0" +dependencies = [ + "anyhow", + "assert_matches", + "async-trait", + "bytes", + "codex-client", + "codex-protocol", + "eventsource-stream", + "futures", + "http", + "pretty_assertions", + "regex-lite", + "serde", + "serde_json", + "thiserror 2.0.17", + "tokio", + "tokio-test", + "tokio-util", + "tracing", +] + [[package]] name = "codex-app-server" version = "0.0.0" @@ -849,7 +886,6 @@ dependencies = [ "codex-login", "codex-protocol", "codex-utils-json-to-toml", - "codex-windows-sandbox", "core_test_support", "mcp-types", "opentelemetry-appender-tracing", @@ -858,6 +894,8 @@ dependencies = [ "serde", "serde_json", "serial_test", + "sha2", + "shlex", "tempfile", "tokio", "toml", @@ -880,6 +918,7 @@ dependencies = [ "serde", "serde_json", "strum_macros 0.27.2", + "thiserror 2.0.17", "ts-rs", "uuid", ] @@ -990,6 +1029,7 @@ dependencies = [ "codex-common", "codex-core", "codex-exec", + "codex-execpolicy", "codex-login", "codex-mcp-server", "codex-process-hardening", @@ -1013,6 +1053,24 @@ dependencies = [ "tracing", ] +[[package]] +name = "codex-client" +version = "0.0.0" +dependencies = [ + "async-trait", + "bytes", + "eventsource-stream", + "futures", + "http", + "rand 0.9.2", + "reqwest", + "serde", + "serde_json", + "thiserror 2.0.17", + "tokio", + "tracing", +] + [[package]] name = "codex-cloud-tasks" version = "0.0.0" @@ -1080,12 +1138,14 @@ dependencies = [ "async-channel", "async-trait", "base64", - "bytes", + "chardetng", "chrono", + "codex-api", "codex-app-server-protocol", "codex-apply-patch", "codex-arg0", "codex-async-utils", + "codex-core", "codex-execpolicy", "codex-file-search", "codex-git", @@ -1096,13 +1156,13 @@ dependencies = [ "codex-utils-pty", "codex-utils-readiness", "codex-utils-string", - "codex-utils-tokenizer", "codex-windows-sandbox", "core-foundation 0.9.4", "core_test_support", "ctor 0.5.0", "dirs", "dunce", + "encoding_rs", "env-flags", "escargot", "eventsource-stream", @@ -1115,16 +1175,19 @@ dependencies = [ "libc", "maplit", "mcp-types", + "once_cell", "openssl-sys", "os_info", "predicates", "pretty_assertions", "rand 0.9.2", + "regex", "regex-lite", "reqwest", "seccompiler", "serde", "serde_json", + "serde_yaml", "serial_test", "sha1", "sha2", @@ -1145,6 +1208,7 @@ dependencies = [ "tracing-test", "tree-sitter", "tree-sitter-bash", + "url", "uuid", "walkdir", "which", @@ -1202,6 +1266,7 @@ dependencies = [ "socket2 0.6.0", "tempfile", "tokio", + "tokio-util", "tracing", "tracing-subscriber", ] @@ -1218,6 +1283,7 @@ dependencies = [ "serde_json", "shlex", "starlark", + "tempfile", "thiserror 2.0.17", ] @@ -1549,6 +1615,7 @@ dependencies = [ "textwrap 0.16.2", "tokio", "tokio-stream", + "tokio-util", "toml", "tracing", "tracing-appender", @@ -1615,21 +1682,9 @@ dependencies = [ name = "codex-utils-string" version = "0.0.0" -[[package]] -name = "codex-utils-tokenizer" -version = "0.0.0" -dependencies = [ - "anyhow", - "codex-utils-cache", - "pretty_assertions", - "thiserror 2.0.17", - "tiktoken-rs", - "tokio", -] - [[package]] name = "codex-windows-sandbox" -version = "0.1.0" +version = "0.0.0" dependencies = [ "anyhow", "codex-protocol", @@ -1638,6 +1693,7 @@ dependencies = [ "rand 0.8.5", "serde", "serde_json", + "tempfile", "windows-sys 0.52.0", ] @@ -1769,11 +1825,13 @@ version = "0.0.0" dependencies = [ "anyhow", "assert_cmd", + "base64", "codex-core", "codex-protocol", "notify", "regex-lite", "serde_json", + "shlex", "tempfile", "tokio", "walkdir", @@ -2448,17 +2506,6 @@ dependencies = [ "once_cell", ] -[[package]] -name = "fancy-regex" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "531e46835a22af56d1e3b66f04844bed63158bc094a628bec1d321d9b4c44bf2" -dependencies = [ - "bit-set", - "regex-automata", - "regex-syntax 0.8.5", -] - [[package]] name = "fastrand" version = "2.3.0" @@ -3250,9 +3297,9 @@ dependencies = [ [[package]] name = "image" -version = "0.25.8" +version = "0.25.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "529feb3e6769d234375c4cf1ee2ce713682b8e76538cb13f9fc23e1400a591e7" +checksum = "e6506c6c10786659413faa717ceebcb8f70731c0a60cbae39795fdf114519c1a" dependencies = [ "bytemuck", "byteorder-lite", @@ -3260,8 +3307,8 @@ dependencies = [ "num-traits", "png", "tiff", - "zune-core", - "zune-jpeg", + "zune-core 0.5.0", + "zune-jpeg 0.5.5", ] [[package]] @@ -3589,9 +3636,9 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "libc" -version = "0.2.175" +version = "0.2.177" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a82ae493e598baaea5209805c49bbf2ea7de956d50d7da0da1164f9c6d28543" +checksum = "2874a2af47a2325c2001a6e6fad9b16a53b802102b528163885171cf92b15976" [[package]] name = "libdbus-sys" @@ -3746,11 +3793,13 @@ dependencies = [ "assert_cmd", "codex-core", "codex-mcp-server", + "core_test_support", "mcp-types", "os_info", "pretty_assertions", "serde", "serde_json", + "shlex", "tokio", "wiremock", ] @@ -4419,6 +4468,12 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" +[[package]] +name = "pastey" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57d6c094ee800037dff99e02cab0eaf3142826586742a270ab3d7a62656bd27a" + [[package]] name = "path-absolutize" version = "3.1.1" @@ -4783,7 +4838,7 @@ dependencies = [ "pin-project-lite", "quinn-proto", "quinn-udp", - "rustc-hash 2.1.1", + "rustc-hash", "rustls", "socket2 0.6.0", "thiserror 2.0.17", @@ -4803,7 +4858,7 @@ dependencies = [ "lru-slab", "rand 0.9.2", "ring", - "rustc-hash 2.1.1", + "rustc-hash", "rustls", "rustls-pki-types", "slab", @@ -4993,9 +5048,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.11.1" +version = "1.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" +checksum = "843bc0191f75f3e22651ae5f1e72939ab2f72a4bc30fa80a066bd66edefc24d4" dependencies = [ "aho-corasick", "memchr", @@ -5005,9 +5060,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.9" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" +checksum = "5276caf25ac86c8d810222b3dbb938e512c55c6831a10f3e6ed1c93b84041f1c" dependencies = [ "aho-corasick", "memchr", @@ -5034,9 +5089,9 @@ checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" [[package]] name = "reqwest" -version = "0.12.23" +version = "0.12.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d429f34c8092b2d42c7c93cec323bb4adeb7c67698f70839adec842ec10c7ceb" +checksum = "9d0946410b9f7b082a427e4ef5c8ff541a88b357bc6c637c40db3a68ac70a36f" dependencies = [ "base64", "bytes", @@ -5097,10 +5152,11 @@ dependencies = [ [[package]] name = "rmcp" -version = "0.8.5" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5947688160b56fb6c827e3c20a72c90392a1d7e9dec74749197aa1780ac42ca" +checksum = "38b18323edc657390a6ed4d7a9110b0dec2dc3ed128eb2a123edfbafabdbddc5" dependencies = [ + "async-trait", "base64", "bytes", "chrono", @@ -5109,7 +5165,7 @@ dependencies = [ "http-body", "http-body-util", "oauth2", - "paste", + "pastey", "pin-project-lite", "process-wrap", "rand 0.9.2", @@ -5131,9 +5187,9 @@ dependencies = [ [[package]] name = "rmcp-macros" -version = "0.8.5" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01263441d3f8635c628e33856c468b96ebbce1af2d3699ea712ca71432d4ee7a" +checksum = "c75d0a62676bf8c8003c4e3c348e2ceb6a7b3e48323681aaf177fdccdac2ce50" dependencies = [ "darling 0.21.3", "proc-macro2", @@ -5148,12 +5204,6 @@ version = "0.1.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "989e6739f80c4ad5b13e0fd7fe89531180375b18520cc8c82080e4dc4035b84f" -[[package]] -name = "rustc-hash" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" - [[package]] name = "rustc-hash" version = "2.1.1" @@ -5698,9 +5748,9 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.14.0" +version = "3.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2c45cd61fefa9db6f254525d46e392b852e0e61d9a1fd36e5bd183450a556d5" +checksum = "4fa237f2807440d238e0364a218270b98f767a00d3dada77b1c53ae88940e2e7" dependencies = [ "base64", "chrono", @@ -5709,8 +5759,7 @@ dependencies = [ "indexmap 2.12.0", "schemars 0.9.0", "schemars 1.0.4", - "serde", - "serde_derive", + "serde_core", "serde_json", "serde_with_macros", "time", @@ -5718,16 +5767,29 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.14.0" +version = "3.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de90945e6565ce0d9a25098082ed4ee4002e047cb59892c318d66821e14bb30f" +checksum = "52a8e3ca0ca629121f70ab50f95249e5a6f925cc0f6ffe8256c45b728875706c" dependencies = [ - "darling 0.20.11", + "darling 0.21.3", "proc-macro2", "quote", "syn 2.0.104", ] +[[package]] +name = "serde_yaml" +version = "0.9.34+deprecated" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" +dependencies = [ + "indexmap 2.12.0", + "itoa", + "ryu", + "serde", + "unsafe-libyaml", +] + [[package]] name = "serial2" version = "0.2.31" @@ -6371,22 +6433,7 @@ dependencies = [ "half", "quick-error", "weezl", - "zune-jpeg", -] - -[[package]] -name = "tiktoken-rs" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a19830747d9034cd9da43a60eaa8e552dfda7712424aebf187b7a60126bae0d" -dependencies = [ - "anyhow", - "base64", - "bstr", - "fancy-regex", - "lazy_static", - "regex", - "rustc-hash 1.1.0", + "zune-jpeg 0.4.19", ] [[package]] @@ -6554,6 +6601,7 @@ dependencies = [ "futures-sink", "futures-util", "pin-project-lite", + "slab", "tokio", ] @@ -6574,18 +6622,18 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "0.7.0" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bade1c3e902f58d73d3f294cd7f20391c1cb2fbcb643b73566bc773971df91e3" +checksum = "f2cdb639ebbc97961c51720f858597f7f24c4fc295327923af55b74c3c724533" dependencies = [ - "serde", + "serde_core", ] [[package]] name = "toml_edit" -version = "0.23.4" +version = "0.23.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7211ff1b8f0d3adae1663b7da9ffe396eabe1ca25f0b0bee42b0da29a9ddce93" +checksum = "6485ef6d0d9b5d0ec17244ff7eb05310113c3f316f2d14200d4de56b3cb98f8d" dependencies = [ "indexmap 2.12.0", "toml_datetime", @@ -6596,18 +6644,18 @@ dependencies = [ [[package]] name = "toml_parser" -version = "1.0.2" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b551886f449aa90d4fe2bdaa9f4a2577ad2dde302c61ecf262d80b116db95c10" +checksum = "c0cbe268d35bdb4bb5a56a2de88d0ad0eb70af5384a99d648cd4b3d04039800e" dependencies = [ "winnow", ] [[package]] name = "toml_writer" -version = "1.0.2" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcc842091f2def52017664b53082ecbbeb5c7731092bad69d2c63050401dfd64" +checksum = "df8b2b54733674ad286d16267dcfc7a71ed5c776e4ac7aa3c3e2561f7c637bf2" [[package]] name = "tonic" @@ -6691,9 +6739,9 @@ checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" [[package]] name = "tracing" -version = "0.1.41" +version = "0.1.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" +checksum = "2d15d90a0b5c19378952d479dc858407149d7bb45a14de0142f6c534b16fc647" dependencies = [ "log", "pin-project-lite", @@ -6715,9 +6763,9 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.30" +version = "0.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81383ab64e72a7a8b8e13130c49e3dab29def6d0c7d76a03087b3cf71c5c6903" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", @@ -6726,9 +6774,9 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.34" +version = "0.1.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9d12581f227e93f094d3af2ae690a574abb8a2b9b7a96e7cfe9647b2b617678" +checksum = "7a04e24fab5c89c6a36eb8558c9656f30d81de51dfa4d3b45f26b21d61fa0a6c" dependencies = [ "once_cell", "valuable", @@ -6957,6 +7005,12 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" +[[package]] +name = "unsafe-libyaml" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" + [[package]] name = "untrusted" version = "0.9.0" @@ -7277,9 +7331,9 @@ dependencies = [ [[package]] name = "webbrowser" -version = "1.0.5" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aaf4f3c0ba838e82b4e5ccc4157003fb8c324ee24c058470ffb82820becbde98" +checksum = "00f1243ef785213e3a32fa0396093424a3a6ea566f9948497e5a2309261a4c97" dependencies = [ "core-foundation 0.10.1", "jni", @@ -7772,9 +7826,9 @@ checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" [[package]] name = "winnow" -version = "0.7.12" +version = "0.7.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3edebf492c8125044983378ecb5766203ad3b4c2f7a922bd7dd207f6d443e95" +checksum = "21a0236b59786fed61e2a80582dd500fe61f18b5dca67a4a067d0bc9039339cf" dependencies = [ "memchr", ] @@ -8071,13 +8125,28 @@ version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f423a2c17029964870cfaabb1f13dfab7d092a62a29a89264f4d36990ca414a" +[[package]] +name = "zune-core" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "111f7d9820f05fd715df3144e254d6fc02ee4088b0644c0ffd0efc9e6d9d2773" + [[package]] name = "zune-jpeg" version = "0.4.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2c9e525af0a6a658e031e95f14b7f889976b74a11ba0eca5a5fc9ac8a1c43a6a" dependencies = [ - "zune-core", + "zune-core 0.4.12", +] + +[[package]] +name = "zune-jpeg" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc6fb7703e32e9a07fb3f757360338b3a567a5054f21b5f52a666752e333d58e" +dependencies = [ + "zune-core 0.5.0", ] [[package]] diff --git a/codex-rs/Cargo.toml b/codex-rs/Cargo.toml index e2aa981381..2339cd4e67 100644 --- a/codex-rs/Cargo.toml +++ b/codex-rs/Cargo.toml @@ -41,7 +41,8 @@ members = [ "utils/pty", "utils/readiness", "utils/string", - "utils/tokenizer", + "codex-client", + "codex-api", ] resolver = "2" @@ -52,11 +53,13 @@ version = "0.0.0" # crates created with `cargo new -w ...` automatically inherit the 2024 # edition. edition = "2024" +license = "Apache-2.0" [workspace.dependencies] # Internal app_test_support = { path = "app-server/tests/common" } codex-ansi-escape = { path = "ansi-escape" } +codex-api = { path = "codex-api" } codex-app-server = { path = "app-server" } codex-app-server-protocol = { path = "app-server-protocol" } codex-apply-patch = { path = "apply-patch" } @@ -64,6 +67,7 @@ codex-arg0 = { path = "arg0" } codex-async-utils = { path = "async-utils" } codex-backend-client = { path = "backend-client" } codex-chatgpt = { path = "chatgpt" } +codex-client = { path = "codex-client" } codex-common = { path = "common" } codex-core = { path = "core" } codex-exec = { path = "exec" } @@ -90,7 +94,6 @@ codex-utils-json-to-toml = { path = "utils/json-to-toml" } codex-utils-pty = { path = "utils/pty" } codex-utils-readiness = { path = "utils/readiness" } codex-utils-string = { path = "utils/string" } -codex-utils-tokenizer = { path = "utils/tokenizer" } codex-windows-sandbox = { path = "windows-sandbox-rs" } core_test_support = { path = "core/tests/common" } mcp-types = { path = "mcp-types" } @@ -110,6 +113,7 @@ async-trait = "0.1.89" axum = { version = "0.8", default-features = false } base64 = "0.22.1" bytes = "1.10.1" +chardetng = "0.1.17" chrono = "0.4.42" clap = "4" clap_complete = "4" @@ -121,6 +125,7 @@ diffy = "0.4.2" dirs = "6" dotenvy = "0.15.7" dunce = "1.0.4" +encoding_rs = "0.8.35" env-flags = "0.1.1" env_logger = "0.11.5" escargot = "0.5" @@ -131,14 +136,14 @@ icu_decimal = "2.1" icu_locale_core = "2.1" icu_provider = { version = "2.1", features = ["sync"] } ignore = "0.4.23" -image = { version = "^0.25.8", default-features = false } +image = { version = "^0.25.9", default-features = false } indexmap = "2.12.0" insta = "1.43.2" itertools = "0.14.0" keyring = { version = "3.6", default-features = false } landlock = "0.4.1" lazy_static = "1" -libc = "0.2.175" +libc = "0.2.177" log = "0.4" lru = "0.12.5" maplit = "1.0.2" @@ -146,7 +151,7 @@ mime_guess = "2.0.5" multimap = "0.10.0" notify = "8.2.0" nucleo-matcher = "0.3.1" -once_cell = "1" +once_cell = "1.20.2" openssl-sys = "*" opentelemetry = "0.30.0" opentelemetry-appender-tracing = "0.30.0" @@ -164,15 +169,17 @@ pulldown-cmark = "0.10" rand = "0.9" ratatui = "0.29.0" ratatui-macros = "0.6.0" +regex = "1.12.2" regex-lite = "0.1.7" reqwest = "0.12" -rmcp = { version = "0.8.5", default-features = false } +rmcp = { version = "0.10.0", default-features = false } schemars = "0.8.22" seccompiler = "0.5.0" sentry = "0.34.0" serde = "1" serde_json = "1" -serde_with = "3.14" +serde_yaml = "0.9" +serde_with = "3.16" serial_test = "3.2.0" sha1 = "0.10.6" sha2 = "0.10" @@ -188,7 +195,6 @@ tempfile = "3.23.0" test-log = "0.2.18" textwrap = "0.16.2" thiserror = "2.0.17" -tiktoken-rs = "0.9" time = "0.3" tiny_http = "0.12" tokio = "1" @@ -196,9 +202,9 @@ tokio-stream = "0.1.17" tokio-test = "0.4" tokio-util = "0.7.16" toml = "0.9.5" -toml_edit = "0.23.4" +toml_edit = "0.23.5" tonic = "0.13.1" -tracing = "0.1.41" +tracing = "0.1.43" tracing-appender = "0.2.3" tracing-subscriber = "0.3.20" tracing-test = "0.2.5" @@ -262,12 +268,7 @@ unwrap_used = "deny" # cargo-shear cannot see the platform-specific openssl-sys usage, so we # silence the false positive here instead of deleting a real dependency. [workspace.metadata.cargo-shear] -ignored = [ - "icu_provider", - "openssl-sys", - "codex-utils-readiness", - "codex-utils-tokenizer", -] +ignored = ["icu_provider", "openssl-sys", "codex-utils-readiness"] [profile.release] lto = "fat" diff --git a/codex-rs/ansi-escape/Cargo.toml b/codex-rs/ansi-escape/Cargo.toml index 4107a72754..a10dbf9134 100644 --- a/codex-rs/ansi-escape/Cargo.toml +++ b/codex-rs/ansi-escape/Cargo.toml @@ -1,7 +1,8 @@ [package] -edition = "2024" name = "codex-ansi-escape" -version = { workspace = true } +version.workspace = true +edition.workspace = true +license.workspace = true [lib] name = "codex_ansi_escape" diff --git a/codex-rs/app-server-protocol/Cargo.toml b/codex-rs/app-server-protocol/Cargo.toml index 4d1afadaa1..c1a251c656 100644 --- a/codex-rs/app-server-protocol/Cargo.toml +++ b/codex-rs/app-server-protocol/Cargo.toml @@ -1,7 +1,8 @@ [package] -edition = "2024" name = "codex-app-server-protocol" -version = { workspace = true } +version.workspace = true +edition.workspace = true +license.workspace = true [lib] name = "codex_app_server_protocol" @@ -19,6 +20,7 @@ schemars = { workspace = true } serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } strum_macros = { workspace = true } +thiserror = { workspace = true } ts-rs = { workspace = true } uuid = { workspace = true, features = ["serde", "v7"] } diff --git a/codex-rs/app-server-protocol/src/protocol/common.rs b/codex-rs/app-server-protocol/src/protocol/common.rs index 650b6f31a6..51d1bd2361 100644 --- a/codex-rs/app-server-protocol/src/protocol/common.rs +++ b/codex-rs/app-server-protocol/src/protocol/common.rs @@ -131,7 +131,7 @@ client_request_definitions! { }, ReviewStart => "review/start" { params: v2::ReviewStartParams, - response: v2::TurnStartResponse, + response: v2::ReviewStartResponse, }, ModelList => "model/list" { @@ -164,6 +164,25 @@ client_request_definitions! { response: v2::FeedbackUploadResponse, }, + /// Execute a command (argv vector) under the server's sandbox. + OneOffCommandExec => "command/exec" { + params: v2::CommandExecParams, + response: v2::CommandExecResponse, + }, + + ConfigRead => "config/read" { + params: v2::ConfigReadParams, + response: v2::ConfigReadResponse, + }, + ConfigValueWrite => "config/value/write" { + params: v2::ConfigValueWriteParams, + response: v2::ConfigWriteResponse, + }, + ConfigBatchWrite => "config/batchWrite" { + params: v2::ConfigBatchWriteParams, + response: v2::ConfigWriteResponse, + }, + GetAccount => "account/read" { params: v2::GetAccountParams, response: v2::GetAccountResponse, @@ -378,7 +397,7 @@ macro_rules! server_notification_definitions { impl TryFrom for ServerNotification { type Error = serde_json::Error; - fn try_from(value: JSONRPCNotification) -> Result { + fn try_from(value: JSONRPCNotification) -> Result { serde_json::from_value(serde_json::to_value(value)?) } } @@ -487,19 +506,25 @@ pub struct FuzzyFileSearchResponse { server_notification_definitions! { /// NEW NOTIFICATIONS + Error => "error" (v2::ErrorNotification), ThreadStarted => "thread/started" (v2::ThreadStartedNotification), + ThreadTokenUsageUpdated => "thread/tokenUsage/updated" (v2::ThreadTokenUsageUpdatedNotification), TurnStarted => "turn/started" (v2::TurnStartedNotification), TurnCompleted => "turn/completed" (v2::TurnCompletedNotification), + TurnDiffUpdated => "turn/diff/updated" (v2::TurnDiffUpdatedNotification), + TurnPlanUpdated => "turn/plan/updated" (v2::TurnPlanUpdatedNotification), ItemStarted => "item/started" (v2::ItemStartedNotification), ItemCompleted => "item/completed" (v2::ItemCompletedNotification), AgentMessageDelta => "item/agentMessage/delta" (v2::AgentMessageDeltaNotification), CommandExecutionOutputDelta => "item/commandExecution/outputDelta" (v2::CommandExecutionOutputDeltaNotification), + FileChangeOutputDelta => "item/fileChange/outputDelta" (v2::FileChangeOutputDeltaNotification), McpToolCallProgress => "item/mcpToolCall/progress" (v2::McpToolCallProgressNotification), AccountUpdated => "account/updated" (v2::AccountUpdatedNotification), AccountRateLimitsUpdated => "account/rateLimits/updated" (v2::AccountRateLimitsUpdatedNotification), ReasoningSummaryTextDelta => "item/reasoning/summaryTextDelta" (v2::ReasoningSummaryTextDeltaNotification), ReasoningSummaryPartAdded => "item/reasoning/summaryPartAdded" (v2::ReasoningSummaryPartAddedNotification), ReasoningTextDelta => "item/reasoning/textDelta" (v2::ReasoningTextDeltaNotification), + ContextCompacted => "thread/compacted" (v2::ContextCompactedNotification), /// Notifies the user of world-writable directories on Windows, which cannot be protected by the sandbox. WindowsWorldWritableWarning => "windows/worldWritableWarning" (v2::WindowsWorldWritableWarningNotification), diff --git a/codex-rs/app-server-protocol/src/protocol/mappers.rs b/codex-rs/app-server-protocol/src/protocol/mappers.rs new file mode 100644 index 0000000000..f708c1fa85 --- /dev/null +++ b/codex-rs/app-server-protocol/src/protocol/mappers.rs @@ -0,0 +1,15 @@ +use crate::protocol::v1; +use crate::protocol::v2; + +impl From for v2::CommandExecParams { + fn from(value: v1::ExecOneOffCommandParams) -> Self { + Self { + command: value.command, + timeout_ms: value + .timeout_ms + .map(|timeout| i64::try_from(timeout).unwrap_or(60_000)), + cwd: value.cwd, + sandbox_policy: value.sandbox_policy.map(std::convert::Into::into), + } + } +} diff --git a/codex-rs/app-server-protocol/src/protocol/mod.rs b/codex-rs/app-server-protocol/src/protocol/mod.rs index 8e2d63e064..e269332438 100644 --- a/codex-rs/app-server-protocol/src/protocol/mod.rs +++ b/codex-rs/app-server-protocol/src/protocol/mod.rs @@ -2,6 +2,7 @@ // Exposes protocol pieces used by `lib.rs` via `pub use protocol::common::*;`. pub mod common; +mod mappers; pub mod thread_history; pub mod v1; pub mod v2; diff --git a/codex-rs/app-server-protocol/src/protocol/thread_history.rs b/codex-rs/app-server-protocol/src/protocol/thread_history.rs index 04cd1190bc..ba1e6261cc 100644 --- a/codex-rs/app-server-protocol/src/protocol/thread_history.rs +++ b/codex-rs/app-server-protocol/src/protocol/thread_history.rs @@ -1,5 +1,6 @@ use crate::protocol::v2::ThreadItem; use crate::protocol::v2::Turn; +use crate::protocol::v2::TurnError; use crate::protocol::v2::TurnStatus; use crate::protocol::v2::UserInput; use codex_protocol::protocol::AgentReasoningEvent; @@ -142,6 +143,7 @@ impl ThreadHistoryBuilder { PendingTurn { id: self.next_turn_id(), items: Vec::new(), + error: None, status: TurnStatus::Completed, } } @@ -190,6 +192,7 @@ impl ThreadHistoryBuilder { struct PendingTurn { id: String, items: Vec, + error: Option, status: TurnStatus, } @@ -198,6 +201,7 @@ impl From for Turn { Self { id: value.id, items: value.items, + error: value.error, status: value.status, } } diff --git a/codex-rs/app-server-protocol/src/protocol/v2.rs b/codex-rs/app-server-protocol/src/protocol/v2.rs index 6b61245bc6..0e2be70c93 100644 --- a/codex-rs/app-server-protocol/src/protocol/v2.rs +++ b/codex-rs/app-server-protocol/src/protocol/v2.rs @@ -11,15 +11,22 @@ use codex_protocol::items::AgentMessageContent as CoreAgentMessageContent; use codex_protocol::items::TurnItem as CoreTurnItem; use codex_protocol::models::ResponseItem; use codex_protocol::parse_command::ParsedCommand as CoreParsedCommand; +use codex_protocol::plan_tool::PlanItemArg as CorePlanItemArg; +use codex_protocol::plan_tool::StepStatus as CorePlanStepStatus; +use codex_protocol::protocol::CodexErrorInfo as CoreCodexErrorInfo; use codex_protocol::protocol::CreditsSnapshot as CoreCreditsSnapshot; use codex_protocol::protocol::RateLimitSnapshot as CoreRateLimitSnapshot; use codex_protocol::protocol::RateLimitWindow as CoreRateLimitWindow; +use codex_protocol::protocol::SessionSource as CoreSessionSource; +use codex_protocol::protocol::TokenUsage as CoreTokenUsage; +use codex_protocol::protocol::TokenUsageInfo as CoreTokenUsageInfo; use codex_protocol::user_input::UserInput as CoreUserInput; use mcp_types::ContentBlock as McpContentBlock; use schemars::JsonSchema; use serde::Deserialize; use serde::Serialize; use serde_json::Value as JsonValue; +use thiserror::Error; use ts_rs::TS; // Macro to declare a camelCased API v2 enum mirroring a core enum which @@ -47,6 +54,72 @@ macro_rules! v2_enum_from_core { }; } +/// This translation layer make sure that we expose codex error code in camel case. +/// +/// When an upstream HTTP status is available (for example, from the Responses API or a provider), +/// it is forwarded in `httpStatusCode` on the relevant `codexErrorInfo` variant. +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub enum CodexErrorInfo { + ContextWindowExceeded, + UsageLimitExceeded, + HttpConnectionFailed { + #[serde(rename = "httpStatusCode")] + #[ts(rename = "httpStatusCode")] + http_status_code: Option, + }, + /// Failed to connect to the response SSE stream. + ResponseStreamConnectionFailed { + #[serde(rename = "httpStatusCode")] + #[ts(rename = "httpStatusCode")] + http_status_code: Option, + }, + InternalServerError, + Unauthorized, + BadRequest, + SandboxError, + /// The response SSE stream disconnected in the middle of a turn before completion. + ResponseStreamDisconnected { + #[serde(rename = "httpStatusCode")] + #[ts(rename = "httpStatusCode")] + http_status_code: Option, + }, + /// Reached the retry limit for responses. + ResponseTooManyFailedAttempts { + #[serde(rename = "httpStatusCode")] + #[ts(rename = "httpStatusCode")] + http_status_code: Option, + }, + Other, +} + +impl From for CodexErrorInfo { + fn from(value: CoreCodexErrorInfo) -> Self { + match value { + CoreCodexErrorInfo::ContextWindowExceeded => CodexErrorInfo::ContextWindowExceeded, + CoreCodexErrorInfo::UsageLimitExceeded => CodexErrorInfo::UsageLimitExceeded, + CoreCodexErrorInfo::HttpConnectionFailed { http_status_code } => { + CodexErrorInfo::HttpConnectionFailed { http_status_code } + } + CoreCodexErrorInfo::ResponseStreamConnectionFailed { http_status_code } => { + CodexErrorInfo::ResponseStreamConnectionFailed { http_status_code } + } + CoreCodexErrorInfo::InternalServerError => CodexErrorInfo::InternalServerError, + CoreCodexErrorInfo::Unauthorized => CodexErrorInfo::Unauthorized, + CoreCodexErrorInfo::BadRequest => CodexErrorInfo::BadRequest, + CoreCodexErrorInfo::SandboxError => CodexErrorInfo::SandboxError, + CoreCodexErrorInfo::ResponseStreamDisconnected { http_status_code } => { + CodexErrorInfo::ResponseStreamDisconnected { http_status_code } + } + CoreCodexErrorInfo::ResponseTooManyFailedAttempts { http_status_code } => { + CodexErrorInfo::ResponseTooManyFailedAttempts { http_status_code } + } + CoreCodexErrorInfo::Other => CodexErrorInfo::Other, + } + } +} + v2_enum_from_core!( pub enum AskForApproval from codex_protocol::protocol::AskForApproval { UnlessTrusted, OnFailure, OnRequest, Never @@ -59,6 +132,133 @@ v2_enum_from_core!( } ); +v2_enum_from_core!( + pub enum ReviewDelivery from codex_protocol::protocol::ReviewDelivery { + Inline, Detached + } +); + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub enum ConfigLayerName { + Mdm, + System, + SessionFlags, + User, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct ConfigLayerMetadata { + pub name: ConfigLayerName, + pub source: String, + pub version: String, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct ConfigLayer { + pub name: ConfigLayerName, + pub source: String, + pub version: String, + pub config: JsonValue, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub enum MergeStrategy { + Replace, + Upsert, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub enum WriteStatus { + Ok, + OkOverridden, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct OverriddenMetadata { + pub message: String, + pub overriding_layer: ConfigLayerMetadata, + pub effective_value: JsonValue, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct ConfigWriteResponse { + pub status: WriteStatus, + pub version: String, + pub overridden_metadata: Option, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub enum ConfigWriteErrorCode { + ConfigLayerReadonly, + ConfigVersionConflict, + ConfigValidationError, + ConfigPathNotFound, + ConfigSchemaUnknownKey, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct ConfigReadParams { + #[serde(default)] + pub include_layers: bool, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct ConfigReadResponse { + pub config: JsonValue, + pub origins: HashMap, + #[serde(skip_serializing_if = "Option::is_none")] + pub layers: Option>, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct ConfigValueWriteParams { + pub file_path: String, + pub key_path: String, + pub value: JsonValue, + pub merge_strategy: MergeStrategy, + pub expected_version: Option, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct ConfigBatchWriteParams { + pub file_path: String, + pub edits: Vec, + pub expected_version: Option, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct ConfigEdit { + pub key_path: String, + pub value: JsonValue, + pub merge_strategy: MergeStrategy, +} + v2_enum_from_core!( pub enum CommandRiskLevel from codex_protocol::approvals::SandboxRiskLevel { Low, @@ -191,6 +391,56 @@ pub enum CommandAction { }, } +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(rename_all = "camelCase", export_to = "v2/")] +#[derive(Default)] +pub enum SessionSource { + Cli, + #[serde(rename = "vscode")] + #[ts(rename = "vscode")] + #[default] + VsCode, + Exec, + AppServer, + #[serde(other)] + Unknown, +} + +impl From for SessionSource { + fn from(value: CoreSessionSource) -> Self { + match value { + CoreSessionSource::Cli => SessionSource::Cli, + CoreSessionSource::VSCode => SessionSource::VsCode, + CoreSessionSource::Exec => SessionSource::Exec, + CoreSessionSource::Mcp => SessionSource::AppServer, + CoreSessionSource::SubAgent(_) => SessionSource::Unknown, + CoreSessionSource::Unknown => SessionSource::Unknown, + } + } +} + +impl From for CoreSessionSource { + fn from(value: SessionSource) -> Self { + match value { + SessionSource::Cli => CoreSessionSource::Cli, + SessionSource::VsCode => CoreSessionSource::VSCode, + SessionSource::Exec => CoreSessionSource::Exec, + SessionSource::AppServer => CoreSessionSource::Mcp, + SessionSource::Unknown => CoreSessionSource::Unknown, + } + } +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct GitInfo { + pub sha: Option, + pub branch: Option, + pub origin_url: Option, +} + impl CommandAction { pub fn into_core(self) -> CoreParsedCommand { match self { @@ -382,6 +632,26 @@ pub struct FeedbackUploadResponse { pub thread_id: String, } +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct CommandExecParams { + pub command: Vec, + #[ts(type = "number | null")] + pub timeout_ms: Option, + pub cwd: Option, + pub sandbox_policy: Option, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct CommandExecResponse { + pub exit_code: i32, + pub stdout: String, + pub stderr: String, +} + // === Threads, Turns, and Items === // Thread APIs #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Default, JsonSchema, TS)] @@ -513,11 +783,21 @@ pub struct Thread { pub id: String, /// Usually the first user message in the thread, if available. pub preview: String, + /// Model provider used for this thread (for example, 'openai'). pub model_provider: String, /// Unix timestamp (in seconds) when the thread was created. + #[ts(type = "number")] pub created_at: i64, /// [UNSTABLE] Path to the thread on disk. pub path: PathBuf, + /// Working directory captured for the thread. + pub cwd: PathBuf, + /// Version of the CLI that created the thread. + pub cli_version: String, + /// Origin of the thread (CLI, VSCode, codex exec, codex app-server, etc.). + pub source: SessionSource, + /// Optional Git metadata captured when the thread was created. + pub git_info: Option, /// Only populated on a `thread/resume` response. /// For all other responses and notifications returning a Thread, /// the turns field will be an empty list. @@ -531,6 +811,63 @@ pub struct AccountUpdatedNotification { pub auth_mode: Option, } +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct ThreadTokenUsageUpdatedNotification { + pub thread_id: String, + pub turn_id: String, + pub token_usage: ThreadTokenUsage, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct ThreadTokenUsage { + pub total: TokenUsageBreakdown, + pub last: TokenUsageBreakdown, + #[ts(type = "number | null")] + pub model_context_window: Option, +} + +impl From for ThreadTokenUsage { + fn from(value: CoreTokenUsageInfo) -> Self { + Self { + total: value.total_token_usage.into(), + last: value.last_token_usage.into(), + model_context_window: value.model_context_window, + } + } +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct TokenUsageBreakdown { + #[ts(type = "number")] + pub total_tokens: i64, + #[ts(type = "number")] + pub input_tokens: i64, + #[ts(type = "number")] + pub cached_input_tokens: i64, + #[ts(type = "number")] + pub output_tokens: i64, + #[ts(type = "number")] + pub reasoning_output_tokens: i64, +} + +impl From for TokenUsageBreakdown { + fn from(value: CoreTokenUsage) -> Self { + Self { + total_tokens: value.total_tokens, + input_tokens: value.input_tokens, + cached_input_tokens: value.cached_input_tokens, + output_tokens: value.output_tokens, + reasoning_output_tokens: value.reasoning_output_tokens, + } + } +} + #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] #[serde(rename_all = "camelCase")] #[ts(export_to = "v2/")] @@ -540,24 +877,36 @@ pub struct Turn { /// For all other responses and notifications returning a Turn, /// the items field will be an empty list. pub items: Vec, - #[serde(flatten)] pub status: TurnStatus, + /// Only populated when the Turn's status is failed. + pub error: Option, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS, Error)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +#[error("{message}")] +pub struct TurnError { + pub message: String, + pub codex_error_info: Option, } #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] #[serde(rename_all = "camelCase")] #[ts(export_to = "v2/")] -pub struct TurnError { - pub message: String, +pub struct ErrorNotification { + pub error: TurnError, + pub thread_id: String, + pub turn_id: String, } #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] -#[serde(tag = "status", rename_all = "camelCase")] -#[ts(tag = "status", export_to = "v2/")] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] pub enum TurnStatus { Completed, Interrupted, - Failed { error: TurnError }, + Failed, InProgress, } @@ -589,9 +938,22 @@ pub struct ReviewStartParams { pub thread_id: String, pub target: ReviewTarget, - /// When true, also append the final review message to the original thread. + /// Where to run the review: inline (default) on the current thread or + /// detached on a new thread (returned in `reviewThreadId`). #[serde(default)] - pub append_to_original_thread: bool, + pub delivery: Option, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct ReviewStartResponse { + pub turn: Turn, + /// Identifies the thread where the review runs. + /// + /// For inline reviews, this is the original thread id. + /// For detached reviews, this is the id of the new review thread. + pub review_thread_id: String, } #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] @@ -701,6 +1063,8 @@ pub enum ThreadItem { command: String, /// The command's working directory. cwd: PathBuf, + /// Identifier for the underlying PTY process (when available). + process_id: Option, status: CommandExecutionStatus, /// A best-effort parsing of the command to understand the action(s) it will perform. /// This returns a list of CommandAction objects because a single shell command may @@ -711,6 +1075,7 @@ pub enum ThreadItem { /// The command's exit code. exit_code: Option, /// The duration of the command execution in milliseconds. + #[ts(type = "number | null")] duration_ms: Option, }, #[serde(rename_all = "camelCase")] @@ -742,7 +1107,10 @@ pub enum ThreadItem { ImageView { id: String, path: String }, #[serde(rename_all = "camelCase")] #[ts(rename_all = "camelCase")] - CodeReview { id: String, review: String }, + EnteredReviewMode { id: String, review: String }, + #[serde(rename_all = "camelCase")] + #[ts(rename_all = "camelCase")] + ExitedReviewMode { id: String, review: String }, } impl From for ThreadItem { @@ -782,6 +1150,7 @@ pub enum CommandExecutionStatus { InProgress, Completed, Failed, + Declined, } #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] @@ -859,6 +1228,7 @@ pub struct ThreadStartedNotification { #[serde(rename_all = "camelCase")] #[ts(export_to = "v2/")] pub struct TurnStartedNotification { + pub thread_id: String, pub turn: Turn, } @@ -875,14 +1245,73 @@ pub struct Usage { #[serde(rename_all = "camelCase")] #[ts(export_to = "v2/")] pub struct TurnCompletedNotification { + pub thread_id: String, pub turn: Turn, } +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +/// Notification that the turn-level unified diff has changed. +/// Contains the latest aggregated diff across all file changes in the turn. +pub struct TurnDiffUpdatedNotification { + pub thread_id: String, + pub turn_id: String, + pub diff: String, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct TurnPlanUpdatedNotification { + pub turn_id: String, + pub explanation: Option, + pub plan: Vec, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct TurnPlanStep { + pub step: String, + pub status: TurnPlanStepStatus, +} + +#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub enum TurnPlanStepStatus { + Pending, + InProgress, + Completed, +} + +impl From for TurnPlanStep { + fn from(value: CorePlanItemArg) -> Self { + Self { + step: value.step, + status: value.status.into(), + } + } +} + +impl From for TurnPlanStepStatus { + fn from(value: CorePlanStepStatus) -> Self { + match value { + CorePlanStepStatus::Pending => Self::Pending, + CorePlanStepStatus::InProgress => Self::InProgress, + CorePlanStepStatus::Completed => Self::Completed, + } + } +} + #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] #[serde(rename_all = "camelCase")] #[ts(export_to = "v2/")] pub struct ItemStartedNotification { pub item: ThreadItem, + pub thread_id: String, + pub turn_id: String, } #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] @@ -890,6 +1319,8 @@ pub struct ItemStartedNotification { #[ts(export_to = "v2/")] pub struct ItemCompletedNotification { pub item: ThreadItem, + pub thread_id: String, + pub turn_id: String, } // Item-specific progress notifications @@ -897,6 +1328,8 @@ pub struct ItemCompletedNotification { #[serde(rename_all = "camelCase")] #[ts(export_to = "v2/")] pub struct AgentMessageDeltaNotification { + pub thread_id: String, + pub turn_id: String, pub item_id: String, pub delta: String, } @@ -905,8 +1338,11 @@ pub struct AgentMessageDeltaNotification { #[serde(rename_all = "camelCase")] #[ts(export_to = "v2/")] pub struct ReasoningSummaryTextDeltaNotification { + pub thread_id: String, + pub turn_id: String, pub item_id: String, pub delta: String, + #[ts(type = "number")] pub summary_index: i64, } @@ -914,7 +1350,10 @@ pub struct ReasoningSummaryTextDeltaNotification { #[serde(rename_all = "camelCase")] #[ts(export_to = "v2/")] pub struct ReasoningSummaryPartAddedNotification { + pub thread_id: String, + pub turn_id: String, pub item_id: String, + #[ts(type = "number")] pub summary_index: i64, } @@ -922,8 +1361,11 @@ pub struct ReasoningSummaryPartAddedNotification { #[serde(rename_all = "camelCase")] #[ts(export_to = "v2/")] pub struct ReasoningTextDeltaNotification { + pub thread_id: String, + pub turn_id: String, pub item_id: String, pub delta: String, + #[ts(type = "number")] pub content_index: i64, } @@ -931,6 +1373,18 @@ pub struct ReasoningTextDeltaNotification { #[serde(rename_all = "camelCase")] #[ts(export_to = "v2/")] pub struct CommandExecutionOutputDeltaNotification { + pub thread_id: String, + pub turn_id: String, + pub item_id: String, + pub delta: String, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct FileChangeOutputDeltaNotification { + pub thread_id: String, + pub turn_id: String, pub item_id: String, pub delta: String, } @@ -939,6 +1393,8 @@ pub struct CommandExecutionOutputDeltaNotification { #[serde(rename_all = "camelCase")] #[ts(export_to = "v2/")] pub struct McpToolCallProgressNotification { + pub thread_id: String, + pub turn_id: String, pub item_id: String, pub message: String, } @@ -952,6 +1408,14 @@ pub struct WindowsWorldWritableWarningNotification { pub failed_scan: bool, } +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct ContextCompactedNotification { + pub thread_id: String, + pub turn_id: String, +} + #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] #[serde(rename_all = "camelCase")] #[ts(export_to = "v2/")] @@ -1036,7 +1500,9 @@ impl From for RateLimitSnapshot { #[ts(export_to = "v2/")] pub struct RateLimitWindow { pub used_percent: i32, + #[ts(type = "number | null")] pub window_duration_mins: Option, + #[ts(type = "number | null")] pub resets_at: Option, } @@ -1091,6 +1557,7 @@ mod tests { use codex_protocol::items::WebSearchItem; use codex_protocol::user_input::UserInput as CoreUserInput; use pretty_assertions::assert_eq; + use serde_json::json; use std::path::PathBuf; #[test] @@ -1176,4 +1643,20 @@ mod tests { } ); } + + #[test] + fn codex_error_info_serializes_http_status_code_in_camel_case() { + let value = CodexErrorInfo::ResponseTooManyFailedAttempts { + http_status_code: Some(401), + }; + + assert_eq!( + serde_json::to_value(value).unwrap(), + json!({ + "responseTooManyFailedAttempts": { + "httpStatusCode": 401 + } + }) + ); + } } diff --git a/codex-rs/app-server-test-client/Cargo.toml b/codex-rs/app-server-test-client/Cargo.toml index 2fd14fb158..25a8813645 100644 --- a/codex-rs/app-server-test-client/Cargo.toml +++ b/codex-rs/app-server-test-client/Cargo.toml @@ -1,7 +1,8 @@ [package] name = "codex-app-server-test-client" -version = { workspace = true } -edition = "2024" +version.workspace = true +edition.workspace = true +license.workspace = true [lints] workspace = true diff --git a/codex-rs/app-server-test-client/src/main.rs b/codex-rs/app-server-test-client/src/main.rs index 7f2b75bd21..8c2a38e46c 100644 --- a/codex-rs/app-server-test-client/src/main.rs +++ b/codex-rs/app-server-test-client/src/main.rs @@ -101,6 +101,15 @@ enum CliCommand { /// Start a V2 turn that should not elicit an ExecCommand approval. #[command(name = "no-trigger-cmd-approval")] NoTriggerCmdApproval, + /// Send two sequential V2 turns in the same thread to test follow-up behavior. + SendFollowUpV2 { + /// Initial user message for the first turn. + #[arg()] + first_message: String, + /// Follow-up user message for the second turn. + #[arg()] + follow_up_message: String, + }, /// Trigger the ChatGPT login flow and wait for completion. TestLogin, /// Fetch the current account rate limits from the Codex app-server. @@ -120,6 +129,10 @@ fn main() -> Result<()> { trigger_patch_approval(codex_bin, user_message) } CliCommand::NoTriggerCmdApproval => no_trigger_cmd_approval(codex_bin), + CliCommand::SendFollowUpV2 { + first_message, + follow_up_message, + } => send_follow_up_v2(codex_bin, first_message, follow_up_message), CliCommand::TestLogin => test_login(codex_bin), CliCommand::GetAccountRateLimits => get_account_rate_limits(codex_bin), } @@ -209,6 +222,44 @@ fn send_message_v2_with_policies( Ok(()) } +fn send_follow_up_v2( + codex_bin: String, + first_message: String, + follow_up_message: String, +) -> Result<()> { + let mut client = CodexClient::spawn(codex_bin)?; + + let initialize = client.initialize()?; + println!("< initialize response: {initialize:?}"); + + let thread_response = client.thread_start(ThreadStartParams::default())?; + println!("< thread/start response: {thread_response:?}"); + + let first_turn_params = TurnStartParams { + thread_id: thread_response.thread.id.clone(), + input: vec![V2UserInput::Text { + text: first_message, + }], + ..Default::default() + }; + let first_turn_response = client.turn_start(first_turn_params)?; + println!("< turn/start response (initial): {first_turn_response:?}"); + client.stream_turn(&thread_response.thread.id, &first_turn_response.turn.id)?; + + let follow_up_params = TurnStartParams { + thread_id: thread_response.thread.id.clone(), + input: vec![V2UserInput::Text { + text: follow_up_message, + }], + ..Default::default() + }; + let follow_up_response = client.turn_start(follow_up_params)?; + println!("< turn/start response (follow-up): {follow_up_response:?}"); + client.stream_turn(&thread_response.thread.id, &follow_up_response.turn.id)?; + + Ok(()) +} + fn test_login(codex_bin: String) -> Result<()> { let mut client = CodexClient::spawn(codex_bin)?; @@ -512,7 +563,9 @@ impl CodexClient { ServerNotification::TurnCompleted(payload) => { if payload.turn.id == turn_id { println!("\n< turn/completed notification: {:?}", payload.turn.status); - if let TurnStatus::Failed { error } = &payload.turn.status { + if payload.turn.status == TurnStatus::Failed + && let Some(error) = payload.turn.error + { println!("[turn error] {}", error.message); } break; diff --git a/codex-rs/app-server/Cargo.toml b/codex-rs/app-server/Cargo.toml index 84eb68c66d..4ffe2d8913 100644 --- a/codex-rs/app-server/Cargo.toml +++ b/codex-rs/app-server/Cargo.toml @@ -1,7 +1,8 @@ [package] -edition = "2024" name = "codex-app-server" -version = { workspace = true } +version.workspace = true +edition.workspace = true +license.workspace = true [[bin]] name = "codex-app-server" @@ -29,6 +30,9 @@ codex-utils-json-to-toml = { workspace = true } chrono = { workspace = true } serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } +sha2 = { workspace = true } +tempfile = { workspace = true } +toml = { workspace = true } tokio = { workspace = true, features = [ "io-std", "macros", @@ -40,7 +44,6 @@ tracing = { workspace = true, features = ["log"] } tracing-subscriber = { workspace = true, features = ["env-filter", "fmt"] } opentelemetry-appender-tracing = { workspace = true } uuid = { workspace = true, features = ["serde", "v7"] } -codex-windows-sandbox.workspace = true [dev-dependencies] app_test_support = { workspace = true } @@ -51,6 +54,5 @@ mcp-types = { workspace = true } os_info = { workspace = true } pretty_assertions = { workspace = true } serial_test = { workspace = true } -tempfile = { workspace = true } -toml = { workspace = true } wiremock = { workspace = true } +shlex = { workspace = true } diff --git a/codex-rs/app-server/README.md b/codex-rs/app-server/README.md index 35653099aa..4e94a2c133 100644 --- a/codex-rs/app-server/README.md +++ b/codex-rs/app-server/README.md @@ -1,6 +1,6 @@ # codex-app-server -`codex app-server` is the interface Codex uses to power rich interfaces such as the [Codex VS Code extension](https://marketplace.visualstudio.com/items?itemName=openai.chatgpt). The message schema is currently unstable, but those who wish to build experimental UIs on top of Codex may find it valuable. +`codex app-server` is the interface Codex uses to power rich interfaces such as the [Codex VS Code extension](https://marketplace.visualstudio.com/items?itemName=openai.chatgpt). ## Table of Contents - [Protocol](#protocol) @@ -9,8 +9,8 @@ - [Initialization](#initialization) - [Core primitives](#core-primitives) - [Thread & turn endpoints](#thread--turn-endpoints) +- [Events (work-in-progress)](#events-work-in-progress) - [Auth endpoints](#auth-endpoints) -- [Events (work-in-progress)](#v2-streaming-events-work-in-progress) ## Protocol @@ -65,7 +65,8 @@ The JSON-RPC API exposes dedicated methods for managing Codex conversations. Thr - `thread/archive` — move a thread’s rollout file into the archived directory; returns `{}` on success. - `turn/start` — add user input to a thread and begin Codex generation; responds with the initial `turn` object and streams `turn/started`, `item/*`, and `turn/completed` notifications. - `turn/interrupt` — request cancellation of an in-flight turn by `(thread_id, turn_id)`; success is an empty `{}` response and the turn finishes with `status: "interrupted"`. -- `review/start` — kick off Codex’s automated reviewer for a thread; responds like `turn/start` and emits a `item/completed` notification with a `codeReview` item when results are ready. +- `review/start` — kick off Codex’s automated reviewer for a thread; responds like `turn/start` and emits `item/started`/`item/completed` notifications with `enteredReviewMode` and `exitedReviewMode` items, plus a final assistant `agentMessage` containing the review. +- `command/exec` — run a single command under the server sandbox without starting a thread/turn (handy for utilities and validation). ### 1) Start or resume a thread @@ -190,49 +191,161 @@ Use `review/start` to run Codex’s reviewer on the currently checked-out projec - `{"type":"baseBranch","branch":"main"}` — diff against the provided branch’s upstream (see prompt for the exact `git merge-base`/`git diff` instructions Codex will run). - `{"type":"commit","sha":"abc1234","title":"Optional subject"}` — review a specific commit. - `{"type":"custom","instructions":"Free-form reviewer instructions"}` — fallback prompt equivalent to the legacy manual review request. -- `appendToOriginalThread` (bool, default `false`) — when `true`, Codex also records a final assistant-style message with the review summary in the original thread. When `false`, only the `codeReview` item is emitted for the review run and no extra message is added to the original thread. +- `delivery` (`"inline"` or `"detached"`, default `"inline"`) — where the review runs: + - `"inline"`: run the review as a new turn on the existing thread. The response’s `reviewThreadId` equals the original `threadId`, and no new `thread/started` notification is emitted. + - `"detached"`: fork a new review thread from the parent conversation and run the review there. The response’s `reviewThreadId` is the id of this new review thread, and the server emits a `thread/started` notification for it before streaming review items. Example request/response: ```json { "method": "review/start", "id": 40, "params": { "threadId": "thr_123", - "appendToOriginalThread": true, + "delivery": "inline", "target": { "type": "commit", "sha": "1234567deadbeef", "title": "Polish tui colors" } } } -{ "id": 40, "result": { "turn": { - "id": "turn_900", - "status": "inProgress", - "items": [ - { "type": "userMessage", "id": "turn_900", "content": [ { "type": "text", "text": "Review commit 1234567: Polish tui colors" } ] } - ], - "error": null -} } } +{ "id": 40, "result": { + "turn": { + "id": "turn_900", + "status": "inProgress", + "items": [ + { "type": "userMessage", "id": "turn_900", "content": [ { "type": "text", "text": "Review commit 1234567: Polish tui colors" } ] } + ], + "error": null + }, + "reviewThreadId": "thr_123" +} } ``` +For a detached review, use `"delivery": "detached"`. The response is the same shape, but `reviewThreadId` will be the id of the new review thread (different from the original `threadId`). The server also emits a `thread/started` notification for that new thread before streaming the review turn. + Codex streams the usual `turn/started` notification followed by an `item/started` -with the same `codeReview` item id so clients can show progress: +with an `enteredReviewMode` item so clients can show progress: ```json { "method": "item/started", "params": { "item": { - "type": "codeReview", + "type": "enteredReviewMode", "id": "turn_900", "review": "current changes" } } } ``` -When the reviewer finishes, the server emits `item/completed` containing the same -`codeReview` item with the final review text: +When the reviewer finishes, the server emits `item/started` and `item/completed` +containing an `exitedReviewMode` item with the final review text: ```json { "method": "item/completed", "params": { "item": { - "type": "codeReview", + "type": "exitedReviewMode", "id": "turn_900", "review": "Looks solid overall...\n\n- Prefer Stylize helpers — app.rs:10-20\n ..." } } } ``` -The `review` string is plain text that already bundles the overall explanation plus a bullet list for each structured finding (matching `ThreadItem::CodeReview` in the generated schema). Use this notification to render the reviewer output in your client. +The `review` string is plain text that already bundles the overall explanation plus a bullet list for each structured finding (matching `ThreadItem::ExitedReviewMode` in the generated schema). Use this notification to render the reviewer output in your client. + +### 7) One-off command execution + +Run a standalone command (argv vector) in the server’s sandbox without creating a thread or turn: + +```json +{ "method": "command/exec", "id": 32, "params": { + "command": ["ls", "-la"], + "cwd": "/Users/me/project", // optional; defaults to server cwd + "sandboxPolicy": { "type": "workspaceWrite" }, // optional; defaults to user config + "timeoutMs": 10000 // optional; ms timeout; defaults to server timeout +} } +{ "id": 32, "result": { "exitCode": 0, "stdout": "...", "stderr": "" } } +``` + +Notes: +- Empty `command` arrays are rejected. +- `sandboxPolicy` accepts the same shape used by `turn/start` (e.g., `dangerFullAccess`, `readOnly`, `workspaceWrite` with flags). +- When omitted, `timeoutMs` falls back to the server default. + +## Events (work-in-progress) + +Event notifications are the server-initiated event stream for thread lifecycles, turn lifecycles, and the items within them. After you start or resume a thread, keep reading stdout for `thread/started`, `turn/*`, and `item/*` notifications. + +### Turn events + +The app-server streams JSON-RPC notifications while a turn is running. Each turn starts with `turn/started` (initial `turn`) and ends with `turn/completed` (final `turn` status). Token usage events stream separately via `thread/tokenUsage/updated`. Clients subscribe to the events they care about, rendering each item incrementally as updates arrive. The per-item lifecycle is always: `item/started` → zero or more item-specific deltas → `item/completed`. + +- `turn/started` — `{ turn }` with the turn id, empty `items`, and `status: "inProgress"`. +- `turn/completed` — `{ turn }` where `turn.status` is `completed`, `interrupted`, or `failed`; failures carry `{ error: { message, codexErrorInfo? } }`. +- `turn/plan/updated` — `{ turnId, explanation?, plan }` whenever the agent shares or changes its plan; each `plan` entry is `{ step, status }` with `status` in `pending`, `inProgress`, or `completed`. + +Today both notifications carry an empty `items` array even when item events were streamed; rely on `item/*` notifications for the canonical item list until this is fixed. + +#### Thread items + +`ThreadItem` is the tagged union carried in turn responses and `item/*` notifications. Currently we support events for the following items: +- `userMessage` — `{id, content}` where `content` is a list of user inputs (`text`, `image`, or `localImage`). +- `agentMessage` — `{id, text}` containing the accumulated agent reply. +- `reasoning` — `{id, summary, content}` where `summary` holds streamed reasoning summaries (applicable for most OpenAI models) and `content` holds raw reasoning blocks (applicable for e.g. open source models). +- `commandExecution` — `{id, command, cwd, status, commandActions, aggregatedOutput?, exitCode?, durationMs?}` for sandboxed commands; `status` is `inProgress`, `completed`, `failed`, or `declined`. +- `fileChange` — `{id, changes, status}` describing proposed edits; `changes` list `{path, kind, diff}` and `status` is `inProgress`, `completed`, `failed`, or `declined`. +- `mcpToolCall` — `{id, server, tool, status, arguments, result?, error?}` describing MCP calls; `status` is `inProgress`, `completed`, or `failed`. +- `webSearch` — `{id, query}` for a web search request issued by the agent. +- `compacted` - `{threadId, turnId}` when codex compacts the conversation history. This can happen automatically. + +All items emit two shared lifecycle events: +- `item/started` — emits the full `item` when a new unit of work begins so the UI can render it immediately; the `item.id` in this payload matches the `itemId` used by deltas. +- `item/completed` — sends the final `item` once that work finishes (e.g., after a tool call or message completes); treat this as the authoritative state. + +There are additional item-specific events: +#### agentMessage +- `item/agentMessage/delta` — appends streamed text for the agent message; concatenate `delta` values for the same `itemId` in order to reconstruct the full reply. +#### reasoning +- `item/reasoning/summaryTextDelta` — streams readable reasoning summaries; `summaryIndex` increments when a new summary section opens. +- `item/reasoning/summaryPartAdded` — marks the boundary between reasoning summary sections for an `itemId`; subsequent `summaryTextDelta` entries share the same `summaryIndex`. +- `item/reasoning/textDelta` — streams raw reasoning text (only applicable for e.g. open source models); use `contentIndex` to group deltas that belong together before showing them in the UI. +#### commandExecution +- `item/commandExecution/outputDelta` — streams stdout/stderr for the command; append deltas in order to render live output alongside `aggregatedOutput` in the final item. +Final `commandExecution` items include parsed `commandActions`, `status`, `exitCode`, and `durationMs` so the UI can summarize what ran and whether it succeeded. +#### fileChange +`fileChange` items contain a `changes` list with `{path, kind, diff}` entries (`kind` is `add`, `delete`, or `update` with an optional `movePath`). The `status` tracks whether apply succeeded (`completed`), failed, or was `declined`. + +### Errors +`error` event is emitted whenever the server hits an error mid-turn (for example, upstream model errors or quota limits). Carries the same `{ error: { message, codexErrorInfo? } }` payload as `turn.status: "failed"` and may precede that terminal notification. + + `codexErrorInfo` maps to the `CodexErrorInfo` enum. Common values: + - `ContextWindowExceeded` + - `UsageLimitExceeded` + - `HttpConnectionFailed { httpStatusCode? }`: upstream HTTP failures including 4xx/5xx + - `ResponseStreamConnectionFailed { httpStatusCode? }`: failure to connect to the response SSE stream + - `ResponseStreamDisconnected { httpStatusCode? }`: disconnect of the response SSE stream in the middle of a turn before completion + - `ResponseTooManyFailedAttempts { httpStatusCode? }` + - `BadRequest` + - `Unauthorized` + - `SandboxError` + - `InternalServerError` + - `Other`: all unclassified errors + +When an upstream HTTP status is available (for example, from the Responses API or a provider), it is forwarded in `httpStatusCode` on the relevant `codexErrorInfo` variant. + +## Approvals + +Certain actions (shell commands or modifying files) may require explicit user approval depending on the user's config. When `turn/start` is used, the app-server drives an approval flow by sending a server-initiated JSON-RPC request to the client. The client must respond to tell Codex whether to proceed. UIs should present these requests inline with the active turn so users can review the proposed command or diff before choosing. + +- Requests include `threadId` and `turnId`—use them to scope UI state to the active conversation. +- Respond with a single `{ "decision": "accept" | "decline" }` payload (plus optional `acceptSettings` on command executions). The server resumes or declines the work and ends the item with `item/completed`. + +### Command execution approvals + +Order of messages: +1. `item/started` — shows the pending `commandExecution` item with `command`, `cwd`, and other fields so you can render the proposed action. +2. `item/commandExecution/requestApproval` (request) — carries the same `itemId`, `threadId`, `turnId`, optionally `reason` or `risk`, plus `parsedCmd` for friendly display. +3. Client response — `{ "decision": "accept", "acceptSettings": { "forSession": false } }` or `{ "decision": "decline" }`. +4. `item/completed` — final `commandExecution` item with `status: "completed" | "failed" | "declined"` and execution output. Render this as the authoritative result. + +### File change approvals + +Order of messages: +1. `item/started` — emits a `fileChange` item with `changes` (diff chunk summaries) and `status: "inProgress"`. Show the proposed edits and paths to the user. +2. `item/fileChange/requestApproval` (request) — includes `itemId`, `threadId`, `turnId`, and an optional `reason`. +3. Client response — `{ "decision": "accept" }` or `{ "decision": "decline" }`. +4. `item/completed` — returns the same `fileChange` item with `status` updated to `completed`, `failed`, or `declined` after the patch attempt. Rely on this to show success/failure and finalize the diff state in your UI. + +UI guidance for IDEs: surface an approval dialog as soon as the request arrives. The turn will proceed after the server receives a response to the approval request. The terminal `item/completed` notification will be sent with the appropriate status. ## Auth endpoints @@ -329,33 +442,3 @@ Field notes: - `codex app-server generate-ts --out ` emits v2 types under `v2/`. - `codex app-server generate-json-schema --out ` outputs `codex_app_server_protocol.schemas.json`. - See [“Authentication and authorization” in the config docs](../../docs/config.md#authentication-and-authorization) for configuration knobs. - - -## Events (work-in-progress) - -Event notifications are the server-initiated event stream for thread lifecycles, turn lifecycles, and the items within them. After you start or resume a thread, keep reading stdout for `thread/started`, `turn/*`, and `item/*` notifications. - -### Turn events - -The app-server streams JSON-RPC notifications while a turn is running. Each turn starts with `turn/started` (initial `turn`) and ends with `turn/completed` (final `turn` plus token `usage`), and clients subscribe to the events they care about, rendering each item incrementally as updates arrive. The per-item lifecycle is always: `item/started` → zero or more item-specific deltas → `item/completed`. - -#### Thread items - -`ThreadItem` is the tagged union carried in turn responses and `item/*` notifications. Currently we support events for the following items: -- `userMessage` — `{id, content}` where `content` is a list of user inputs (`text`, `image`, or `localImage`). -- `agentMessage` — `{id, text}` containing the accumulated agent reply. -- `reasoning` — `{id, summary, content}` where `summary` holds streamed reasoning summaries (applicable for most OpenAI models) and `content` holds raw reasoning blocks (applicable for e.g. open source models). -- `mcpToolCall` — `{id, server, tool, status, arguments, result?, error?}` describing MCP calls; `status` is `inProgress`, `completed`, or `failed`. -- `webSearch` — `{id, query}` for a web search request issued by the agent. - -All items emit two shared lifecycle events: -- `item/started` — emits the full `item` when a new unit of work begins so the UI can render it immediately; the `item.id` in this payload matches the `itemId` used by deltas. -- `item/completed` — sends the final `item` once that work finishes (e.g., after a tool call or message completes); treat this as the authoritative state. - -There are additional item-specific events: -#### agentMessage -- `item/agentMessage/delta` — appends streamed text for the agent message; concatenate `delta` values for the same `itemId` in order to reconstruct the full reply. -#### reasoning -- `item/reasoning/summaryTextDelta` — streams readable reasoning summaries; `summaryIndex` increments when a new summary section opens. -- `item/reasoning/summaryPartAdded` — marks the boundary between reasoning summary sections for an `itemId`; subsequent `summaryTextDelta` entries share the same `summaryIndex`. -- `item/reasoning/textDelta` — streams raw reasoning text (only applicable for e.g. open source models); use `contentIndex` to group deltas that belong together before showing them in the UI. diff --git a/codex-rs/app-server/src/bespoke_event_handling.rs b/codex-rs/app-server/src/bespoke_event_handling.rs index d4b00e67f9..b4dd16b9a6 100644 --- a/codex-rs/app-server/src/bespoke_event_handling.rs +++ b/codex-rs/app-server/src/bespoke_event_handling.rs @@ -8,13 +8,17 @@ use codex_app_server_protocol::AgentMessageDeltaNotification; use codex_app_server_protocol::ApplyPatchApprovalParams; use codex_app_server_protocol::ApplyPatchApprovalResponse; use codex_app_server_protocol::ApprovalDecision; +use codex_app_server_protocol::CodexErrorInfo as V2CodexErrorInfo; use codex_app_server_protocol::CommandAction as V2ParsedCommand; use codex_app_server_protocol::CommandExecutionOutputDeltaNotification; use codex_app_server_protocol::CommandExecutionRequestApprovalParams; use codex_app_server_protocol::CommandExecutionRequestApprovalResponse; use codex_app_server_protocol::CommandExecutionStatus; +use codex_app_server_protocol::ContextCompactedNotification; +use codex_app_server_protocol::ErrorNotification; use codex_app_server_protocol::ExecCommandApprovalParams; use codex_app_server_protocol::ExecCommandApprovalResponse; +use codex_app_server_protocol::FileChangeOutputDeltaNotification; use codex_app_server_protocol::FileChangeRequestApprovalParams; use codex_app_server_protocol::FileChangeRequestApprovalResponse; use codex_app_server_protocol::FileUpdateChange; @@ -33,10 +37,15 @@ use codex_app_server_protocol::SandboxCommandAssessment as V2SandboxCommandAsses use codex_app_server_protocol::ServerNotification; use codex_app_server_protocol::ServerRequestPayload; use codex_app_server_protocol::ThreadItem; +use codex_app_server_protocol::ThreadTokenUsage; +use codex_app_server_protocol::ThreadTokenUsageUpdatedNotification; use codex_app_server_protocol::Turn; use codex_app_server_protocol::TurnCompletedNotification; +use codex_app_server_protocol::TurnDiffUpdatedNotification; use codex_app_server_protocol::TurnError; use codex_app_server_protocol::TurnInterruptResponse; +use codex_app_server_protocol::TurnPlanStep; +use codex_app_server_protocol::TurnPlanUpdatedNotification; use codex_app_server_protocol::TurnStatus; use codex_core::CodexConversation; use codex_core::parse_command::shlex_join; @@ -50,8 +59,12 @@ use codex_core::protocol::McpToolCallBeginEvent; use codex_core::protocol::McpToolCallEndEvent; use codex_core::protocol::Op; use codex_core::protocol::ReviewDecision; +use codex_core::protocol::TokenCountEvent; +use codex_core::protocol::TurnDiffEvent; use codex_core::review_format::format_review_findings_block; +use codex_core::review_prompts; use codex_protocol::ConversationId; +use codex_protocol::plan_tool::UpdatePlanArgs; use codex_protocol::protocol::ReviewOutputEvent; use std::collections::HashMap; use std::convert::TryFrom; @@ -71,10 +84,19 @@ pub(crate) async fn apply_bespoke_event_handling( turn_summary_store: TurnSummaryStore, api_version: ApiVersion, ) { - let Event { id: event_id, msg } = event; + let Event { + id: event_turn_id, + msg, + } = event; match msg { EventMsg::TaskComplete(_ev) => { - handle_turn_complete(conversation_id, event_id, &outgoing, &turn_summary_store).await; + handle_turn_complete( + conversation_id, + event_turn_id, + &outgoing, + &turn_summary_store, + ) + .await; } EventMsg::ApplyPatchApprovalRequest(ApplyPatchApprovalRequestEvent { call_id, @@ -95,7 +117,7 @@ pub(crate) async fn apply_bespoke_event_handling( .send_request(ServerRequestPayload::ApplyPatchApproval(params)) .await; tokio::spawn(async move { - on_patch_approval_response(event_id, rx, conversation).await; + on_patch_approval_response(event_turn_id, rx, conversation).await; }); } ApiVersion::V2 => { @@ -115,7 +137,11 @@ pub(crate) async fn apply_bespoke_event_handling( changes: patch_changes.clone(), status: PatchApplyStatus::InProgress, }; - let notification = ItemStartedNotification { item }; + let notification = ItemStartedNotification { + thread_id: conversation_id.to_string(), + turn_id: event_turn_id.clone(), + item, + }; outgoing .send_server_notification(ServerNotification::ItemStarted(notification)) .await; @@ -133,7 +159,7 @@ pub(crate) async fn apply_bespoke_event_handling( .await; tokio::spawn(async move { on_file_change_request_approval_response( - event_id, + event_turn_id, conversation_id, item_id, patch_changes, @@ -169,16 +195,24 @@ pub(crate) async fn apply_bespoke_event_handling( .send_request(ServerRequestPayload::ExecCommandApproval(params)) .await; tokio::spawn(async move { - on_exec_approval_response(event_id, rx, conversation).await; + on_exec_approval_response(event_turn_id, rx, conversation).await; }); } ApiVersion::V2 => { + let item_id = call_id.clone(); + let command_actions = parsed_cmd + .iter() + .cloned() + .map(V2ParsedCommand::from) + .collect::>(); + let command_string = shlex_join(&command); + let params = CommandExecutionRequestApprovalParams { thread_id: conversation_id.to_string(), turn_id: turn_id.clone(), // Until we migrate the core to be aware of a first class CommandExecutionItem // and emit the corresponding EventMsg, we repurpose the call_id as the item_id. - item_id: call_id.clone(), + item_id: item_id.clone(), reason, risk: risk.map(V2SandboxCommandAssessment::from), }; @@ -188,26 +222,48 @@ pub(crate) async fn apply_bespoke_event_handling( )) .await; tokio::spawn(async move { - on_command_execution_request_approval_response(event_id, rx, conversation) - .await; + on_command_execution_request_approval_response( + event_turn_id, + conversation_id, + item_id, + command_string, + cwd, + command_actions, + rx, + conversation, + outgoing, + ) + .await; }); } }, // TODO(celia): properly construct McpToolCall TurnItem in core. EventMsg::McpToolCallBegin(begin_event) => { - let notification = construct_mcp_tool_call_notification(begin_event).await; + let notification = construct_mcp_tool_call_notification( + begin_event, + conversation_id.to_string(), + event_turn_id.clone(), + ) + .await; outgoing .send_server_notification(ServerNotification::ItemStarted(notification)) .await; } EventMsg::McpToolCallEnd(end_event) => { - let notification = construct_mcp_tool_call_end_notification(end_event).await; + let notification = construct_mcp_tool_call_end_notification( + end_event, + conversation_id.to_string(), + event_turn_id.clone(), + ) + .await; outgoing .send_server_notification(ServerNotification::ItemCompleted(notification)) .await; } EventMsg::AgentMessageContentDelta(event) => { let notification = AgentMessageDeltaNotification { + thread_id: conversation_id.to_string(), + turn_id: event_turn_id.clone(), item_id: event.item_id, delta: event.delta, }; @@ -215,8 +271,19 @@ pub(crate) async fn apply_bespoke_event_handling( .send_server_notification(ServerNotification::AgentMessageDelta(notification)) .await; } + EventMsg::ContextCompacted(..) => { + let notification = ContextCompactedNotification { + thread_id: conversation_id.to_string(), + turn_id: event_turn_id.clone(), + }; + outgoing + .send_server_notification(ServerNotification::ContextCompacted(notification)) + .await; + } EventMsg::ReasoningContentDelta(event) => { let notification = ReasoningSummaryTextDeltaNotification { + thread_id: conversation_id.to_string(), + turn_id: event_turn_id.clone(), item_id: event.item_id, delta: event.delta, summary_index: event.summary_index, @@ -229,6 +296,8 @@ pub(crate) async fn apply_bespoke_event_handling( } EventMsg::ReasoningRawContentDelta(event) => { let notification = ReasoningTextDeltaNotification { + thread_id: conversation_id.to_string(), + turn_id: event_turn_id.clone(), item_id: event.item_id, delta: event.delta, content_index: event.content_index, @@ -239,6 +308,8 @@ pub(crate) async fn apply_bespoke_event_handling( } EventMsg::AgentReasoningSectionBreak(event) => { let notification = ReasoningSummaryPartAddedNotification { + thread_id: conversation_id.to_string(), + turn_id: event_turn_id.clone(), item_id: event.item_id, summary_index: event.summary_index, }; @@ -249,57 +320,131 @@ pub(crate) async fn apply_bespoke_event_handling( .await; } EventMsg::TokenCount(token_count_event) => { - if let Some(rate_limits) = token_count_event.rate_limits { - outgoing - .send_server_notification(ServerNotification::AccountRateLimitsUpdated( - AccountRateLimitsUpdatedNotification { - rate_limits: rate_limits.into(), - }, - )) - .await; - } + handle_token_count_event(conversation_id, event_turn_id, token_count_event, &outgoing) + .await; } EventMsg::Error(ev) => { - handle_error(conversation_id, ev.message, &turn_summary_store).await; + let turn_error = TurnError { + message: ev.message, + codex_error_info: ev.codex_error_info.map(V2CodexErrorInfo::from), + }; + handle_error(conversation_id, turn_error.clone(), &turn_summary_store).await; + outgoing + .send_server_notification(ServerNotification::Error(ErrorNotification { + error: turn_error, + thread_id: conversation_id.to_string(), + turn_id: event_turn_id.clone(), + })) + .await; } - EventMsg::EnteredReviewMode(review_request) => { - let notification = ItemStartedNotification { - item: ThreadItem::CodeReview { - id: event_id.clone(), - review: review_request.user_facing_hint, - }, + EventMsg::StreamError(ev) => { + // We don't need to update the turn summary store for stream errors as they are intermediate error states for retries, + // but we notify the client. + let turn_error = TurnError { + message: ev.message, + codex_error_info: ev.codex_error_info.map(V2CodexErrorInfo::from), }; outgoing - .send_server_notification(ServerNotification::ItemStarted(notification)) + .send_server_notification(ServerNotification::Error(ErrorNotification { + error: turn_error, + thread_id: conversation_id.to_string(), + turn_id: event_turn_id.clone(), + })) + .await; + } + EventMsg::ViewImageToolCall(view_image_event) => { + let item = ThreadItem::ImageView { + id: view_image_event.call_id.clone(), + path: view_image_event.path.to_string_lossy().into_owned(), + }; + let started = ItemStartedNotification { + thread_id: conversation_id.to_string(), + turn_id: event_turn_id.clone(), + item: item.clone(), + }; + outgoing + .send_server_notification(ServerNotification::ItemStarted(started)) + .await; + let completed = ItemCompletedNotification { + thread_id: conversation_id.to_string(), + turn_id: event_turn_id.clone(), + item, + }; + outgoing + .send_server_notification(ServerNotification::ItemCompleted(completed)) + .await; + } + EventMsg::EnteredReviewMode(review_request) => { + let review = review_request + .user_facing_hint + .unwrap_or_else(|| review_prompts::user_facing_hint(&review_request.target)); + let item = ThreadItem::EnteredReviewMode { + id: event_turn_id.clone(), + review, + }; + let started = ItemStartedNotification { + thread_id: conversation_id.to_string(), + turn_id: event_turn_id.clone(), + item: item.clone(), + }; + outgoing + .send_server_notification(ServerNotification::ItemStarted(started)) + .await; + let completed = ItemCompletedNotification { + thread_id: conversation_id.to_string(), + turn_id: event_turn_id.clone(), + item, + }; + outgoing + .send_server_notification(ServerNotification::ItemCompleted(completed)) .await; } EventMsg::ItemStarted(item_started_event) => { let item: ThreadItem = item_started_event.item.clone().into(); - let notification = ItemStartedNotification { item }; + let notification = ItemStartedNotification { + thread_id: conversation_id.to_string(), + turn_id: event_turn_id.clone(), + item, + }; outgoing .send_server_notification(ServerNotification::ItemStarted(notification)) .await; } EventMsg::ItemCompleted(item_completed_event) => { let item: ThreadItem = item_completed_event.item.clone().into(); - let notification = ItemCompletedNotification { item }; + let notification = ItemCompletedNotification { + thread_id: conversation_id.to_string(), + turn_id: event_turn_id.clone(), + item, + }; outgoing .send_server_notification(ServerNotification::ItemCompleted(notification)) .await; } EventMsg::ExitedReviewMode(review_event) => { - let review_text = match review_event.review_output { + let review = match review_event.review_output { Some(output) => render_review_output_text(&output), None => REVIEW_FALLBACK_MESSAGE.to_string(), }; - let notification = ItemCompletedNotification { - item: ThreadItem::CodeReview { - id: event_id, - review: review_text, - }, + let item = ThreadItem::ExitedReviewMode { + id: event_turn_id.clone(), + review, + }; + let started = ItemStartedNotification { + thread_id: conversation_id.to_string(), + turn_id: event_turn_id.clone(), + item: item.clone(), }; outgoing - .send_server_notification(ServerNotification::ItemCompleted(notification)) + .send_server_notification(ServerNotification::ItemStarted(started)) + .await; + let completed = ItemCompletedNotification { + thread_id: conversation_id.to_string(), + turn_id: event_turn_id.clone(), + item, + }; + outgoing + .send_server_notification(ServerNotification::ItemCompleted(completed)) .await; } EventMsg::PatchApplyBegin(patch_begin_event) => { @@ -318,7 +463,11 @@ pub(crate) async fn apply_bespoke_event_handling( changes: convert_patch_changes(&patch_begin_event.changes), status: PatchApplyStatus::InProgress, }; - let notification = ItemStartedNotification { item }; + let notification = ItemStartedNotification { + thread_id: conversation_id.to_string(), + turn_id: event_turn_id.clone(), + item, + }; outgoing .send_server_notification(ServerNotification::ItemStarted(notification)) .await; @@ -340,41 +489,82 @@ pub(crate) async fn apply_bespoke_event_handling( item_id, changes, status, + event_turn_id.clone(), outgoing.as_ref(), &turn_summary_store, ) .await; } EventMsg::ExecCommandBegin(exec_command_begin_event) => { + let item_id = exec_command_begin_event.call_id.clone(); + let command_actions = exec_command_begin_event + .parsed_cmd + .into_iter() + .map(V2ParsedCommand::from) + .collect::>(); + let command = shlex_join(&exec_command_begin_event.command); + let cwd = exec_command_begin_event.cwd; + let process_id = exec_command_begin_event.process_id; + let item = ThreadItem::CommandExecution { - id: exec_command_begin_event.call_id.clone(), - command: shlex_join(&exec_command_begin_event.command), - cwd: exec_command_begin_event.cwd, + id: item_id, + command, + cwd, + process_id, status: CommandExecutionStatus::InProgress, - command_actions: exec_command_begin_event - .parsed_cmd - .into_iter() - .map(V2ParsedCommand::from) - .collect(), + command_actions, aggregated_output: None, exit_code: None, duration_ms: None, }; - let notification = ItemStartedNotification { item }; + let notification = ItemStartedNotification { + thread_id: conversation_id.to_string(), + turn_id: event_turn_id.clone(), + item, + }; outgoing .send_server_notification(ServerNotification::ItemStarted(notification)) .await; } EventMsg::ExecCommandOutputDelta(exec_command_output_delta_event) => { - let notification = CommandExecutionOutputDeltaNotification { - item_id: exec_command_output_delta_event.call_id.clone(), - delta: String::from_utf8_lossy(&exec_command_output_delta_event.chunk).to_string(), + let item_id = exec_command_output_delta_event.call_id.clone(); + let delta = String::from_utf8_lossy(&exec_command_output_delta_event.chunk).to_string(); + // The underlying EventMsg::ExecCommandOutputDelta is used for shell, unified_exec, + // and apply_patch tool calls. We represent apply_patch with the FileChange item, and + // everything else with the CommandExecution item. + // + // We need to detect which item type it is so we can emit the right notification. + // We already have state tracking FileChange items on item/started, so let's use that. + let is_file_change = { + let map = turn_summary_store.lock().await; + map.get(&conversation_id) + .is_some_and(|summary| summary.file_change_started.contains(&item_id)) }; - outgoing - .send_server_notification(ServerNotification::CommandExecutionOutputDelta( - notification, - )) - .await; + if is_file_change { + let notification = FileChangeOutputDeltaNotification { + thread_id: conversation_id.to_string(), + turn_id: event_turn_id.clone(), + item_id, + delta, + }; + outgoing + .send_server_notification(ServerNotification::FileChangeOutputDelta( + notification, + )) + .await; + } else { + let notification = CommandExecutionOutputDeltaNotification { + thread_id: conversation_id.to_string(), + turn_id: event_turn_id.clone(), + item_id, + delta, + }; + outgoing + .send_server_notification(ServerNotification::CommandExecutionOutputDelta( + notification, + )) + .await; + } } EventMsg::ExecCommandEnd(exec_command_end_event) => { let ExecCommandEndEvent { @@ -382,6 +572,7 @@ pub(crate) async fn apply_bespoke_event_handling( command, cwd, parsed_cmd, + process_id, aggregated_output, exit_code, duration, @@ -393,6 +584,10 @@ pub(crate) async fn apply_bespoke_event_handling( } else { CommandExecutionStatus::Failed }; + let command_actions = parsed_cmd + .into_iter() + .map(V2ParsedCommand::from) + .collect::>(); let aggregated_output = if aggregated_output.is_empty() { None @@ -406,14 +601,19 @@ pub(crate) async fn apply_bespoke_event_handling( id: call_id, command: shlex_join(&command), cwd, + process_id, status, - command_actions: parsed_cmd.into_iter().map(V2ParsedCommand::from).collect(), + command_actions, aggregated_output, exit_code: Some(exit_code), duration_ms: Some(duration_ms), }; - let notification = ItemCompletedNotification { item }; + let notification = ItemCompletedNotification { + thread_id: conversation_id.to_string(), + turn_id: event_turn_id.clone(), + item, + }; outgoing .send_server_notification(ServerNotification::ItemCompleted(notification)) .await; @@ -441,23 +641,92 @@ pub(crate) async fn apply_bespoke_event_handling( } } - handle_turn_interrupted(conversation_id, event_id, &outgoing, &turn_summary_store) - .await; + handle_turn_interrupted( + conversation_id, + event_turn_id, + &outgoing, + &turn_summary_store, + ) + .await; + } + EventMsg::TurnDiff(turn_diff_event) => { + handle_turn_diff( + conversation_id, + &event_turn_id, + turn_diff_event, + api_version, + outgoing.as_ref(), + ) + .await; + } + EventMsg::PlanUpdate(plan_update_event) => { + handle_turn_plan_update( + &event_turn_id, + plan_update_event, + api_version, + outgoing.as_ref(), + ) + .await; } _ => {} } } +async fn handle_turn_diff( + conversation_id: ConversationId, + event_turn_id: &str, + turn_diff_event: TurnDiffEvent, + api_version: ApiVersion, + outgoing: &OutgoingMessageSender, +) { + if let ApiVersion::V2 = api_version { + let notification = TurnDiffUpdatedNotification { + thread_id: conversation_id.to_string(), + turn_id: event_turn_id.to_string(), + diff: turn_diff_event.unified_diff, + }; + outgoing + .send_server_notification(ServerNotification::TurnDiffUpdated(notification)) + .await; + } +} + +async fn handle_turn_plan_update( + event_turn_id: &str, + plan_update_event: UpdatePlanArgs, + api_version: ApiVersion, + outgoing: &OutgoingMessageSender, +) { + if let ApiVersion::V2 = api_version { + let notification = TurnPlanUpdatedNotification { + turn_id: event_turn_id.to_string(), + explanation: plan_update_event.explanation, + plan: plan_update_event + .plan + .into_iter() + .map(TurnPlanStep::from) + .collect(), + }; + outgoing + .send_server_notification(ServerNotification::TurnPlanUpdated(notification)) + .await; + } +} + async fn emit_turn_completed_with_status( - event_id: String, + conversation_id: ConversationId, + event_turn_id: String, status: TurnStatus, + error: Option, outgoing: &OutgoingMessageSender, ) { let notification = TurnCompletedNotification { + thread_id: conversation_id.to_string(), turn: Turn { - id: event_id, + id: event_turn_id, items: vec![], + error, status, }, }; @@ -471,6 +740,7 @@ async fn complete_file_change_item( item_id: String, changes: Vec, status: PatchApplyStatus, + turn_id: String, outgoing: &OutgoingMessageSender, turn_summary_store: &TurnSummaryStore, ) { @@ -486,7 +756,44 @@ async fn complete_file_change_item( changes, status, }; - let notification = ItemCompletedNotification { item }; + let notification = ItemCompletedNotification { + thread_id: conversation_id.to_string(), + turn_id, + item, + }; + outgoing + .send_server_notification(ServerNotification::ItemCompleted(notification)) + .await; +} + +#[allow(clippy::too_many_arguments)] +async fn complete_command_execution_item( + conversation_id: ConversationId, + turn_id: String, + item_id: String, + command: String, + cwd: PathBuf, + process_id: Option, + command_actions: Vec, + status: CommandExecutionStatus, + outgoing: &OutgoingMessageSender, +) { + let item = ThreadItem::CommandExecution { + id: item_id, + command, + cwd, + process_id, + status, + command_actions, + aggregated_output: None, + exit_code: None, + duration_ms: None, + }; + let notification = ItemCompletedNotification { + thread_id: conversation_id.to_string(), + turn_id, + item, + }; outgoing .send_server_notification(ServerNotification::ItemCompleted(notification)) .await; @@ -502,45 +809,77 @@ async fn find_and_remove_turn_summary( async fn handle_turn_complete( conversation_id: ConversationId, - event_id: String, + event_turn_id: String, outgoing: &OutgoingMessageSender, turn_summary_store: &TurnSummaryStore, ) { let turn_summary = find_and_remove_turn_summary(conversation_id, turn_summary_store).await; - let status = if let Some(message) = turn_summary.last_error_message { - TurnStatus::Failed { - error: TurnError { message }, - } - } else { - TurnStatus::Completed + let (status, error) = match turn_summary.last_error { + Some(error) => (TurnStatus::Failed, Some(error)), + None => (TurnStatus::Completed, None), }; - emit_turn_completed_with_status(event_id, status, outgoing).await; + emit_turn_completed_with_status(conversation_id, event_turn_id, status, error, outgoing).await; } async fn handle_turn_interrupted( conversation_id: ConversationId, - event_id: String, + event_turn_id: String, outgoing: &OutgoingMessageSender, turn_summary_store: &TurnSummaryStore, ) { find_and_remove_turn_summary(conversation_id, turn_summary_store).await; - emit_turn_completed_with_status(event_id, TurnStatus::Interrupted, outgoing).await; + emit_turn_completed_with_status( + conversation_id, + event_turn_id, + TurnStatus::Interrupted, + None, + outgoing, + ) + .await; +} + +async fn handle_token_count_event( + conversation_id: ConversationId, + turn_id: String, + token_count_event: TokenCountEvent, + outgoing: &OutgoingMessageSender, +) { + let TokenCountEvent { info, rate_limits } = token_count_event; + if let Some(token_usage) = info.map(ThreadTokenUsage::from) { + let notification = ThreadTokenUsageUpdatedNotification { + thread_id: conversation_id.to_string(), + turn_id, + token_usage, + }; + outgoing + .send_server_notification(ServerNotification::ThreadTokenUsageUpdated(notification)) + .await; + } + if let Some(rate_limits) = rate_limits { + outgoing + .send_server_notification(ServerNotification::AccountRateLimitsUpdated( + AccountRateLimitsUpdatedNotification { + rate_limits: rate_limits.into(), + }, + )) + .await; + } } async fn handle_error( conversation_id: ConversationId, - message: String, + error: TurnError, turn_summary_store: &TurnSummaryStore, ) { let mut map = turn_summary_store.lock().await; - map.entry(conversation_id).or_default().last_error_message = Some(message); + map.entry(conversation_id).or_default().last_error = Some(error); } async fn on_patch_approval_response( - event_id: String, + event_turn_id: String, receiver: oneshot::Receiver, codex: Arc, ) { @@ -551,7 +890,7 @@ async fn on_patch_approval_response( error!("request failed: {err:?}"); if let Err(submit_err) = codex .submit(Op::PatchApproval { - id: event_id.clone(), + id: event_turn_id.clone(), decision: ReviewDecision::Denied, }) .await @@ -572,7 +911,7 @@ async fn on_patch_approval_response( if let Err(err) = codex .submit(Op::PatchApproval { - id: event_id, + id: event_turn_id, decision: response.decision, }) .await @@ -582,7 +921,7 @@ async fn on_patch_approval_response( } async fn on_exec_approval_response( - event_id: String, + event_turn_id: String, receiver: oneshot::Receiver, conversation: Arc, ) { @@ -608,7 +947,7 @@ async fn on_exec_approval_response( if let Err(err) = conversation .submit(Op::ExecApproval { - id: event_id, + id: event_turn_id, decision: response.decision, }) .await @@ -681,7 +1020,7 @@ fn format_file_change_diff(change: &CoreFileChange) -> String { #[allow(clippy::too_many_arguments)] async fn on_file_change_request_approval_response( - event_id: String, + event_turn_id: String, conversation_id: ConversationId, item_id: String, changes: Vec, @@ -726,6 +1065,7 @@ async fn on_file_change_request_approval_response( item_id, changes, status, + event_turn_id.clone(), outgoing.as_ref(), &turn_summary_store, ) @@ -734,7 +1074,7 @@ async fn on_file_change_request_approval_response( if let Err(err) = codex .submit(Op::PatchApproval { - id: event_id, + id: event_turn_id, decision, }) .await @@ -743,45 +1083,75 @@ async fn on_file_change_request_approval_response( } } +#[allow(clippy::too_many_arguments)] async fn on_command_execution_request_approval_response( - event_id: String, + event_turn_id: String, + conversation_id: ConversationId, + item_id: String, + command: String, + cwd: PathBuf, + command_actions: Vec, receiver: oneshot::Receiver, conversation: Arc, + outgoing: Arc, ) { let response = receiver.await; - let value = match response { - Ok(value) => value, + let (decision, completion_status) = match response { + Ok(value) => { + let response = serde_json::from_value::(value) + .unwrap_or_else(|err| { + error!("failed to deserialize CommandExecutionRequestApprovalResponse: {err}"); + CommandExecutionRequestApprovalResponse { + decision: ApprovalDecision::Decline, + accept_settings: None, + } + }); + + let CommandExecutionRequestApprovalResponse { + decision, + accept_settings, + } = response; + + let (decision, completion_status) = match (decision, accept_settings) { + (ApprovalDecision::Accept, Some(settings)) if settings.for_session => { + (ReviewDecision::ApprovedForSession, None) + } + (ApprovalDecision::Accept, _) => (ReviewDecision::Approved, None), + (ApprovalDecision::Decline, _) => ( + ReviewDecision::Denied, + Some(CommandExecutionStatus::Declined), + ), + (ApprovalDecision::Cancel, _) => ( + ReviewDecision::Abort, + Some(CommandExecutionStatus::Declined), + ), + }; + (decision, completion_status) + } Err(err) => { error!("request failed: {err:?}"); - return; + (ReviewDecision::Denied, Some(CommandExecutionStatus::Failed)) } }; - let response = serde_json::from_value::(value) - .unwrap_or_else(|err| { - error!("failed to deserialize CommandExecutionRequestApprovalResponse: {err}"); - CommandExecutionRequestApprovalResponse { - decision: ApprovalDecision::Decline, - accept_settings: None, - } - }); + if let Some(status) = completion_status { + complete_command_execution_item( + conversation_id, + event_turn_id.clone(), + item_id.clone(), + command.clone(), + cwd.clone(), + None, + command_actions.clone(), + status, + outgoing.as_ref(), + ) + .await; + } - let CommandExecutionRequestApprovalResponse { - decision, - accept_settings, - } = response; - - let decision = match (decision, accept_settings) { - (ApprovalDecision::Accept, Some(settings)) if settings.for_session => { - ReviewDecision::ApprovedForSession - } - (ApprovalDecision::Accept, _) => ReviewDecision::Approved, - (ApprovalDecision::Decline, _) => ReviewDecision::Denied, - (ApprovalDecision::Cancel, _) => ReviewDecision::Abort, - }; if let Err(err) = conversation .submit(Op::ExecApproval { - id: event_id, + id: event_turn_id, decision, }) .await @@ -793,6 +1163,8 @@ async fn on_command_execution_request_approval_response( /// similar to handle_mcp_tool_call_begin in exec async fn construct_mcp_tool_call_notification( begin_event: McpToolCallBeginEvent, + thread_id: String, + turn_id: String, ) -> ItemStartedNotification { let item = ThreadItem::McpToolCall { id: begin_event.call_id, @@ -803,12 +1175,18 @@ async fn construct_mcp_tool_call_notification( result: None, error: None, }; - ItemStartedNotification { item } + ItemStartedNotification { + thread_id, + turn_id, + item, + } } /// simiilar to handle_mcp_tool_call_end in exec async fn construct_mcp_tool_call_end_notification( end_event: McpToolCallEndEvent, + thread_id: String, + turn_id: String, ) -> ItemCompletedNotification { let status = if end_event.is_success() { McpToolCallStatus::Completed @@ -841,7 +1219,11 @@ async fn construct_mcp_tool_call_end_notification( result, error, }; - ItemCompletedNotification { item } + ItemCompletedNotification { + thread_id, + turn_id, + item, + } } #[cfg(test)] @@ -853,7 +1235,15 @@ mod tests { use anyhow::Result; use anyhow::anyhow; use anyhow::bail; + use codex_app_server_protocol::TurnPlanStepStatus; + use codex_core::protocol::CreditsSnapshot; use codex_core::protocol::McpInvocation; + use codex_core::protocol::RateLimitSnapshot; + use codex_core::protocol::RateLimitWindow; + use codex_core::protocol::TokenUsage; + use codex_core::protocol::TokenUsageInfo; + use codex_protocol::plan_tool::PlanItemArg; + use codex_protocol::plan_tool::StepStatus; use mcp_types::CallToolResult; use mcp_types::ContentBlock; use mcp_types::TextContent; @@ -873,24 +1263,38 @@ mod tests { let conversation_id = ConversationId::new(); let turn_summary_store = new_turn_summary_store(); - handle_error(conversation_id, "boom".to_string(), &turn_summary_store).await; + handle_error( + conversation_id, + TurnError { + message: "boom".to_string(), + codex_error_info: Some(V2CodexErrorInfo::InternalServerError), + }, + &turn_summary_store, + ) + .await; let turn_summary = find_and_remove_turn_summary(conversation_id, &turn_summary_store).await; - assert_eq!(turn_summary.last_error_message, Some("boom".to_string())); + assert_eq!( + turn_summary.last_error, + Some(TurnError { + message: "boom".to_string(), + codex_error_info: Some(V2CodexErrorInfo::InternalServerError), + }) + ); Ok(()) } #[tokio::test] async fn test_handle_turn_complete_emits_completed_without_error() -> Result<()> { let conversation_id = ConversationId::new(); - let event_id = "complete1".to_string(); + let event_turn_id = "complete1".to_string(); let (tx, mut rx) = mpsc::channel(CHANNEL_CAPACITY); let outgoing = Arc::new(OutgoingMessageSender::new(tx)); let turn_summary_store = new_turn_summary_store(); handle_turn_complete( conversation_id, - event_id.clone(), + event_turn_id.clone(), &outgoing, &turn_summary_store, ) @@ -902,8 +1306,9 @@ mod tests { .ok_or_else(|| anyhow!("should send one notification"))?; match msg { OutgoingMessage::AppServerNotification(ServerNotification::TurnCompleted(n)) => { - assert_eq!(n.turn.id, event_id); + assert_eq!(n.turn.id, event_turn_id); assert_eq!(n.turn.status, TurnStatus::Completed); + assert_eq!(n.turn.error, None); } other => bail!("unexpected message: {other:?}"), } @@ -914,15 +1319,23 @@ mod tests { #[tokio::test] async fn test_handle_turn_interrupted_emits_interrupted_with_error() -> Result<()> { let conversation_id = ConversationId::new(); - let event_id = "interrupt1".to_string(); + let event_turn_id = "interrupt1".to_string(); let turn_summary_store = new_turn_summary_store(); - handle_error(conversation_id, "oops".to_string(), &turn_summary_store).await; + handle_error( + conversation_id, + TurnError { + message: "oops".to_string(), + codex_error_info: None, + }, + &turn_summary_store, + ) + .await; let (tx, mut rx) = mpsc::channel(CHANNEL_CAPACITY); let outgoing = Arc::new(OutgoingMessageSender::new(tx)); handle_turn_interrupted( conversation_id, - event_id.clone(), + event_turn_id.clone(), &outgoing, &turn_summary_store, ) @@ -934,8 +1347,9 @@ mod tests { .ok_or_else(|| anyhow!("should send one notification"))?; match msg { OutgoingMessage::AppServerNotification(ServerNotification::TurnCompleted(n)) => { - assert_eq!(n.turn.id, event_id); + assert_eq!(n.turn.id, event_turn_id); assert_eq!(n.turn.status, TurnStatus::Interrupted); + assert_eq!(n.turn.error, None); } other => bail!("unexpected message: {other:?}"), } @@ -946,15 +1360,23 @@ mod tests { #[tokio::test] async fn test_handle_turn_complete_emits_failed_with_error() -> Result<()> { let conversation_id = ConversationId::new(); - let event_id = "complete_err1".to_string(); + let event_turn_id = "complete_err1".to_string(); let turn_summary_store = new_turn_summary_store(); - handle_error(conversation_id, "bad".to_string(), &turn_summary_store).await; + handle_error( + conversation_id, + TurnError { + message: "bad".to_string(), + codex_error_info: Some(V2CodexErrorInfo::Other), + }, + &turn_summary_store, + ) + .await; let (tx, mut rx) = mpsc::channel(CHANNEL_CAPACITY); let outgoing = Arc::new(OutgoingMessageSender::new(tx)); handle_turn_complete( conversation_id, - event_id.clone(), + event_turn_id.clone(), &outgoing, &turn_summary_store, ) @@ -966,14 +1388,14 @@ mod tests { .ok_or_else(|| anyhow!("should send one notification"))?; match msg { OutgoingMessage::AppServerNotification(ServerNotification::TurnCompleted(n)) => { - assert_eq!(n.turn.id, event_id); + assert_eq!(n.turn.id, event_turn_id); + assert_eq!(n.turn.status, TurnStatus::Failed); assert_eq!( - n.turn.status, - TurnStatus::Failed { - error: TurnError { - message: "bad".to_string(), - } - } + n.turn.error, + Some(TurnError { + message: "bad".to_string(), + codex_error_info: Some(V2CodexErrorInfo::Other), + }) ); } other => bail!("unexpected message: {other:?}"), @@ -982,6 +1404,155 @@ mod tests { Ok(()) } + #[tokio::test] + async fn test_handle_turn_plan_update_emits_notification_for_v2() -> Result<()> { + let (tx, mut rx) = mpsc::channel(CHANNEL_CAPACITY); + let outgoing = OutgoingMessageSender::new(tx); + let update = UpdatePlanArgs { + explanation: Some("need plan".to_string()), + plan: vec![ + PlanItemArg { + step: "first".to_string(), + status: StepStatus::Pending, + }, + PlanItemArg { + step: "second".to_string(), + status: StepStatus::Completed, + }, + ], + }; + + handle_turn_plan_update("turn-123", update, ApiVersion::V2, &outgoing).await; + + let msg = rx + .recv() + .await + .ok_or_else(|| anyhow!("should send one notification"))?; + match msg { + OutgoingMessage::AppServerNotification(ServerNotification::TurnPlanUpdated(n)) => { + assert_eq!(n.turn_id, "turn-123"); + assert_eq!(n.explanation.as_deref(), Some("need plan")); + assert_eq!(n.plan.len(), 2); + assert_eq!(n.plan[0].step, "first"); + assert_eq!(n.plan[0].status, TurnPlanStepStatus::Pending); + assert_eq!(n.plan[1].step, "second"); + assert_eq!(n.plan[1].status, TurnPlanStepStatus::Completed); + } + other => bail!("unexpected message: {other:?}"), + } + assert!(rx.try_recv().is_err(), "no extra messages expected"); + Ok(()) + } + + #[tokio::test] + async fn test_handle_token_count_event_emits_usage_and_rate_limits() -> Result<()> { + let conversation_id = ConversationId::new(); + let turn_id = "turn-123".to_string(); + let (tx, mut rx) = mpsc::channel(CHANNEL_CAPACITY); + let outgoing = Arc::new(OutgoingMessageSender::new(tx)); + + let info = TokenUsageInfo { + total_token_usage: TokenUsage { + input_tokens: 100, + cached_input_tokens: 25, + output_tokens: 50, + reasoning_output_tokens: 9, + total_tokens: 200, + }, + last_token_usage: TokenUsage { + input_tokens: 10, + cached_input_tokens: 5, + output_tokens: 7, + reasoning_output_tokens: 1, + total_tokens: 23, + }, + model_context_window: Some(4096), + }; + let rate_limits = RateLimitSnapshot { + primary: Some(RateLimitWindow { + used_percent: 42.5, + window_minutes: Some(15), + resets_at: Some(1700000000), + }), + secondary: None, + credits: Some(CreditsSnapshot { + has_credits: true, + unlimited: false, + balance: Some("5".to_string()), + }), + }; + + handle_token_count_event( + conversation_id, + turn_id.clone(), + TokenCountEvent { + info: Some(info), + rate_limits: Some(rate_limits), + }, + &outgoing, + ) + .await; + + let first = rx + .recv() + .await + .ok_or_else(|| anyhow!("expected usage notification"))?; + match first { + OutgoingMessage::AppServerNotification( + ServerNotification::ThreadTokenUsageUpdated(payload), + ) => { + assert_eq!(payload.thread_id, conversation_id.to_string()); + assert_eq!(payload.turn_id, turn_id); + let usage = payload.token_usage; + assert_eq!(usage.total.total_tokens, 200); + assert_eq!(usage.total.cached_input_tokens, 25); + assert_eq!(usage.last.output_tokens, 7); + assert_eq!(usage.model_context_window, Some(4096)); + } + other => bail!("unexpected notification: {other:?}"), + } + + let second = rx + .recv() + .await + .ok_or_else(|| anyhow!("expected rate limit notification"))?; + match second { + OutgoingMessage::AppServerNotification( + ServerNotification::AccountRateLimitsUpdated(payload), + ) => { + assert!(payload.rate_limits.primary.is_some()); + assert!(payload.rate_limits.credits.is_some()); + } + other => bail!("unexpected notification: {other:?}"), + } + Ok(()) + } + + #[tokio::test] + async fn test_handle_token_count_event_without_usage_info() -> Result<()> { + let conversation_id = ConversationId::new(); + let turn_id = "turn-456".to_string(); + let (tx, mut rx) = mpsc::channel(CHANNEL_CAPACITY); + let outgoing = Arc::new(OutgoingMessageSender::new(tx)); + + handle_token_count_event( + conversation_id, + turn_id.clone(), + TokenCountEvent { + info: None, + rate_limits: None, + }, + &outgoing, + ) + .await; + + assert!( + rx.try_recv().is_err(), + "no notifications should be emitted when token usage info is absent" + ); + Ok(()) + } + #[tokio::test] async fn test_construct_mcp_tool_call_begin_notification_with_args() { let begin_event = McpToolCallBeginEvent { @@ -993,9 +1564,18 @@ mod tests { }, }; - let notification = construct_mcp_tool_call_notification(begin_event.clone()).await; + let thread_id = ConversationId::new().to_string(); + let turn_id = "turn_1".to_string(); + let notification = construct_mcp_tool_call_notification( + begin_event.clone(), + thread_id.clone(), + turn_id.clone(), + ) + .await; let expected = ItemStartedNotification { + thread_id, + turn_id, item: ThreadItem::McpToolCall { id: begin_event.call_id, server: begin_event.invocation.server, @@ -1022,7 +1602,15 @@ mod tests { // Turn 1 on conversation A let a_turn1 = "a_turn1".to_string(); - handle_error(conversation_a, "a1".to_string(), &turn_summary_store).await; + handle_error( + conversation_a, + TurnError { + message: "a1".to_string(), + codex_error_info: Some(V2CodexErrorInfo::BadRequest), + }, + &turn_summary_store, + ) + .await; handle_turn_complete( conversation_a, a_turn1.clone(), @@ -1033,7 +1621,15 @@ mod tests { // Turn 1 on conversation B let b_turn1 = "b_turn1".to_string(); - handle_error(conversation_b, "b1".to_string(), &turn_summary_store).await; + handle_error( + conversation_b, + TurnError { + message: "b1".to_string(), + codex_error_info: None, + }, + &turn_summary_store, + ) + .await; handle_turn_complete( conversation_b, b_turn1.clone(), @@ -1060,13 +1656,13 @@ mod tests { match msg { OutgoingMessage::AppServerNotification(ServerNotification::TurnCompleted(n)) => { assert_eq!(n.turn.id, a_turn1); + assert_eq!(n.turn.status, TurnStatus::Failed); assert_eq!( - n.turn.status, - TurnStatus::Failed { - error: TurnError { - message: "a1".to_string(), - } - } + n.turn.error, + Some(TurnError { + message: "a1".to_string(), + codex_error_info: Some(V2CodexErrorInfo::BadRequest), + }) ); } other => bail!("unexpected message: {other:?}"), @@ -1080,13 +1676,13 @@ mod tests { match msg { OutgoingMessage::AppServerNotification(ServerNotification::TurnCompleted(n)) => { assert_eq!(n.turn.id, b_turn1); + assert_eq!(n.turn.status, TurnStatus::Failed); assert_eq!( - n.turn.status, - TurnStatus::Failed { - error: TurnError { - message: "b1".to_string(), - } - } + n.turn.error, + Some(TurnError { + message: "b1".to_string(), + codex_error_info: None, + }) ); } other => bail!("unexpected message: {other:?}"), @@ -1101,6 +1697,7 @@ mod tests { OutgoingMessage::AppServerNotification(ServerNotification::TurnCompleted(n)) => { assert_eq!(n.turn.id, a_turn2); assert_eq!(n.turn.status, TurnStatus::Completed); + assert_eq!(n.turn.error, None); } other => bail!("unexpected message: {other:?}"), } @@ -1120,9 +1717,18 @@ mod tests { }, }; - let notification = construct_mcp_tool_call_notification(begin_event.clone()).await; + let thread_id = ConversationId::new().to_string(); + let turn_id = "turn_2".to_string(); + let notification = construct_mcp_tool_call_notification( + begin_event.clone(), + thread_id.clone(), + turn_id.clone(), + ) + .await; let expected = ItemStartedNotification { + thread_id, + turn_id, item: ThreadItem::McpToolCall { id: begin_event.call_id, server: begin_event.invocation.server, @@ -1161,9 +1767,18 @@ mod tests { result: Ok(result), }; - let notification = construct_mcp_tool_call_end_notification(end_event.clone()).await; + let thread_id = ConversationId::new().to_string(); + let turn_id = "turn_3".to_string(); + let notification = construct_mcp_tool_call_end_notification( + end_event.clone(), + thread_id.clone(), + turn_id.clone(), + ) + .await; let expected = ItemCompletedNotification { + thread_id, + turn_id, item: ThreadItem::McpToolCall { id: end_event.call_id, server: end_event.invocation.server, @@ -1194,9 +1809,18 @@ mod tests { result: Err("boom".to_string()), }; - let notification = construct_mcp_tool_call_end_notification(end_event.clone()).await; + let thread_id = ConversationId::new().to_string(); + let turn_id = "turn_4".to_string(); + let notification = construct_mcp_tool_call_end_notification( + end_event.clone(), + thread_id.clone(), + turn_id.clone(), + ) + .await; let expected = ItemCompletedNotification { + thread_id, + turn_id, item: ThreadItem::McpToolCall { id: end_event.call_id, server: end_event.invocation.server, @@ -1212,4 +1836,61 @@ mod tests { assert_eq!(notification, expected); } + + #[tokio::test] + async fn test_handle_turn_diff_emits_v2_notification() -> Result<()> { + let (tx, mut rx) = mpsc::channel(CHANNEL_CAPACITY); + let outgoing = OutgoingMessageSender::new(tx); + let unified_diff = "--- a\n+++ b\n".to_string(); + let conversation_id = ConversationId::new(); + + handle_turn_diff( + conversation_id, + "turn-1", + TurnDiffEvent { + unified_diff: unified_diff.clone(), + }, + ApiVersion::V2, + &outgoing, + ) + .await; + + let msg = rx + .recv() + .await + .ok_or_else(|| anyhow!("should send one notification"))?; + match msg { + OutgoingMessage::AppServerNotification(ServerNotification::TurnDiffUpdated( + notification, + )) => { + assert_eq!(notification.thread_id, conversation_id.to_string()); + assert_eq!(notification.turn_id, "turn-1"); + assert_eq!(notification.diff, unified_diff); + } + other => bail!("unexpected message: {other:?}"), + } + assert!(rx.try_recv().is_err(), "no extra messages expected"); + Ok(()) + } + + #[tokio::test] + async fn test_handle_turn_diff_is_noop_for_v1() -> Result<()> { + let (tx, mut rx) = mpsc::channel(CHANNEL_CAPACITY); + let outgoing = OutgoingMessageSender::new(tx); + let conversation_id = ConversationId::new(); + + handle_turn_diff( + conversation_id, + "turn-1", + TurnDiffEvent { + unified_diff: "diff".to_string(), + }, + ApiVersion::V1, + &outgoing, + ) + .await; + + assert!(rx.try_recv().is_err(), "no messages expected"); + Ok(()) + } } diff --git a/codex-rs/app-server/src/codex_message_processor.rs b/codex-rs/app-server/src/codex_message_processor.rs index 8afa2075da..d10790b61a 100644 --- a/codex-rs/app-server/src/codex_message_processor.rs +++ b/codex-rs/app-server/src/codex_message_processor.rs @@ -21,9 +21,9 @@ use codex_app_server_protocol::CancelLoginAccountParams; use codex_app_server_protocol::CancelLoginAccountResponse; use codex_app_server_protocol::CancelLoginChatGptResponse; use codex_app_server_protocol::ClientRequest; +use codex_app_server_protocol::CommandExecParams; use codex_app_server_protocol::ConversationGitInfo; use codex_app_server_protocol::ConversationSummary; -use codex_app_server_protocol::ExecOneOffCommandParams; use codex_app_server_protocol::ExecOneOffCommandResponse; use codex_app_server_protocol::FeedbackUploadParams; use codex_app_server_protocol::FeedbackUploadResponse; @@ -39,6 +39,7 @@ use codex_app_server_protocol::GetConversationSummaryResponse; use codex_app_server_protocol::GetUserAgentResponse; use codex_app_server_protocol::GetUserSavedConfigResponse; use codex_app_server_protocol::GitDiffToRemoteResponse; +use codex_app_server_protocol::GitInfo as ApiGitInfo; use codex_app_server_protocol::InputItem as WireInputItem; use codex_app_server_protocol::InterruptConversationParams; use codex_app_server_protocol::JSONRPCErrorError; @@ -60,8 +61,10 @@ use codex_app_server_protocol::RemoveConversationSubscriptionResponse; use codex_app_server_protocol::RequestId; use codex_app_server_protocol::ResumeConversationParams; use codex_app_server_protocol::ResumeConversationResponse; +use codex_app_server_protocol::ReviewDelivery as ApiReviewDelivery; use codex_app_server_protocol::ReviewStartParams; -use codex_app_server_protocol::ReviewTarget; +use codex_app_server_protocol::ReviewStartResponse; +use codex_app_server_protocol::ReviewTarget as ApiReviewTarget; use codex_app_server_protocol::SandboxMode; use codex_app_server_protocol::SendUserMessageParams; use codex_app_server_protocol::SendUserMessageResponse; @@ -83,6 +86,7 @@ use codex_app_server_protocol::ThreadStartParams; use codex_app_server_protocol::ThreadStartResponse; use codex_app_server_protocol::ThreadStartedNotification; use codex_app_server_protocol::Turn; +use codex_app_server_protocol::TurnError; use codex_app_server_protocol::TurnInterruptParams; use codex_app_server_protocol::TurnStartParams; use codex_app_server_protocol::TurnStartResponse; @@ -91,7 +95,6 @@ use codex_app_server_protocol::TurnStatus; use codex_app_server_protocol::UserInfoResponse; use codex_app_server_protocol::UserInput as V2UserInput; use codex_app_server_protocol::UserSavedConfig; -use codex_app_server_protocol::WindowsWorldWritableWarningNotification; use codex_app_server_protocol::build_turns_from_event_msgs; use codex_backend_client::Client as BackendClient; use codex_core::AuthManager; @@ -115,12 +118,13 @@ use codex_core::exec::ExecParams; use codex_core::exec_env::create_env; use codex_core::features::Feature; use codex_core::find_conversation_path_by_id_str; -use codex_core::get_platform_sandbox; use codex_core::git_info::git_diff_to_remote; use codex_core::parse_cursor; use codex_core::protocol::EventMsg; use codex_core::protocol::Op; +use codex_core::protocol::ReviewDelivery as CoreReviewDelivery; use codex_core::protocol::ReviewRequest; +use codex_core::protocol::ReviewTarget as CoreReviewTarget; use codex_core::protocol::SessionConfiguredEvent; use codex_core::read_head_for_summary; use codex_feedback::CodexFeedback; @@ -131,7 +135,7 @@ use codex_protocol::ConversationId; use codex_protocol::config_types::ForcedLoginMethod; use codex_protocol::items::TurnItem; use codex_protocol::models::ResponseItem; -use codex_protocol::protocol::GitInfo; +use codex_protocol::protocol::GitInfo as CoreGitInfo; use codex_protocol::protocol::RateLimitSnapshot as CoreRateLimitSnapshot; use codex_protocol::protocol::RolloutItem; use codex_protocol::protocol::SessionMetaLine; @@ -161,8 +165,8 @@ pub(crate) type PendingInterrupts = Arc, pub(crate) file_change_started: HashSet, + pub(crate) last_error: Option, } pub(crate) type TurnSummaryStore = Arc>>; @@ -251,8 +255,7 @@ impl CodexMessageProcessor { } fn review_request_from_target( - target: ReviewTarget, - append_to_original_thread: bool, + target: ApiReviewTarget, ) -> Result<(ReviewRequest, String), JSONRPCErrorError> { fn invalid_request(message: String) -> JSONRPCErrorError { JSONRPCErrorError { @@ -262,77 +265,52 @@ impl CodexMessageProcessor { } } - match target { - // TODO(jif) those messages will be extracted in a follow-up PR. - ReviewTarget::UncommittedChanges => Ok(( - ReviewRequest { - prompt: "Review the current code changes (staged, unstaged, and untracked files) and provide prioritized findings.".to_string(), - user_facing_hint: "current changes".to_string(), - append_to_original_thread, - }, - "Review uncommitted changes".to_string(), - )), - ReviewTarget::BaseBranch { branch } => { + let cleaned_target = match target { + ApiReviewTarget::UncommittedChanges => ApiReviewTarget::UncommittedChanges, + ApiReviewTarget::BaseBranch { branch } => { let branch = branch.trim().to_string(); if branch.is_empty() { return Err(invalid_request("branch must not be empty".to_string())); } - let prompt = format!("Review the code changes against the base branch '{branch}'. Start by finding the merge diff between the current branch and {branch}'s upstream e.g. (`git merge-base HEAD \"$(git rev-parse --abbrev-ref \"{branch}@{{upstream}}\")\"`), then run `git diff` against that SHA to see what changes we would merge into the {branch} branch. Provide prioritized, actionable findings."); - let hint = format!("changes against '{branch}'"); - let display = format!("Review changes against base branch '{branch}'"); - Ok(( - ReviewRequest { - prompt, - user_facing_hint: hint, - append_to_original_thread, - }, - display, - )) + ApiReviewTarget::BaseBranch { branch } } - ReviewTarget::Commit { sha, title } => { + ApiReviewTarget::Commit { sha, title } => { let sha = sha.trim().to_string(); if sha.is_empty() { return Err(invalid_request("sha must not be empty".to_string())); } - let brief_title = title + let title = title .map(|t| t.trim().to_string()) .filter(|t| !t.is_empty()); - let prompt = if let Some(title) = brief_title.clone() { - format!("Review the code changes introduced by commit {sha} (\"{title}\"). Provide prioritized, actionable findings.") - } else { - format!("Review the code changes introduced by commit {sha}. Provide prioritized, actionable findings.") - }; - let short_sha = sha.chars().take(7).collect::(); - let hint = format!("commit {short_sha}"); - let display = if let Some(title) = brief_title { - format!("Review commit {short_sha}: {title}") - } else { - format!("Review commit {short_sha}") - }; - Ok(( - ReviewRequest { - prompt, - user_facing_hint: hint, - append_to_original_thread, - }, - display, - )) + ApiReviewTarget::Commit { sha, title } } - ReviewTarget::Custom { instructions } => { + ApiReviewTarget::Custom { instructions } => { let trimmed = instructions.trim().to_string(); if trimmed.is_empty() { - return Err(invalid_request("instructions must not be empty".to_string())); + return Err(invalid_request( + "instructions must not be empty".to_string(), + )); + } + ApiReviewTarget::Custom { + instructions: trimmed, } - Ok(( - ReviewRequest { - prompt: trimmed.clone(), - user_facing_hint: trimmed.clone(), - append_to_original_thread, - }, - trimmed, - )) } - } + }; + + let core_target = match cleaned_target { + ApiReviewTarget::UncommittedChanges => CoreReviewTarget::UncommittedChanges, + ApiReviewTarget::BaseBranch { branch } => CoreReviewTarget::BaseBranch { branch }, + ApiReviewTarget::Commit { sha, title } => CoreReviewTarget::Commit { sha, title }, + ApiReviewTarget::Custom { instructions } => CoreReviewTarget::Custom { instructions }, + }; + + let hint = codex_core::review_prompts::user_facing_hint(&core_target); + let review_request = ReviewRequest { + target: core_target, + user_facing_hint: Some(hint.clone()), + }; + + Ok((review_request, hint)) } pub async fn process_request(&mut self, request: ClientRequest) { @@ -468,9 +446,17 @@ impl CodexMessageProcessor { ClientRequest::FuzzyFileSearch { request_id, params } => { self.fuzzy_file_search(request_id, params).await; } - ClientRequest::ExecOneOffCommand { request_id, params } => { + ClientRequest::OneOffCommandExec { request_id, params } => { self.exec_one_off_command(request_id, params).await; } + ClientRequest::ExecOneOffCommand { request_id, params } => { + self.exec_one_off_command(request_id, params.into()).await; + } + ClientRequest::ConfigRead { .. } + | ClientRequest::ConfigValueWrite { .. } + | ClientRequest::ConfigBatchWrite { .. } => { + warn!("Config request reached CodexMessageProcessor unexpectedly"); + } ClientRequest::GetAccountRateLimits { request_id, params: _, @@ -1150,7 +1136,7 @@ impl CodexMessageProcessor { } } - async fn exec_one_off_command(&self, request_id: RequestId, params: ExecOneOffCommandParams) { + async fn exec_one_off_command(&self, request_id: RequestId, params: CommandExecParams) { tracing::debug!("ExecOneOffCommand params: {params:?}"); if params.command.is_empty() { @@ -1165,11 +1151,13 @@ impl CodexMessageProcessor { let cwd = params.cwd.unwrap_or_else(|| self.config.cwd.clone()); let env = create_env(&self.config.shell_environment_policy); - let timeout_ms = params.timeout_ms; + let timeout_ms = params + .timeout_ms + .and_then(|timeout_ms| u64::try_from(timeout_ms).ok()); let exec_params = ExecParams { command: params.command, cwd, - timeout_ms, + expiration: timeout_ms.into(), env, with_escalated_permissions: None, justification: None, @@ -1178,15 +1166,9 @@ impl CodexMessageProcessor { let effective_policy = params .sandbox_policy + .map(|policy| policy.to_core()) .unwrap_or_else(|| self.config.sandbox_policy.clone()); - let sandbox_type = match &effective_policy { - codex_core::protocol::SandboxPolicy::DangerFullAccess => { - codex_core::exec::SandboxType::None - } - _ => get_platform_sandbox().unwrap_or(codex_core::exec::SandboxType::None), - }; - tracing::debug!("Sandbox type: {sandbox_type:?}"); let codex_linux_sandbox_exe = self.config.codex_linux_sandbox_exe.clone(); let outgoing = self.outgoing.clone(); let req_id = request_id; @@ -1195,7 +1177,6 @@ impl CodexMessageProcessor { tokio::spawn(async move { match codex_core::exec::process_exec_tool_call( exec_params, - sandbox_type, &effective_policy, sandbox_cwd.as_path(), &codex_linux_sandbox_exe, @@ -1275,10 +1256,6 @@ impl CodexMessageProcessor { return; } }; - if cfg!(windows) && config.features.enabled(Feature::WindowsSandbox) { - self.handle_windows_world_writable_warning(config.cwd.clone()) - .await; - } match self.conversation_manager.new_conversation(config).await { Ok(conversation_id) => { @@ -1998,10 +1975,6 @@ impl CodexMessageProcessor { return; } }; - if cfg!(windows) && config.features.enabled(Feature::WindowsSandbox) { - self.handle_windows_world_writable_warning(config.cwd.clone()) - .await; - } let conversation_history = if let Some(path) = path { match RolloutRecorder::get_rollout_history(&path).await { @@ -2456,6 +2429,7 @@ impl CodexMessageProcessor { let turn = Turn { id: turn_id.clone(), items: vec![], + error: None, status: TurnStatus::InProgress, }; @@ -2463,7 +2437,10 @@ impl CodexMessageProcessor { self.outgoing.send_response(request_id, response).await; // Emit v2 turn/started notification. - let notif = TurnStartedNotification { turn }; + let notif = TurnStartedNotification { + thread_id: params.thread_id, + turn, + }; self.outgoing .send_server_notification(ServerNotification::TurnStarted(notif)) .await; @@ -2479,60 +2456,221 @@ impl CodexMessageProcessor { } } - async fn review_start(&self, request_id: RequestId, params: ReviewStartParams) { + fn build_review_turn(turn_id: String, display_text: &str) -> Turn { + let items = if display_text.is_empty() { + Vec::new() + } else { + vec![ThreadItem::UserMessage { + id: turn_id.clone(), + content: vec![V2UserInput::Text { + text: display_text.to_string(), + }], + }] + }; + + Turn { + id: turn_id, + items, + error: None, + status: TurnStatus::InProgress, + } + } + + async fn emit_review_started( + &self, + request_id: &RequestId, + turn: Turn, + parent_thread_id: String, + review_thread_id: String, + ) { + let response = ReviewStartResponse { + turn: turn.clone(), + review_thread_id, + }; + self.outgoing + .send_response(request_id.clone(), response) + .await; + + let notif = TurnStartedNotification { + thread_id: parent_thread_id, + turn, + }; + self.outgoing + .send_server_notification(ServerNotification::TurnStarted(notif)) + .await; + } + + async fn start_inline_review( + &self, + request_id: &RequestId, + parent_conversation: Arc, + review_request: ReviewRequest, + display_text: &str, + parent_thread_id: String, + ) -> std::result::Result<(), JSONRPCErrorError> { + let turn_id = parent_conversation + .submit(Op::Review { review_request }) + .await; + + match turn_id { + Ok(turn_id) => { + let turn = Self::build_review_turn(turn_id, display_text); + self.emit_review_started( + request_id, + turn, + parent_thread_id.clone(), + parent_thread_id, + ) + .await; + Ok(()) + } + Err(err) => Err(JSONRPCErrorError { + code: INTERNAL_ERROR_CODE, + message: format!("failed to start review: {err}"), + data: None, + }), + } + } + + async fn start_detached_review( + &mut self, + request_id: &RequestId, + parent_conversation_id: ConversationId, + review_request: ReviewRequest, + display_text: &str, + ) -> std::result::Result<(), JSONRPCErrorError> { + let rollout_path = find_conversation_path_by_id_str( + &self.config.codex_home, + &parent_conversation_id.to_string(), + ) + .await + .map_err(|err| JSONRPCErrorError { + code: INTERNAL_ERROR_CODE, + message: format!("failed to locate conversation id {parent_conversation_id}: {err}"), + data: None, + })? + .ok_or_else(|| JSONRPCErrorError { + code: INVALID_REQUEST_ERROR_CODE, + message: format!("no rollout found for conversation id {parent_conversation_id}"), + data: None, + })?; + + let mut config = self.config.as_ref().clone(); + config.model = self.config.review_model.clone(); + + let NewConversation { + conversation_id, + conversation, + session_configured, + .. + } = self + .conversation_manager + .fork_conversation(usize::MAX, config, rollout_path) + .await + .map_err(|err| JSONRPCErrorError { + code: INTERNAL_ERROR_CODE, + message: format!("error creating detached review conversation: {err}"), + data: None, + })?; + + if let Err(err) = self + .attach_conversation_listener(conversation_id, false, ApiVersion::V2) + .await + { + tracing::warn!( + "failed to attach listener for review conversation {}: {}", + conversation_id, + err.message + ); + } + + let rollout_path = conversation.rollout_path(); + let fallback_provider = self.config.model_provider_id.as_str(); + match read_summary_from_rollout(rollout_path.as_path(), fallback_provider).await { + Ok(summary) => { + let thread = summary_to_thread(summary); + let notif = ThreadStartedNotification { thread }; + self.outgoing + .send_server_notification(ServerNotification::ThreadStarted(notif)) + .await; + } + Err(err) => { + tracing::warn!( + "failed to load summary for review conversation {}: {}", + session_configured.session_id, + err + ); + } + } + + let turn_id = conversation + .submit(Op::Review { review_request }) + .await + .map_err(|err| JSONRPCErrorError { + code: INTERNAL_ERROR_CODE, + message: format!("failed to start detached review turn: {err}"), + data: None, + })?; + + let turn = Self::build_review_turn(turn_id, display_text); + let review_thread_id = conversation_id.to_string(); + self.emit_review_started(request_id, turn, review_thread_id.clone(), review_thread_id) + .await; + + Ok(()) + } + + async fn review_start(&mut self, request_id: RequestId, params: ReviewStartParams) { let ReviewStartParams { thread_id, target, - append_to_original_thread, + delivery, } = params; - let (_, conversation) = match self.conversation_from_thread_id(&thread_id).await { - Ok(v) => v, - Err(error) => { - self.outgoing.send_error(request_id, error).await; - return; - } - }; - - let (review_request, display_text) = - match Self::review_request_from_target(target, append_to_original_thread) { - Ok(value) => value, - Err(err) => { - self.outgoing.send_error(request_id, err).await; + let (parent_conversation_id, parent_conversation) = + match self.conversation_from_thread_id(&thread_id).await { + Ok(v) => v, + Err(error) => { + self.outgoing.send_error(request_id, error).await; return; } }; - let turn_id = conversation.submit(Op::Review { review_request }).await; - - match turn_id { - Ok(turn_id) => { - let mut items = Vec::new(); - if !display_text.is_empty() { - items.push(ThreadItem::UserMessage { - id: turn_id.clone(), - content: vec![V2UserInput::Text { text: display_text }], - }); - } - let turn = Turn { - id: turn_id.clone(), - items, - status: TurnStatus::InProgress, - }; - let response = TurnStartResponse { turn: turn.clone() }; - self.outgoing.send_response(request_id, response).await; - - let notif = TurnStartedNotification { turn }; - self.outgoing - .send_server_notification(ServerNotification::TurnStarted(notif)) - .await; - } + let (review_request, display_text) = match Self::review_request_from_target(target) { + Ok(value) => value, Err(err) => { - let error = JSONRPCErrorError { - code: INTERNAL_ERROR_CODE, - message: format!("failed to start review: {err}"), - data: None, - }; - self.outgoing.send_error(request_id, error).await; + self.outgoing.send_error(request_id, err).await; + return; + } + }; + + let delivery = delivery.unwrap_or(ApiReviewDelivery::Inline).to_core(); + match delivery { + CoreReviewDelivery::Inline => { + if let Err(err) = self + .start_inline_review( + &request_id, + parent_conversation, + review_request, + display_text.as_str(), + thread_id.clone(), + ) + .await + { + self.outgoing.send_error(request_id, err).await; + } + } + CoreReviewDelivery::Detached => { + if let Err(err) = self + .start_detached_review( + &request_id, + parent_conversation_id, + review_request, + display_text.as_str(), + ) + .await + { + self.outgoing.send_error(request_id, err).await; + } } } } @@ -2784,6 +2922,7 @@ impl CodexMessageProcessor { } else { None }; + let session_source = self.conversation_manager.session_source(); let upload_result = tokio::task::spawn_blocking(move || { let rollout_path_ref = validated_rollout_path.as_deref(); @@ -2792,6 +2931,7 @@ impl CodexMessageProcessor { reason.as_deref(), include_logs, rollout_path_ref, + Some(session_source), ) }) .await; @@ -2835,53 +2975,6 @@ impl CodexMessageProcessor { Err(_) => None, } } - - /// On Windows, when using the experimental sandbox, we need to warn the user about world-writable directories. - async fn handle_windows_world_writable_warning(&self, cwd: PathBuf) { - if !cfg!(windows) { - return; - } - - if !self.config.features.enabled(Feature::WindowsSandbox) { - return; - } - - if !matches!( - self.config.sandbox_policy, - codex_protocol::protocol::SandboxPolicy::WorkspaceWrite { .. } - | codex_protocol::protocol::SandboxPolicy::ReadOnly - ) { - return; - } - - if self - .config - .notices - .hide_world_writable_warning - .unwrap_or(false) - { - return; - } - - // This function is stubbed out to return None on non-Windows platforms - if let Some((sample_paths, extra_count, failed_scan)) = - codex_windows_sandbox::world_writable_warning_details( - self.config.codex_home.as_path(), - cwd, - ) - { - tracing::warn!("world writable warning: {sample_paths:?} {extra_count} {failed_scan}"); - self.outgoing - .send_server_notification(ServerNotification::WindowsWorldWritableWarning( - WindowsWorldWritableWarningNotification { - sample_paths, - extra_count, - failed_scan, - }, - )) - .await; - } - } } async fn derive_config_from_params( @@ -2960,7 +3053,7 @@ fn extract_conversation_summary( path: PathBuf, head: &[serde_json::Value], session_meta: &SessionMeta, - git: Option<&GitInfo>, + git: Option<&CoreGitInfo>, fallback_provider: &str, ) -> Option { let preview = head @@ -3001,7 +3094,7 @@ fn extract_conversation_summary( }) } -fn map_git_info(git_info: &GitInfo) -> ConversationGitInfo { +fn map_git_info(git_info: &CoreGitInfo) -> ConversationGitInfo { ConversationGitInfo { sha: git_info.commit_hash.clone(), branch: git_info.branch.clone(), @@ -3024,10 +3117,18 @@ fn summary_to_thread(summary: ConversationSummary) -> Thread { preview, timestamp, model_provider, - .. + cwd, + cli_version, + source, + git_info, } = summary; let created_at = parse_datetime(timestamp.as_deref()); + let git_info = git_info.map(|info| ApiGitInfo { + sha: info.sha, + branch: info.branch, + origin_url: info.origin_url, + }); Thread { id: conversation_id.to_string(), @@ -3035,6 +3136,10 @@ fn summary_to_thread(summary: ConversationSummary) -> Thread { model_provider, created_at: created_at.map(|dt| dt.timestamp()).unwrap_or(0), path, + cwd, + cli_version, + source: source.into(), + git_info, turns: Vec::new(), } } diff --git a/codex-rs/app-server/src/config_api.rs b/codex-rs/app-server/src/config_api.rs new file mode 100644 index 0000000000..68bbdd8c66 --- /dev/null +++ b/codex-rs/app-server/src/config_api.rs @@ -0,0 +1,974 @@ +use crate::error_code::INTERNAL_ERROR_CODE; +use crate::error_code::INVALID_REQUEST_ERROR_CODE; +use anyhow::anyhow; +use codex_app_server_protocol::ConfigBatchWriteParams; +use codex_app_server_protocol::ConfigLayer; +use codex_app_server_protocol::ConfigLayerMetadata; +use codex_app_server_protocol::ConfigLayerName; +use codex_app_server_protocol::ConfigReadParams; +use codex_app_server_protocol::ConfigReadResponse; +use codex_app_server_protocol::ConfigValueWriteParams; +use codex_app_server_protocol::ConfigWriteErrorCode; +use codex_app_server_protocol::ConfigWriteResponse; +use codex_app_server_protocol::JSONRPCErrorError; +use codex_app_server_protocol::MergeStrategy; +use codex_app_server_protocol::OverriddenMetadata; +use codex_app_server_protocol::WriteStatus; +use codex_core::config::ConfigToml; +use codex_core::config_loader::LoadedConfigLayers; +use codex_core::config_loader::LoaderOverrides; +use codex_core::config_loader::load_config_layers_with_overrides; +use codex_core::config_loader::merge_toml_values; +use serde_json::Value as JsonValue; +use serde_json::json; +use sha2::Digest; +use sha2::Sha256; +use std::collections::HashMap; +use std::path::Path; +use std::path::PathBuf; +use tempfile::NamedTempFile; +use tokio::task; +use toml::Value as TomlValue; + +const SESSION_FLAGS_SOURCE: &str = "--config"; +const MDM_SOURCE: &str = "com.openai.codex/config_toml_base64"; +const CONFIG_FILE_NAME: &str = "config.toml"; + +#[derive(Clone)] +pub(crate) struct ConfigApi { + codex_home: PathBuf, + cli_overrides: Vec<(String, TomlValue)>, + loader_overrides: LoaderOverrides, +} + +impl ConfigApi { + pub(crate) fn new(codex_home: PathBuf, cli_overrides: Vec<(String, TomlValue)>) -> Self { + Self { + codex_home, + cli_overrides, + loader_overrides: LoaderOverrides::default(), + } + } + + #[cfg(test)] + fn with_overrides( + codex_home: PathBuf, + cli_overrides: Vec<(String, TomlValue)>, + loader_overrides: LoaderOverrides, + ) -> Self { + Self { + codex_home, + cli_overrides, + loader_overrides, + } + } + + pub(crate) async fn read( + &self, + params: ConfigReadParams, + ) -> Result { + let layers = self + .load_layers_state() + .await + .map_err(|err| internal_error("failed to read configuration layers", err))?; + + let effective = layers.effective_config(); + validate_config(&effective).map_err(|err| internal_error("invalid configuration", err))?; + + let response = ConfigReadResponse { + config: to_json_value(&effective), + origins: layers.origins(), + layers: params.include_layers.then(|| layers.layers_high_to_low()), + }; + + Ok(response) + } + + pub(crate) async fn write_value( + &self, + params: ConfigValueWriteParams, + ) -> Result { + let edits = vec![(params.key_path, params.value, params.merge_strategy)]; + self.apply_edits(params.file_path, params.expected_version, edits) + .await + } + + pub(crate) async fn batch_write( + &self, + params: ConfigBatchWriteParams, + ) -> Result { + let edits = params + .edits + .into_iter() + .map(|edit| (edit.key_path, edit.value, edit.merge_strategy)) + .collect(); + + self.apply_edits(params.file_path, params.expected_version, edits) + .await + } + + async fn apply_edits( + &self, + file_path: String, + expected_version: Option, + edits: Vec<(String, JsonValue, MergeStrategy)>, + ) -> Result { + let allowed_path = self.codex_home.join(CONFIG_FILE_NAME); + if !paths_match(&allowed_path, &file_path) { + return Err(config_write_error( + ConfigWriteErrorCode::ConfigLayerReadonly, + "Only writes to the user config are allowed", + )); + } + + let layers = self + .load_layers_state() + .await + .map_err(|err| internal_error("failed to load configuration", err))?; + + if let Some(expected) = expected_version.as_deref() + && expected != layers.user.version + { + return Err(config_write_error( + ConfigWriteErrorCode::ConfigVersionConflict, + "Configuration was modified since last read. Fetch latest version and retry.", + )); + } + + let mut user_config = layers.user.config.clone(); + let mut mutated = false; + let mut parsed_segments = Vec::new(); + + for (key_path, value, strategy) in edits.into_iter() { + let segments = parse_key_path(&key_path).map_err(|message| { + config_write_error(ConfigWriteErrorCode::ConfigValidationError, message) + })?; + let parsed_value = parse_value(value).map_err(|message| { + config_write_error(ConfigWriteErrorCode::ConfigValidationError, message) + })?; + + let changed = apply_merge(&mut user_config, &segments, parsed_value.as_ref(), strategy) + .map_err(|err| match err { + MergeError::PathNotFound => config_write_error( + ConfigWriteErrorCode::ConfigPathNotFound, + "Path not found", + ), + MergeError::Validation(message) => { + config_write_error(ConfigWriteErrorCode::ConfigValidationError, message) + } + })?; + + mutated |= changed; + parsed_segments.push(segments); + } + + validate_config(&user_config).map_err(|err| { + config_write_error( + ConfigWriteErrorCode::ConfigValidationError, + format!("Invalid configuration: {err}"), + ) + })?; + + let updated_layers = layers.with_user_config(user_config.clone()); + let effective = updated_layers.effective_config(); + validate_config(&effective).map_err(|err| { + config_write_error( + ConfigWriteErrorCode::ConfigValidationError, + format!("Invalid configuration: {err}"), + ) + })?; + + if mutated { + self.persist_user_config(&user_config) + .await + .map_err(|err| internal_error("failed to persist config.toml", err))?; + } + + let overridden = first_overridden_edit(&updated_layers, &effective, &parsed_segments); + let status = overridden + .as_ref() + .map(|_| WriteStatus::OkOverridden) + .unwrap_or(WriteStatus::Ok); + + Ok(ConfigWriteResponse { + status, + version: updated_layers.user.version.clone(), + overridden_metadata: overridden, + }) + } + + async fn load_layers_state(&self) -> std::io::Result { + let LoadedConfigLayers { + base, + managed_config, + managed_preferences, + } = load_config_layers_with_overrides(&self.codex_home, self.loader_overrides.clone()) + .await?; + + let user = LayerState::new( + ConfigLayerName::User, + self.codex_home.join(CONFIG_FILE_NAME), + base, + ); + + let session_flags = LayerState::new( + ConfigLayerName::SessionFlags, + PathBuf::from(SESSION_FLAGS_SOURCE), + { + let mut root = TomlValue::Table(toml::map::Map::new()); + for (path, value) in self.cli_overrides.iter() { + apply_override(&mut root, path, value.clone()); + } + root + }, + ); + + let system = managed_config.map(|cfg| { + LayerState::new( + ConfigLayerName::System, + system_config_path(&self.codex_home), + cfg, + ) + }); + + let mdm = managed_preferences + .map(|cfg| LayerState::new(ConfigLayerName::Mdm, PathBuf::from(MDM_SOURCE), cfg)); + + Ok(LayersState { + user, + session_flags, + system, + mdm, + }) + } + + async fn persist_user_config(&self, user_config: &TomlValue) -> anyhow::Result<()> { + let codex_home = self.codex_home.clone(); + let serialized = toml::to_string_pretty(user_config)?; + + task::spawn_blocking(move || -> anyhow::Result<()> { + std::fs::create_dir_all(&codex_home)?; + + let target = codex_home.join(CONFIG_FILE_NAME); + let tmp = NamedTempFile::new_in(&codex_home)?; + std::fs::write(tmp.path(), serialized.as_bytes())?; + tmp.persist(&target)?; + Ok(()) + }) + .await + .map_err(|err| anyhow!("config persistence task panicked: {err}"))??; + + Ok(()) + } +} + +fn parse_value(value: JsonValue) -> Result, String> { + if value.is_null() { + return Ok(None); + } + + serde_json::from_value::(value) + .map(Some) + .map_err(|err| format!("invalid value: {err}")) +} + +fn parse_key_path(path: &str) -> Result, String> { + if path.trim().is_empty() { + return Err("keyPath must not be empty".to_string()); + } + Ok(path + .split('.') + .map(std::string::ToString::to_string) + .collect()) +} + +fn apply_override(target: &mut TomlValue, path: &str, value: TomlValue) { + use toml::value::Table; + + let segments: Vec<&str> = path.split('.').collect(); + let mut current = target; + + for (idx, segment) in segments.iter().enumerate() { + let is_last = idx == segments.len() - 1; + + if is_last { + match current { + TomlValue::Table(table) => { + table.insert(segment.to_string(), value); + } + _ => { + let mut table = Table::new(); + table.insert(segment.to_string(), value); + *current = TomlValue::Table(table); + } + } + return; + } + + match current { + TomlValue::Table(table) => { + current = table + .entry((*segment).to_string()) + .or_insert_with(|| TomlValue::Table(Table::new())); + } + _ => { + *current = TomlValue::Table(Table::new()); + if let TomlValue::Table(tbl) = current { + current = tbl + .entry((*segment).to_string()) + .or_insert_with(|| TomlValue::Table(Table::new())); + } + } + } + } +} + +#[derive(Debug)] +enum MergeError { + PathNotFound, + Validation(String), +} + +fn apply_merge( + root: &mut TomlValue, + segments: &[String], + value: Option<&TomlValue>, + strategy: MergeStrategy, +) -> Result { + let Some(value) = value else { + return clear_path(root, segments); + }; + + let Some((last, parents)) = segments.split_last() else { + return Err(MergeError::Validation( + "keyPath must not be empty".to_string(), + )); + }; + + let mut current = root; + + for segment in parents { + match current { + TomlValue::Table(table) => { + current = table + .entry(segment.clone()) + .or_insert_with(|| TomlValue::Table(toml::map::Map::new())); + } + _ => { + *current = TomlValue::Table(toml::map::Map::new()); + if let TomlValue::Table(table) = current { + current = table + .entry(segment.clone()) + .or_insert_with(|| TomlValue::Table(toml::map::Map::new())); + } + } + } + } + + let table = current.as_table_mut().ok_or_else(|| { + MergeError::Validation("cannot set value on non-table parent".to_string()) + })?; + + if matches!(strategy, MergeStrategy::Upsert) + && let Some(existing) = table.get_mut(last) + && matches!(existing, TomlValue::Table(_)) + && matches!(value, TomlValue::Table(_)) + { + merge_toml_values(existing, value); + return Ok(true); + } + + let changed = table + .get(last) + .map(|existing| Some(existing) != Some(value)) + .unwrap_or(true); + table.insert(last.clone(), value.clone()); + Ok(changed) +} + +fn clear_path(root: &mut TomlValue, segments: &[String]) -> Result { + let Some((last, parents)) = segments.split_last() else { + return Err(MergeError::Validation( + "keyPath must not be empty".to_string(), + )); + }; + + let mut current = root; + for segment in parents { + match current { + TomlValue::Table(table) => { + current = table.get_mut(segment).ok_or(MergeError::PathNotFound)?; + } + _ => return Err(MergeError::PathNotFound), + } + } + + let Some(parent) = current.as_table_mut() else { + return Err(MergeError::PathNotFound); + }; + + Ok(parent.remove(last).is_some()) +} + +#[derive(Clone)] +struct LayerState { + name: ConfigLayerName, + source: PathBuf, + config: TomlValue, + version: String, +} + +impl LayerState { + fn new(name: ConfigLayerName, source: PathBuf, config: TomlValue) -> Self { + let version = version_for_toml(&config); + Self { + name, + source, + config, + version, + } + } + + fn metadata(&self) -> ConfigLayerMetadata { + ConfigLayerMetadata { + name: self.name.clone(), + source: self.source.display().to_string(), + version: self.version.clone(), + } + } + + fn as_layer(&self) -> ConfigLayer { + ConfigLayer { + name: self.name.clone(), + source: self.source.display().to_string(), + version: self.version.clone(), + config: to_json_value(&self.config), + } + } +} + +#[derive(Clone)] +struct LayersState { + user: LayerState, + session_flags: LayerState, + system: Option, + mdm: Option, +} + +impl LayersState { + fn with_user_config(self, user_config: TomlValue) -> Self { + Self { + user: LayerState::new(self.user.name, self.user.source, user_config), + session_flags: self.session_flags, + system: self.system, + mdm: self.mdm, + } + } + + fn effective_config(&self) -> TomlValue { + let mut merged = self.user.config.clone(); + merge_toml_values(&mut merged, &self.session_flags.config); + if let Some(system) = &self.system { + merge_toml_values(&mut merged, &system.config); + } + if let Some(mdm) = &self.mdm { + merge_toml_values(&mut merged, &mdm.config); + } + merged + } + + fn origins(&self) -> HashMap { + let mut origins = HashMap::new(); + let mut path = Vec::new(); + + record_origins( + &self.user.config, + &self.user.metadata(), + &mut path, + &mut origins, + ); + record_origins( + &self.session_flags.config, + &self.session_flags.metadata(), + &mut path, + &mut origins, + ); + if let Some(system) = &self.system { + record_origins(&system.config, &system.metadata(), &mut path, &mut origins); + } + if let Some(mdm) = &self.mdm { + record_origins(&mdm.config, &mdm.metadata(), &mut path, &mut origins); + } + + origins + } + + fn layers_high_to_low(&self) -> Vec { + let mut layers = Vec::new(); + if let Some(mdm) = &self.mdm { + layers.push(mdm.as_layer()); + } + if let Some(system) = &self.system { + layers.push(system.as_layer()); + } + layers.push(self.session_flags.as_layer()); + layers.push(self.user.as_layer()); + layers + } +} + +fn record_origins( + value: &TomlValue, + meta: &ConfigLayerMetadata, + path: &mut Vec, + origins: &mut HashMap, +) { + match value { + TomlValue::Table(table) => { + for (key, val) in table { + path.push(key.clone()); + record_origins(val, meta, path, origins); + path.pop(); + } + } + TomlValue::Array(items) => { + for (idx, item) in items.iter().enumerate() { + path.push(idx.to_string()); + record_origins(item, meta, path, origins); + path.pop(); + } + } + _ => { + if !path.is_empty() { + origins.insert(path.join("."), meta.clone()); + } + } + } +} + +fn to_json_value(value: &TomlValue) -> JsonValue { + serde_json::to_value(value).unwrap_or(JsonValue::Null) +} + +fn validate_config(value: &TomlValue) -> Result<(), toml::de::Error> { + let _: ConfigToml = value.clone().try_into()?; + Ok(()) +} + +fn version_for_toml(value: &TomlValue) -> String { + let json = to_json_value(value); + let canonical = canonical_json(&json); + let serialized = serde_json::to_vec(&canonical).unwrap_or_default(); + let mut hasher = Sha256::new(); + hasher.update(serialized); + let hash = hasher.finalize(); + let hex = hash + .iter() + .map(|byte| format!("{byte:02x}")) + .collect::(); + format!("sha256:{hex}") +} + +fn canonical_json(value: &JsonValue) -> JsonValue { + match value { + JsonValue::Object(map) => { + let mut sorted = serde_json::Map::new(); + let mut keys = map.keys().cloned().collect::>(); + keys.sort(); + for key in keys { + if let Some(val) = map.get(&key) { + sorted.insert(key, canonical_json(val)); + } + } + JsonValue::Object(sorted) + } + JsonValue::Array(items) => JsonValue::Array(items.iter().map(canonical_json).collect()), + other => other.clone(), + } +} + +fn paths_match(expected: &Path, provided: &str) -> bool { + let provided_path = PathBuf::from(provided); + if let (Ok(expanded_expected), Ok(expanded_provided)) = + (expected.canonicalize(), provided_path.canonicalize()) + { + return expanded_expected == expanded_provided; + } + + expected == provided_path +} + +fn value_at_path<'a>(root: &'a TomlValue, segments: &[String]) -> Option<&'a TomlValue> { + let mut current = root; + for segment in segments { + match current { + TomlValue::Table(table) => { + current = table.get(segment)?; + } + TomlValue::Array(items) => { + let idx: usize = segment.parse().ok()?; + current = items.get(idx)?; + } + _ => return None, + } + } + Some(current) +} + +fn override_message(layer: &ConfigLayerName) -> String { + match layer { + ConfigLayerName::Mdm => "Overridden by managed policy (mdm)".to_string(), + ConfigLayerName::System => "Overridden by managed config (system)".to_string(), + ConfigLayerName::SessionFlags => "Overridden by session flags".to_string(), + ConfigLayerName::User => "Overridden by user config".to_string(), + } +} + +fn compute_override_metadata( + layers: &LayersState, + effective: &TomlValue, + segments: &[String], +) -> Option { + let user_value = value_at_path(&layers.user.config, segments); + let effective_value = value_at_path(effective, segments); + + if user_value.is_some() && user_value == effective_value { + return None; + } + + if user_value.is_none() && effective_value.is_none() { + return None; + } + + let effective_layer = find_effective_layer(layers, segments); + let overriding_layer = effective_layer.unwrap_or_else(|| layers.user.metadata()); + let message = override_message(&overriding_layer.name); + + Some(OverriddenMetadata { + message, + overriding_layer, + effective_value: effective_value + .map(to_json_value) + .unwrap_or(JsonValue::Null), + }) +} + +fn first_overridden_edit( + layers: &LayersState, + effective: &TomlValue, + edits: &[Vec], +) -> Option { + for segments in edits { + if let Some(meta) = compute_override_metadata(layers, effective, segments) { + return Some(meta); + } + } + None +} + +fn find_effective_layer(layers: &LayersState, segments: &[String]) -> Option { + let check = + |state: &LayerState| value_at_path(&state.config, segments).map(|_| state.metadata()); + + if let Some(mdm) = &layers.mdm + && let Some(meta) = check(mdm) + { + return Some(meta); + } + if let Some(system) = &layers.system + && let Some(meta) = check(system) + { + return Some(meta); + } + if let Some(meta) = check(&layers.session_flags) { + return Some(meta); + } + check(&layers.user) +} + +fn system_config_path(codex_home: &Path) -> PathBuf { + if let Ok(path) = std::env::var("CODEX_MANAGED_CONFIG_PATH") { + return PathBuf::from(path); + } + + #[cfg(unix)] + { + let _ = codex_home; + PathBuf::from("/etc/codex/managed_config.toml") + } + + #[cfg(not(unix))] + { + codex_home.join("managed_config.toml") + } +} + +fn internal_error(context: &str, err: E) -> JSONRPCErrorError { + JSONRPCErrorError { + code: INTERNAL_ERROR_CODE, + message: format!("{context}: {err}"), + data: None, + } +} + +fn config_write_error(code: ConfigWriteErrorCode, message: impl Into) -> JSONRPCErrorError { + JSONRPCErrorError { + code: INVALID_REQUEST_ERROR_CODE, + message: message.into(), + data: Some(json!({ + "config_write_error_code": code, + })), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use pretty_assertions::assert_eq; + use tempfile::tempdir; + + #[tokio::test] + async fn read_includes_origins_and_layers() { + let tmp = tempdir().expect("tempdir"); + std::fs::write(tmp.path().join(CONFIG_FILE_NAME), "model = \"user\"").unwrap(); + + let managed_path = tmp.path().join("managed_config.toml"); + std::fs::write(&managed_path, "approval_policy = \"never\"").unwrap(); + + let api = ConfigApi::with_overrides( + tmp.path().to_path_buf(), + vec![], + LoaderOverrides { + managed_config_path: Some(managed_path), + #[cfg(target_os = "macos")] + managed_preferences_base64: None, + }, + ); + + let response = api + .read(ConfigReadParams { + include_layers: true, + }) + .await + .expect("response"); + + assert_eq!( + response.config.get("approval_policy"), + Some(&json!("never")) + ); + + assert_eq!( + response + .origins + .get("approval_policy") + .expect("origin") + .name, + ConfigLayerName::System + ); + let layers = response.layers.expect("layers present"); + assert_eq!(layers.first().unwrap().name, ConfigLayerName::System); + assert_eq!(layers.get(1).unwrap().name, ConfigLayerName::SessionFlags); + assert_eq!(layers.last().unwrap().name, ConfigLayerName::User); + } + + #[tokio::test] + async fn write_value_reports_override() { + let tmp = tempdir().expect("tempdir"); + std::fs::write( + tmp.path().join(CONFIG_FILE_NAME), + "approval_policy = \"on-request\"", + ) + .unwrap(); + + let managed_path = tmp.path().join("managed_config.toml"); + std::fs::write(&managed_path, "approval_policy = \"never\"").unwrap(); + + let api = ConfigApi::with_overrides( + tmp.path().to_path_buf(), + vec![], + LoaderOverrides { + managed_config_path: Some(managed_path), + #[cfg(target_os = "macos")] + managed_preferences_base64: None, + }, + ); + + let result = api + .write_value(ConfigValueWriteParams { + file_path: tmp.path().join(CONFIG_FILE_NAME).display().to_string(), + key_path: "approval_policy".to_string(), + value: json!("never"), + merge_strategy: MergeStrategy::Replace, + expected_version: None, + }) + .await + .expect("result"); + + let read_after = api + .read(ConfigReadParams { + include_layers: true, + }) + .await + .expect("read"); + let config_object = read_after.config.as_object().expect("object"); + assert_eq!(config_object.get("approval_policy"), Some(&json!("never"))); + assert_eq!( + read_after + .origins + .get("approval_policy") + .expect("origin") + .name, + ConfigLayerName::System + ); + assert_eq!(result.status, WriteStatus::Ok); + assert!(result.overridden_metadata.is_none()); + } + + #[tokio::test] + async fn version_conflict_rejected() { + let tmp = tempdir().expect("tempdir"); + std::fs::write(tmp.path().join(CONFIG_FILE_NAME), "model = \"user\"").unwrap(); + + let api = ConfigApi::new(tmp.path().to_path_buf(), vec![]); + let error = api + .write_value(ConfigValueWriteParams { + file_path: tmp.path().join(CONFIG_FILE_NAME).display().to_string(), + key_path: "model".to_string(), + value: json!("gpt-5"), + merge_strategy: MergeStrategy::Replace, + expected_version: Some("sha256:bogus".to_string()), + }) + .await + .expect_err("should fail"); + + assert_eq!(error.code, INVALID_REQUEST_ERROR_CODE); + assert_eq!( + error + .data + .as_ref() + .and_then(|d| d.get("config_write_error_code")) + .and_then(serde_json::Value::as_str), + Some("configVersionConflict") + ); + } + + #[tokio::test] + async fn invalid_user_value_rejected_even_if_overridden_by_managed() { + let tmp = tempdir().expect("tempdir"); + std::fs::write(tmp.path().join(CONFIG_FILE_NAME), "model = \"user\"").unwrap(); + + let managed_path = tmp.path().join("managed_config.toml"); + std::fs::write(&managed_path, "approval_policy = \"never\"").unwrap(); + + let api = ConfigApi::with_overrides( + tmp.path().to_path_buf(), + vec![], + LoaderOverrides { + managed_config_path: Some(managed_path), + #[cfg(target_os = "macos")] + managed_preferences_base64: None, + }, + ); + + let error = api + .write_value(ConfigValueWriteParams { + file_path: tmp.path().join(CONFIG_FILE_NAME).display().to_string(), + key_path: "approval_policy".to_string(), + value: json!("bogus"), + merge_strategy: MergeStrategy::Replace, + expected_version: None, + }) + .await + .expect_err("should fail validation"); + + assert_eq!(error.code, INVALID_REQUEST_ERROR_CODE); + assert_eq!( + error + .data + .as_ref() + .and_then(|d| d.get("config_write_error_code")) + .and_then(serde_json::Value::as_str), + Some("configValidationError") + ); + + let contents = + std::fs::read_to_string(tmp.path().join(CONFIG_FILE_NAME)).expect("read config"); + assert_eq!(contents.trim(), "model = \"user\""); + } + + #[tokio::test] + async fn read_reports_managed_overrides_user_and_session_flags() { + let tmp = tempdir().expect("tempdir"); + std::fs::write(tmp.path().join(CONFIG_FILE_NAME), "model = \"user\"").unwrap(); + + let managed_path = tmp.path().join("managed_config.toml"); + std::fs::write(&managed_path, "model = \"system\"").unwrap(); + + let cli_overrides = vec![( + "model".to_string(), + TomlValue::String("session".to_string()), + )]; + + let api = ConfigApi::with_overrides( + tmp.path().to_path_buf(), + cli_overrides, + LoaderOverrides { + managed_config_path: Some(managed_path), + #[cfg(target_os = "macos")] + managed_preferences_base64: None, + }, + ); + + let response = api + .read(ConfigReadParams { + include_layers: true, + }) + .await + .expect("response"); + + assert_eq!(response.config.get("model"), Some(&json!("system"))); + assert_eq!( + response.origins.get("model").expect("origin").name, + ConfigLayerName::System + ); + let layers = response.layers.expect("layers"); + assert_eq!(layers.first().unwrap().name, ConfigLayerName::System); + assert_eq!(layers.get(1).unwrap().name, ConfigLayerName::SessionFlags); + assert_eq!(layers.get(2).unwrap().name, ConfigLayerName::User); + } + + #[tokio::test] + async fn write_value_reports_managed_override() { + let tmp = tempdir().expect("tempdir"); + std::fs::write(tmp.path().join(CONFIG_FILE_NAME), "").unwrap(); + + let managed_path = tmp.path().join("managed_config.toml"); + std::fs::write(&managed_path, "approval_policy = \"never\"").unwrap(); + + let api = ConfigApi::with_overrides( + tmp.path().to_path_buf(), + vec![], + LoaderOverrides { + managed_config_path: Some(managed_path), + #[cfg(target_os = "macos")] + managed_preferences_base64: None, + }, + ); + + let result = api + .write_value(ConfigValueWriteParams { + file_path: tmp.path().join(CONFIG_FILE_NAME).display().to_string(), + key_path: "approval_policy".to_string(), + value: json!("on-request"), + merge_strategy: MergeStrategy::Replace, + expected_version: None, + }) + .await + .expect("result"); + + assert_eq!(result.status, WriteStatus::OkOverridden); + let overridden = result.overridden_metadata.expect("overridden metadata"); + assert_eq!(overridden.overriding_layer.name, ConfigLayerName::System); + assert_eq!(overridden.effective_value, json!("never")); + } +} diff --git a/codex-rs/app-server/src/lib.rs b/codex-rs/app-server/src/lib.rs index 9ad6f50b21..2aee5bc93e 100644 --- a/codex-rs/app-server/src/lib.rs +++ b/codex-rs/app-server/src/lib.rs @@ -18,6 +18,7 @@ use tokio::io::AsyncWriteExt; use tokio::io::BufReader; use tokio::io::{self}; use tokio::sync::mpsc; +use toml::Value as TomlValue; use tracing::Level; use tracing::debug; use tracing::error; @@ -30,6 +31,7 @@ use tracing_subscriber::util::SubscriberInitExt; mod bespoke_event_handling; mod codex_message_processor; +mod config_api; mod error_code; mod fuzzy_file_search; mod message_processor; @@ -80,11 +82,12 @@ pub async fn run_main( format!("error parsing -c overrides: {e}"), ) })?; - let config = Config::load_with_cli_overrides(cli_kv_overrides, ConfigOverrides::default()) - .await - .map_err(|e| { - std::io::Error::new(ErrorKind::InvalidData, format!("error loading config: {e}")) - })?; + let config = + Config::load_with_cli_overrides(cli_kv_overrides.clone(), ConfigOverrides::default()) + .await + .map_err(|e| { + std::io::Error::new(ErrorKind::InvalidData, format!("error loading config: {e}")) + })?; let feedback = CodexFeedback::new(); @@ -121,10 +124,12 @@ pub async fn run_main( // Task: process incoming messages. let processor_handle = tokio::spawn({ let outgoing_message_sender = OutgoingMessageSender::new(outgoing_tx); + let cli_overrides: Vec<(String, TomlValue)> = cli_kv_overrides.clone(); let mut processor = MessageProcessor::new( outgoing_message_sender, codex_linux_sandbox_exe, std::sync::Arc::new(config), + cli_overrides, feedback.clone(), ); async move { diff --git a/codex-rs/app-server/src/message_processor.rs b/codex-rs/app-server/src/message_processor.rs index 403263b896..90560e9b3c 100644 --- a/codex-rs/app-server/src/message_processor.rs +++ b/codex-rs/app-server/src/message_processor.rs @@ -1,16 +1,22 @@ use std::path::PathBuf; +use std::sync::Arc; use crate::codex_message_processor::CodexMessageProcessor; +use crate::config_api::ConfigApi; use crate::error_code::INVALID_REQUEST_ERROR_CODE; use crate::outgoing_message::OutgoingMessageSender; use codex_app_server_protocol::ClientInfo; use codex_app_server_protocol::ClientRequest; +use codex_app_server_protocol::ConfigBatchWriteParams; +use codex_app_server_protocol::ConfigReadParams; +use codex_app_server_protocol::ConfigValueWriteParams; use codex_app_server_protocol::InitializeResponse; use codex_app_server_protocol::JSONRPCError; use codex_app_server_protocol::JSONRPCErrorError; use codex_app_server_protocol::JSONRPCNotification; use codex_app_server_protocol::JSONRPCRequest; use codex_app_server_protocol::JSONRPCResponse; +use codex_app_server_protocol::RequestId; use codex_core::AuthManager; use codex_core::ConversationManager; use codex_core::config::Config; @@ -18,11 +24,12 @@ use codex_core::default_client::USER_AGENT_SUFFIX; use codex_core::default_client::get_codex_user_agent; use codex_feedback::CodexFeedback; use codex_protocol::protocol::SessionSource; -use std::sync::Arc; +use toml::Value as TomlValue; pub(crate) struct MessageProcessor { outgoing: Arc, codex_message_processor: CodexMessageProcessor, + config_api: ConfigApi, initialized: bool, } @@ -33,6 +40,7 @@ impl MessageProcessor { outgoing: OutgoingMessageSender, codex_linux_sandbox_exe: Option, config: Arc, + cli_overrides: Vec<(String, TomlValue)>, feedback: CodexFeedback, ) -> Self { let outgoing = Arc::new(outgoing); @@ -50,13 +58,15 @@ impl MessageProcessor { conversation_manager, outgoing.clone(), codex_linux_sandbox_exe, - config, + Arc::clone(&config), feedback, ); + let config_api = ConfigApi::new(config.codex_home.clone(), cli_overrides); Self { outgoing, codex_message_processor, + config_api, initialized: false, } } @@ -134,9 +144,20 @@ impl MessageProcessor { } } - self.codex_message_processor - .process_request(codex_request) - .await; + match codex_request { + ClientRequest::ConfigRead { request_id, params } => { + self.handle_config_read(request_id, params).await; + } + ClientRequest::ConfigValueWrite { request_id, params } => { + self.handle_config_value_write(request_id, params).await; + } + ClientRequest::ConfigBatchWrite { request_id, params } => { + self.handle_config_batch_write(request_id, params).await; + } + other => { + self.codex_message_processor.process_request(other).await; + } + } } pub(crate) async fn process_notification(&self, notification: JSONRPCNotification) { @@ -156,4 +177,33 @@ impl MessageProcessor { pub(crate) fn process_error(&mut self, err: JSONRPCError) { tracing::error!("<- error: {:?}", err); } + + async fn handle_config_read(&self, request_id: RequestId, params: ConfigReadParams) { + match self.config_api.read(params).await { + Ok(response) => self.outgoing.send_response(request_id, response).await, + Err(error) => self.outgoing.send_error(request_id, error).await, + } + } + + async fn handle_config_value_write( + &self, + request_id: RequestId, + params: ConfigValueWriteParams, + ) { + match self.config_api.write_value(params).await { + Ok(response) => self.outgoing.send_response(request_id, response).await, + Err(error) => self.outgoing.send_error(request_id, error).await, + } + } + + async fn handle_config_batch_write( + &self, + request_id: RequestId, + params: ConfigBatchWriteParams, + ) { + match self.config_api.batch_write(params).await { + Ok(response) => self.outgoing.send_response(request_id, response).await, + Err(error) => self.outgoing.send_error(request_id, error).await, + } + } } diff --git a/codex-rs/app-server/tests/common/Cargo.toml b/codex-rs/app-server/tests/common/Cargo.toml index 6240f755ee..380f70d9b1 100644 --- a/codex-rs/app-server/tests/common/Cargo.toml +++ b/codex-rs/app-server/tests/common/Cargo.toml @@ -1,7 +1,8 @@ [package] -edition = "2024" name = "app_test_support" -version = { workspace = true } +version.workspace = true +edition.workspace = true +license.workspace = true [lib] path = "lib.rs" @@ -24,3 +25,5 @@ tokio = { workspace = true, features = [ ] } uuid = { workspace = true } wiremock = { workspace = true } +core_test_support = { path = "../../../core/tests/common" } +shlex = { workspace = true } diff --git a/codex-rs/app-server/tests/common/lib.rs b/codex-rs/app-server/tests/common/lib.rs index dc3d24ccae..6fd54a66dc 100644 --- a/codex-rs/app-server/tests/common/lib.rs +++ b/codex-rs/app-server/tests/common/lib.rs @@ -9,12 +9,15 @@ pub use auth_fixtures::ChatGptIdTokenClaims; pub use auth_fixtures::encode_id_token; pub use auth_fixtures::write_chatgpt_auth; use codex_app_server_protocol::JSONRPCResponse; +pub use core_test_support::format_with_current_shell; +pub use core_test_support::format_with_current_shell_display; pub use mcp_process::McpProcess; pub use mock_model_server::create_mock_chat_completions_server; pub use mock_model_server::create_mock_chat_completions_server_unchecked; pub use responses::create_apply_patch_sse_response; +pub use responses::create_exec_command_sse_response; pub use responses::create_final_assistant_message_sse_response; -pub use responses::create_shell_sse_response; +pub use responses::create_shell_command_sse_response; pub use rollout::create_fake_rollout; use serde::de::DeserializeOwned; diff --git a/codex-rs/app-server/tests/common/mcp_process.rs b/codex-rs/app-server/tests/common/mcp_process.rs index 920a6fa01d..e2da40bf2d 100644 --- a/codex-rs/app-server/tests/common/mcp_process.rs +++ b/codex-rs/app-server/tests/common/mcp_process.rs @@ -18,6 +18,9 @@ use codex_app_server_protocol::CancelLoginAccountParams; use codex_app_server_protocol::CancelLoginChatGptParams; use codex_app_server_protocol::ClientInfo; use codex_app_server_protocol::ClientNotification; +use codex_app_server_protocol::ConfigBatchWriteParams; +use codex_app_server_protocol::ConfigReadParams; +use codex_app_server_protocol::ConfigValueWriteParams; use codex_app_server_protocol::FeedbackUploadParams; use codex_app_server_protocol::GetAccountParams; use codex_app_server_protocol::GetAuthStatusParams; @@ -401,6 +404,30 @@ impl McpProcess { self.send_request("logoutChatGpt", None).await } + pub async fn send_config_read_request( + &mut self, + params: ConfigReadParams, + ) -> anyhow::Result { + let params = Some(serde_json::to_value(params)?); + self.send_request("config/read", params).await + } + + pub async fn send_config_value_write_request( + &mut self, + params: ConfigValueWriteParams, + ) -> anyhow::Result { + let params = Some(serde_json::to_value(params)?); + self.send_request("config/value/write", params).await + } + + pub async fn send_config_batch_write_request( + &mut self, + params: ConfigBatchWriteParams, + ) -> anyhow::Result { + let params = Some(serde_json::to_value(params)?); + self.send_request("config/batchWrite", params).await + } + /// Send an `account/logout` JSON-RPC request. pub async fn send_logout_account_request(&mut self) -> anyhow::Result { self.send_request("account/logout", None).await diff --git a/codex-rs/app-server/tests/common/responses.rs b/codex-rs/app-server/tests/common/responses.rs index 9a827fb986..d3d1f40cdc 100644 --- a/codex-rs/app-server/tests/common/responses.rs +++ b/codex-rs/app-server/tests/common/responses.rs @@ -1,17 +1,18 @@ use serde_json::json; use std::path::Path; -pub fn create_shell_sse_response( +pub fn create_shell_command_sse_response( command: Vec, workdir: Option<&Path>, timeout_ms: Option, call_id: &str, ) -> anyhow::Result { - // The `arguments`` for the `shell` tool is a serialized JSON object. + // The `arguments` for the `shell_command` tool is a serialized JSON object. + let command_str = shlex::try_join(command.iter().map(String::as_str))?; let tool_call_arguments = serde_json::to_string(&json!({ - "command": command, + "command": command_str, "workdir": workdir.map(|w| w.to_string_lossy()), - "timeout": timeout_ms + "timeout_ms": timeout_ms }))?; let tool_call = json!({ "choices": [ @@ -21,7 +22,7 @@ pub fn create_shell_sse_response( { "id": call_id, "function": { - "name": "shell", + "name": "shell_command", "arguments": tool_call_arguments } } @@ -62,10 +63,10 @@ pub fn create_apply_patch_sse_response( patch_content: &str, call_id: &str, ) -> anyhow::Result { - // Use shell command to call apply_patch with heredoc format - let shell_command = format!("apply_patch <<'EOF'\n{patch_content}\nEOF"); + // Use shell_command to call apply_patch with heredoc format + let command = format!("apply_patch <<'EOF'\n{patch_content}\nEOF"); let tool_call_arguments = serde_json::to_string(&json!({ - "command": ["bash", "-lc", shell_command] + "command": command }))?; let tool_call = json!({ @@ -76,7 +77,46 @@ pub fn create_apply_patch_sse_response( { "id": call_id, "function": { - "name": "shell", + "name": "shell_command", + "arguments": tool_call_arguments + } + } + ] + }, + "finish_reason": "tool_calls" + } + ] + }); + + let sse = format!( + "data: {}\n\ndata: DONE\n\n", + serde_json::to_string(&tool_call)? + ); + Ok(sse) +} + +pub fn create_exec_command_sse_response(call_id: &str) -> anyhow::Result { + let (cmd, args) = if cfg!(windows) { + ("cmd.exe", vec!["/d", "/c", "echo hi"]) + } else { + ("/bin/sh", vec!["-c", "echo hi"]) + }; + let command = std::iter::once(cmd.to_string()) + .chain(args.into_iter().map(str::to_string)) + .collect::>(); + let tool_call_arguments = serde_json::to_string(&json!({ + "cmd": command.join(" "), + "yield_time_ms": 500 + }))?; + let tool_call = json!({ + "choices": [ + { + "delta": { + "tool_calls": [ + { + "id": call_id, + "function": { + "name": "exec_command", "arguments": tool_call_arguments } } diff --git a/codex-rs/app-server/tests/common/rollout.rs b/codex-rs/app-server/tests/common/rollout.rs index 203b989d3e..ee505d8887 100644 --- a/codex-rs/app-server/tests/common/rollout.rs +++ b/codex-rs/app-server/tests/common/rollout.rs @@ -1,6 +1,8 @@ use anyhow::Result; use codex_protocol::ConversationId; +use codex_protocol::protocol::GitInfo; use codex_protocol::protocol::SessionMeta; +use codex_protocol::protocol::SessionMetaLine; use codex_protocol::protocol::SessionSource; use serde_json::json; use std::fs; @@ -22,6 +24,7 @@ pub fn create_fake_rollout( meta_rfc3339: &str, preview: &str, model_provider: Option<&str>, + git_info: Option, ) -> Result { let uuid = Uuid::new_v4(); let uuid_str = uuid.to_string(); @@ -37,7 +40,7 @@ pub fn create_fake_rollout( let file_path = dir.join(format!("rollout-{filename_ts}-{uuid}.jsonl")); // Build JSONL lines - let payload = serde_json::to_value(SessionMeta { + let meta = SessionMeta { id: conversation_id, timestamp: meta_rfc3339.to_string(), cwd: PathBuf::from("/"), @@ -47,6 +50,10 @@ pub fn create_fake_rollout( source: SessionSource::Cli, model_provider: model_provider.map(str::to_string), name: None, + }; + let payload = serde_json::to_value(SessionMetaLine { + meta, + git: git_info, })?; let lines = [ diff --git a/codex-rs/app-server/tests/suite/codex_message_processor_flow.rs b/codex-rs/app-server/tests/suite/codex_message_processor_flow.rs index 1feda42841..a64aca8051 100644 --- a/codex-rs/app-server/tests/suite/codex_message_processor_flow.rs +++ b/codex-rs/app-server/tests/suite/codex_message_processor_flow.rs @@ -2,7 +2,8 @@ use anyhow::Result; use app_test_support::McpProcess; use app_test_support::create_final_assistant_message_sse_response; use app_test_support::create_mock_chat_completions_server; -use app_test_support::create_shell_sse_response; +use app_test_support::create_shell_command_sse_response; +use app_test_support::format_with_current_shell; use app_test_support::to_response; use codex_app_server_protocol::AddConversationListenerParams; use codex_app_server_protocol::AddConversationSubscriptionResponse; @@ -56,7 +57,7 @@ async fn test_codex_jsonrpc_conversation_flow() -> Result<()> { // Create a mock model server that immediately ends each turn. // Two turns are expected: initial session configure + one user message. let responses = vec![ - create_shell_sse_response( + create_shell_command_sse_response( vec!["ls".to_string()], Some(&working_directory), Some(5000), @@ -175,7 +176,7 @@ async fn test_send_user_turn_changes_approval_policy_behavior() -> Result<()> { // Mock server will request a python shell call for the first and second turn, then finish. let responses = vec![ - create_shell_sse_response( + create_shell_command_sse_response( vec![ "python3".to_string(), "-c".to_string(), @@ -186,7 +187,7 @@ async fn test_send_user_turn_changes_approval_policy_behavior() -> Result<()> { "call1", )?, create_final_assistant_message_sse_response("done 1")?, - create_shell_sse_response( + create_shell_command_sse_response( vec![ "python3".to_string(), "-c".to_string(), @@ -267,11 +268,7 @@ async fn test_send_user_turn_changes_approval_policy_behavior() -> Result<()> { ExecCommandApprovalParams { conversation_id, call_id: "call1".to_string(), - command: vec![ - "python3".to_string(), - "-c".to_string(), - "print(42)".to_string(), - ], + command: format_with_current_shell("python3 -c 'print(42)'"), cwd: working_directory.clone(), reason: None, risk: None, @@ -353,23 +350,15 @@ async fn test_send_user_turn_updates_sandbox_and_cwd_between_turns() -> Result<( std::fs::create_dir(&second_cwd)?; let responses = vec![ - create_shell_sse_response( - vec![ - "bash".to_string(), - "-lc".to_string(), - "echo first turn".to_string(), - ], + create_shell_command_sse_response( + vec!["echo".to_string(), "first".to_string(), "turn".to_string()], None, Some(5000), "call-first", )?, create_final_assistant_message_sse_response("done first")?, - create_shell_sse_response( - vec![ - "bash".to_string(), - "-lc".to_string(), - "echo second turn".to_string(), - ], + create_shell_command_sse_response( + vec!["echo".to_string(), "second".to_string(), "turn".to_string()], None, Some(5000), "call-second", @@ -481,13 +470,9 @@ async fn test_send_user_turn_updates_sandbox_and_cwd_between_turns() -> Result<( exec_begin.cwd, second_cwd, "exec turn should run from updated cwd" ); + let expected_command = format_with_current_shell("echo second turn"); assert_eq!( - exec_begin.command, - vec![ - "bash".to_string(), - "-lc".to_string(), - "echo second turn".to_string() - ], + exec_begin.command, expected_command, "exec turn should run expected command" ); diff --git a/codex-rs/app-server/tests/suite/interrupt.rs b/codex-rs/app-server/tests/suite/interrupt.rs index 86b0a3f3f8..d8e6182be8 100644 --- a/codex-rs/app-server/tests/suite/interrupt.rs +++ b/codex-rs/app-server/tests/suite/interrupt.rs @@ -19,7 +19,7 @@ use tokio::time::timeout; use app_test_support::McpProcess; use app_test_support::create_mock_chat_completions_server; -use app_test_support::create_shell_sse_response; +use app_test_support::create_shell_command_sse_response; use app_test_support::to_response; const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); @@ -56,7 +56,7 @@ async fn shell_command_interruption() -> anyhow::Result<()> { std::fs::create_dir(&working_directory)?; // Create mock server with a single SSE response: the long sleep command - let server = create_mock_chat_completions_server(vec![create_shell_sse_response( + let server = create_mock_chat_completions_server(vec![create_shell_command_sse_response( shell_command.clone(), Some(&working_directory), Some(10_000), // 10 seconds timeout in ms diff --git a/codex-rs/app-server/tests/suite/list_resume.rs b/codex-rs/app-server/tests/suite/list_resume.rs index 30be93a2e6..1e89c06848 100644 --- a/codex-rs/app-server/tests/suite/list_resume.rs +++ b/codex-rs/app-server/tests/suite/list_resume.rs @@ -31,6 +31,7 @@ async fn test_list_and_resume_conversations() -> Result<()> { "2025-01-02T12:00:00Z", "Hello A", Some("openai"), + None, )?; create_fake_rollout( codex_home.path(), @@ -38,6 +39,7 @@ async fn test_list_and_resume_conversations() -> Result<()> { "2025-01-01T13:00:00Z", "Hello B", Some("openai"), + None, )?; create_fake_rollout( codex_home.path(), @@ -45,6 +47,7 @@ async fn test_list_and_resume_conversations() -> Result<()> { "2025-01-01T12:00:00Z", "Hello C", None, + None, )?; let mut mcp = McpProcess::new(codex_home.path()).await?; @@ -105,6 +108,7 @@ async fn test_list_and_resume_conversations() -> Result<()> { "2025-01-01T11:30:00Z", "Hello TP", Some("test-provider"), + None, )?; // Filtering by model provider should return only matching sessions. diff --git a/codex-rs/app-server/tests/suite/send_message.rs b/codex-rs/app-server/tests/suite/send_message.rs index 8d2b36af2f..39b3a31a8a 100644 --- a/codex-rs/app-server/tests/suite/send_message.rs +++ b/codex-rs/app-server/tests/suite/send_message.rs @@ -272,40 +272,45 @@ async fn read_raw_response_item( mcp: &mut McpProcess, conversation_id: ConversationId, ) -> ResponseItem { - let raw_notification: JSONRPCNotification = timeout( - DEFAULT_READ_TIMEOUT, - mcp.read_stream_until_notification_message("codex/event/raw_response_item"), - ) - .await - .expect("codex/event/raw_response_item notification timeout") - .expect("codex/event/raw_response_item notification resp"); + loop { + let raw_notification: JSONRPCNotification = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_notification_message("codex/event/raw_response_item"), + ) + .await + .expect("codex/event/raw_response_item notification timeout") + .expect("codex/event/raw_response_item notification resp"); - let serde_json::Value::Object(params) = raw_notification - .params - .expect("codex/event/raw_response_item should have params") - else { - panic!("codex/event/raw_response_item should have params"); - }; + let serde_json::Value::Object(params) = raw_notification + .params + .expect("codex/event/raw_response_item should have params") + else { + panic!("codex/event/raw_response_item should have params"); + }; - let conversation_id_value = params - .get("conversationId") - .and_then(|value| value.as_str()) - .expect("raw response item should include conversationId"); + let conversation_id_value = params + .get("conversationId") + .and_then(|value| value.as_str()) + .expect("raw response item should include conversationId"); - assert_eq!( - conversation_id_value, - conversation_id.to_string(), - "raw response item conversation mismatch" - ); + assert_eq!( + conversation_id_value, + conversation_id.to_string(), + "raw response item conversation mismatch" + ); - let msg_value = params - .get("msg") - .cloned() - .expect("raw response item should include msg payload"); + let msg_value = params + .get("msg") + .cloned() + .expect("raw response item should include msg payload"); - let event: RawResponseItemEvent = - serde_json::from_value(msg_value).expect("deserialize raw response item"); - event.item + // Ghost snapshots are produced concurrently and may arrive before the model reply. + let event: RawResponseItemEvent = + serde_json::from_value(msg_value).expect("deserialize raw response item"); + if !matches!(event.item, ResponseItem::GhostSnapshot { .. }) { + return event.item; + } + } } fn assert_instructions_message(item: &ResponseItem) { diff --git a/codex-rs/app-server/tests/suite/v2/config_rpc.rs b/codex-rs/app-server/tests/suite/v2/config_rpc.rs new file mode 100644 index 0000000000..343a13c3c4 --- /dev/null +++ b/codex-rs/app-server/tests/suite/v2/config_rpc.rs @@ -0,0 +1,347 @@ +use anyhow::Result; +use app_test_support::McpProcess; +use app_test_support::to_response; +use codex_app_server_protocol::ConfigBatchWriteParams; +use codex_app_server_protocol::ConfigEdit; +use codex_app_server_protocol::ConfigLayerName; +use codex_app_server_protocol::ConfigReadParams; +use codex_app_server_protocol::ConfigReadResponse; +use codex_app_server_protocol::ConfigValueWriteParams; +use codex_app_server_protocol::ConfigWriteResponse; +use codex_app_server_protocol::JSONRPCError; +use codex_app_server_protocol::JSONRPCResponse; +use codex_app_server_protocol::MergeStrategy; +use codex_app_server_protocol::RequestId; +use codex_app_server_protocol::WriteStatus; +use pretty_assertions::assert_eq; +use serde_json::json; +use tempfile::TempDir; +use tokio::time::timeout; + +const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); + +fn write_config(codex_home: &TempDir, contents: &str) -> Result<()> { + Ok(std::fs::write( + codex_home.path().join("config.toml"), + contents, + )?) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn config_read_returns_effective_and_layers() -> Result<()> { + let codex_home = TempDir::new()?; + write_config( + &codex_home, + r#" +model = "gpt-user" +sandbox_mode = "workspace-write" +"#, + )?; + + let mut mcp = McpProcess::new(codex_home.path()).await?; + timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; + + let request_id = mcp + .send_config_read_request(ConfigReadParams { + include_layers: true, + }) + .await?; + let resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(request_id)), + ) + .await??; + let ConfigReadResponse { + config, + origins, + layers, + } = to_response(resp)?; + + assert_eq!(config.get("model"), Some(&json!("gpt-user"))); + assert_eq!( + origins.get("model").expect("origin").name, + ConfigLayerName::User + ); + let layers = layers.expect("layers present"); + assert_eq!(layers.len(), 2); + assert_eq!(layers[0].name, ConfigLayerName::SessionFlags); + assert_eq!(layers[1].name, ConfigLayerName::User); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn config_read_includes_system_layer_and_overrides() -> Result<()> { + let codex_home = TempDir::new()?; + write_config( + &codex_home, + r#" +model = "gpt-user" +approval_policy = "on-request" +sandbox_mode = "workspace-write" + +[sandbox_workspace_write] +writable_roots = ["/user"] +network_access = true +"#, + )?; + + let managed_path = codex_home.path().join("managed_config.toml"); + std::fs::write( + &managed_path, + r#" +model = "gpt-system" +approval_policy = "never" + +[sandbox_workspace_write] +writable_roots = ["/system"] +"#, + )?; + + let managed_path_str = managed_path.display().to_string(); + + let mut mcp = McpProcess::new_with_env( + codex_home.path(), + &[("CODEX_MANAGED_CONFIG_PATH", Some(&managed_path_str))], + ) + .await?; + timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; + + let request_id = mcp + .send_config_read_request(ConfigReadParams { + include_layers: true, + }) + .await?; + let resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(request_id)), + ) + .await??; + let ConfigReadResponse { + config, + origins, + layers, + } = to_response(resp)?; + + assert_eq!(config.get("model"), Some(&json!("gpt-system"))); + assert_eq!( + origins.get("model").expect("origin").name, + ConfigLayerName::System + ); + + assert_eq!(config.get("approval_policy"), Some(&json!("never"))); + assert_eq!( + origins.get("approval_policy").expect("origin").name, + ConfigLayerName::System + ); + + assert_eq!(config.get("sandbox_mode"), Some(&json!("workspace-write"))); + assert_eq!( + origins.get("sandbox_mode").expect("origin").name, + ConfigLayerName::User + ); + + assert_eq!( + config + .get("sandbox_workspace_write") + .and_then(|v| v.get("writable_roots")), + Some(&json!(["/system"])) + ); + assert_eq!( + origins + .get("sandbox_workspace_write.writable_roots.0") + .expect("origin") + .name, + ConfigLayerName::System + ); + + assert_eq!( + config + .get("sandbox_workspace_write") + .and_then(|v| v.get("network_access")), + Some(&json!(true)) + ); + assert_eq!( + origins + .get("sandbox_workspace_write.network_access") + .expect("origin") + .name, + ConfigLayerName::User + ); + + let layers = layers.expect("layers present"); + assert_eq!(layers.len(), 3); + assert_eq!(layers[0].name, ConfigLayerName::System); + assert_eq!(layers[1].name, ConfigLayerName::SessionFlags); + assert_eq!(layers[2].name, ConfigLayerName::User); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn config_value_write_replaces_value() -> Result<()> { + let codex_home = TempDir::new()?; + write_config( + &codex_home, + r#" +model = "gpt-old" +"#, + )?; + + let mut mcp = McpProcess::new(codex_home.path()).await?; + timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; + + let read_id = mcp + .send_config_read_request(ConfigReadParams { + include_layers: false, + }) + .await?; + let read_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(read_id)), + ) + .await??; + let read: ConfigReadResponse = to_response(read_resp)?; + let expected_version = read.origins.get("model").map(|m| m.version.clone()); + + let write_id = mcp + .send_config_value_write_request(ConfigValueWriteParams { + file_path: codex_home.path().join("config.toml").display().to_string(), + key_path: "model".to_string(), + value: json!("gpt-new"), + merge_strategy: MergeStrategy::Replace, + expected_version, + }) + .await?; + let write_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(write_id)), + ) + .await??; + let write: ConfigWriteResponse = to_response(write_resp)?; + + assert_eq!(write.status, WriteStatus::Ok); + assert!(write.overridden_metadata.is_none()); + + let verify_id = mcp + .send_config_read_request(ConfigReadParams { + include_layers: false, + }) + .await?; + let verify_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(verify_id)), + ) + .await??; + let verify: ConfigReadResponse = to_response(verify_resp)?; + assert_eq!(verify.config.get("model"), Some(&json!("gpt-new"))); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn config_value_write_rejects_version_conflict() -> Result<()> { + let codex_home = TempDir::new()?; + write_config( + &codex_home, + r#" +model = "gpt-old" +"#, + )?; + + let mut mcp = McpProcess::new(codex_home.path()).await?; + timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; + + let write_id = mcp + .send_config_value_write_request(ConfigValueWriteParams { + file_path: codex_home.path().join("config.toml").display().to_string(), + key_path: "model".to_string(), + value: json!("gpt-new"), + merge_strategy: MergeStrategy::Replace, + expected_version: Some("sha256:stale".to_string()), + }) + .await?; + + let err: JSONRPCError = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_error_message(RequestId::Integer(write_id)), + ) + .await??; + let code = err + .error + .data + .as_ref() + .and_then(|d| d.get("config_write_error_code")) + .and_then(|v| v.as_str()); + assert_eq!(code, Some("configVersionConflict")); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn config_batch_write_applies_multiple_edits() -> Result<()> { + let codex_home = TempDir::new()?; + write_config(&codex_home, "")?; + + let mut mcp = McpProcess::new(codex_home.path()).await?; + timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; + + let batch_id = mcp + .send_config_batch_write_request(ConfigBatchWriteParams { + file_path: codex_home.path().join("config.toml").display().to_string(), + edits: vec![ + ConfigEdit { + key_path: "sandbox_mode".to_string(), + value: json!("workspace-write"), + merge_strategy: MergeStrategy::Replace, + }, + ConfigEdit { + key_path: "sandbox_workspace_write".to_string(), + value: json!({ + "writable_roots": ["/tmp"], + "network_access": false + }), + merge_strategy: MergeStrategy::Replace, + }, + ], + expected_version: None, + }) + .await?; + let batch_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(batch_id)), + ) + .await??; + let batch_write: ConfigWriteResponse = to_response(batch_resp)?; + assert_eq!(batch_write.status, WriteStatus::Ok); + + let read_id = mcp + .send_config_read_request(ConfigReadParams { + include_layers: false, + }) + .await?; + let read_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(read_id)), + ) + .await??; + let read: ConfigReadResponse = to_response(read_resp)?; + assert_eq!( + read.config.get("sandbox_mode"), + Some(&json!("workspace-write")) + ); + assert_eq!( + read.config + .get("sandbox_workspace_write") + .and_then(|v| v.get("writable_roots")), + Some(&json!(["/tmp"])) + ); + assert_eq!( + read.config + .get("sandbox_workspace_write") + .and_then(|v| v.get("network_access")), + Some(&json!(false)) + ); + + Ok(()) +} diff --git a/codex-rs/app-server/tests/suite/v2/mod.rs b/codex-rs/app-server/tests/suite/v2/mod.rs index a8594e7ca2..16d2142b2e 100644 --- a/codex-rs/app-server/tests/suite/v2/mod.rs +++ b/codex-rs/app-server/tests/suite/v2/mod.rs @@ -1,4 +1,5 @@ mod account; +mod config_rpc; mod model_list; mod rate_limits; mod review; diff --git a/codex-rs/app-server/tests/suite/v2/review.rs b/codex-rs/app-server/tests/suite/v2/review.rs index cdb3acd088..3ad987a388 100644 --- a/codex-rs/app-server/tests/suite/v2/review.rs +++ b/codex-rs/app-server/tests/suite/v2/review.rs @@ -9,12 +9,13 @@ use codex_app_server_protocol::JSONRPCError; use codex_app_server_protocol::JSONRPCNotification; use codex_app_server_protocol::JSONRPCResponse; use codex_app_server_protocol::RequestId; +use codex_app_server_protocol::ReviewDelivery; use codex_app_server_protocol::ReviewStartParams; +use codex_app_server_protocol::ReviewStartResponse; use codex_app_server_protocol::ReviewTarget; use codex_app_server_protocol::ThreadItem; use codex_app_server_protocol::ThreadStartParams; use codex_app_server_protocol::ThreadStartResponse; -use codex_app_server_protocol::TurnStartResponse; use codex_app_server_protocol::TurnStatus; use serde_json::json; use tempfile::TempDir; @@ -59,7 +60,7 @@ async fn review_start_runs_review_turn_and_emits_code_review_item() -> Result<() let review_req = mcp .send_review_start_request(ReviewStartParams { thread_id: thread_id.clone(), - append_to_original_thread: true, + delivery: Some(ReviewDelivery::Inline), target: ReviewTarget::Commit { sha: "1234567deadbeef".to_string(), title: Some("Tidy UI colors".to_string()), @@ -71,43 +72,43 @@ async fn review_start_runs_review_turn_and_emits_code_review_item() -> Result<() mcp.read_stream_until_response_message(RequestId::Integer(review_req)), ) .await??; - let TurnStartResponse { turn } = to_response::(review_resp)?; + let ReviewStartResponse { + turn, + review_thread_id, + } = to_response::(review_resp)?; + assert_eq!(review_thread_id, thread_id.clone()); let turn_id = turn.id.clone(); assert_eq!(turn.status, TurnStatus::InProgress); - assert_eq!(turn.items.len(), 1); - match &turn.items[0] { - ThreadItem::UserMessage { content, .. } => { - assert_eq!(content.len(), 1); - assert!(matches!( - &content[0], - codex_app_server_protocol::UserInput::Text { .. } - )); - } - other => panic!("expected user message, got {other:?}"), - } - let _started: JSONRPCNotification = timeout( - DEFAULT_READ_TIMEOUT, - mcp.read_stream_until_notification_message("turn/started"), - ) - .await??; - let item_started: JSONRPCNotification = timeout( - DEFAULT_READ_TIMEOUT, - mcp.read_stream_until_notification_message("item/started"), - ) - .await??; - let started: ItemStartedNotification = - serde_json::from_value(item_started.params.expect("params must be present"))?; - match started.item { - ThreadItem::CodeReview { id, review } => { - assert_eq!(id, turn_id); - assert_eq!(review, "commit 1234567"); + // Confirm we see the EnteredReviewMode marker on the main thread. + let mut saw_entered_review_mode = false; + for _ in 0..10 { + let item_started: JSONRPCNotification = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_notification_message("item/started"), + ) + .await??; + let started: ItemStartedNotification = + serde_json::from_value(item_started.params.expect("params must be present"))?; + match started.item { + ThreadItem::EnteredReviewMode { id, review } => { + assert_eq!(id, turn_id); + assert_eq!(review, "commit 1234567: Tidy UI colors"); + saw_entered_review_mode = true; + break; + } + _ => continue, } - other => panic!("expected code review item, got {other:?}"), } + assert!( + saw_entered_review_mode, + "did not observe enteredReviewMode item" + ); + // Confirm we see the ExitedReviewMode marker (with review text) + // on the same turn. Ignore any other items the stream surfaces. let mut review_body: Option = None; - for _ in 0..5 { + for _ in 0..10 { let review_notif: JSONRPCNotification = timeout( DEFAULT_READ_TIMEOUT, mcp.read_stream_until_notification_message("item/completed"), @@ -116,13 +117,12 @@ async fn review_start_runs_review_turn_and_emits_code_review_item() -> Result<() let completed: ItemCompletedNotification = serde_json::from_value(review_notif.params.expect("params must be present"))?; match completed.item { - ThreadItem::CodeReview { id, review } => { + ThreadItem::ExitedReviewMode { id, review } => { assert_eq!(id, turn_id); review_body = Some(review); break; } - ThreadItem::UserMessage { .. } => continue, - other => panic!("unexpected item/completed payload: {other:?}"), + _ => continue, } } @@ -146,7 +146,7 @@ async fn review_start_rejects_empty_base_branch() -> Result<()> { let request_id = mcp .send_review_start_request(ReviewStartParams { thread_id, - append_to_original_thread: true, + delivery: Some(ReviewDelivery::Inline), target: ReviewTarget::BaseBranch { branch: " ".to_string(), }, @@ -167,6 +167,56 @@ async fn review_start_rejects_empty_base_branch() -> Result<()> { Ok(()) } +#[tokio::test] +async fn review_start_with_detached_delivery_returns_new_thread_id() -> Result<()> { + let review_payload = json!({ + "findings": [], + "overall_correctness": "ok", + "overall_explanation": "detached review", + "overall_confidence_score": 0.5 + }) + .to_string(); + let responses = vec![create_final_assistant_message_sse_response( + &review_payload, + )?]; + let server = create_mock_chat_completions_server_unchecked(responses).await; + + let codex_home = TempDir::new()?; + create_config_toml(codex_home.path(), &server.uri())?; + + let mut mcp = McpProcess::new(codex_home.path()).await?; + timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; + + let thread_id = start_default_thread(&mut mcp).await?; + + let review_req = mcp + .send_review_start_request(ReviewStartParams { + thread_id: thread_id.clone(), + delivery: Some(ReviewDelivery::Detached), + target: ReviewTarget::Custom { + instructions: "detached review".to_string(), + }, + }) + .await?; + let review_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(review_req)), + ) + .await??; + let ReviewStartResponse { + turn, + review_thread_id, + } = to_response::(review_resp)?; + + assert_eq!(turn.status, TurnStatus::InProgress); + assert_ne!( + review_thread_id, thread_id, + "detached review should run on a different thread" + ); + + Ok(()) +} + #[tokio::test] async fn review_start_rejects_empty_commit_sha() -> Result<()> { let server = create_mock_chat_completions_server_unchecked(vec![]).await; @@ -180,7 +230,7 @@ async fn review_start_rejects_empty_commit_sha() -> Result<()> { let request_id = mcp .send_review_start_request(ReviewStartParams { thread_id, - append_to_original_thread: true, + delivery: Some(ReviewDelivery::Inline), target: ReviewTarget::Commit { sha: "\t".to_string(), title: None, @@ -215,7 +265,7 @@ async fn review_start_rejects_empty_custom_instructions() -> Result<()> { let request_id = mcp .send_review_start_request(ReviewStartParams { thread_id, - append_to_original_thread: true, + delivery: Some(ReviewDelivery::Inline), target: ReviewTarget::Custom { instructions: "\n\n".to_string(), }, diff --git a/codex-rs/app-server/tests/suite/v2/thread_list.rs b/codex-rs/app-server/tests/suite/v2/thread_list.rs index 464fb4eee8..57299ef97e 100644 --- a/codex-rs/app-server/tests/suite/v2/thread_list.rs +++ b/codex-rs/app-server/tests/suite/v2/thread_list.rs @@ -2,10 +2,14 @@ use anyhow::Result; use app_test_support::McpProcess; use app_test_support::create_fake_rollout; use app_test_support::to_response; +use codex_app_server_protocol::GitInfo as ApiGitInfo; use codex_app_server_protocol::JSONRPCResponse; use codex_app_server_protocol::RequestId; +use codex_app_server_protocol::SessionSource; use codex_app_server_protocol::ThreadListParams; use codex_app_server_protocol::ThreadListResponse; +use codex_protocol::protocol::GitInfo as CoreGitInfo; +use std::path::PathBuf; use tempfile::TempDir; use tokio::time::timeout; @@ -24,7 +28,7 @@ async fn thread_list_basic_empty() -> Result<()> { .send_thread_list_request(ThreadListParams { cursor: None, limit: Some(10), - model_providers: None, + model_providers: Some(vec!["mock_provider".to_string()]), }) .await?; let list_resp: JSONRPCResponse = timeout( @@ -63,6 +67,7 @@ async fn thread_list_pagination_next_cursor_none_on_last_page() -> Result<()> { "2025-01-02T12:00:00Z", "Hello", Some("mock_provider"), + None, )?; let _b = create_fake_rollout( codex_home.path(), @@ -70,6 +75,7 @@ async fn thread_list_pagination_next_cursor_none_on_last_page() -> Result<()> { "2025-01-01T13:00:00Z", "Hello", Some("mock_provider"), + None, )?; let _c = create_fake_rollout( codex_home.path(), @@ -77,6 +83,7 @@ async fn thread_list_pagination_next_cursor_none_on_last_page() -> Result<()> { "2025-01-01T12:00:00Z", "Hello", Some("mock_provider"), + None, )?; let mut mcp = McpProcess::new(codex_home.path()).await?; @@ -104,6 +111,10 @@ async fn thread_list_pagination_next_cursor_none_on_last_page() -> Result<()> { assert_eq!(thread.preview, "Hello"); assert_eq!(thread.model_provider, "mock_provider"); assert!(thread.created_at > 0); + assert_eq!(thread.cwd, PathBuf::from("/")); + assert_eq!(thread.cli_version, "0.0.0"); + assert_eq!(thread.source, SessionSource::Cli); + assert_eq!(thread.git_info, None); } let cursor1 = cursor1.expect("expected nextCursor on first page"); @@ -129,6 +140,10 @@ async fn thread_list_pagination_next_cursor_none_on_last_page() -> Result<()> { assert_eq!(thread.preview, "Hello"); assert_eq!(thread.model_provider, "mock_provider"); assert!(thread.created_at > 0); + assert_eq!(thread.cwd, PathBuf::from("/")); + assert_eq!(thread.cli_version, "0.0.0"); + assert_eq!(thread.source, SessionSource::Cli); + assert_eq!(thread.git_info, None); } assert_eq!(cursor2, None, "expected nextCursor to be null on last page"); @@ -147,6 +162,7 @@ async fn thread_list_respects_provider_filter() -> Result<()> { "2025-01-02T10:00:00Z", "X", Some("mock_provider"), + None, )?; // mock_provider let _b = create_fake_rollout( codex_home.path(), @@ -154,6 +170,7 @@ async fn thread_list_respects_provider_filter() -> Result<()> { "2025-01-02T11:00:00Z", "X", Some("other_provider"), + None, )?; let mut mcp = McpProcess::new(codex_home.path()).await?; @@ -180,6 +197,63 @@ async fn thread_list_respects_provider_filter() -> Result<()> { assert_eq!(thread.model_provider, "other_provider"); let expected_ts = chrono::DateTime::parse_from_rfc3339("2025-01-02T11:00:00Z")?.timestamp(); assert_eq!(thread.created_at, expected_ts); + assert_eq!(thread.cwd, PathBuf::from("/")); + assert_eq!(thread.cli_version, "0.0.0"); + assert_eq!(thread.source, SessionSource::Cli); + assert_eq!(thread.git_info, None); + + Ok(()) +} + +#[tokio::test] +async fn thread_list_includes_git_info() -> Result<()> { + let codex_home = TempDir::new()?; + create_minimal_config(codex_home.path())?; + + let git_info = CoreGitInfo { + commit_hash: Some("abc123".to_string()), + branch: Some("main".to_string()), + repository_url: Some("https://example.com/repo.git".to_string()), + }; + let conversation_id = create_fake_rollout( + codex_home.path(), + "2025-02-01T09-00-00", + "2025-02-01T09:00:00Z", + "Git info preview", + Some("mock_provider"), + Some(git_info), + )?; + + let mut mcp = McpProcess::new(codex_home.path()).await?; + timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; + + let list_id = mcp + .send_thread_list_request(ThreadListParams { + cursor: None, + limit: Some(10), + model_providers: Some(vec!["mock_provider".to_string()]), + }) + .await?; + let resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(list_id)), + ) + .await??; + let ThreadListResponse { data, .. } = to_response::(resp)?; + let thread = data + .iter() + .find(|t| t.id == conversation_id) + .expect("expected thread for created rollout"); + + let expected_git = ApiGitInfo { + sha: Some("abc123".to_string()), + branch: Some("main".to_string()), + origin_url: Some("https://example.com/repo.git".to_string()), + }; + assert_eq!(thread.git_info, Some(expected_git)); + assert_eq!(thread.source, SessionSource::Cli); + assert_eq!(thread.cwd, PathBuf::from("/")); + assert_eq!(thread.cli_version, "0.0.0"); Ok(()) } diff --git a/codex-rs/app-server/tests/suite/v2/thread_resume.rs b/codex-rs/app-server/tests/suite/v2/thread_resume.rs index e22b711ea3..be8562e2fb 100644 --- a/codex-rs/app-server/tests/suite/v2/thread_resume.rs +++ b/codex-rs/app-server/tests/suite/v2/thread_resume.rs @@ -5,6 +5,7 @@ use app_test_support::create_mock_chat_completions_server; use app_test_support::to_response; use codex_app_server_protocol::JSONRPCResponse; use codex_app_server_protocol::RequestId; +use codex_app_server_protocol::SessionSource; use codex_app_server_protocol::ThreadItem; use codex_app_server_protocol::ThreadResumeParams; use codex_app_server_protocol::ThreadResumeResponse; @@ -14,6 +15,7 @@ use codex_app_server_protocol::TurnStatus; use codex_app_server_protocol::UserInput; use codex_protocol::models::ContentItem; use codex_protocol::models::ResponseItem; +use std::path::PathBuf; use tempfile::TempDir; use tokio::time::timeout; @@ -75,6 +77,7 @@ async fn thread_resume_returns_rollout_history() -> Result<()> { "2025-01-05T12:00:00Z", preview, Some("mock_provider"), + None, )?; let mut mcp = McpProcess::new(codex_home.path()).await?; @@ -97,6 +100,10 @@ async fn thread_resume_returns_rollout_history() -> Result<()> { assert_eq!(thread.preview, preview); assert_eq!(thread.model_provider, "mock_provider"); assert!(thread.path.is_absolute()); + assert_eq!(thread.cwd, PathBuf::from("/")); + assert_eq!(thread.cli_version, "0.0.0"); + assert_eq!(thread.source, SessionSource::Cli); + assert_eq!(thread.git_info, None); assert_eq!( thread.turns.len(), diff --git a/codex-rs/app-server/tests/suite/v2/turn_interrupt.rs b/codex-rs/app-server/tests/suite/v2/turn_interrupt.rs index beafec9cfc..f68ffb899c 100644 --- a/codex-rs/app-server/tests/suite/v2/turn_interrupt.rs +++ b/codex-rs/app-server/tests/suite/v2/turn_interrupt.rs @@ -3,7 +3,7 @@ use anyhow::Result; use app_test_support::McpProcess; use app_test_support::create_mock_chat_completions_server; -use app_test_support::create_shell_sse_response; +use app_test_support::create_shell_command_sse_response; use app_test_support::to_response; use codex_app_server_protocol::JSONRPCNotification; use codex_app_server_protocol::JSONRPCResponse; @@ -41,7 +41,7 @@ async fn turn_interrupt_aborts_running_turn() -> Result<()> { std::fs::create_dir(&working_directory)?; // Mock server: long-running shell command then (after abort) nothing else needed. - let server = create_mock_chat_completions_server(vec![create_shell_sse_response( + let server = create_mock_chat_completions_server(vec![create_shell_command_sse_response( shell_command.clone(), Some(&working_directory), Some(10_000), @@ -88,10 +88,11 @@ async fn turn_interrupt_aborts_running_turn() -> Result<()> { // Give the command a brief moment to start. tokio::time::sleep(std::time::Duration::from_secs(1)).await; + let thread_id = thread.id.clone(); // Interrupt the in-progress turn by id (v2 API). let interrupt_id = mcp .send_turn_interrupt_request(TurnInterruptParams { - thread_id: thread.id, + thread_id: thread_id.clone(), turn_id: turn.id, }) .await?; @@ -112,6 +113,7 @@ async fn turn_interrupt_aborts_running_turn() -> Result<()> { .params .expect("turn/completed params must be present"), )?; + assert_eq!(completed.thread_id, thread_id); assert_eq!(completed.turn.status, TurnStatus::Interrupted); Ok(()) diff --git a/codex-rs/app-server/tests/suite/v2/turn_start.rs b/codex-rs/app-server/tests/suite/v2/turn_start.rs index b644315d97..03ee279e51 100644 --- a/codex-rs/app-server/tests/suite/v2/turn_start.rs +++ b/codex-rs/app-server/tests/suite/v2/turn_start.rs @@ -1,13 +1,17 @@ use anyhow::Result; use app_test_support::McpProcess; use app_test_support::create_apply_patch_sse_response; +use app_test_support::create_exec_command_sse_response; use app_test_support::create_final_assistant_message_sse_response; use app_test_support::create_mock_chat_completions_server; use app_test_support::create_mock_chat_completions_server_unchecked; -use app_test_support::create_shell_sse_response; +use app_test_support::create_shell_command_sse_response; +use app_test_support::format_with_current_shell_display; use app_test_support::to_response; use codex_app_server_protocol::ApprovalDecision; +use codex_app_server_protocol::CommandExecutionRequestApprovalResponse; use codex_app_server_protocol::CommandExecutionStatus; +use codex_app_server_protocol::FileChangeOutputDeltaNotification; use codex_app_server_protocol::FileChangeRequestApprovalResponse; use codex_app_server_protocol::ItemCompletedNotification; use codex_app_server_protocol::ItemStartedNotification; @@ -93,6 +97,7 @@ async fn turn_start_emits_notifications_and_accepts_model_override() -> Result<( .await??; let started: TurnStartedNotification = serde_json::from_value(notif.params.expect("params must be present"))?; + assert_eq!(started.thread_id, thread.id); assert_eq!( started.turn.status, codex_app_server_protocol::TurnStatus::InProgress @@ -136,6 +141,7 @@ async fn turn_start_emits_notifications_and_accepts_model_override() -> Result<( .params .expect("turn/completed params must be present"), )?; + assert_eq!(completed.thread_id, thread.id); assert_eq!(completed.turn.status, TurnStatus::Completed); Ok(()) @@ -203,7 +209,7 @@ async fn turn_start_exec_approval_toggle_v2() -> Result<()> { // Mock server: first turn requests a shell call (elicitation), then completes. // Second turn same, but we'll set approval_policy=never to avoid elicitation. let responses = vec![ - create_shell_sse_response( + create_shell_command_sse_response( vec![ "python3".to_string(), "-c".to_string(), @@ -214,7 +220,7 @@ async fn turn_start_exec_approval_toggle_v2() -> Result<()> { "call1", )?, create_final_assistant_message_sse_response("done 1")?, - create_shell_sse_response( + create_shell_command_sse_response( vec![ "python3".to_string(), "-c".to_string(), @@ -328,6 +334,145 @@ async fn turn_start_exec_approval_toggle_v2() -> Result<()> { Ok(()) } +#[tokio::test] +async fn turn_start_exec_approval_decline_v2() -> Result<()> { + skip_if_no_network!(Ok(())); + + let tmp = TempDir::new()?; + let codex_home = tmp.path().to_path_buf(); + let workspace = tmp.path().join("workspace"); + std::fs::create_dir(&workspace)?; + + let responses = vec![ + create_shell_command_sse_response( + vec![ + "python3".to_string(), + "-c".to_string(), + "print(42)".to_string(), + ], + None, + Some(5000), + "call-decline", + )?, + create_final_assistant_message_sse_response("done")?, + ]; + let server = create_mock_chat_completions_server(responses).await; + create_config_toml(codex_home.as_path(), &server.uri(), "untrusted")?; + + let mut mcp = McpProcess::new(codex_home.as_path()).await?; + timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; + + let start_id = mcp + .send_thread_start_request(ThreadStartParams { + model: Some("mock-model".to_string()), + ..Default::default() + }) + .await?; + let start_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(start_id)), + ) + .await??; + let ThreadStartResponse { thread, .. } = to_response::(start_resp)?; + + let turn_id = mcp + .send_turn_start_request(TurnStartParams { + thread_id: thread.id.clone(), + input: vec![V2UserInput::Text { + text: "run python".to_string(), + }], + cwd: Some(workspace.clone()), + ..Default::default() + }) + .await?; + let turn_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(turn_id)), + ) + .await??; + let TurnStartResponse { turn } = to_response::(turn_resp)?; + + let started_command_execution = timeout(DEFAULT_READ_TIMEOUT, async { + loop { + let started_notif = mcp + .read_stream_until_notification_message("item/started") + .await?; + let started: ItemStartedNotification = + serde_json::from_value(started_notif.params.clone().expect("item/started params"))?; + if let ThreadItem::CommandExecution { .. } = started.item { + return Ok::(started.item); + } + } + }) + .await??; + let ThreadItem::CommandExecution { id, status, .. } = started_command_execution else { + unreachable!("loop ensures we break on command execution items"); + }; + assert_eq!(id, "call-decline"); + assert_eq!(status, CommandExecutionStatus::InProgress); + + let server_req = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_request_message(), + ) + .await??; + let ServerRequest::CommandExecutionRequestApproval { request_id, params } = server_req else { + panic!("expected CommandExecutionRequestApproval request") + }; + assert_eq!(params.item_id, "call-decline"); + assert_eq!(params.thread_id, thread.id); + assert_eq!(params.turn_id, turn.id); + + mcp.send_response( + request_id, + serde_json::to_value(CommandExecutionRequestApprovalResponse { + decision: ApprovalDecision::Decline, + accept_settings: None, + })?, + ) + .await?; + + let completed_command_execution = timeout(DEFAULT_READ_TIMEOUT, async { + loop { + let completed_notif = mcp + .read_stream_until_notification_message("item/completed") + .await?; + let completed: ItemCompletedNotification = serde_json::from_value( + completed_notif + .params + .clone() + .expect("item/completed params"), + )?; + if let ThreadItem::CommandExecution { .. } = completed.item { + return Ok::(completed.item); + } + } + }) + .await??; + let ThreadItem::CommandExecution { + id, + status, + exit_code, + aggregated_output, + .. + } = completed_command_execution + else { + unreachable!("loop ensures we break on command execution items"); + }; + assert_eq!(id, "call-decline"); + assert_eq!(status, CommandExecutionStatus::Declined); + assert!(exit_code.is_none()); + assert!(aggregated_output.is_none()); + + timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_notification_message("codex/event/task_complete"), + ) + .await??; + + Ok(()) +} + #[tokio::test] async fn turn_start_updates_sandbox_and_cwd_between_turns_v2() -> Result<()> { skip_if_no_network!(Ok(())); @@ -343,23 +488,15 @@ async fn turn_start_updates_sandbox_and_cwd_between_turns_v2() -> Result<()> { std::fs::create_dir(&second_cwd)?; let responses = vec![ - create_shell_sse_response( - vec![ - "bash".to_string(), - "-lc".to_string(), - "echo first turn".to_string(), - ], + create_shell_command_sse_response( + vec!["echo".to_string(), "first".to_string(), "turn".to_string()], None, Some(5000), "call-first", )?, create_final_assistant_message_sse_response("done first")?, - create_shell_sse_response( - vec![ - "bash".to_string(), - "-lc".to_string(), - "echo second turn".to_string(), - ], + create_shell_command_sse_response( + vec!["echo".to_string(), "second".to_string(), "turn".to_string()], None, Some(5000), "call-second", @@ -465,7 +602,8 @@ async fn turn_start_updates_sandbox_and_cwd_between_turns_v2() -> Result<()> { unreachable!("loop ensures we break on command execution items"); }; assert_eq!(cwd, second_cwd); - assert_eq!(command, "bash -lc 'echo second turn'"); + let expected_command = format_with_current_shell_display("echo second turn"); + assert_eq!(command, expected_command); assert_eq!(status, CommandExecutionStatus::InProgress); timeout( @@ -588,6 +726,26 @@ async fn turn_start_file_change_approval_v2() -> Result<()> { ) .await?; + let output_delta_notif = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_notification_message("item/fileChange/outputDelta"), + ) + .await??; + let output_delta: FileChangeOutputDeltaNotification = serde_json::from_value( + output_delta_notif + .params + .clone() + .expect("item/fileChange/outputDelta params"), + )?; + assert_eq!(output_delta.thread_id, thread.id); + assert_eq!(output_delta.turn_id, turn.id); + assert_eq!(output_delta.item_id, "patch-call"); + assert!( + !output_delta.delta.is_empty(), + "expected delta to be non-empty, got: {}", + output_delta.delta + ); + let completed_file_change = timeout(DEFAULT_READ_TIMEOUT, async { loop { let completed_notif = mcp @@ -771,6 +929,134 @@ async fn turn_start_file_change_approval_decline_v2() -> Result<()> { Ok(()) } +#[tokio::test] +#[cfg_attr(windows, ignore = "process id reporting differs on Windows")] +async fn command_execution_notifications_include_process_id() -> Result<()> { + skip_if_no_network!(Ok(())); + + let responses = vec![ + create_exec_command_sse_response("uexec-1")?, + create_final_assistant_message_sse_response("done")?, + ]; + let server = create_mock_chat_completions_server(responses).await; + let codex_home = TempDir::new()?; + create_config_toml(codex_home.path(), &server.uri(), "never")?; + let config_toml = codex_home.path().join("config.toml"); + let mut config_contents = std::fs::read_to_string(&config_toml)?; + config_contents.push_str( + r#" +[features] +unified_exec = true +"#, + ); + std::fs::write(&config_toml, config_contents)?; + + let mut mcp = McpProcess::new(codex_home.path()).await?; + timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; + + let start_id = mcp + .send_thread_start_request(ThreadStartParams { + model: Some("mock-model".to_string()), + ..Default::default() + }) + .await?; + let start_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(start_id)), + ) + .await??; + let ThreadStartResponse { thread, .. } = to_response::(start_resp)?; + + let turn_id = mcp + .send_turn_start_request(TurnStartParams { + thread_id: thread.id.clone(), + input: vec![V2UserInput::Text { + text: "run a command".to_string(), + }], + ..Default::default() + }) + .await?; + let turn_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(turn_id)), + ) + .await??; + let TurnStartResponse { turn: _turn } = to_response::(turn_resp)?; + + let started_command = timeout(DEFAULT_READ_TIMEOUT, async { + loop { + let notif = mcp + .read_stream_until_notification_message("item/started") + .await?; + let started: ItemStartedNotification = serde_json::from_value( + notif + .params + .clone() + .expect("item/started should include params"), + )?; + if let ThreadItem::CommandExecution { .. } = started.item { + return Ok::(started.item); + } + } + }) + .await??; + let ThreadItem::CommandExecution { + id, + process_id: started_process_id, + status, + .. + } = started_command + else { + unreachable!("loop ensures we break on command execution items"); + }; + assert_eq!(id, "uexec-1"); + assert_eq!(status, CommandExecutionStatus::InProgress); + let started_process_id = started_process_id.expect("process id should be present"); + + let completed_command = timeout(DEFAULT_READ_TIMEOUT, async { + loop { + let notif = mcp + .read_stream_until_notification_message("item/completed") + .await?; + let completed: ItemCompletedNotification = serde_json::from_value( + notif + .params + .clone() + .expect("item/completed should include params"), + )?; + if let ThreadItem::CommandExecution { .. } = completed.item { + return Ok::(completed.item); + } + } + }) + .await??; + let ThreadItem::CommandExecution { + id: completed_id, + process_id: completed_process_id, + status: completed_status, + exit_code, + .. + } = completed_command + else { + unreachable!("loop ensures we break on command execution items"); + }; + assert_eq!(completed_id, "uexec-1"); + assert_eq!(completed_status, CommandExecutionStatus::Completed); + assert_eq!(exit_code, Some(0)); + assert_eq!( + completed_process_id.as_deref(), + Some(started_process_id.as_str()) + ); + + timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_notification_message("turn/completed"), + ) + .await??; + + Ok(()) +} + // Helper to create a config.toml pointing at the mock model server. fn create_config_toml( codex_home: &Path, diff --git a/codex-rs/apply-patch/Cargo.toml b/codex-rs/apply-patch/Cargo.toml index a239cd6310..1a918ce93b 100644 --- a/codex-rs/apply-patch/Cargo.toml +++ b/codex-rs/apply-patch/Cargo.toml @@ -1,7 +1,8 @@ [package] -edition = "2024" name = "codex-apply-patch" -version = { workspace = true } +version.workspace = true +edition.workspace = true +license.workspace = true [lib] name = "codex_apply_patch" diff --git a/codex-rs/apply-patch/src/lib.rs b/codex-rs/apply-patch/src/lib.rs index ac2f409791..867d19a2e8 100644 --- a/codex-rs/apply-patch/src/lib.rs +++ b/codex-rs/apply-patch/src/lib.rs @@ -31,6 +31,13 @@ pub const APPLY_PATCH_TOOL_INSTRUCTIONS: &str = include_str!("../apply_patch_too const APPLY_PATCH_COMMANDS: [&str; 2] = ["apply_patch", "applypatch"]; +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum ApplyPatchShell { + Unix, + PowerShell, + Cmd, +} + #[derive(Debug, Error, PartialEq)] pub enum ApplyPatchError { #[error(transparent)] @@ -96,6 +103,57 @@ pub struct ApplyPatchArgs { pub workdir: Option, } +fn classify_shell_name(shell: &str) -> Option { + std::path::Path::new(shell) + .file_stem() + .and_then(|name| name.to_str()) + .map(str::to_ascii_lowercase) +} + +fn classify_shell(shell: &str, flag: &str) -> Option { + classify_shell_name(shell).and_then(|name| match name.as_str() { + "bash" | "zsh" | "sh" if flag == "-lc" => Some(ApplyPatchShell::Unix), + "pwsh" | "powershell" if flag.eq_ignore_ascii_case("-command") => { + Some(ApplyPatchShell::PowerShell) + } + "cmd" if flag.eq_ignore_ascii_case("/c") => Some(ApplyPatchShell::Cmd), + _ => None, + }) +} + +fn can_skip_flag(shell: &str, flag: &str) -> bool { + classify_shell_name(shell).is_some_and(|name| { + matches!(name.as_str(), "pwsh" | "powershell") && flag.eq_ignore_ascii_case("-noprofile") + }) +} + +fn parse_shell_script(argv: &[String]) -> Option<(ApplyPatchShell, &str)> { + match argv { + [shell, flag, script] => classify_shell(shell, flag).map(|shell_type| { + let script = script.as_str(); + (shell_type, script) + }), + [shell, skip_flag, flag, script] if can_skip_flag(shell, skip_flag) => { + classify_shell(shell, flag).map(|shell_type| { + let script = script.as_str(); + (shell_type, script) + }) + } + _ => None, + } +} + +fn extract_apply_patch_from_shell( + shell: ApplyPatchShell, + script: &str, +) -> std::result::Result<(String, Option), ExtractHeredocError> { + match shell { + ApplyPatchShell::Unix | ApplyPatchShell::PowerShell | ApplyPatchShell::Cmd => { + extract_apply_patch_from_bash(script) + } + } +} + pub fn maybe_parse_apply_patch(argv: &[String]) -> MaybeApplyPatch { match argv { // Direct invocation: apply_patch @@ -103,9 +161,9 @@ pub fn maybe_parse_apply_patch(argv: &[String]) -> MaybeApplyPatch { Ok(source) => MaybeApplyPatch::Body(source), Err(e) => MaybeApplyPatch::PatchParseError(e), }, - // Bash heredoc form: (optional `cd &&`) apply_patch <<'EOF' ... - [bash, flag, script] if bash == "bash" && flag == "-lc" => { - match extract_apply_patch_from_bash(script) { + // Shell heredoc form: (optional `cd &&`) apply_patch <<'EOF' ... + _ => match parse_shell_script(argv) { + Some((shell, script)) => match extract_apply_patch_from_shell(shell, script) { Ok((body, workdir)) => match parse_patch(&body) { Ok(mut source) => { source.workdir = workdir; @@ -117,9 +175,9 @@ pub fn maybe_parse_apply_patch(argv: &[String]) -> MaybeApplyPatch { MaybeApplyPatch::NotApplyPatch } Err(e) => MaybeApplyPatch::ShellParseError(e), - } - } - _ => MaybeApplyPatch::NotApplyPatch, + }, + None => MaybeApplyPatch::NotApplyPatch, + }, } } @@ -214,24 +272,17 @@ impl ApplyPatchAction { /// cwd must be an absolute path so that we can resolve relative paths in the /// patch. pub fn maybe_parse_apply_patch_verified(argv: &[String], cwd: &Path) -> MaybeApplyPatchVerified { - // Detect a raw patch body passed directly as the command or as the body of a bash -lc + // Detect a raw patch body passed directly as the command or as the body of a shell // script. In these cases, report an explicit error rather than applying the patch. - match argv { - [body] => { - if parse_patch(body).is_ok() { - return MaybeApplyPatchVerified::CorrectnessError( - ApplyPatchError::ImplicitInvocation, - ); - } - } - [bash, flag, script] if bash == "bash" && flag == "-lc" => { - if parse_patch(script).is_ok() { - return MaybeApplyPatchVerified::CorrectnessError( - ApplyPatchError::ImplicitInvocation, - ); - } - } - _ => {} + if let [body] = argv + && parse_patch(body).is_ok() + { + return MaybeApplyPatchVerified::CorrectnessError(ApplyPatchError::ImplicitInvocation); + } + if let Some((_, script)) = parse_shell_script(argv) + && parse_patch(script).is_ok() + { + return MaybeApplyPatchVerified::CorrectnessError(ApplyPatchError::ImplicitInvocation); } match maybe_parse_apply_patch(argv) { @@ -863,6 +914,22 @@ mod tests { strs_to_strings(&["bash", "-lc", script]) } + fn args_powershell(script: &str) -> Vec { + strs_to_strings(&["powershell.exe", "-Command", script]) + } + + fn args_powershell_no_profile(script: &str) -> Vec { + strs_to_strings(&["powershell.exe", "-NoProfile", "-Command", script]) + } + + fn args_pwsh(script: &str) -> Vec { + strs_to_strings(&["pwsh", "-NoProfile", "-Command", script]) + } + + fn args_cmd(script: &str) -> Vec { + strs_to_strings(&["cmd.exe", "/c", script]) + } + fn heredoc_script(prefix: &str) -> String { format!( "{prefix}apply_patch <<'PATCH'\n*** Begin Patch\n*** Add File: foo\n+hi\n*** End Patch\nPATCH" @@ -882,8 +949,7 @@ mod tests { }] } - fn assert_match(script: &str, expected_workdir: Option<&str>) { - let args = args_bash(script); + fn assert_match_args(args: Vec, expected_workdir: Option<&str>) { match maybe_parse_apply_patch(&args) { MaybeApplyPatch::Body(ApplyPatchArgs { hunks, workdir, .. }) => { assert_eq!(workdir.as_deref(), expected_workdir); @@ -893,6 +959,11 @@ mod tests { } } + fn assert_match(script: &str, expected_workdir: Option<&str>) { + let args = args_bash(script); + assert_match_args(args, expected_workdir); + } + fn assert_not_match(script: &str) { let args = args_bash(script); assert_matches!( @@ -1006,6 +1077,28 @@ PATCH"#, } } + #[test] + fn test_powershell_heredoc() { + let script = heredoc_script(""); + assert_match_args(args_powershell(&script), None); + } + #[test] + fn test_powershell_heredoc_no_profile() { + let script = heredoc_script(""); + assert_match_args(args_powershell_no_profile(&script), None); + } + #[test] + fn test_pwsh_heredoc() { + let script = heredoc_script(""); + assert_match_args(args_pwsh(&script), None); + } + + #[test] + fn test_cmd_heredoc_with_cd() { + let script = heredoc_script("cd foo && "); + assert_match_args(args_cmd(&script), Some("foo")); + } + #[test] fn test_heredoc_with_leading_cd() { assert_match(&heredoc_script("cd foo && "), Some("foo")); diff --git a/codex-rs/arg0/Cargo.toml b/codex-rs/arg0/Cargo.toml index 10d09e4a4b..c82bdd58da 100644 --- a/codex-rs/arg0/Cargo.toml +++ b/codex-rs/arg0/Cargo.toml @@ -1,7 +1,8 @@ [package] -edition = "2024" name = "codex-arg0" -version = { workspace = true } +version.workspace = true +edition.workspace = true +license.workspace = true [lib] name = "codex_arg0" diff --git a/codex-rs/async-utils/Cargo.toml b/codex-rs/async-utils/Cargo.toml index 5203db0f5d..891af17a5f 100644 --- a/codex-rs/async-utils/Cargo.toml +++ b/codex-rs/async-utils/Cargo.toml @@ -1,7 +1,8 @@ [package] -edition.workspace = true name = "codex-async-utils" version.workspace = true +edition.workspace = true +license.workspace = true [lints] workspace = true diff --git a/codex-rs/backend-client/Cargo.toml b/codex-rs/backend-client/Cargo.toml index 0cf802399c..ec5546a670 100644 --- a/codex-rs/backend-client/Cargo.toml +++ b/codex-rs/backend-client/Cargo.toml @@ -1,7 +1,8 @@ [package] name = "codex-backend-client" -version = "0.0.0" -edition = "2024" +version.workspace = true +edition.workspace = true +license.workspace = true publish = false [lib] diff --git a/codex-rs/chatgpt/Cargo.toml b/codex-rs/chatgpt/Cargo.toml index c46046b1f6..b58cd62340 100644 --- a/codex-rs/chatgpt/Cargo.toml +++ b/codex-rs/chatgpt/Cargo.toml @@ -1,7 +1,8 @@ [package] -edition = "2024" name = "codex-chatgpt" -version = { workspace = true } +version.workspace = true +edition.workspace = true +license.workspace = true [lints] workspace = true diff --git a/codex-rs/cli/Cargo.toml b/codex-rs/cli/Cargo.toml index deddc068cd..6c80a12595 100644 --- a/codex-rs/cli/Cargo.toml +++ b/codex-rs/cli/Cargo.toml @@ -1,7 +1,8 @@ [package] -edition = "2024" name = "codex-cli" -version = { workspace = true } +version.workspace = true +edition.workspace = true +license.workspace = true [[bin]] name = "codex" @@ -26,6 +27,7 @@ codex-cloud-tasks = { path = "../cloud-tasks" } codex-common = { workspace = true, features = ["cli"] } codex-core = { workspace = true } codex-exec = { workspace = true } +codex-execpolicy = { workspace = true } codex-login = { workspace = true } codex-mcp-server = { workspace = true } codex-process-hardening = { workspace = true } diff --git a/codex-rs/cli/src/main.rs b/codex-rs/cli/src/main.rs index f0c4b2e34c..581a902067 100644 --- a/codex-rs/cli/src/main.rs +++ b/codex-rs/cli/src/main.rs @@ -18,6 +18,9 @@ use codex_cli::login::run_logout; use codex_cloud_tasks::Cli as CloudTasksCli; use codex_common::CliConfigOverrides; use codex_exec::Cli as ExecCli; +use codex_exec::Command as ExecCommand; +use codex_exec::ReviewArgs; +use codex_execpolicy::ExecPolicyCheckCommand; use codex_responses_api_proxy::Args as ResponsesApiProxyArgs; use codex_tui::AppExitInfo; use codex_tui::Cli as TuiCli; @@ -71,6 +74,9 @@ enum Subcommand { #[clap(visible_alias = "e")] Exec(ExecCli), + /// Run a code review non-interactively. + Review(ReviewArgs), + /// Manage login. Login(LoginCommand), @@ -93,6 +99,10 @@ enum Subcommand { #[clap(visible_alias = "debug")] Sandbox(SandboxArgs), + /// Execpolicy tooling. + #[clap(hide = true)] + Execpolicy(ExecpolicyCommand), + /// Apply the latest diff produced by Codex agent as a `git apply` to your local working tree. #[clap(visible_alias = "a")] Apply(ApplyCommand), @@ -175,6 +185,19 @@ enum SandboxCommand { Windows(WindowsCommand), } +#[derive(Debug, Parser)] +struct ExecpolicyCommand { + #[command(subcommand)] + sub: ExecpolicySubcommand, +} + +#[derive(Debug, clap::Subcommand)] +enum ExecpolicySubcommand { + /// Check execpolicy files against a command. + #[clap(name = "check")] + Check(ExecPolicyCheckCommand), +} + #[derive(Debug, Parser)] struct LoginCommand { #[clap(skip)] @@ -340,6 +363,10 @@ fn run_update_action(action: UpdateAction) -> anyhow::Result<()> { Ok(()) } +fn run_execpolicycheck(cmd: ExecPolicyCheckCommand) -> anyhow::Result<()> { + cmd.run() +} + #[derive(Debug, Default, Parser, Clone)] struct FeatureToggles { /// Enable a feature (repeatable). Equivalent to `-c features.=true`. @@ -440,6 +467,15 @@ async fn cli_main(codex_linux_sandbox_exe: Option) -> anyhow::Result<() ); codex_exec::run_main(exec_cli, codex_linux_sandbox_exe).await?; } + Some(Subcommand::Review(review_args)) => { + let mut exec_cli = ExecCli::try_parse_from(["codex", "exec"])?; + exec_cli.command = Some(ExecCommand::Review(review_args)); + prepend_config_flags( + &mut exec_cli.config_overrides, + root_config_overrides.clone(), + ); + codex_exec::run_main(exec_cli, codex_linux_sandbox_exe).await?; + } Some(Subcommand::McpServer) => { codex_mcp_server::run_main(codex_linux_sandbox_exe, root_config_overrides).await?; } @@ -575,6 +611,9 @@ async fn cli_main(codex_linux_sandbox_exe: Option) -> anyhow::Result<() .await?; } }, + Some(Subcommand::Execpolicy(ExecpolicyCommand { sub })) => match sub { + ExecpolicySubcommand::Check(cmd) => run_execpolicycheck(cmd)?, + }, Some(Subcommand::Apply(mut apply_cli)) => { prepend_config_flags( &mut apply_cli.config_overrides, diff --git a/codex-rs/cli/src/mcp_cmd.rs b/codex-rs/cli/src/mcp_cmd.rs index ec37c3a6bd..93f22e705c 100644 --- a/codex-rs/cli/src/mcp_cmd.rs +++ b/codex-rs/cli/src/mcp_cmd.rs @@ -79,6 +79,7 @@ pub struct GetArgs { } #[derive(Debug, clap::Parser)] +#[command(override_usage = "codex mcp add [OPTIONS] (--url | -- ...)")] pub struct AddArgs { /// Name for the MCP server configuration. pub name: String, diff --git a/codex-rs/cli/tests/execpolicy.rs b/codex-rs/cli/tests/execpolicy.rs new file mode 100644 index 0000000000..c6bca85bc6 --- /dev/null +++ b/codex-rs/cli/tests/execpolicy.rs @@ -0,0 +1,58 @@ +use std::fs; + +use assert_cmd::Command; +use pretty_assertions::assert_eq; +use serde_json::json; +use tempfile::TempDir; + +#[test] +fn execpolicy_check_matches_expected_json() -> Result<(), Box> { + let codex_home = TempDir::new()?; + let policy_path = codex_home.path().join("policy.codexpolicy"); + fs::write( + &policy_path, + r#" +prefix_rule( + pattern = ["git", "push"], + decision = "forbidden", +) +"#, + )?; + + let output = Command::cargo_bin("codex")? + .env("CODEX_HOME", codex_home.path()) + .args([ + "execpolicy", + "check", + "--policy", + policy_path + .to_str() + .expect("policy path should be valid UTF-8"), + "git", + "push", + "origin", + "main", + ]) + .output()?; + + assert!(output.status.success()); + let result: serde_json::Value = serde_json::from_slice(&output.stdout)?; + assert_eq!( + result, + json!({ + "match": { + "decision": "forbidden", + "matchedRules": [ + { + "prefixRuleMatch": { + "matchedPrefix": ["git", "push"], + "decision": "forbidden" + } + } + ] + } + }) + ); + + Ok(()) +} diff --git a/codex-rs/cloud-tasks-client/Cargo.toml b/codex-rs/cloud-tasks-client/Cargo.toml index 1a4eaa7aa8..15a206079a 100644 --- a/codex-rs/cloud-tasks-client/Cargo.toml +++ b/codex-rs/cloud-tasks-client/Cargo.toml @@ -1,7 +1,8 @@ [package] name = "codex-cloud-tasks-client" -version = { workspace = true } -edition = "2024" +version.workspace = true +edition.workspace = true +license.workspace = true [lib] name = "codex_cloud_tasks_client" diff --git a/codex-rs/cloud-tasks/Cargo.toml b/codex-rs/cloud-tasks/Cargo.toml index 46044fbb8d..c9edf5b4ad 100644 --- a/codex-rs/cloud-tasks/Cargo.toml +++ b/codex-rs/cloud-tasks/Cargo.toml @@ -1,7 +1,8 @@ [package] -edition = "2024" name = "codex-cloud-tasks" -version = { workspace = true } +version.workspace = true +edition.workspace = true +license.workspace = true [lib] name = "codex_cloud_tasks" diff --git a/codex-rs/codex-api/Cargo.toml b/codex-rs/codex-api/Cargo.toml new file mode 100644 index 0000000000..f79416c96e --- /dev/null +++ b/codex-rs/codex-api/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "codex-api" +version.workspace = true +edition.workspace = true +license.workspace = true + +[dependencies] +async-trait = { workspace = true } +bytes = { workspace = true } +codex-client = { workspace = true } +codex-protocol = { workspace = true } +futures = { workspace = true } +http = { workspace = true } +serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true } +thiserror = { workspace = true } +tokio = { workspace = true, features = ["macros", "rt", "sync", "time"] } +tracing = { workspace = true } +eventsource-stream = { workspace = true } +regex-lite = { workspace = true } +tokio-util = { workspace = true, features = ["codec"] } + +[dev-dependencies] +anyhow = { workspace = true } +assert_matches = { workspace = true } +pretty_assertions = { workspace = true } +tokio-test = { workspace = true } + +[lints] +workspace = true diff --git a/codex-rs/codex-api/README.md b/codex-rs/codex-api/README.md new file mode 100644 index 0000000000..98db0bec65 --- /dev/null +++ b/codex-rs/codex-api/README.md @@ -0,0 +1,32 @@ +# codex-api + +Typed clients for Codex/OpenAI APIs built on top of the generic transport in `codex-client`. + +- Hosts the request/response models and prompt helpers for Responses, Chat Completions, and Compact APIs. +- Owns provider configuration (base URLs, headers, query params), auth header injection, retry tuning, and stream idle settings. +- Parses SSE streams into `ResponseEvent`/`ResponseStream`, including rate-limit snapshots and API-specific error mapping. +- Serves as the wire-level layer consumed by `codex-core`; higher layers handle auth refresh and business logic. + +## Core interface + +The public interface of this crate is intentionally small and uniform: + +- **Prompted endpoints (Chat + Responses)** + - Input: a single `Prompt` plus endpoint-specific options. + - `Prompt` (re-exported as `codex_api::Prompt`) carries: + - `instructions: String` – the fully-resolved system prompt for this turn. + - `input: Vec` – conversation history and user/tool messages. + - `tools: Vec` – JSON tools compatible with the target API. + - `parallel_tool_calls: bool`. + - `output_schema: Option` – used to build `text.format` when present. + - Output: a `ResponseStream` of `ResponseEvent` (both re-exported from `common`). + +- **Compaction endpoint** + - Input: `CompactionInput<'a>` (re-exported as `codex_api::CompactionInput`): + - `model: &str`. + - `input: &[ResponseItem]` – history to compact. + - `instructions: &str` – fully-resolved compaction instructions. + - Output: `Vec`. + - `CompactClient::compact_input(&CompactionInput, extra_headers)` wraps the JSON encoding and retry/telemetry wiring. + +All HTTP details (URLs, headers, retry/backoff policies, SSE framing) are encapsulated in `codex-api` and `codex-client`. Callers construct prompts/inputs using protocol types and work with typed streams of `ResponseEvent` or compacted `ResponseItem` values. diff --git a/codex-rs/codex-api/src/auth.rs b/codex-rs/codex-api/src/auth.rs new file mode 100644 index 0000000000..6c26963cba --- /dev/null +++ b/codex-rs/codex-api/src/auth.rs @@ -0,0 +1,27 @@ +use codex_client::Request; + +/// Provides bearer and account identity information for API requests. +/// +/// Implementations should be cheap and non-blocking; any asynchronous +/// refresh or I/O should be handled by higher layers before requests +/// reach this interface. +pub trait AuthProvider: Send + Sync { + fn bearer_token(&self) -> Option; + fn account_id(&self) -> Option { + None + } +} + +pub(crate) fn add_auth_headers(auth: &A, mut req: Request) -> Request { + if let Some(token) = auth.bearer_token() + && let Ok(header) = format!("Bearer {token}").parse() + { + let _ = req.headers.insert(http::header::AUTHORIZATION, header); + } + if let Some(account_id) = auth.account_id() + && let Ok(header) = account_id.parse() + { + let _ = req.headers.insert("ChatGPT-Account-ID", header); + } + req +} diff --git a/codex-rs/codex-api/src/common.rs b/codex-rs/codex-api/src/common.rs new file mode 100644 index 0000000000..addab02dc7 --- /dev/null +++ b/codex-rs/codex-api/src/common.rs @@ -0,0 +1,167 @@ +use crate::error::ApiError; +use codex_protocol::config_types::ReasoningEffort as ReasoningEffortConfig; +use codex_protocol::config_types::ReasoningSummary as ReasoningSummaryConfig; +use codex_protocol::config_types::Verbosity as VerbosityConfig; +use codex_protocol::models::ResponseItem; +use codex_protocol::protocol::RateLimitSnapshot; +use codex_protocol::protocol::TokenUsage; +use futures::Stream; +use serde::Serialize; +use serde_json::Value; +use std::pin::Pin; +use std::task::Context; +use std::task::Poll; +use tokio::sync::mpsc; + +/// Canonical prompt input for Chat and Responses endpoints. +#[derive(Debug, Clone)] +pub struct Prompt { + /// Fully-resolved system instructions for this turn. + pub instructions: String, + /// Conversation history and user/tool messages. + pub input: Vec, + /// JSON-encoded tool definitions compatible with the target API. + // TODO(jif) have a proper type here + pub tools: Vec, + /// Whether parallel tool calls are permitted. + pub parallel_tool_calls: bool, + /// Optional output schema used to build the `text.format` controls. + pub output_schema: Option, +} + +/// Canonical input payload for the compaction endpoint. +#[derive(Debug, Clone, Serialize)] +pub struct CompactionInput<'a> { + pub model: &'a str, + pub input: &'a [ResponseItem], + pub instructions: &'a str, +} + +#[derive(Debug)] +pub enum ResponseEvent { + Created, + OutputItemDone(ResponseItem), + OutputItemAdded(ResponseItem), + Completed { + response_id: String, + token_usage: Option, + }, + OutputTextDelta(String), + ReasoningSummaryDelta { + delta: String, + summary_index: i64, + }, + ReasoningContentDelta { + delta: String, + content_index: i64, + }, + ReasoningSummaryPartAdded { + summary_index: i64, + }, + RateLimits(RateLimitSnapshot), +} + +#[derive(Debug, Serialize, Clone)] +pub struct Reasoning { + #[serde(skip_serializing_if = "Option::is_none")] + pub effort: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub summary: Option, +} + +#[derive(Debug, Serialize, Default, Clone)] +#[serde(rename_all = "snake_case")] +pub enum TextFormatType { + #[default] + JsonSchema, +} + +#[derive(Debug, Serialize, Default, Clone)] +pub struct TextFormat { + /// Format type used by the OpenAI text controls. + pub r#type: TextFormatType, + /// When true, the server is expected to strictly validate responses. + pub strict: bool, + /// JSON schema for the desired output. + pub schema: Value, + /// Friendly name for the format, used in telemetry/debugging. + pub name: String, +} + +/// Controls the `text` field for the Responses API, combining verbosity and +/// optional JSON schema output formatting. +#[derive(Debug, Serialize, Default, Clone)] +pub struct TextControls { + #[serde(skip_serializing_if = "Option::is_none")] + pub verbosity: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub format: Option, +} + +#[derive(Debug, Serialize, Default, Clone)] +#[serde(rename_all = "lowercase")] +pub enum OpenAiVerbosity { + Low, + #[default] + Medium, + High, +} + +impl From for OpenAiVerbosity { + fn from(v: VerbosityConfig) -> Self { + match v { + VerbosityConfig::Low => OpenAiVerbosity::Low, + VerbosityConfig::Medium => OpenAiVerbosity::Medium, + VerbosityConfig::High => OpenAiVerbosity::High, + } + } +} + +#[derive(Debug, Serialize)] +pub struct ResponsesApiRequest<'a> { + pub model: &'a str, + pub instructions: &'a str, + pub input: &'a [ResponseItem], + pub tools: &'a [serde_json::Value], + pub tool_choice: &'static str, + pub parallel_tool_calls: bool, + pub reasoning: Option, + pub store: bool, + pub stream: bool, + pub include: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub prompt_cache_key: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub text: Option, +} + +pub fn create_text_param_for_request( + verbosity: Option, + output_schema: &Option, +) -> Option { + if verbosity.is_none() && output_schema.is_none() { + return None; + } + + Some(TextControls { + verbosity: verbosity.map(std::convert::Into::into), + format: output_schema.as_ref().map(|schema| TextFormat { + r#type: TextFormatType::JsonSchema, + strict: true, + schema: schema.clone(), + name: "codex_output_schema".to_string(), + }), + }) +} + +pub struct ResponseStream { + pub rx_event: mpsc::Receiver>, +} + +impl Stream for ResponseStream { + type Item = Result; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + self.rx_event.poll_recv(cx) + } +} diff --git a/codex-rs/codex-api/src/endpoint/chat.rs b/codex-rs/codex-api/src/endpoint/chat.rs new file mode 100644 index 0000000000..4ad133dda4 --- /dev/null +++ b/codex-rs/codex-api/src/endpoint/chat.rs @@ -0,0 +1,266 @@ +use crate::ChatRequest; +use crate::auth::AuthProvider; +use crate::common::Prompt as ApiPrompt; +use crate::common::ResponseEvent; +use crate::common::ResponseStream; +use crate::endpoint::streaming::StreamingClient; +use crate::error::ApiError; +use crate::provider::Provider; +use crate::provider::WireApi; +use crate::sse::chat::spawn_chat_stream; +use crate::telemetry::SseTelemetry; +use codex_client::HttpTransport; +use codex_client::RequestTelemetry; +use codex_protocol::models::ContentItem; +use codex_protocol::models::ReasoningItemContent; +use codex_protocol::models::ResponseItem; +use codex_protocol::protocol::SessionSource; +use futures::Stream; +use http::HeaderMap; +use serde_json::Value; +use std::collections::VecDeque; +use std::pin::Pin; +use std::sync::Arc; +use std::task::Context; +use std::task::Poll; + +pub struct ChatClient { + streaming: StreamingClient, +} + +impl ChatClient { + pub fn new(transport: T, provider: Provider, auth: A) -> Self { + Self { + streaming: StreamingClient::new(transport, provider, auth), + } + } + + pub fn with_telemetry( + self, + request: Option>, + sse: Option>, + ) -> Self { + Self { + streaming: self.streaming.with_telemetry(request, sse), + } + } + + pub async fn stream_request(&self, request: ChatRequest) -> Result { + self.stream(request.body, request.headers).await + } + + pub async fn stream_prompt( + &self, + model: &str, + prompt: &ApiPrompt, + conversation_id: Option, + session_source: Option, + ) -> Result { + use crate::requests::ChatRequestBuilder; + + let request = + ChatRequestBuilder::new(model, &prompt.instructions, &prompt.input, &prompt.tools) + .conversation_id(conversation_id) + .session_source(session_source) + .build(self.streaming.provider())?; + + self.stream_request(request).await + } + + fn path(&self) -> &'static str { + match self.streaming.provider().wire { + WireApi::Chat => "chat/completions", + _ => "responses", + } + } + + pub async fn stream( + &self, + body: Value, + extra_headers: HeaderMap, + ) -> Result { + self.streaming + .stream(self.path(), body, extra_headers, spawn_chat_stream) + .await + } +} + +#[derive(Copy, Clone, Eq, PartialEq)] +pub enum AggregateMode { + AggregatedOnly, + Streaming, +} + +/// Stream adapter that merges token deltas into a single assistant message per turn. +pub struct AggregatedStream { + inner: ResponseStream, + cumulative: String, + cumulative_reasoning: String, + pending: VecDeque, + mode: AggregateMode, +} + +impl Stream for AggregatedStream { + type Item = Result; + + fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let this = self.get_mut(); + + if let Some(ev) = this.pending.pop_front() { + return Poll::Ready(Some(Ok(ev))); + } + + loop { + match Pin::new(&mut this.inner).poll_next(cx) { + Poll::Pending => return Poll::Pending, + Poll::Ready(None) => return Poll::Ready(None), + Poll::Ready(Some(Err(e))) => return Poll::Ready(Some(Err(e))), + Poll::Ready(Some(Ok(ResponseEvent::OutputItemDone(item)))) => { + let is_assistant_message = matches!( + &item, + ResponseItem::Message { role, .. } if role == "assistant" + ); + + if is_assistant_message { + match this.mode { + AggregateMode::AggregatedOnly => { + if this.cumulative.is_empty() + && let ResponseItem::Message { content, .. } = &item + && let Some(text) = content.iter().find_map(|c| match c { + ContentItem::OutputText { text } => Some(text), + _ => None, + }) + { + this.cumulative.push_str(text); + } + continue; + } + AggregateMode::Streaming => { + if this.cumulative.is_empty() { + return Poll::Ready(Some(Ok(ResponseEvent::OutputItemDone( + item, + )))); + } else { + continue; + } + } + } + } + + return Poll::Ready(Some(Ok(ResponseEvent::OutputItemDone(item)))); + } + Poll::Ready(Some(Ok(ResponseEvent::RateLimits(snapshot)))) => { + return Poll::Ready(Some(Ok(ResponseEvent::RateLimits(snapshot)))); + } + Poll::Ready(Some(Ok(ResponseEvent::Completed { + response_id, + token_usage, + }))) => { + let mut emitted_any = false; + + if !this.cumulative_reasoning.is_empty() { + let aggregated_reasoning = ResponseItem::Reasoning { + id: String::new(), + summary: Vec::new(), + content: Some(vec![ReasoningItemContent::ReasoningText { + text: std::mem::take(&mut this.cumulative_reasoning), + }]), + encrypted_content: None, + }; + this.pending + .push_back(ResponseEvent::OutputItemDone(aggregated_reasoning)); + emitted_any = true; + } + + if !this.cumulative.is_empty() { + let aggregated_message = ResponseItem::Message { + id: None, + role: "assistant".to_string(), + content: vec![ContentItem::OutputText { + text: std::mem::take(&mut this.cumulative), + }], + }; + this.pending + .push_back(ResponseEvent::OutputItemDone(aggregated_message)); + emitted_any = true; + } + + if emitted_any { + this.pending.push_back(ResponseEvent::Completed { + response_id: response_id.clone(), + token_usage: token_usage.clone(), + }); + if let Some(ev) = this.pending.pop_front() { + return Poll::Ready(Some(Ok(ev))); + } + } + + return Poll::Ready(Some(Ok(ResponseEvent::Completed { + response_id, + token_usage, + }))); + } + Poll::Ready(Some(Ok(ResponseEvent::Created))) => { + continue; + } + Poll::Ready(Some(Ok(ResponseEvent::OutputTextDelta(delta)))) => { + this.cumulative.push_str(&delta); + if matches!(this.mode, AggregateMode::Streaming) { + return Poll::Ready(Some(Ok(ResponseEvent::OutputTextDelta(delta)))); + } else { + continue; + } + } + Poll::Ready(Some(Ok(ResponseEvent::ReasoningContentDelta { + delta, + content_index, + }))) => { + this.cumulative_reasoning.push_str(&delta); + if matches!(this.mode, AggregateMode::Streaming) { + return Poll::Ready(Some(Ok(ResponseEvent::ReasoningContentDelta { + delta, + content_index, + }))); + } else { + continue; + } + } + Poll::Ready(Some(Ok(ResponseEvent::ReasoningSummaryDelta { .. }))) => continue, + Poll::Ready(Some(Ok(ResponseEvent::ReasoningSummaryPartAdded { .. }))) => { + continue; + } + Poll::Ready(Some(Ok(ResponseEvent::OutputItemAdded(item)))) => { + return Poll::Ready(Some(Ok(ResponseEvent::OutputItemAdded(item)))); + } + } + } + } +} + +pub trait AggregateStreamExt { + fn aggregate(self) -> AggregatedStream; + + fn streaming_mode(self) -> ResponseStream; +} + +impl AggregateStreamExt for ResponseStream { + fn aggregate(self) -> AggregatedStream { + AggregatedStream::new(self, AggregateMode::AggregatedOnly) + } + + fn streaming_mode(self) -> ResponseStream { + self + } +} + +impl AggregatedStream { + fn new(inner: ResponseStream, mode: AggregateMode) -> Self { + AggregatedStream { + inner, + cumulative: String::new(), + cumulative_reasoning: String::new(), + pending: VecDeque::new(), + mode, + } + } +} diff --git a/codex-rs/codex-api/src/endpoint/compact.rs b/codex-rs/codex-api/src/endpoint/compact.rs new file mode 100644 index 0000000000..2b02ebd0f0 --- /dev/null +++ b/codex-rs/codex-api/src/endpoint/compact.rs @@ -0,0 +1,162 @@ +use crate::auth::AuthProvider; +use crate::auth::add_auth_headers; +use crate::common::CompactionInput; +use crate::error::ApiError; +use crate::provider::Provider; +use crate::provider::WireApi; +use crate::telemetry::run_with_request_telemetry; +use codex_client::HttpTransport; +use codex_client::RequestTelemetry; +use codex_protocol::models::ResponseItem; +use http::HeaderMap; +use http::Method; +use serde::Deserialize; +use serde_json::to_value; +use std::sync::Arc; + +pub struct CompactClient { + transport: T, + provider: Provider, + auth: A, + request_telemetry: Option>, +} + +impl CompactClient { + pub fn new(transport: T, provider: Provider, auth: A) -> Self { + Self { + transport, + provider, + auth, + request_telemetry: None, + } + } + + pub fn with_telemetry(mut self, request: Option>) -> Self { + self.request_telemetry = request; + self + } + + fn path(&self) -> Result<&'static str, ApiError> { + match self.provider.wire { + WireApi::Compact | WireApi::Responses => Ok("responses/compact"), + WireApi::Chat => Err(ApiError::Stream( + "compact endpoint requires responses wire api".to_string(), + )), + } + } + + pub async fn compact( + &self, + body: serde_json::Value, + extra_headers: HeaderMap, + ) -> Result, ApiError> { + let path = self.path()?; + let builder = || { + let mut req = self.provider.build_request(Method::POST, path); + req.headers.extend(extra_headers.clone()); + req.body = Some(body.clone()); + add_auth_headers(&self.auth, req) + }; + + let resp = run_with_request_telemetry( + self.provider.retry.to_policy(), + self.request_telemetry.clone(), + builder, + |req| self.transport.execute(req), + ) + .await?; + let parsed: CompactHistoryResponse = + serde_json::from_slice(&resp.body).map_err(|e| ApiError::Stream(e.to_string()))?; + Ok(parsed.output) + } + + pub async fn compact_input( + &self, + input: &CompactionInput<'_>, + extra_headers: HeaderMap, + ) -> Result, ApiError> { + let body = to_value(input) + .map_err(|e| ApiError::Stream(format!("failed to encode compaction input: {e}")))?; + self.compact(body, extra_headers).await + } +} + +#[derive(Debug, Deserialize)] +struct CompactHistoryResponse { + output: Vec, +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::provider::RetryConfig; + use async_trait::async_trait; + use codex_client::Request; + use codex_client::Response; + use codex_client::StreamResponse; + use codex_client::TransportError; + use http::HeaderMap; + use std::time::Duration; + + #[derive(Clone, Default)] + struct DummyTransport; + + #[async_trait] + impl HttpTransport for DummyTransport { + async fn execute(&self, _req: Request) -> Result { + Err(TransportError::Build("execute should not run".to_string())) + } + + async fn stream(&self, _req: Request) -> Result { + Err(TransportError::Build("stream should not run".to_string())) + } + } + + #[derive(Clone, Default)] + struct DummyAuth; + + impl AuthProvider for DummyAuth { + fn bearer_token(&self) -> Option { + None + } + } + + fn provider(wire: WireApi) -> Provider { + Provider { + name: "test".to_string(), + base_url: "https://example.com/v1".to_string(), + query_params: None, + wire, + headers: HeaderMap::new(), + retry: RetryConfig { + max_attempts: 1, + base_delay: Duration::from_millis(1), + retry_429: false, + retry_5xx: true, + retry_transport: true, + }, + stream_idle_timeout: Duration::from_secs(1), + } + } + + #[tokio::test] + async fn errors_when_wire_is_chat() { + let client = CompactClient::new(DummyTransport, provider(WireApi::Chat), DummyAuth); + let input = CompactionInput { + model: "gpt-test", + input: &[], + instructions: "inst", + }; + let err = client + .compact_input(&input, HeaderMap::new()) + .await + .expect_err("expected wire mismatch to fail"); + + match err { + ApiError::Stream(msg) => { + assert_eq!(msg, "compact endpoint requires responses wire api"); + } + other => panic!("unexpected error: {other:?}"), + } + } +} diff --git a/codex-rs/codex-api/src/endpoint/mod.rs b/codex-rs/codex-api/src/endpoint/mod.rs new file mode 100644 index 0000000000..104b4c2640 --- /dev/null +++ b/codex-rs/codex-api/src/endpoint/mod.rs @@ -0,0 +1,4 @@ +pub mod chat; +pub mod compact; +pub mod responses; +mod streaming; diff --git a/codex-rs/codex-api/src/endpoint/responses.rs b/codex-rs/codex-api/src/endpoint/responses.rs new file mode 100644 index 0000000000..d3a314d763 --- /dev/null +++ b/codex-rs/codex-api/src/endpoint/responses.rs @@ -0,0 +1,107 @@ +use crate::auth::AuthProvider; +use crate::common::Prompt as ApiPrompt; +use crate::common::Reasoning; +use crate::common::ResponseStream; +use crate::common::TextControls; +use crate::endpoint::streaming::StreamingClient; +use crate::error::ApiError; +use crate::provider::Provider; +use crate::provider::WireApi; +use crate::requests::ResponsesRequest; +use crate::requests::ResponsesRequestBuilder; +use crate::sse::spawn_response_stream; +use crate::telemetry::SseTelemetry; +use codex_client::HttpTransport; +use codex_client::RequestTelemetry; +use codex_protocol::protocol::SessionSource; +use http::HeaderMap; +use serde_json::Value; +use std::sync::Arc; + +pub struct ResponsesClient { + streaming: StreamingClient, +} + +#[derive(Default)] +pub struct ResponsesOptions { + pub reasoning: Option, + pub include: Vec, + pub prompt_cache_key: Option, + pub text: Option, + pub store_override: Option, + pub conversation_id: Option, + pub session_source: Option, +} + +impl ResponsesClient { + pub fn new(transport: T, provider: Provider, auth: A) -> Self { + Self { + streaming: StreamingClient::new(transport, provider, auth), + } + } + + pub fn with_telemetry( + self, + request: Option>, + sse: Option>, + ) -> Self { + Self { + streaming: self.streaming.with_telemetry(request, sse), + } + } + + pub async fn stream_request( + &self, + request: ResponsesRequest, + ) -> Result { + self.stream(request.body, request.headers).await + } + + pub async fn stream_prompt( + &self, + model: &str, + prompt: &ApiPrompt, + options: ResponsesOptions, + ) -> Result { + let ResponsesOptions { + reasoning, + include, + prompt_cache_key, + text, + store_override, + conversation_id, + session_source, + } = options; + + let request = ResponsesRequestBuilder::new(model, &prompt.instructions, &prompt.input) + .tools(&prompt.tools) + .parallel_tool_calls(prompt.parallel_tool_calls) + .reasoning(reasoning) + .include(include) + .prompt_cache_key(prompt_cache_key) + .text(text) + .conversation(conversation_id) + .session_source(session_source) + .store_override(store_override) + .build(self.streaming.provider())?; + + self.stream_request(request).await + } + + fn path(&self) -> &'static str { + match self.streaming.provider().wire { + WireApi::Responses | WireApi::Compact => "responses", + WireApi::Chat => "chat/completions", + } + } + + pub async fn stream( + &self, + body: Value, + extra_headers: HeaderMap, + ) -> Result { + self.streaming + .stream(self.path(), body, extra_headers, spawn_response_stream) + .await + } +} diff --git a/codex-rs/codex-api/src/endpoint/streaming.rs b/codex-rs/codex-api/src/endpoint/streaming.rs new file mode 100644 index 0000000000..156d4084bc --- /dev/null +++ b/codex-rs/codex-api/src/endpoint/streaming.rs @@ -0,0 +1,82 @@ +use crate::auth::AuthProvider; +use crate::auth::add_auth_headers; +use crate::common::ResponseStream; +use crate::error::ApiError; +use crate::provider::Provider; +use crate::telemetry::SseTelemetry; +use crate::telemetry::run_with_request_telemetry; +use codex_client::HttpTransport; +use codex_client::RequestTelemetry; +use codex_client::StreamResponse; +use http::HeaderMap; +use http::Method; +use serde_json::Value; +use std::sync::Arc; +use std::time::Duration; + +pub(crate) struct StreamingClient { + transport: T, + provider: Provider, + auth: A, + request_telemetry: Option>, + sse_telemetry: Option>, +} + +impl StreamingClient { + pub(crate) fn new(transport: T, provider: Provider, auth: A) -> Self { + Self { + transport, + provider, + auth, + request_telemetry: None, + sse_telemetry: None, + } + } + + pub(crate) fn with_telemetry( + mut self, + request: Option>, + sse: Option>, + ) -> Self { + self.request_telemetry = request; + self.sse_telemetry = sse; + self + } + + pub(crate) fn provider(&self) -> &Provider { + &self.provider + } + + pub(crate) async fn stream( + &self, + path: &str, + body: Value, + extra_headers: HeaderMap, + spawner: fn(StreamResponse, Duration, Option>) -> ResponseStream, + ) -> Result { + let builder = || { + let mut req = self.provider.build_request(Method::POST, path); + req.headers.extend(extra_headers.clone()); + req.headers.insert( + http::header::ACCEPT, + http::HeaderValue::from_static("text/event-stream"), + ); + req.body = Some(body.clone()); + add_auth_headers(&self.auth, req) + }; + + let stream_response = run_with_request_telemetry( + self.provider.retry.to_policy(), + self.request_telemetry.clone(), + builder, + |req| self.transport.stream(req), + ) + .await?; + + Ok(spawner( + stream_response, + self.provider.stream_idle_timeout, + self.sse_telemetry.clone(), + )) + } +} diff --git a/codex-rs/codex-api/src/error.rs b/codex-rs/codex-api/src/error.rs new file mode 100644 index 0000000000..60118e8723 --- /dev/null +++ b/codex-rs/codex-api/src/error.rs @@ -0,0 +1,34 @@ +use crate::rate_limits::RateLimitError; +use codex_client::TransportError; +use http::StatusCode; +use std::time::Duration; +use thiserror::Error; + +#[derive(Debug, Error)] +pub enum ApiError { + #[error(transparent)] + Transport(#[from] TransportError), + #[error("api error {status}: {message}")] + Api { status: StatusCode, message: String }, + #[error("stream error: {0}")] + Stream(String), + #[error("context window exceeded")] + ContextWindowExceeded, + #[error("quota exceeded")] + QuotaExceeded, + #[error("usage not included")] + UsageNotIncluded, + #[error("retryable error: {message}")] + Retryable { + message: String, + delay: Option, + }, + #[error("rate limit: {0}")] + RateLimit(String), +} + +impl From for ApiError { + fn from(err: RateLimitError) -> Self { + Self::RateLimit(err.to_string()) + } +} diff --git a/codex-rs/codex-api/src/lib.rs b/codex-rs/codex-api/src/lib.rs new file mode 100644 index 0000000000..acde4b4589 --- /dev/null +++ b/codex-rs/codex-api/src/lib.rs @@ -0,0 +1,35 @@ +pub mod auth; +pub mod common; +pub mod endpoint; +pub mod error; +pub mod provider; +pub mod rate_limits; +pub mod requests; +pub mod sse; +pub mod telemetry; + +pub use codex_client::RequestTelemetry; +pub use codex_client::ReqwestTransport; +pub use codex_client::TransportError; + +pub use crate::auth::AuthProvider; +pub use crate::common::CompactionInput; +pub use crate::common::Prompt; +pub use crate::common::ResponseEvent; +pub use crate::common::ResponseStream; +pub use crate::common::ResponsesApiRequest; +pub use crate::common::create_text_param_for_request; +pub use crate::endpoint::chat::AggregateStreamExt; +pub use crate::endpoint::chat::ChatClient; +pub use crate::endpoint::compact::CompactClient; +pub use crate::endpoint::responses::ResponsesClient; +pub use crate::endpoint::responses::ResponsesOptions; +pub use crate::error::ApiError; +pub use crate::provider::Provider; +pub use crate::provider::WireApi; +pub use crate::requests::ChatRequest; +pub use crate::requests::ChatRequestBuilder; +pub use crate::requests::ResponsesRequest; +pub use crate::requests::ResponsesRequestBuilder; +pub use crate::sse::stream_from_fixture; +pub use crate::telemetry::SseTelemetry; diff --git a/codex-rs/codex-api/src/provider.rs b/codex-rs/codex-api/src/provider.rs new file mode 100644 index 0000000000..8bd5fc9093 --- /dev/null +++ b/codex-rs/codex-api/src/provider.rs @@ -0,0 +1,118 @@ +use codex_client::Request; +use codex_client::RetryOn; +use codex_client::RetryPolicy; +use http::Method; +use http::header::HeaderMap; +use std::collections::HashMap; +use std::time::Duration; + +/// Wire-level APIs supported by a `Provider`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum WireApi { + Responses, + Chat, + Compact, +} + +/// High-level retry configuration for a provider. +/// +/// This is converted into a `RetryPolicy` used by `codex-client` to drive +/// transport-level retries for both unary and streaming calls. +#[derive(Debug, Clone)] +pub struct RetryConfig { + pub max_attempts: u64, + pub base_delay: Duration, + pub retry_429: bool, + pub retry_5xx: bool, + pub retry_transport: bool, +} + +impl RetryConfig { + pub fn to_policy(&self) -> RetryPolicy { + RetryPolicy { + max_attempts: self.max_attempts, + base_delay: self.base_delay, + retry_on: RetryOn { + retry_429: self.retry_429, + retry_5xx: self.retry_5xx, + retry_transport: self.retry_transport, + }, + } + } +} + +/// HTTP endpoint configuration used to talk to a concrete API deployment. +/// +/// Encapsulates base URL, default headers, query params, retry policy, and +/// stream idle timeout, plus helper methods for building requests. +#[derive(Debug, Clone)] +pub struct Provider { + pub name: String, + pub base_url: String, + pub query_params: Option>, + pub wire: WireApi, + pub headers: HeaderMap, + pub retry: RetryConfig, + pub stream_idle_timeout: Duration, +} + +impl Provider { + pub fn url_for_path(&self, path: &str) -> String { + let base = self.base_url.trim_end_matches('/'); + let path = path.trim_start_matches('/'); + let mut url = if path.is_empty() { + base.to_string() + } else { + format!("{base}/{path}") + }; + + if let Some(params) = &self.query_params + && !params.is_empty() + { + let qs = params + .iter() + .map(|(k, v)| format!("{k}={v}")) + .collect::>() + .join("&"); + url.push('?'); + url.push_str(&qs); + } + + url + } + + pub fn build_request(&self, method: Method, path: &str) -> Request { + Request { + method, + url: self.url_for_path(path), + headers: self.headers.clone(), + body: None, + timeout: None, + } + } + + pub fn is_azure_responses_endpoint(&self) -> bool { + if self.wire != WireApi::Responses { + return false; + } + + if self.name.eq_ignore_ascii_case("azure") { + return true; + } + + self.base_url.to_ascii_lowercase().contains("openai.azure.") + || matches_azure_responses_base_url(&self.base_url) + } +} + +fn matches_azure_responses_base_url(base_url: &str) -> bool { + const AZURE_MARKERS: [&str; 5] = [ + "cognitiveservices.azure.", + "aoai.azure.", + "azure-api.", + "azurefd.", + "windows.net/openai", + ]; + let base = base_url.to_ascii_lowercase(); + AZURE_MARKERS.iter().any(|marker| base.contains(marker)) +} diff --git a/codex-rs/codex-api/src/rate_limits.rs b/codex-rs/codex-api/src/rate_limits.rs new file mode 100644 index 0000000000..69092063f6 --- /dev/null +++ b/codex-rs/codex-api/src/rate_limits.rs @@ -0,0 +1,105 @@ +use codex_protocol::protocol::CreditsSnapshot; +use codex_protocol::protocol::RateLimitSnapshot; +use codex_protocol::protocol::RateLimitWindow; +use http::HeaderMap; +use std::fmt::Display; + +#[derive(Debug)] +pub struct RateLimitError { + pub message: String, +} + +impl Display for RateLimitError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.message) + } +} + +/// Parses the bespoke Codex rate-limit headers into a `RateLimitSnapshot`. +pub fn parse_rate_limit(headers: &HeaderMap) -> Option { + let primary = parse_rate_limit_window( + headers, + "x-codex-primary-used-percent", + "x-codex-primary-window-minutes", + "x-codex-primary-reset-at", + ); + + let secondary = parse_rate_limit_window( + headers, + "x-codex-secondary-used-percent", + "x-codex-secondary-window-minutes", + "x-codex-secondary-reset-at", + ); + + let credits = parse_credits_snapshot(headers); + + Some(RateLimitSnapshot { + primary, + secondary, + credits, + }) +} + +fn parse_rate_limit_window( + headers: &HeaderMap, + used_percent_header: &str, + window_minutes_header: &str, + resets_at_header: &str, +) -> Option { + let used_percent: Option = parse_header_f64(headers, used_percent_header); + + used_percent.and_then(|used_percent| { + let window_minutes = parse_header_i64(headers, window_minutes_header); + let resets_at = parse_header_i64(headers, resets_at_header); + + let has_data = used_percent != 0.0 + || window_minutes.is_some_and(|minutes| minutes != 0) + || resets_at.is_some(); + + has_data.then_some(RateLimitWindow { + used_percent, + window_minutes, + resets_at, + }) + }) +} + +fn parse_credits_snapshot(headers: &HeaderMap) -> Option { + let has_credits = parse_header_bool(headers, "x-codex-credits-has-credits")?; + let unlimited = parse_header_bool(headers, "x-codex-credits-unlimited")?; + let balance = parse_header_str(headers, "x-codex-credits-balance") + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(std::string::ToString::to_string); + Some(CreditsSnapshot { + has_credits, + unlimited, + balance, + }) +} + +fn parse_header_f64(headers: &HeaderMap, name: &str) -> Option { + parse_header_str(headers, name)? + .parse::() + .ok() + .filter(|v| v.is_finite()) +} + +fn parse_header_i64(headers: &HeaderMap, name: &str) -> Option { + parse_header_str(headers, name)?.parse::().ok() +} + +fn parse_header_bool(headers: &HeaderMap, name: &str) -> Option { + let raw = parse_header_str(headers, name)?; + if raw.eq_ignore_ascii_case("true") || raw == "1" { + Some(true) + } else if raw.eq_ignore_ascii_case("false") || raw == "0" { + Some(false) + } else { + None + } +} + +fn parse_header_str<'a>(headers: &'a HeaderMap, name: &str) -> Option<&'a str> { + headers.get(name)?.to_str().ok() +} diff --git a/codex-rs/codex-api/src/requests/chat.rs b/codex-rs/codex-api/src/requests/chat.rs new file mode 100644 index 0000000000..c1ba89f37a --- /dev/null +++ b/codex-rs/codex-api/src/requests/chat.rs @@ -0,0 +1,388 @@ +use crate::error::ApiError; +use crate::provider::Provider; +use crate::requests::headers::build_conversation_headers; +use crate::requests::headers::insert_header; +use crate::requests::headers::subagent_header; +use codex_protocol::models::ContentItem; +use codex_protocol::models::FunctionCallOutputContentItem; +use codex_protocol::models::ReasoningItemContent; +use codex_protocol::models::ResponseItem; +use codex_protocol::protocol::SessionSource; +use http::HeaderMap; +use serde_json::Value; +use serde_json::json; +use std::collections::HashMap; + +/// Assembled request body plus headers for Chat Completions streaming calls. +pub struct ChatRequest { + pub body: Value, + pub headers: HeaderMap, +} + +pub struct ChatRequestBuilder<'a> { + model: &'a str, + instructions: &'a str, + input: &'a [ResponseItem], + tools: &'a [Value], + conversation_id: Option, + session_source: Option, +} + +impl<'a> ChatRequestBuilder<'a> { + pub fn new( + model: &'a str, + instructions: &'a str, + input: &'a [ResponseItem], + tools: &'a [Value], + ) -> Self { + Self { + model, + instructions, + input, + tools, + conversation_id: None, + session_source: None, + } + } + + pub fn conversation_id(mut self, id: Option) -> Self { + self.conversation_id = id; + self + } + + pub fn session_source(mut self, source: Option) -> Self { + self.session_source = source; + self + } + + pub fn build(self, _provider: &Provider) -> Result { + let mut messages = Vec::::new(); + messages.push(json!({"role": "system", "content": self.instructions})); + + let input = self.input; + let mut reasoning_by_anchor_index: HashMap = HashMap::new(); + let mut last_emitted_role: Option<&str> = None; + for item in input { + match item { + ResponseItem::Message { role, .. } => last_emitted_role = Some(role.as_str()), + ResponseItem::FunctionCall { .. } | ResponseItem::LocalShellCall { .. } => { + last_emitted_role = Some("assistant") + } + ResponseItem::FunctionCallOutput { .. } => last_emitted_role = Some("tool"), + ResponseItem::Reasoning { .. } | ResponseItem::Other => {} + ResponseItem::CustomToolCall { .. } => {} + ResponseItem::CustomToolCallOutput { .. } => {} + ResponseItem::WebSearchCall { .. } => {} + ResponseItem::GhostSnapshot { .. } => {} + ResponseItem::CompactionSummary { .. } => {} + } + } + + let mut last_user_index: Option = None; + for (idx, item) in input.iter().enumerate() { + if let ResponseItem::Message { role, .. } = item + && role == "user" + { + last_user_index = Some(idx); + } + } + + if !matches!(last_emitted_role, Some("user")) { + for (idx, item) in input.iter().enumerate() { + if let Some(u_idx) = last_user_index + && idx <= u_idx + { + continue; + } + + if let ResponseItem::Reasoning { + content: Some(items), + .. + } = item + { + let mut text = String::new(); + for entry in items { + match entry { + ReasoningItemContent::ReasoningText { text: segment } + | ReasoningItemContent::Text { text: segment } => { + text.push_str(segment) + } + } + } + if text.trim().is_empty() { + continue; + } + + let mut attached = false; + if idx > 0 + && let ResponseItem::Message { role, .. } = &input[idx - 1] + && role == "assistant" + { + reasoning_by_anchor_index + .entry(idx - 1) + .and_modify(|v| v.push_str(&text)) + .or_insert(text.clone()); + attached = true; + } + + if !attached && idx + 1 < input.len() { + match &input[idx + 1] { + ResponseItem::FunctionCall { .. } + | ResponseItem::LocalShellCall { .. } => { + reasoning_by_anchor_index + .entry(idx + 1) + .and_modify(|v| v.push_str(&text)) + .or_insert(text.clone()); + } + ResponseItem::Message { role, .. } if role == "assistant" => { + reasoning_by_anchor_index + .entry(idx + 1) + .and_modify(|v| v.push_str(&text)) + .or_insert(text.clone()); + } + _ => {} + } + } + } + } + } + + let mut last_assistant_text: Option = None; + + for (idx, item) in input.iter().enumerate() { + match item { + ResponseItem::Message { role, content, .. } => { + let mut text = String::new(); + let mut items: Vec = Vec::new(); + let mut saw_image = false; + + for c in content { + match c { + ContentItem::InputText { text: t } + | ContentItem::OutputText { text: t } => { + text.push_str(t); + items.push(json!({"type":"text","text": t})); + } + ContentItem::InputImage { image_url } => { + saw_image = true; + items.push( + json!({"type":"image_url","image_url": {"url": image_url}}), + ); + } + } + } + + if role == "assistant" { + if let Some(prev) = &last_assistant_text + && prev == &text + { + continue; + } + last_assistant_text = Some(text.clone()); + } + + let content_value = if role == "assistant" { + json!(text) + } else if saw_image { + json!(items) + } else { + json!(text) + }; + + let mut msg = json!({"role": role, "content": content_value}); + if role == "assistant" + && let Some(reasoning) = reasoning_by_anchor_index.get(&idx) + && let Some(obj) = msg.as_object_mut() + { + obj.insert("reasoning".to_string(), json!(reasoning)); + } + messages.push(msg); + } + ResponseItem::FunctionCall { + name, + arguments, + call_id, + .. + } => { + let mut msg = json!({ + "role": "assistant", + "content": null, + "tool_calls": [{ + "id": call_id, + "type": "function", + "function": { + "name": name, + "arguments": arguments, + } + }] + }); + if let Some(reasoning) = reasoning_by_anchor_index.get(&idx) + && let Some(obj) = msg.as_object_mut() + { + obj.insert("reasoning".to_string(), json!(reasoning)); + } + messages.push(msg); + } + ResponseItem::LocalShellCall { + id, + call_id: _, + status, + action, + } => { + let mut msg = json!({ + "role": "assistant", + "content": null, + "tool_calls": [{ + "id": id.clone().unwrap_or_default(), + "type": "local_shell_call", + "status": status, + "action": action, + }] + }); + if let Some(reasoning) = reasoning_by_anchor_index.get(&idx) + && let Some(obj) = msg.as_object_mut() + { + obj.insert("reasoning".to_string(), json!(reasoning)); + } + messages.push(msg); + } + ResponseItem::FunctionCallOutput { call_id, output } => { + let content_value = if let Some(items) = &output.content_items { + let mapped: Vec = items + .iter() + .map(|it| match it { + FunctionCallOutputContentItem::InputText { text } => { + json!({"type":"text","text": text}) + } + FunctionCallOutputContentItem::InputImage { image_url } => { + json!({"type":"image_url","image_url": {"url": image_url}}) + } + }) + .collect(); + json!(mapped) + } else { + json!(output.content) + }; + + messages.push(json!({ + "role": "tool", + "tool_call_id": call_id, + "content": content_value, + })); + } + ResponseItem::CustomToolCall { + id, + call_id: _, + name, + input, + status: _, + } => { + messages.push(json!({ + "role": "assistant", + "content": null, + "tool_calls": [{ + "id": id, + "type": "custom", + "custom": { + "name": name, + "input": input, + } + }] + })); + } + ResponseItem::CustomToolCallOutput { call_id, output } => { + messages.push(json!({ + "role": "tool", + "tool_call_id": call_id, + "content": output, + })); + } + ResponseItem::GhostSnapshot { .. } => { + continue; + } + ResponseItem::Reasoning { .. } + | ResponseItem::WebSearchCall { .. } + | ResponseItem::Other + | ResponseItem::CompactionSummary { .. } => { + continue; + } + } + } + + let payload = json!({ + "model": self.model, + "messages": messages, + "stream": true, + "tools": self.tools, + }); + + let mut headers = build_conversation_headers(self.conversation_id); + if let Some(subagent) = subagent_header(&self.session_source) { + insert_header(&mut headers, "x-openai-subagent", &subagent); + } + + Ok(ChatRequest { + body: payload, + headers, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::provider::RetryConfig; + use crate::provider::WireApi; + use codex_protocol::protocol::SessionSource; + use codex_protocol::protocol::SubAgentSource; + use http::HeaderValue; + use pretty_assertions::assert_eq; + use std::time::Duration; + + fn provider() -> Provider { + Provider { + name: "openai".to_string(), + base_url: "https://api.openai.com/v1".to_string(), + query_params: None, + wire: WireApi::Chat, + headers: HeaderMap::new(), + retry: RetryConfig { + max_attempts: 1, + base_delay: Duration::from_millis(10), + retry_429: false, + retry_5xx: true, + retry_transport: true, + }, + stream_idle_timeout: Duration::from_secs(1), + } + } + + #[test] + fn attaches_conversation_and_subagent_headers() { + let prompt_input = vec![ResponseItem::Message { + id: None, + role: "user".to_string(), + content: vec![ContentItem::InputText { + text: "hi".to_string(), + }], + }]; + let req = ChatRequestBuilder::new("gpt-test", "inst", &prompt_input, &[]) + .conversation_id(Some("conv-1".into())) + .session_source(Some(SessionSource::SubAgent(SubAgentSource::Review))) + .build(&provider()) + .expect("request"); + + assert_eq!( + req.headers.get("conversation_id"), + Some(&HeaderValue::from_static("conv-1")) + ); + assert_eq!( + req.headers.get("session_id"), + Some(&HeaderValue::from_static("conv-1")) + ); + assert_eq!( + req.headers.get("x-openai-subagent"), + Some(&HeaderValue::from_static("review")) + ); + } +} diff --git a/codex-rs/codex-api/src/requests/headers.rs b/codex-rs/codex-api/src/requests/headers.rs new file mode 100644 index 0000000000..4d8a17d18d --- /dev/null +++ b/codex-rs/codex-api/src/requests/headers.rs @@ -0,0 +1,36 @@ +use codex_protocol::protocol::SessionSource; +use http::HeaderMap; +use http::HeaderValue; + +pub(crate) fn build_conversation_headers(conversation_id: Option) -> HeaderMap { + let mut headers = HeaderMap::new(); + if let Some(id) = conversation_id { + insert_header(&mut headers, "conversation_id", &id); + insert_header(&mut headers, "session_id", &id); + } + headers +} + +pub(crate) fn subagent_header(source: &Option) -> Option { + let SessionSource::SubAgent(sub) = source.as_ref()? else { + return None; + }; + match sub { + codex_protocol::protocol::SubAgentSource::Other(label) => Some(label.clone()), + other => Some( + serde_json::to_value(other) + .ok() + .and_then(|v| v.as_str().map(std::string::ToString::to_string)) + .unwrap_or_else(|| "other".to_string()), + ), + } +} + +pub(crate) fn insert_header(headers: &mut HeaderMap, name: &str, value: &str) { + if let (Ok(header_name), Ok(header_value)) = ( + name.parse::(), + HeaderValue::from_str(value), + ) { + headers.insert(header_name, header_value); + } +} diff --git a/codex-rs/codex-api/src/requests/mod.rs b/codex-rs/codex-api/src/requests/mod.rs new file mode 100644 index 0000000000..f0ab23a25f --- /dev/null +++ b/codex-rs/codex-api/src/requests/mod.rs @@ -0,0 +1,8 @@ +pub mod chat; +pub(crate) mod headers; +pub mod responses; + +pub use chat::ChatRequest; +pub use chat::ChatRequestBuilder; +pub use responses::ResponsesRequest; +pub use responses::ResponsesRequestBuilder; diff --git a/codex-rs/codex-api/src/requests/responses.rs b/codex-rs/codex-api/src/requests/responses.rs new file mode 100644 index 0000000000..543b79bbe9 --- /dev/null +++ b/codex-rs/codex-api/src/requests/responses.rs @@ -0,0 +1,247 @@ +use crate::common::Reasoning; +use crate::common::ResponsesApiRequest; +use crate::common::TextControls; +use crate::error::ApiError; +use crate::provider::Provider; +use crate::requests::headers::build_conversation_headers; +use crate::requests::headers::insert_header; +use crate::requests::headers::subagent_header; +use codex_protocol::models::ResponseItem; +use codex_protocol::protocol::SessionSource; +use http::HeaderMap; +use serde_json::Value; + +/// Assembled request body plus headers for a Responses stream request. +pub struct ResponsesRequest { + pub body: Value, + pub headers: HeaderMap, +} + +#[derive(Default)] +pub struct ResponsesRequestBuilder<'a> { + model: Option<&'a str>, + instructions: Option<&'a str>, + input: Option<&'a [ResponseItem]>, + tools: Option<&'a [Value]>, + parallel_tool_calls: bool, + reasoning: Option, + include: Vec, + prompt_cache_key: Option, + text: Option, + conversation_id: Option, + session_source: Option, + store_override: Option, + headers: HeaderMap, +} + +impl<'a> ResponsesRequestBuilder<'a> { + pub fn new(model: &'a str, instructions: &'a str, input: &'a [ResponseItem]) -> Self { + Self { + model: Some(model), + instructions: Some(instructions), + input: Some(input), + ..Default::default() + } + } + + pub fn tools(mut self, tools: &'a [Value]) -> Self { + self.tools = Some(tools); + self + } + + pub fn parallel_tool_calls(mut self, enabled: bool) -> Self { + self.parallel_tool_calls = enabled; + self + } + + pub fn reasoning(mut self, reasoning: Option) -> Self { + self.reasoning = reasoning; + self + } + + pub fn include(mut self, include: Vec) -> Self { + self.include = include; + self + } + + pub fn prompt_cache_key(mut self, key: Option) -> Self { + self.prompt_cache_key = key; + self + } + + pub fn text(mut self, text: Option) -> Self { + self.text = text; + self + } + + pub fn conversation(mut self, conversation_id: Option) -> Self { + self.conversation_id = conversation_id; + self + } + + pub fn session_source(mut self, source: Option) -> Self { + self.session_source = source; + self + } + + pub fn store_override(mut self, store: Option) -> Self { + self.store_override = store; + self + } + + pub fn extra_headers(mut self, headers: HeaderMap) -> Self { + self.headers = headers; + self + } + + pub fn build(self, provider: &Provider) -> Result { + let model = self + .model + .ok_or_else(|| ApiError::Stream("missing model for responses request".into()))?; + let instructions = self + .instructions + .ok_or_else(|| ApiError::Stream("missing instructions for responses request".into()))?; + let input = self + .input + .ok_or_else(|| ApiError::Stream("missing input for responses request".into()))?; + let tools = self.tools.unwrap_or_default(); + + let store = self + .store_override + .unwrap_or_else(|| provider.is_azure_responses_endpoint()); + + let req = ResponsesApiRequest { + model, + instructions, + input, + tools, + tool_choice: "auto", + parallel_tool_calls: self.parallel_tool_calls, + reasoning: self.reasoning, + store, + stream: true, + include: self.include, + prompt_cache_key: self.prompt_cache_key, + text: self.text, + }; + + let mut body = serde_json::to_value(&req) + .map_err(|e| ApiError::Stream(format!("failed to encode responses request: {e}")))?; + + if store && provider.is_azure_responses_endpoint() { + attach_item_ids(&mut body, input); + } + + let mut headers = self.headers; + headers.extend(build_conversation_headers(self.conversation_id)); + if let Some(subagent) = subagent_header(&self.session_source) { + insert_header(&mut headers, "x-openai-subagent", &subagent); + } + + Ok(ResponsesRequest { body, headers }) + } +} + +fn attach_item_ids(payload_json: &mut Value, original_items: &[ResponseItem]) { + let Some(input_value) = payload_json.get_mut("input") else { + return; + }; + let Value::Array(items) = input_value else { + return; + }; + + for (value, item) in items.iter_mut().zip(original_items.iter()) { + if let ResponseItem::Reasoning { id, .. } + | ResponseItem::Message { id: Some(id), .. } + | ResponseItem::WebSearchCall { id: Some(id), .. } + | ResponseItem::FunctionCall { id: Some(id), .. } + | ResponseItem::LocalShellCall { id: Some(id), .. } + | ResponseItem::CustomToolCall { id: Some(id), .. } = item + { + if id.is_empty() { + continue; + } + + if let Some(obj) = value.as_object_mut() { + obj.insert("id".to_string(), Value::String(id.clone())); + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::provider::RetryConfig; + use crate::provider::WireApi; + use codex_protocol::protocol::SubAgentSource; + use http::HeaderValue; + use pretty_assertions::assert_eq; + use std::time::Duration; + + fn provider(name: &str, base_url: &str) -> Provider { + Provider { + name: name.to_string(), + base_url: base_url.to_string(), + query_params: None, + wire: WireApi::Responses, + headers: HeaderMap::new(), + retry: RetryConfig { + max_attempts: 1, + base_delay: Duration::from_millis(50), + retry_429: false, + retry_5xx: true, + retry_transport: true, + }, + stream_idle_timeout: Duration::from_secs(5), + } + } + + #[test] + fn azure_default_store_attaches_ids_and_headers() { + let provider = provider("azure", "https://example.openai.azure.com/v1"); + let input = vec![ + ResponseItem::Message { + id: Some("m1".into()), + role: "assistant".into(), + content: Vec::new(), + }, + ResponseItem::Message { + id: None, + role: "assistant".into(), + content: Vec::new(), + }, + ]; + + let request = ResponsesRequestBuilder::new("gpt-test", "inst", &input) + .conversation(Some("conv-1".into())) + .session_source(Some(SessionSource::SubAgent(SubAgentSource::Review))) + .build(&provider) + .expect("request"); + + assert_eq!(request.body.get("store"), Some(&Value::Bool(true))); + + let ids: Vec> = request + .body + .get("input") + .and_then(|v| v.as_array()) + .into_iter() + .flatten() + .map(|item| item.get("id").and_then(|v| v.as_str().map(str::to_string))) + .collect(); + assert_eq!(ids, vec![Some("m1".to_string()), None]); + + assert_eq!( + request.headers.get("conversation_id"), + Some(&HeaderValue::from_static("conv-1")) + ); + assert_eq!( + request.headers.get("session_id"), + Some(&HeaderValue::from_static("conv-1")) + ); + assert_eq!( + request.headers.get("x-openai-subagent"), + Some(&HeaderValue::from_static("review")) + ); + } +} diff --git a/codex-rs/codex-api/src/sse/chat.rs b/codex-rs/codex-api/src/sse/chat.rs new file mode 100644 index 0000000000..7f50bb634a --- /dev/null +++ b/codex-rs/codex-api/src/sse/chat.rs @@ -0,0 +1,504 @@ +use crate::common::ResponseEvent; +use crate::common::ResponseStream; +use crate::error::ApiError; +use crate::telemetry::SseTelemetry; +use codex_client::StreamResponse; +use codex_protocol::models::ContentItem; +use codex_protocol::models::ReasoningItemContent; +use codex_protocol::models::ResponseItem; +use eventsource_stream::Eventsource; +use futures::Stream; +use futures::StreamExt; +use std::collections::HashMap; +use std::time::Duration; +use tokio::sync::mpsc; +use tokio::time::Instant; +use tokio::time::timeout; +use tracing::debug; +use tracing::trace; + +pub(crate) fn spawn_chat_stream( + stream_response: StreamResponse, + idle_timeout: Duration, + telemetry: Option>, +) -> ResponseStream { + let (tx_event, rx_event) = mpsc::channel::>(1600); + tokio::spawn(async move { + process_chat_sse(stream_response.bytes, tx_event, idle_timeout, telemetry).await; + }); + ResponseStream { rx_event } +} + +pub async fn process_chat_sse( + stream: S, + tx_event: mpsc::Sender>, + idle_timeout: Duration, + telemetry: Option>, +) where + S: Stream> + Unpin, +{ + let mut stream = stream.eventsource(); + + #[derive(Default, Debug)] + struct ToolCallState { + name: Option, + arguments: String, + } + + let mut tool_calls: HashMap = HashMap::new(); + let mut tool_call_order: Vec = Vec::new(); + let mut assistant_item: Option = None; + let mut reasoning_item: Option = None; + let mut completed_sent = false; + + loop { + let start = Instant::now(); + let response = timeout(idle_timeout, stream.next()).await; + if let Some(t) = telemetry.as_ref() { + t.on_sse_poll(&response, start.elapsed()); + } + let sse = match response { + Ok(Some(Ok(sse))) => sse, + Ok(Some(Err(e))) => { + let _ = tx_event.send(Err(ApiError::Stream(e.to_string()))).await; + return; + } + Ok(None) => { + if let Some(reasoning) = reasoning_item { + let _ = tx_event + .send(Ok(ResponseEvent::OutputItemDone(reasoning))) + .await; + } + + if let Some(assistant) = assistant_item { + let _ = tx_event + .send(Ok(ResponseEvent::OutputItemDone(assistant))) + .await; + } + if !completed_sent { + let _ = tx_event + .send(Ok(ResponseEvent::Completed { + response_id: String::new(), + token_usage: None, + })) + .await; + } + return; + } + Err(_) => { + let _ = tx_event + .send(Err(ApiError::Stream("idle timeout waiting for SSE".into()))) + .await; + return; + } + }; + + trace!("SSE event: {}", sse.data); + + if sse.data.trim().is_empty() { + continue; + } + + let value: serde_json::Value = match serde_json::from_str(&sse.data) { + Ok(val) => val, + Err(err) => { + debug!( + "Failed to parse ChatCompletions SSE event: {err}, data: {}", + &sse.data + ); + continue; + } + }; + + let Some(choices) = value.get("choices").and_then(|c| c.as_array()) else { + continue; + }; + + for choice in choices { + if let Some(delta) = choice.get("delta") { + if let Some(reasoning) = delta.get("reasoning") { + if let Some(text) = reasoning.as_str() { + append_reasoning_text(&tx_event, &mut reasoning_item, text.to_string()) + .await; + } else if let Some(text) = reasoning.get("text").and_then(|v| v.as_str()) { + append_reasoning_text(&tx_event, &mut reasoning_item, text.to_string()) + .await; + } else if let Some(text) = reasoning.get("content").and_then(|v| v.as_str()) { + append_reasoning_text(&tx_event, &mut reasoning_item, text.to_string()) + .await; + } + } + + if let Some(content) = delta.get("content") { + if content.is_array() { + for item in content.as_array().unwrap_or(&vec![]) { + if let Some(text) = item.get("text").and_then(|t| t.as_str()) { + append_assistant_text( + &tx_event, + &mut assistant_item, + text.to_string(), + ) + .await; + } + } + } else if let Some(text) = content.as_str() { + append_assistant_text(&tx_event, &mut assistant_item, text.to_string()) + .await; + } + } + + if let Some(tool_call_values) = delta.get("tool_calls").and_then(|c| c.as_array()) { + for tool_call in tool_call_values { + let id = tool_call + .get("id") + .and_then(|i| i.as_str()) + .map(str::to_string) + .unwrap_or_else(|| format!("tool-call-{}", tool_call_order.len())); + + let call_state = tool_calls.entry(id.clone()).or_default(); + if !tool_call_order.contains(&id) { + tool_call_order.push(id.clone()); + } + + if let Some(func) = tool_call.get("function") { + if let Some(fname) = func.get("name").and_then(|n| n.as_str()) { + call_state.name = Some(fname.to_string()); + } + if let Some(arguments) = func.get("arguments").and_then(|a| a.as_str()) + { + call_state.arguments.push_str(arguments); + } + } + } + } + } + + if let Some(message) = choice.get("message") + && let Some(reasoning) = message.get("reasoning") + { + if let Some(text) = reasoning.as_str() { + append_reasoning_text(&tx_event, &mut reasoning_item, text.to_string()).await; + } else if let Some(text) = reasoning.get("text").and_then(|v| v.as_str()) { + append_reasoning_text(&tx_event, &mut reasoning_item, text.to_string()).await; + } else if let Some(text) = reasoning.get("content").and_then(|v| v.as_str()) { + append_reasoning_text(&tx_event, &mut reasoning_item, text.to_string()).await; + } + } + + let finish_reason = choice.get("finish_reason").and_then(|r| r.as_str()); + if finish_reason == Some("stop") { + if let Some(reasoning) = reasoning_item.take() { + let _ = tx_event + .send(Ok(ResponseEvent::OutputItemDone(reasoning))) + .await; + } + + if let Some(assistant) = assistant_item.take() { + let _ = tx_event + .send(Ok(ResponseEvent::OutputItemDone(assistant))) + .await; + } + if !completed_sent { + let _ = tx_event + .send(Ok(ResponseEvent::Completed { + response_id: String::new(), + token_usage: None, + })) + .await; + completed_sent = true; + } + continue; + } + + if finish_reason == Some("length") { + let _ = tx_event.send(Err(ApiError::ContextWindowExceeded)).await; + return; + } + + if finish_reason == Some("tool_calls") { + if let Some(reasoning) = reasoning_item.take() { + let _ = tx_event + .send(Ok(ResponseEvent::OutputItemDone(reasoning))) + .await; + } + + for call_id in tool_call_order.drain(..) { + let state = tool_calls.remove(&call_id).unwrap_or_default(); + let item = ResponseItem::FunctionCall { + id: None, + name: state.name.unwrap_or_default(), + arguments: state.arguments, + call_id: call_id.clone(), + }; + let _ = tx_event.send(Ok(ResponseEvent::OutputItemDone(item))).await; + } + } + } + } +} + +async fn append_assistant_text( + tx_event: &mpsc::Sender>, + assistant_item: &mut Option, + text: String, +) { + if assistant_item.is_none() { + let item = ResponseItem::Message { + id: None, + role: "assistant".to_string(), + content: vec![], + }; + *assistant_item = Some(item.clone()); + let _ = tx_event + .send(Ok(ResponseEvent::OutputItemAdded(item))) + .await; + } + + if let Some(ResponseItem::Message { content, .. }) = assistant_item { + content.push(ContentItem::OutputText { text: text.clone() }); + let _ = tx_event + .send(Ok(ResponseEvent::OutputTextDelta(text.clone()))) + .await; + } +} + +async fn append_reasoning_text( + tx_event: &mpsc::Sender>, + reasoning_item: &mut Option, + text: String, +) { + if reasoning_item.is_none() { + let item = ResponseItem::Reasoning { + id: String::new(), + summary: Vec::new(), + content: Some(vec![]), + encrypted_content: None, + }; + *reasoning_item = Some(item.clone()); + let _ = tx_event + .send(Ok(ResponseEvent::OutputItemAdded(item))) + .await; + } + + if let Some(ResponseItem::Reasoning { + content: Some(content), + .. + }) = reasoning_item + { + let content_index = content.len() as i64; + content.push(ReasoningItemContent::ReasoningText { text: text.clone() }); + + let _ = tx_event + .send(Ok(ResponseEvent::ReasoningContentDelta { + delta: text.clone(), + content_index, + })) + .await; + } +} + +#[cfg(test)] +mod tests { + use super::*; + use assert_matches::assert_matches; + use codex_protocol::models::ResponseItem; + use futures::TryStreamExt; + use serde_json::json; + use tokio::sync::mpsc; + use tokio_util::io::ReaderStream; + + fn build_body(events: &[serde_json::Value]) -> String { + let mut body = String::new(); + for e in events { + body.push_str(&format!("event: message\ndata: {e}\n\n")); + } + body + } + + async fn collect_events(body: &str) -> Vec { + let reader = ReaderStream::new(std::io::Cursor::new(body.to_string())) + .map_err(|err| codex_client::TransportError::Network(err.to_string())); + let (tx, mut rx) = mpsc::channel::>(16); + tokio::spawn(process_chat_sse( + reader, + tx, + Duration::from_millis(1000), + None, + )); + + let mut out = Vec::new(); + while let Some(ev) = rx.recv().await { + out.push(ev.expect("stream error")); + } + out + } + + #[tokio::test] + async fn emits_multiple_tool_calls() { + let delta_a = json!({ + "choices": [{ + "delta": { + "tool_calls": [{ + "id": "call_a", + "function": { "name": "do_a", "arguments": "{\"foo\":1}" } + }] + } + }] + }); + + let delta_b = json!({ + "choices": [{ + "delta": { + "tool_calls": [{ + "id": "call_b", + "function": { "name": "do_b", "arguments": "{\"bar\":2}" } + }] + } + }] + }); + + let finish = json!({ + "choices": [{ + "finish_reason": "tool_calls" + }] + }); + + let body = build_body(&[delta_a, delta_b, finish]); + let events = collect_events(&body).await; + assert_eq!(events.len(), 3); + + assert_matches!( + &events[0], + ResponseEvent::OutputItemDone(ResponseItem::FunctionCall { call_id, name, arguments, .. }) + if call_id == "call_a" && name == "do_a" && arguments == "{\"foo\":1}" + ); + assert_matches!( + &events[1], + ResponseEvent::OutputItemDone(ResponseItem::FunctionCall { call_id, name, arguments, .. }) + if call_id == "call_b" && name == "do_b" && arguments == "{\"bar\":2}" + ); + assert_matches!(events[2], ResponseEvent::Completed { .. }); + } + + #[tokio::test] + async fn concatenates_tool_call_arguments_across_deltas() { + let delta_name = json!({ + "choices": [{ + "delta": { + "tool_calls": [{ + "id": "call_a", + "function": { "name": "do_a" } + }] + } + }] + }); + + let delta_args_1 = json!({ + "choices": [{ + "delta": { + "tool_calls": [{ + "id": "call_a", + "function": { "arguments": "{ \"foo\":" } + }] + } + }] + }); + + let delta_args_2 = json!({ + "choices": [{ + "delta": { + "tool_calls": [{ + "id": "call_a", + "function": { "arguments": "1}" } + }] + } + }] + }); + + let finish = json!({ + "choices": [{ + "finish_reason": "tool_calls" + }] + }); + + let body = build_body(&[delta_name, delta_args_1, delta_args_2, finish]); + let events = collect_events(&body).await; + assert_matches!( + &events[..], + [ + ResponseEvent::OutputItemDone(ResponseItem::FunctionCall { call_id, name, arguments, .. }), + ResponseEvent::Completed { .. } + ] if call_id == "call_a" && name == "do_a" && arguments == "{ \"foo\":1}" + ); + } + + #[tokio::test] + async fn emits_tool_calls_even_when_content_and_reasoning_present() { + let delta_content_and_tools = json!({ + "choices": [{ + "delta": { + "content": [{"text": "hi"}], + "reasoning": "because", + "tool_calls": [{ + "id": "call_a", + "function": { "name": "do_a", "arguments": "{}" } + }] + } + }] + }); + + let finish = json!({ + "choices": [{ + "finish_reason": "tool_calls" + }] + }); + + let body = build_body(&[delta_content_and_tools, finish]); + let events = collect_events(&body).await; + + assert_matches!( + &events[..], + [ + ResponseEvent::OutputItemAdded(ResponseItem::Reasoning { .. }), + ResponseEvent::ReasoningContentDelta { .. }, + ResponseEvent::OutputItemAdded(ResponseItem::Message { .. }), + ResponseEvent::OutputTextDelta(delta), + ResponseEvent::OutputItemDone(ResponseItem::Reasoning { .. }), + ResponseEvent::OutputItemDone(ResponseItem::FunctionCall { call_id, name, .. }), + ResponseEvent::OutputItemDone(ResponseItem::Message { .. }), + ResponseEvent::Completed { .. } + ] if delta == "hi" && call_id == "call_a" && name == "do_a" + ); + } + + #[tokio::test] + async fn drops_partial_tool_calls_on_stop_finish_reason() { + let delta_tool = json!({ + "choices": [{ + "delta": { + "tool_calls": [{ + "id": "call_a", + "function": { "name": "do_a", "arguments": "{}" } + }] + } + }] + }); + + let finish_stop = json!({ + "choices": [{ + "finish_reason": "stop" + }] + }); + + let body = build_body(&[delta_tool, finish_stop]); + let events = collect_events(&body).await; + + assert!(!events.iter().any(|ev| { + matches!( + ev, + ResponseEvent::OutputItemDone(ResponseItem::FunctionCall { .. }) + ) + })); + assert_matches!(events.last(), Some(ResponseEvent::Completed { .. })); + } +} diff --git a/codex-rs/codex-api/src/sse/mod.rs b/codex-rs/codex-api/src/sse/mod.rs new file mode 100644 index 0000000000..e3ab770c43 --- /dev/null +++ b/codex-rs/codex-api/src/sse/mod.rs @@ -0,0 +1,6 @@ +pub mod chat; +pub mod responses; + +pub use responses::process_sse; +pub use responses::spawn_response_stream; +pub use responses::stream_from_fixture; diff --git a/codex-rs/codex-api/src/sse/responses.rs b/codex-rs/codex-api/src/sse/responses.rs new file mode 100644 index 0000000000..5dbec7b77a --- /dev/null +++ b/codex-rs/codex-api/src/sse/responses.rs @@ -0,0 +1,672 @@ +use crate::common::ResponseEvent; +use crate::common::ResponseStream; +use crate::error::ApiError; +use crate::rate_limits::parse_rate_limit; +use crate::telemetry::SseTelemetry; +use codex_client::ByteStream; +use codex_client::StreamResponse; +use codex_client::TransportError; +use codex_protocol::models::ResponseItem; +use codex_protocol::protocol::TokenUsage; +use eventsource_stream::Eventsource; +use futures::StreamExt; +use futures::TryStreamExt; +use serde::Deserialize; +use serde_json::Value; +use std::io::BufRead; +use std::path::Path; +use std::sync::Arc; +use std::time::Duration; +use tokio::sync::mpsc; +use tokio::time::Instant; +use tokio::time::timeout; +use tokio_util::io::ReaderStream; +use tracing::debug; +use tracing::trace; + +/// Streams SSE events from an on-disk fixture for tests. +pub fn stream_from_fixture( + path: impl AsRef, + idle_timeout: Duration, +) -> Result { + let file = + std::fs::File::open(path.as_ref()).map_err(|err| ApiError::Stream(err.to_string()))?; + let mut content = String::new(); + for line in std::io::BufReader::new(file).lines() { + let line = line.map_err(|err| ApiError::Stream(err.to_string()))?; + content.push_str(&line); + content.push_str("\n\n"); + } + + let reader = std::io::Cursor::new(content); + let stream = ReaderStream::new(reader).map_err(|err| TransportError::Network(err.to_string())); + let (tx_event, rx_event) = mpsc::channel::>(1600); + tokio::spawn(process_sse(Box::pin(stream), tx_event, idle_timeout, None)); + Ok(ResponseStream { rx_event }) +} + +pub fn spawn_response_stream( + stream_response: StreamResponse, + idle_timeout: Duration, + telemetry: Option>, +) -> ResponseStream { + let rate_limits = parse_rate_limit(&stream_response.headers); + let (tx_event, rx_event) = mpsc::channel::>(1600); + tokio::spawn(async move { + if let Some(snapshot) = rate_limits { + let _ = tx_event.send(Ok(ResponseEvent::RateLimits(snapshot))).await; + } + process_sse(stream_response.bytes, tx_event, idle_timeout, telemetry).await; + }); + + ResponseStream { rx_event } +} + +#[derive(Debug, Deserialize)] +#[allow(dead_code)] +struct Error { + r#type: Option, + code: Option, + message: Option, + plan_type: Option, + resets_at: Option, +} + +#[derive(Debug, Deserialize)] +#[allow(dead_code)] +struct ResponseCompleted { + id: String, + #[serde(default)] + usage: Option, +} + +#[derive(Debug, Deserialize)] +struct ResponseCompletedUsage { + input_tokens: i64, + input_tokens_details: Option, + output_tokens: i64, + output_tokens_details: Option, + total_tokens: i64, +} + +impl From for TokenUsage { + fn from(val: ResponseCompletedUsage) -> Self { + TokenUsage { + input_tokens: val.input_tokens, + cached_input_tokens: val + .input_tokens_details + .map(|d| d.cached_tokens) + .unwrap_or(0), + output_tokens: val.output_tokens, + reasoning_output_tokens: val + .output_tokens_details + .map(|d| d.reasoning_tokens) + .unwrap_or(0), + total_tokens: val.total_tokens, + } + } +} + +#[derive(Debug, Deserialize)] +struct ResponseCompletedInputTokensDetails { + cached_tokens: i64, +} + +#[derive(Debug, Deserialize)] +struct ResponseCompletedOutputTokensDetails { + reasoning_tokens: i64, +} + +#[derive(Deserialize, Debug)] +struct SseEvent { + #[serde(rename = "type")] + kind: String, + response: Option, + item: Option, + delta: Option, + summary_index: Option, + content_index: Option, +} + +pub async fn process_sse( + stream: ByteStream, + tx_event: mpsc::Sender>, + idle_timeout: Duration, + telemetry: Option>, +) { + let mut stream = stream.eventsource(); + let mut response_completed: Option = None; + let mut response_error: Option = None; + + loop { + let start = Instant::now(); + let response = timeout(idle_timeout, stream.next()).await; + if let Some(t) = telemetry.as_ref() { + t.on_sse_poll(&response, start.elapsed()); + } + let sse = match response { + Ok(Some(Ok(sse))) => sse, + Ok(Some(Err(e))) => { + debug!("SSE Error: {e:#}"); + let _ = tx_event.send(Err(ApiError::Stream(e.to_string()))).await; + return; + } + Ok(None) => { + match response_completed.take() { + Some(ResponseCompleted { id, usage }) => { + let event = ResponseEvent::Completed { + response_id: id, + token_usage: usage.map(Into::into), + }; + let _ = tx_event.send(Ok(event)).await; + } + None => { + let error = response_error.unwrap_or(ApiError::Stream( + "stream closed before response.completed".into(), + )); + let _ = tx_event.send(Err(error)).await; + } + } + return; + } + Err(_) => { + let _ = tx_event + .send(Err(ApiError::Stream("idle timeout waiting for SSE".into()))) + .await; + return; + } + }; + + let raw = sse.data.clone(); + trace!("SSE event: {raw}"); + + let event: SseEvent = match serde_json::from_str(&sse.data) { + Ok(event) => event, + Err(e) => { + debug!("Failed to parse SSE event: {e}, data: {}", &sse.data); + continue; + } + }; + + match event.kind.as_str() { + "response.output_item.done" => { + let Some(item_val) = event.item else { continue }; + let Ok(item) = serde_json::from_value::(item_val) else { + debug!("failed to parse ResponseItem from output_item.done"); + continue; + }; + + let event = ResponseEvent::OutputItemDone(item); + if tx_event.send(Ok(event)).await.is_err() { + return; + } + } + "response.output_text.delta" => { + if let Some(delta) = event.delta { + let event = ResponseEvent::OutputTextDelta(delta); + if tx_event.send(Ok(event)).await.is_err() { + return; + } + } + } + "response.reasoning_summary_text.delta" => { + if let (Some(delta), Some(summary_index)) = (event.delta, event.summary_index) { + let event = ResponseEvent::ReasoningSummaryDelta { + delta, + summary_index, + }; + if tx_event.send(Ok(event)).await.is_err() { + return; + } + } + } + "response.reasoning_text.delta" => { + if let (Some(delta), Some(content_index)) = (event.delta, event.content_index) { + let event = ResponseEvent::ReasoningContentDelta { + delta, + content_index, + }; + if tx_event.send(Ok(event)).await.is_err() { + return; + } + } + } + "response.created" => { + if event.response.is_some() { + let _ = tx_event.send(Ok(ResponseEvent::Created {})).await; + } + } + "response.failed" => { + if let Some(resp_val) = event.response { + response_error = + Some(ApiError::Stream("response.failed event received".into())); + + if let Some(error) = resp_val.get("error") + && let Ok(error) = serde_json::from_value::(error.clone()) + { + if is_context_window_error(&error) { + response_error = Some(ApiError::ContextWindowExceeded); + } else if is_quota_exceeded_error(&error) { + response_error = Some(ApiError::QuotaExceeded); + } else if is_usage_not_included(&error) { + response_error = Some(ApiError::UsageNotIncluded); + } else { + let delay = try_parse_retry_after(&error); + let message = error.message.clone().unwrap_or_default(); + response_error = Some(ApiError::Retryable { message, delay }); + } + } + } + } + "response.completed" => { + if let Some(resp_val) = event.response { + match serde_json::from_value::(resp_val) { + Ok(r) => { + response_completed = Some(r); + } + Err(e) => { + let error = format!("failed to parse ResponseCompleted: {e}"); + debug!(error); + response_error = Some(ApiError::Stream(error)); + continue; + } + }; + }; + } + "response.output_item.added" => { + let Some(item_val) = event.item else { continue }; + let Ok(item) = serde_json::from_value::(item_val) else { + debug!("failed to parse ResponseItem from output_item.done"); + continue; + }; + + let event = ResponseEvent::OutputItemAdded(item); + if tx_event.send(Ok(event)).await.is_err() { + return; + } + } + "response.reasoning_summary_part.added" => { + if let Some(summary_index) = event.summary_index { + let event = ResponseEvent::ReasoningSummaryPartAdded { summary_index }; + if tx_event.send(Ok(event)).await.is_err() { + return; + } + } + } + _ => {} + } + } +} + +fn try_parse_retry_after(err: &Error) -> Option { + if err.code.as_deref() != Some("rate_limit_exceeded") { + return None; + } + + let re = rate_limit_regex(); + if let Some(message) = &err.message + && let Some(captures) = re.captures(message) + { + let seconds = captures.get(1); + let unit = captures.get(2); + + if let (Some(value), Some(unit)) = (seconds, unit) { + let value = value.as_str().parse::().ok()?; + let unit = unit.as_str().to_ascii_lowercase(); + + if unit == "s" || unit.starts_with("second") { + return Some(Duration::from_secs_f64(value)); + } else if unit == "ms" { + return Some(Duration::from_millis(value as u64)); + } + } + } + None +} + +fn is_context_window_error(error: &Error) -> bool { + error.code.as_deref() == Some("context_length_exceeded") +} + +fn is_quota_exceeded_error(error: &Error) -> bool { + error.code.as_deref() == Some("insufficient_quota") +} + +fn is_usage_not_included(error: &Error) -> bool { + error.code.as_deref() == Some("usage_not_included") +} + +fn rate_limit_regex() -> &'static regex_lite::Regex { + static RE: std::sync::OnceLock = std::sync::OnceLock::new(); + #[expect(clippy::unwrap_used)] + RE.get_or_init(|| { + regex_lite::Regex::new(r"(?i)try again in\s*(\d+(?:\.\d+)?)\s*(s|ms|seconds?)").unwrap() + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use assert_matches::assert_matches; + use codex_protocol::models::ResponseItem; + use pretty_assertions::assert_eq; + use serde_json::json; + use tokio::sync::mpsc; + use tokio_test::io::Builder as IoBuilder; + + async fn collect_events(chunks: &[&[u8]]) -> Vec> { + let mut builder = IoBuilder::new(); + for chunk in chunks { + builder.read(chunk); + } + + let reader = builder.build(); + let stream = + ReaderStream::new(reader).map_err(|err| TransportError::Network(err.to_string())); + let (tx, mut rx) = mpsc::channel::>(16); + tokio::spawn(process_sse(Box::pin(stream), tx, idle_timeout(), None)); + + let mut events = Vec::new(); + while let Some(ev) = rx.recv().await { + events.push(ev); + } + events + } + + async fn run_sse(events: Vec) -> Vec { + let mut body = String::new(); + for e in events { + let kind = e + .get("type") + .and_then(|v| v.as_str()) + .expect("fixture event missing type"); + if e.as_object().map(|o| o.len() == 1).unwrap_or(false) { + body.push_str(&format!("event: {kind}\n\n")); + } else { + body.push_str(&format!("event: {kind}\ndata: {e}\n\n")); + } + } + + let (tx, mut rx) = mpsc::channel::>(8); + let stream = ReaderStream::new(std::io::Cursor::new(body)) + .map_err(|err| TransportError::Network(err.to_string())); + tokio::spawn(process_sse(Box::pin(stream), tx, idle_timeout(), None)); + + let mut out = Vec::new(); + while let Some(ev) = rx.recv().await { + out.push(ev.expect("channel closed")); + } + out + } + + fn idle_timeout() -> Duration { + Duration::from_millis(1000) + } + + #[tokio::test] + async fn parses_items_and_completed() { + let item1 = json!({ + "type": "response.output_item.done", + "item": { + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": "Hello"}] + } + }) + .to_string(); + + let item2 = json!({ + "type": "response.output_item.done", + "item": { + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": "World"}] + } + }) + .to_string(); + + let completed = json!({ + "type": "response.completed", + "response": { "id": "resp1" } + }) + .to_string(); + + let sse1 = format!("event: response.output_item.done\ndata: {item1}\n\n"); + let sse2 = format!("event: response.output_item.done\ndata: {item2}\n\n"); + let sse3 = format!("event: response.completed\ndata: {completed}\n\n"); + + let events = collect_events(&[sse1.as_bytes(), sse2.as_bytes(), sse3.as_bytes()]).await; + + assert_eq!(events.len(), 3); + + assert_matches!( + &events[0], + Ok(ResponseEvent::OutputItemDone(ResponseItem::Message { role, .. })) + if role == "assistant" + ); + + assert_matches!( + &events[1], + Ok(ResponseEvent::OutputItemDone(ResponseItem::Message { role, .. })) + if role == "assistant" + ); + + match &events[2] { + Ok(ResponseEvent::Completed { + response_id, + token_usage, + }) => { + assert_eq!(response_id, "resp1"); + assert!(token_usage.is_none()); + } + other => panic!("unexpected third event: {other:?}"), + } + } + + #[tokio::test] + async fn error_when_missing_completed() { + let item1 = json!({ + "type": "response.output_item.done", + "item": { + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": "Hello"}] + } + }) + .to_string(); + + let sse1 = format!("event: response.output_item.done\ndata: {item1}\n\n"); + + let events = collect_events(&[sse1.as_bytes()]).await; + + assert_eq!(events.len(), 2); + + assert_matches!(events[0], Ok(ResponseEvent::OutputItemDone(_))); + + match &events[1] { + Err(ApiError::Stream(msg)) => { + assert_eq!(msg, "stream closed before response.completed") + } + other => panic!("unexpected second event: {other:?}"), + } + } + + #[tokio::test] + async fn error_when_error_event() { + let raw_error = r#"{"type":"response.failed","sequence_number":3,"response":{"id":"resp_689bcf18d7f08194bf3440ba62fe05d803fee0cdac429894","object":"response","created_at":1755041560,"status":"failed","background":false,"error":{"code":"rate_limit_exceeded","message":"Rate limit reached for gpt-5.1 in organization org-AAA on tokens per min (TPM): Limit 30000, Used 22999, Requested 12528. Please try again in 11.054s. Visit https://platform.openai.com/account/rate-limits to learn more."}, "usage":null,"user":null,"metadata":{}}}"#; + + let sse1 = format!("event: response.failed\ndata: {raw_error}\n\n"); + + let events = collect_events(&[sse1.as_bytes()]).await; + + assert_eq!(events.len(), 1); + + match &events[0] { + Err(ApiError::Retryable { message, delay }) => { + assert_eq!( + message, + "Rate limit reached for gpt-5.1 in organization org-AAA on tokens per min (TPM): Limit 30000, Used 22999, Requested 12528. Please try again in 11.054s. Visit https://platform.openai.com/account/rate-limits to learn more." + ); + assert_eq!(*delay, Some(Duration::from_secs_f64(11.054))); + } + other => panic!("unexpected second event: {other:?}"), + } + } + + #[tokio::test] + async fn context_window_error_is_fatal() { + let raw_error = r#"{"type":"response.failed","sequence_number":3,"response":{"id":"resp_5c66275b97b9baef1ed95550adb3b7ec13b17aafd1d2f11b","object":"response","created_at":1759510079,"status":"failed","background":false,"error":{"code":"context_length_exceeded","message":"Your input exceeds the context window of this model. Please adjust your input and try again."},"usage":null,"user":null,"metadata":{}}}"#; + + let sse1 = format!("event: response.failed\ndata: {raw_error}\n\n"); + + let events = collect_events(&[sse1.as_bytes()]).await; + + assert_eq!(events.len(), 1); + + assert_matches!(events[0], Err(ApiError::ContextWindowExceeded)); + } + + #[tokio::test] + async fn context_window_error_with_newline_is_fatal() { + let raw_error = r#"{"type":"response.failed","sequence_number":4,"response":{"id":"resp_fatal_newline","object":"response","created_at":1759510080,"status":"failed","background":false,"error":{"code":"context_length_exceeded","message":"Your input exceeds the context window of this model. Please adjust your input and try\nagain."},"usage":null,"user":null,"metadata":{}}}"#; + + let sse1 = format!("event: response.failed\ndata: {raw_error}\n\n"); + + let events = collect_events(&[sse1.as_bytes()]).await; + + assert_eq!(events.len(), 1); + + assert_matches!(events[0], Err(ApiError::ContextWindowExceeded)); + } + + #[tokio::test] + async fn quota_exceeded_error_is_fatal() { + let raw_error = r#"{"type":"response.failed","sequence_number":3,"response":{"id":"resp_fatal_quota","object":"response","created_at":1759771626,"status":"failed","background":false,"error":{"code":"insufficient_quota","message":"You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."},"incomplete_details":null}}"#; + + let sse1 = format!("event: response.failed\ndata: {raw_error}\n\n"); + + let events = collect_events(&[sse1.as_bytes()]).await; + + assert_eq!(events.len(), 1); + + assert_matches!(events[0], Err(ApiError::QuotaExceeded)); + } + + #[tokio::test] + async fn table_driven_event_kinds() { + struct TestCase { + name: &'static str, + event: serde_json::Value, + expect_first: fn(&ResponseEvent) -> bool, + expected_len: usize, + } + + fn is_created(ev: &ResponseEvent) -> bool { + matches!(ev, ResponseEvent::Created) + } + fn is_output(ev: &ResponseEvent) -> bool { + matches!(ev, ResponseEvent::OutputItemDone(_)) + } + fn is_completed(ev: &ResponseEvent) -> bool { + matches!(ev, ResponseEvent::Completed { .. }) + } + + let completed = json!({ + "type": "response.completed", + "response": { + "id": "c", + "usage": { + "input_tokens": 0, + "input_tokens_details": null, + "output_tokens": 0, + "output_tokens_details": null, + "total_tokens": 0 + }, + "output": [] + } + }); + + let cases = vec![ + TestCase { + name: "created", + event: json!({"type": "response.created", "response": {}}), + expect_first: is_created, + expected_len: 2, + }, + TestCase { + name: "output_item.done", + event: json!({ + "type": "response.output_item.done", + "item": { + "type": "message", + "role": "assistant", + "content": [ + {"type": "output_text", "text": "hi"} + ] + } + }), + expect_first: is_output, + expected_len: 2, + }, + TestCase { + name: "unknown", + event: json!({"type": "response.new_tool_event"}), + expect_first: is_completed, + expected_len: 1, + }, + ]; + + for case in cases { + let mut evs = vec![case.event]; + evs.push(completed.clone()); + + let out = run_sse(evs).await; + assert_eq!(out.len(), case.expected_len, "case {}", case.name); + assert!( + (case.expect_first)(&out[0]), + "first event mismatch in case {}", + case.name + ); + } + } + + #[test] + fn test_try_parse_retry_after() { + let err = Error { + r#type: None, + message: Some("Rate limit reached for gpt-5.1 in organization org- on tokens per min (TPM): Limit 1, Used 1, Requested 19304. Please try again in 28ms. Visit https://platform.openai.com/account/rate-limits to learn more.".to_string()), + code: Some("rate_limit_exceeded".to_string()), + plan_type: None, + resets_at: None, + }; + + let delay = try_parse_retry_after(&err); + assert_eq!(delay, Some(Duration::from_millis(28))); + } + + #[test] + fn test_try_parse_retry_after_no_delay() { + let err = Error { + r#type: None, + message: Some("Rate limit reached for gpt-5.1 in organization on tokens per min (TPM): Limit 30000, Used 6899, Requested 24050. Please try again in 1.898s. Visit https://platform.openai.com/account/rate-limits to learn more.".to_string()), + code: Some("rate_limit_exceeded".to_string()), + plan_type: None, + resets_at: None, + }; + let delay = try_parse_retry_after(&err); + assert_eq!(delay, Some(Duration::from_secs_f64(1.898))); + } + + #[test] + fn test_try_parse_retry_after_azure() { + let err = Error { + r#type: None, + message: Some("Rate limit exceeded. Try again in 35 seconds.".to_string()), + code: Some("rate_limit_exceeded".to_string()), + plan_type: None, + resets_at: None, + }; + let delay = try_parse_retry_after(&err); + assert_eq!(delay, Some(Duration::from_secs(35))); + } +} diff --git a/codex-rs/codex-api/src/telemetry.rs b/codex-rs/codex-api/src/telemetry.rs new file mode 100644 index 0000000000..d6a38b2af3 --- /dev/null +++ b/codex-rs/codex-api/src/telemetry.rs @@ -0,0 +1,84 @@ +use codex_client::Request; +use codex_client::RequestTelemetry; +use codex_client::Response; +use codex_client::RetryPolicy; +use codex_client::StreamResponse; +use codex_client::TransportError; +use codex_client::run_with_retry; +use http::StatusCode; +use std::future::Future; +use std::sync::Arc; +use std::time::Duration; +use tokio::time::Instant; + +/// Generic telemetry. +pub trait SseTelemetry: Send + Sync { + fn on_sse_poll( + &self, + result: &Result< + Option< + Result< + eventsource_stream::Event, + eventsource_stream::EventStreamError, + >, + >, + tokio::time::error::Elapsed, + >, + duration: Duration, + ); +} + +pub(crate) trait WithStatus { + fn status(&self) -> StatusCode; +} + +fn http_status(err: &TransportError) -> Option { + match err { + TransportError::Http { status, .. } => Some(*status), + _ => None, + } +} + +impl WithStatus for Response { + fn status(&self) -> StatusCode { + self.status + } +} + +impl WithStatus for StreamResponse { + fn status(&self) -> StatusCode { + self.status + } +} + +pub(crate) async fn run_with_request_telemetry( + policy: RetryPolicy, + telemetry: Option>, + make_request: impl FnMut() -> Request, + send: F, +) -> Result +where + T: WithStatus, + F: Clone + Fn(Request) -> Fut, + Fut: Future>, +{ + // Wraps `run_with_retry` to attach per-attempt request telemetry for both + // unary and streaming HTTP calls. + run_with_retry(policy, make_request, move |req, attempt| { + let telemetry = telemetry.clone(); + let send = send.clone(); + async move { + let start = Instant::now(); + let result = send(req).await; + if let Some(t) = telemetry.as_ref() { + let (status, err) = match &result { + Ok(resp) => (Some(resp.status()), None), + Err(err) => (http_status(err), Some(err)), + }; + t.on_request(attempt, status, err, start.elapsed()); + } + result + } + }) + .await +} diff --git a/codex-rs/codex-api/tests/clients.rs b/codex-rs/codex-api/tests/clients.rs new file mode 100644 index 0000000000..3dafaf74fa --- /dev/null +++ b/codex-rs/codex-api/tests/clients.rs @@ -0,0 +1,315 @@ +use std::sync::Arc; +use std::sync::Mutex; +use std::time::Duration; + +use anyhow::Result; +use async_trait::async_trait; +use bytes::Bytes; +use codex_api::AuthProvider; +use codex_api::ChatClient; +use codex_api::Provider; +use codex_api::ResponsesClient; +use codex_api::ResponsesOptions; +use codex_api::WireApi; +use codex_client::HttpTransport; +use codex_client::Request; +use codex_client::Response; +use codex_client::StreamResponse; +use codex_client::TransportError; +use codex_protocol::models::ContentItem; +use codex_protocol::models::ResponseItem; +use http::HeaderMap; +use http::StatusCode; +use pretty_assertions::assert_eq; +use serde_json::Value; + +fn assert_path_ends_with(requests: &[Request], suffix: &str) { + assert_eq!(requests.len(), 1); + let url = &requests[0].url; + assert!( + url.ends_with(suffix), + "expected url to end with {suffix}, got {url}" + ); +} + +#[derive(Debug, Default, Clone)] +struct RecordingState { + stream_requests: Arc>>, +} + +impl RecordingState { + fn record(&self, req: Request) { + let mut guard = self + .stream_requests + .lock() + .unwrap_or_else(|err| panic!("mutex poisoned: {err}")); + guard.push(req); + } + + fn take_stream_requests(&self) -> Vec { + let mut guard = self + .stream_requests + .lock() + .unwrap_or_else(|err| panic!("mutex poisoned: {err}")); + std::mem::take(&mut *guard) + } +} + +#[derive(Clone)] +struct RecordingTransport { + state: RecordingState, +} + +impl RecordingTransport { + fn new(state: RecordingState) -> Self { + Self { state } + } +} + +#[async_trait] +impl HttpTransport for RecordingTransport { + async fn execute(&self, _req: Request) -> Result { + Err(TransportError::Build("execute should not run".to_string())) + } + + async fn stream(&self, req: Request) -> Result { + self.state.record(req); + + let stream = futures::stream::iter(Vec::>::new()); + Ok(StreamResponse { + status: StatusCode::OK, + headers: HeaderMap::new(), + bytes: Box::pin(stream), + }) + } +} + +#[derive(Clone, Default)] +struct NoAuth; + +impl AuthProvider for NoAuth { + fn bearer_token(&self) -> Option { + None + } +} + +#[derive(Clone)] +struct StaticAuth { + token: String, + account_id: String, +} + +impl StaticAuth { + fn new(token: &str, account_id: &str) -> Self { + Self { + token: token.to_string(), + account_id: account_id.to_string(), + } + } +} + +impl AuthProvider for StaticAuth { + fn bearer_token(&self) -> Option { + Some(self.token.clone()) + } + + fn account_id(&self) -> Option { + Some(self.account_id.clone()) + } +} + +fn provider(name: &str, wire: WireApi) -> Provider { + Provider { + name: name.to_string(), + base_url: "https://example.com/v1".to_string(), + query_params: None, + wire, + headers: HeaderMap::new(), + retry: codex_api::provider::RetryConfig { + max_attempts: 1, + base_delay: Duration::from_millis(1), + retry_429: false, + retry_5xx: false, + retry_transport: true, + }, + stream_idle_timeout: Duration::from_millis(10), + } +} + +#[derive(Clone)] +struct FlakyTransport { + state: Arc>, +} + +impl Default for FlakyTransport { + fn default() -> Self { + Self::new() + } +} + +impl FlakyTransport { + fn new() -> Self { + Self { + state: Arc::new(Mutex::new(0)), + } + } + + fn attempts(&self) -> i64 { + *self + .state + .lock() + .unwrap_or_else(|err| panic!("mutex poisoned: {err}")) + } +} + +#[async_trait] +impl HttpTransport for FlakyTransport { + async fn execute(&self, _req: Request) -> Result { + Err(TransportError::Build("execute should not run".to_string())) + } + + async fn stream(&self, _req: Request) -> Result { + let mut attempts = self + .state + .lock() + .unwrap_or_else(|err| panic!("mutex poisoned: {err}")); + *attempts += 1; + + if *attempts == 1 { + return Err(TransportError::Network("first attempt fails".to_string())); + } + + let stream = futures::stream::iter(vec![Ok(Bytes::from( + r#"event: message +data: {"id":"resp-1","output":[{"type":"message","role":"assistant","content":[{"type":"output_text","text":"hi"}]}]} + +"#, + ))]); + + Ok(StreamResponse { + status: StatusCode::OK, + headers: HeaderMap::new(), + bytes: Box::pin(stream), + }) + } +} + +#[tokio::test] +async fn chat_client_uses_chat_completions_path_for_chat_wire() -> Result<()> { + let state = RecordingState::default(); + let transport = RecordingTransport::new(state.clone()); + let client = ChatClient::new(transport, provider("openai", WireApi::Chat), NoAuth); + + let body = serde_json::json!({ "echo": true }); + let _stream = client.stream(body, HeaderMap::new()).await?; + + let requests = state.take_stream_requests(); + assert_path_ends_with(&requests, "/chat/completions"); + Ok(()) +} + +#[tokio::test] +async fn chat_client_uses_responses_path_for_responses_wire() -> Result<()> { + let state = RecordingState::default(); + let transport = RecordingTransport::new(state.clone()); + let client = ChatClient::new(transport, provider("openai", WireApi::Responses), NoAuth); + + let body = serde_json::json!({ "echo": true }); + let _stream = client.stream(body, HeaderMap::new()).await?; + + let requests = state.take_stream_requests(); + assert_path_ends_with(&requests, "/responses"); + Ok(()) +} + +#[tokio::test] +async fn responses_client_uses_responses_path_for_responses_wire() -> Result<()> { + let state = RecordingState::default(); + let transport = RecordingTransport::new(state.clone()); + let client = ResponsesClient::new(transport, provider("openai", WireApi::Responses), NoAuth); + + let body = serde_json::json!({ "echo": true }); + let _stream = client.stream(body, HeaderMap::new()).await?; + + let requests = state.take_stream_requests(); + assert_path_ends_with(&requests, "/responses"); + Ok(()) +} + +#[tokio::test] +async fn responses_client_uses_chat_path_for_chat_wire() -> Result<()> { + let state = RecordingState::default(); + let transport = RecordingTransport::new(state.clone()); + let client = ResponsesClient::new(transport, provider("openai", WireApi::Chat), NoAuth); + + let body = serde_json::json!({ "echo": true }); + let _stream = client.stream(body, HeaderMap::new()).await?; + + let requests = state.take_stream_requests(); + assert_path_ends_with(&requests, "/chat/completions"); + Ok(()) +} + +#[tokio::test] +async fn streaming_client_adds_auth_headers() -> Result<()> { + let state = RecordingState::default(); + let transport = RecordingTransport::new(state.clone()); + let auth = StaticAuth::new("secret-token", "acct-1"); + let client = ResponsesClient::new(transport, provider("openai", WireApi::Responses), auth); + + let body = serde_json::json!({ "model": "gpt-test" }); + let _stream = client.stream(body, HeaderMap::new()).await?; + + let requests = state.take_stream_requests(); + assert_eq!(requests.len(), 1); + let req = &requests[0]; + + let auth_header = req.headers.get(http::header::AUTHORIZATION); + assert!(auth_header.is_some(), "missing auth header"); + assert_eq!( + auth_header.unwrap().to_str().ok(), + Some("Bearer secret-token") + ); + + let account_header = req.headers.get("ChatGPT-Account-ID"); + assert!(account_header.is_some(), "missing account header"); + assert_eq!(account_header.unwrap().to_str().ok(), Some("acct-1")); + + let accept_header = req.headers.get(http::header::ACCEPT); + assert!(accept_header.is_some(), "missing Accept header"); + assert_eq!( + accept_header.unwrap().to_str().ok(), + Some("text/event-stream") + ); + Ok(()) +} + +#[tokio::test] +async fn streaming_client_retries_on_transport_error() -> Result<()> { + let transport = FlakyTransport::new(); + + let mut provider = provider("openai", WireApi::Responses); + provider.retry.max_attempts = 2; + + let client = ResponsesClient::new(transport.clone(), provider, NoAuth); + + let prompt = codex_api::Prompt { + instructions: "Say hi".to_string(), + input: vec![ResponseItem::Message { + id: None, + role: "user".to_string(), + content: vec![ContentItem::InputText { + text: "hi".to_string(), + }], + }], + tools: Vec::::new(), + parallel_tool_calls: false, + output_schema: None, + }; + + let options = ResponsesOptions::default(); + + let _stream = client.stream_prompt("gpt-test", &prompt, options).await?; + assert_eq!(transport.attempts(), 2); + Ok(()) +} diff --git a/codex-rs/codex-api/tests/sse_end_to_end.rs b/codex-rs/codex-api/tests/sse_end_to_end.rs new file mode 100644 index 0000000000..b91cf3a5d8 --- /dev/null +++ b/codex-rs/codex-api/tests/sse_end_to_end.rs @@ -0,0 +1,229 @@ +use std::time::Duration; + +use anyhow::Result; +use async_trait::async_trait; +use bytes::Bytes; +use codex_api::AggregateStreamExt; +use codex_api::AuthProvider; +use codex_api::Provider; +use codex_api::ResponseEvent; +use codex_api::ResponsesClient; +use codex_api::WireApi; +use codex_client::HttpTransport; +use codex_client::Request; +use codex_client::Response; +use codex_client::StreamResponse; +use codex_client::TransportError; +use codex_protocol::models::ContentItem; +use codex_protocol::models::ResponseItem; +use futures::StreamExt; +use http::HeaderMap; +use http::StatusCode; +use pretty_assertions::assert_eq; +use serde_json::Value; + +#[derive(Clone)] +struct FixtureSseTransport { + body: String, +} + +impl FixtureSseTransport { + fn new(body: String) -> Self { + Self { body } + } +} + +#[async_trait] +impl HttpTransport for FixtureSseTransport { + async fn execute(&self, _req: Request) -> Result { + Err(TransportError::Build("execute should not run".to_string())) + } + + async fn stream(&self, _req: Request) -> Result { + let stream = futures::stream::iter(vec![Ok::(Bytes::from( + self.body.clone(), + ))]); + Ok(StreamResponse { + status: StatusCode::OK, + headers: HeaderMap::new(), + bytes: Box::pin(stream), + }) + } +} + +#[derive(Clone, Default)] +struct NoAuth; + +impl AuthProvider for NoAuth { + fn bearer_token(&self) -> Option { + None + } +} + +fn provider(name: &str, wire: WireApi) -> Provider { + Provider { + name: name.to_string(), + base_url: "https://example.com/v1".to_string(), + query_params: None, + wire, + headers: HeaderMap::new(), + retry: codex_api::provider::RetryConfig { + max_attempts: 1, + base_delay: Duration::from_millis(1), + retry_429: false, + retry_5xx: false, + retry_transport: true, + }, + stream_idle_timeout: Duration::from_millis(50), + } +} + +fn build_responses_body(events: Vec) -> String { + let mut body = String::new(); + for e in events { + let kind = e + .get("type") + .and_then(|v| v.as_str()) + .unwrap_or_else(|| panic!("fixture event missing type in SSE fixture: {e}")); + if e.as_object().map(|o| o.len() == 1).unwrap_or(false) { + body.push_str(&format!("event: {kind}\n\n")); + } else { + body.push_str(&format!("event: {kind}\ndata: {e}\n\n")); + } + } + body +} + +#[tokio::test] +async fn responses_stream_parses_items_and_completed_end_to_end() -> Result<()> { + let item1 = serde_json::json!({ + "type": "response.output_item.done", + "item": { + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": "Hello"}] + } + }); + + let item2 = serde_json::json!({ + "type": "response.output_item.done", + "item": { + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": "World"}] + } + }); + + let completed = serde_json::json!({ + "type": "response.completed", + "response": { "id": "resp1" } + }); + + let body = build_responses_body(vec![item1, item2, completed]); + let transport = FixtureSseTransport::new(body); + let client = ResponsesClient::new(transport, provider("openai", WireApi::Responses), NoAuth); + + let mut stream = client + .stream(serde_json::json!({"echo": true}), HeaderMap::new()) + .await?; + + let mut events = Vec::new(); + while let Some(ev) = stream.next().await { + events.push(ev?); + } + + let events: Vec = events + .into_iter() + .filter(|ev| !matches!(ev, ResponseEvent::RateLimits(_))) + .collect(); + + assert_eq!(events.len(), 3); + + match &events[0] { + ResponseEvent::OutputItemDone(ResponseItem::Message { role, .. }) => { + assert_eq!(role, "assistant"); + } + other => panic!("unexpected first event: {other:?}"), + } + + match &events[1] { + ResponseEvent::OutputItemDone(ResponseItem::Message { role, .. }) => { + assert_eq!(role, "assistant"); + } + other => panic!("unexpected second event: {other:?}"), + } + + match &events[2] { + ResponseEvent::Completed { + response_id, + token_usage, + } => { + assert_eq!(response_id, "resp1"); + assert!(token_usage.is_none()); + } + other => panic!("unexpected third event: {other:?}"), + } + + Ok(()) +} + +#[tokio::test] +async fn responses_stream_aggregates_output_text_deltas() -> Result<()> { + let delta1 = serde_json::json!({ + "type": "response.output_text.delta", + "delta": "Hello, " + }); + + let delta2 = serde_json::json!({ + "type": "response.output_text.delta", + "delta": "world" + }); + + let completed = serde_json::json!({ + "type": "response.completed", + "response": { "id": "resp-agg" } + }); + + let body = build_responses_body(vec![delta1, delta2, completed]); + let transport = FixtureSseTransport::new(body); + let client = ResponsesClient::new(transport, provider("openai", WireApi::Responses), NoAuth); + + let stream = client + .stream(serde_json::json!({"echo": true}), HeaderMap::new()) + .await?; + + let mut stream = stream.aggregate(); + let mut events = Vec::new(); + while let Some(ev) = stream.next().await { + events.push(ev?); + } + + let events: Vec = events + .into_iter() + .filter(|ev| !matches!(ev, ResponseEvent::RateLimits(_))) + .collect(); + + assert_eq!(events.len(), 2); + + match &events[0] { + ResponseEvent::OutputItemDone(ResponseItem::Message { content, .. }) => { + let mut aggregated = String::new(); + for item in content { + if let ContentItem::OutputText { text } = item { + aggregated.push_str(text); + } + } + assert_eq!(aggregated, "Hello, world"); + } + other => panic!("unexpected first event: {other:?}"), + } + + match &events[1] { + ResponseEvent::Completed { response_id, .. } => { + assert_eq!(response_id, "resp-agg"); + } + other => panic!("unexpected second event: {other:?}"), + } + + Ok(()) +} diff --git a/codex-rs/codex-backend-openapi-models/Cargo.toml b/codex-rs/codex-backend-openapi-models/Cargo.toml index 1a600495b9..f9bad4a494 100644 --- a/codex-rs/codex-backend-openapi-models/Cargo.toml +++ b/codex-rs/codex-backend-openapi-models/Cargo.toml @@ -1,7 +1,8 @@ [package] name = "codex-backend-openapi-models" -version = { workspace = true } -edition = "2024" +version.workspace = true +edition.workspace = true +license.workspace = true [lib] name = "codex_backend_openapi_models" diff --git a/codex-rs/codex-client/Cargo.toml b/codex-rs/codex-client/Cargo.toml new file mode 100644 index 0000000000..d1ff23ef36 --- /dev/null +++ b/codex-rs/codex-client/Cargo.toml @@ -0,0 +1,22 @@ +[package] +edition.workspace = true +license.workspace = true +name = "codex-client" +version.workspace = true + +[dependencies] +async-trait = { workspace = true } +bytes = { workspace = true } +eventsource-stream = { workspace = true } +futures = { workspace = true } +http = { workspace = true } +rand = { workspace = true } +reqwest = { workspace = true, features = ["json", "stream"] } +serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true } +thiserror = { workspace = true } +tokio = { workspace = true, features = ["macros", "rt", "time", "sync"] } +tracing = { workspace = true } + +[lints] +workspace = true diff --git a/codex-rs/codex-client/README.md b/codex-rs/codex-client/README.md new file mode 100644 index 0000000000..045ee7b343 --- /dev/null +++ b/codex-rs/codex-client/README.md @@ -0,0 +1,8 @@ +# codex-client + +Generic transport layer that wraps HTTP requests, retries, and streaming primitives without any Codex/OpenAI awareness. + +- Defines `HttpTransport` and a default `ReqwestTransport` plus thin `Request`/`Response` types. +- Provides retry utilities (`RetryPolicy`, `RetryOn`, `run_with_retry`, `backoff`) that callers plug into for unary and streaming calls. +- Supplies the `sse_stream` helper to turn byte streams into raw SSE `data:` frames with idle timeouts and surfaced stream errors. +- Consumed by higher-level crates like `codex-api`; it stays neutral on endpoints, headers, or API-specific error shapes. diff --git a/codex-rs/codex-client/src/error.rs b/codex-rs/codex-client/src/error.rs new file mode 100644 index 0000000000..086b91a505 --- /dev/null +++ b/codex-rs/codex-client/src/error.rs @@ -0,0 +1,29 @@ +use http::HeaderMap; +use http::StatusCode; +use thiserror::Error; + +#[derive(Debug, Error)] +pub enum TransportError { + #[error("http {status}: {body:?}")] + Http { + status: StatusCode, + headers: Option, + body: Option, + }, + #[error("retry limit reached")] + RetryLimit, + #[error("timeout")] + Timeout, + #[error("network error: {0}")] + Network(String), + #[error("request build error: {0}")] + Build(String), +} + +#[derive(Debug, Error)] +pub enum StreamError { + #[error("stream failed: {0}")] + Stream(String), + #[error("timeout")] + Timeout, +} diff --git a/codex-rs/codex-client/src/lib.rs b/codex-rs/codex-client/src/lib.rs new file mode 100644 index 0000000000..3ac00a21a8 --- /dev/null +++ b/codex-rs/codex-client/src/lib.rs @@ -0,0 +1,21 @@ +mod error; +mod request; +mod retry; +mod sse; +mod telemetry; +mod transport; + +pub use crate::error::StreamError; +pub use crate::error::TransportError; +pub use crate::request::Request; +pub use crate::request::Response; +pub use crate::retry::RetryOn; +pub use crate::retry::RetryPolicy; +pub use crate::retry::backoff; +pub use crate::retry::run_with_retry; +pub use crate::sse::sse_stream; +pub use crate::telemetry::RequestTelemetry; +pub use crate::transport::ByteStream; +pub use crate::transport::HttpTransport; +pub use crate::transport::ReqwestTransport; +pub use crate::transport::StreamResponse; diff --git a/codex-rs/codex-client/src/request.rs b/codex-rs/codex-client/src/request.rs new file mode 100644 index 0000000000..f3d205de99 --- /dev/null +++ b/codex-rs/codex-client/src/request.rs @@ -0,0 +1,39 @@ +use bytes::Bytes; +use http::Method; +use reqwest::header::HeaderMap; +use serde::Serialize; +use serde_json::Value; +use std::time::Duration; + +#[derive(Debug, Clone)] +pub struct Request { + pub method: Method, + pub url: String, + pub headers: HeaderMap, + pub body: Option, + pub timeout: Option, +} + +impl Request { + pub fn new(method: Method, url: String) -> Self { + Self { + method, + url, + headers: HeaderMap::new(), + body: None, + timeout: None, + } + } + + pub fn with_json(mut self, body: &T) -> Self { + self.body = serde_json::to_value(body).ok(); + self + } +} + +#[derive(Debug, Clone)] +pub struct Response { + pub status: http::StatusCode, + pub headers: HeaderMap, + pub body: Bytes, +} diff --git a/codex-rs/codex-client/src/retry.rs b/codex-rs/codex-client/src/retry.rs new file mode 100644 index 0000000000..c7bdd34b1e --- /dev/null +++ b/codex-rs/codex-client/src/retry.rs @@ -0,0 +1,73 @@ +use crate::error::TransportError; +use crate::request::Request; +use rand::Rng; +use std::future::Future; +use std::time::Duration; +use tokio::time::sleep; + +#[derive(Debug, Clone)] +pub struct RetryPolicy { + pub max_attempts: u64, + pub base_delay: Duration, + pub retry_on: RetryOn, +} + +#[derive(Debug, Clone)] +pub struct RetryOn { + pub retry_429: bool, + pub retry_5xx: bool, + pub retry_transport: bool, +} + +impl RetryOn { + pub fn should_retry(&self, err: &TransportError, attempt: u64, max_attempts: u64) -> bool { + if attempt >= max_attempts { + return false; + } + match err { + TransportError::Http { status, .. } => { + (self.retry_429 && status.as_u16() == 429) + || (self.retry_5xx && status.is_server_error()) + } + TransportError::Timeout | TransportError::Network(_) => self.retry_transport, + _ => false, + } + } +} + +pub fn backoff(base: Duration, attempt: u64) -> Duration { + if attempt == 0 { + return base; + } + let exp = 2u64.saturating_pow(attempt as u32 - 1); + let millis = base.as_millis() as u64; + let raw = millis.saturating_mul(exp); + let jitter: f64 = rand::rng().random_range(0.9..1.1); + Duration::from_millis((raw as f64 * jitter) as u64) +} + +pub async fn run_with_retry( + policy: RetryPolicy, + mut make_req: impl FnMut() -> Request, + op: F, +) -> Result +where + F: Fn(Request, u64) -> Fut, + Fut: Future>, +{ + for attempt in 0..=policy.max_attempts { + let req = make_req(); + match op(req, attempt).await { + Ok(resp) => return Ok(resp), + Err(err) + if policy + .retry_on + .should_retry(&err, attempt, policy.max_attempts) => + { + sleep(backoff(policy.base_delay, attempt + 1)).await; + } + Err(err) => return Err(err), + } + } + Err(TransportError::RetryLimit) +} diff --git a/codex-rs/codex-client/src/sse.rs b/codex-rs/codex-client/src/sse.rs new file mode 100644 index 0000000000..f3aba3a2c5 --- /dev/null +++ b/codex-rs/codex-client/src/sse.rs @@ -0,0 +1,48 @@ +use crate::error::StreamError; +use crate::transport::ByteStream; +use eventsource_stream::Eventsource; +use futures::StreamExt; +use tokio::sync::mpsc; +use tokio::time::Duration; +use tokio::time::timeout; + +/// Minimal SSE helper that forwards raw `data:` frames as UTF-8 strings. +/// +/// Errors and idle timeouts are sent as `Err(StreamError)` before the task exits. +pub fn sse_stream( + stream: ByteStream, + idle_timeout: Duration, + tx: mpsc::Sender>, +) { + tokio::spawn(async move { + let mut stream = stream + .map(|res| res.map_err(|e| StreamError::Stream(e.to_string()))) + .eventsource(); + + loop { + match timeout(idle_timeout, stream.next()).await { + Ok(Some(Ok(ev))) => { + if tx.send(Ok(ev.data.clone())).await.is_err() { + return; + } + } + Ok(Some(Err(e))) => { + let _ = tx.send(Err(StreamError::Stream(e.to_string()))).await; + return; + } + Ok(None) => { + let _ = tx + .send(Err(StreamError::Stream( + "stream closed before completion".into(), + ))) + .await; + return; + } + Err(_) => { + let _ = tx.send(Err(StreamError::Timeout)).await; + return; + } + } + } + }); +} diff --git a/codex-rs/codex-client/src/telemetry.rs b/codex-rs/codex-client/src/telemetry.rs new file mode 100644 index 0000000000..457d47f4fc --- /dev/null +++ b/codex-rs/codex-client/src/telemetry.rs @@ -0,0 +1,14 @@ +use crate::error::TransportError; +use http::StatusCode; +use std::time::Duration; + +/// API specific telemetry. +pub trait RequestTelemetry: Send + Sync { + fn on_request( + &self, + attempt: u64, + status: Option, + error: Option<&TransportError>, + duration: Duration, + ); +} diff --git a/codex-rs/codex-client/src/transport.rs b/codex-rs/codex-client/src/transport.rs new file mode 100644 index 0000000000..5edc9a7b77 --- /dev/null +++ b/codex-rs/codex-client/src/transport.rs @@ -0,0 +1,119 @@ +use crate::error::TransportError; +use crate::request::Request; +use crate::request::Response; +use async_trait::async_trait; +use bytes::Bytes; +use futures::StreamExt; +use futures::stream::BoxStream; +use http::HeaderMap; +use http::Method; +use http::StatusCode; +use tracing::Level; +use tracing::enabled; +use tracing::trace; + +pub type ByteStream = BoxStream<'static, Result>; + +pub struct StreamResponse { + pub status: StatusCode, + pub headers: HeaderMap, + pub bytes: ByteStream, +} + +#[async_trait] +pub trait HttpTransport: Send + Sync { + async fn execute(&self, req: Request) -> Result; + async fn stream(&self, req: Request) -> Result; +} + +#[derive(Clone, Debug)] +pub struct ReqwestTransport { + client: reqwest::Client, +} + +impl ReqwestTransport { + pub fn new(client: reqwest::Client) -> Self { + Self { client } + } + + fn build(&self, req: Request) -> Result { + let mut builder = self + .client + .request( + Method::from_bytes(req.method.as_str().as_bytes()).unwrap_or(Method::GET), + &req.url, + ) + .headers(req.headers); + if let Some(timeout) = req.timeout { + builder = builder.timeout(timeout); + } + if let Some(body) = req.body { + builder = builder.json(&body); + } + Ok(builder) + } + + fn map_error(err: reqwest::Error) -> TransportError { + if err.is_timeout() { + TransportError::Timeout + } else { + TransportError::Network(err.to_string()) + } + } +} + +#[async_trait] +impl HttpTransport for ReqwestTransport { + async fn execute(&self, req: Request) -> Result { + let builder = self.build(req)?; + let resp = builder.send().await.map_err(Self::map_error)?; + let status = resp.status(); + let headers = resp.headers().clone(); + let bytes = resp.bytes().await.map_err(Self::map_error)?; + if !status.is_success() { + let body = String::from_utf8(bytes.to_vec()).ok(); + return Err(TransportError::Http { + status, + headers: Some(headers), + body, + }); + } + Ok(Response { + status, + headers, + body: bytes, + }) + } + + async fn stream(&self, req: Request) -> Result { + if enabled!(Level::TRACE) { + trace!( + "{} to {}: {}", + req.method, + req.url, + req.body.as_ref().unwrap_or_default() + ); + } + + let builder = self.build(req)?; + let resp = builder.send().await.map_err(Self::map_error)?; + let status = resp.status(); + let headers = resp.headers().clone(); + if !status.is_success() { + let body = resp.text().await.ok(); + return Err(TransportError::Http { + status, + headers: Some(headers), + body, + }); + } + let stream = resp + .bytes_stream() + .map(|result| result.map_err(Self::map_error)); + Ok(StreamResponse { + status, + headers, + bytes: Box::pin(stream), + }) + } +} diff --git a/codex-rs/common/Cargo.toml b/codex-rs/common/Cargo.toml index cff9c4b307..377d054483 100644 --- a/codex-rs/common/Cargo.toml +++ b/codex-rs/common/Cargo.toml @@ -1,7 +1,8 @@ [package] -edition = "2024" name = "codex-common" -version = { workspace = true } +version.workspace = true +edition.workspace = true +license.workspace = true [lints] workspace = true diff --git a/codex-rs/core/Cargo.toml b/codex-rs/core/Cargo.toml index bc7b3668d0..8a329d0672 100644 --- a/codex-rs/core/Cargo.toml +++ b/codex-rs/core/Cargo.toml @@ -1,7 +1,8 @@ [package] -edition = "2024" name = "codex-core" -version = { workspace = true } +version.workspace = true +edition.workspace = true +license.workspace = true [lib] doctest = false @@ -17,11 +18,12 @@ askama = { workspace = true } async-channel = { workspace = true } async-trait = { workspace = true } base64 = { workspace = true } -bytes = { workspace = true } chrono = { workspace = true, features = ["serde"] } +chardetng = { workspace = true } codex-app-server-protocol = { workspace = true } codex-apply-patch = { workspace = true } codex-async-utils = { workspace = true } +codex-api = { workspace = true } codex-execpolicy = { workspace = true } codex-file-search = { workspace = true } codex-git = { workspace = true } @@ -32,11 +34,11 @@ codex-rmcp-client = { workspace = true } codex-utils-pty = { workspace = true } codex-utils-readiness = { workspace = true } codex-utils-string = { workspace = true } -codex-utils-tokenizer = { workspace = true } codex-windows-sandbox = { package = "codex-windows-sandbox", path = "../windows-sandbox-rs" } dirs = { workspace = true } dunce = { workspace = true } env-flags = { workspace = true } +encoding_rs = { workspace = true } eventsource-stream = { workspace = true } futures = { workspace = true } http = { workspace = true } @@ -50,11 +52,15 @@ regex-lite = { workspace = true } reqwest = { workspace = true, features = ["json", "stream"] } serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } +serde_yaml = { workspace = true } sha1 = { workspace = true } sha2 = { workspace = true } shlex = { workspace = true } similar = { workspace = true } strum_macros = { workspace = true } +url = { workspace = true } +once_cell = { workspace = true } +regex = { workspace = true } tempfile = { workspace = true } test-case = "3.3.1" test-log = { workspace = true } @@ -82,6 +88,9 @@ uuid = { workspace = true, features = ["serde", "v4", "v5"] } which = { workspace = true } wildmatch = { workspace = true } +[features] +deterministic_process_ids = [] + [target.'cfg(target_os = "linux")'.dependencies] landlock = { workspace = true } @@ -110,6 +119,7 @@ keyring = { workspace = true, features = ["sync-secret-service"] } assert_cmd = { workspace = true } assert_matches = { workspace = true } codex-arg0 = { workspace = true } +codex-core = { path = ".", features = ["deterministic_process_ids"] } core_test_support = { workspace = true } ctor = { workspace = true } escargot = { workspace = true } diff --git a/codex-rs/core/src/api_bridge.rs b/codex-rs/core/src/api_bridge.rs new file mode 100644 index 0000000000..79fd67d650 --- /dev/null +++ b/codex-rs/core/src/api_bridge.rs @@ -0,0 +1,162 @@ +use chrono::DateTime; +use chrono::Utc; +use codex_api::AuthProvider as ApiAuthProvider; +use codex_api::TransportError; +use codex_api::error::ApiError; +use codex_api::rate_limits::parse_rate_limit; +use http::HeaderMap; +use serde::Deserialize; + +use crate::auth::CodexAuth; +use crate::error::CodexErr; +use crate::error::RetryLimitReachedError; +use crate::error::UnexpectedResponseError; +use crate::error::UsageLimitReachedError; +use crate::model_provider_info::ModelProviderInfo; +use crate::token_data::PlanType; + +pub(crate) fn map_api_error(err: ApiError) -> CodexErr { + match err { + ApiError::ContextWindowExceeded => CodexErr::ContextWindowExceeded, + ApiError::QuotaExceeded => CodexErr::QuotaExceeded, + ApiError::UsageNotIncluded => CodexErr::UsageNotIncluded, + ApiError::Retryable { message, delay } => CodexErr::Stream(message, delay), + ApiError::Stream(msg) => CodexErr::Stream(msg, None), + ApiError::Api { status, message } => CodexErr::UnexpectedStatus(UnexpectedResponseError { + status, + body: message, + request_id: None, + }), + ApiError::Transport(transport) => match transport { + TransportError::Http { + status, + headers, + body, + } => { + let body_text = body.unwrap_or_default(); + + if status == http::StatusCode::BAD_REQUEST { + if body_text + .contains("The image data you provided does not represent a valid image") + { + CodexErr::InvalidImageRequest() + } else { + CodexErr::InvalidRequest(body_text) + } + } else if status == http::StatusCode::INTERNAL_SERVER_ERROR { + CodexErr::InternalServerError + } else if status == http::StatusCode::TOO_MANY_REQUESTS { + if let Ok(err) = serde_json::from_str::(&body_text) { + if err.error.error_type.as_deref() == Some("usage_limit_reached") { + let rate_limits = headers.as_ref().and_then(parse_rate_limit); + let resets_at = err + .error + .resets_at + .and_then(|seconds| DateTime::::from_timestamp(seconds, 0)); + return CodexErr::UsageLimitReached(UsageLimitReachedError { + plan_type: err.error.plan_type, + resets_at, + rate_limits, + }); + } else if err.error.error_type.as_deref() == Some("usage_not_included") { + return CodexErr::UsageNotIncluded; + } + } + + CodexErr::RetryLimit(RetryLimitReachedError { + status, + request_id: extract_request_id(headers.as_ref()), + }) + } else { + CodexErr::UnexpectedStatus(UnexpectedResponseError { + status, + body: body_text, + request_id: extract_request_id(headers.as_ref()), + }) + } + } + TransportError::RetryLimit => CodexErr::RetryLimit(RetryLimitReachedError { + status: http::StatusCode::INTERNAL_SERVER_ERROR, + request_id: None, + }), + TransportError::Timeout => CodexErr::Timeout, + TransportError::Network(msg) | TransportError::Build(msg) => { + CodexErr::Stream(msg, None) + } + }, + ApiError::RateLimit(msg) => CodexErr::Stream(msg, None), + } +} + +fn extract_request_id(headers: Option<&HeaderMap>) -> Option { + headers.and_then(|map| { + ["cf-ray", "x-request-id", "x-oai-request-id"] + .iter() + .find_map(|name| { + map.get(*name) + .and_then(|v| v.to_str().ok()) + .map(str::to_string) + }) + }) +} + +pub(crate) async fn auth_provider_from_auth( + auth: Option, + provider: &ModelProviderInfo, +) -> crate::error::Result { + if let Some(api_key) = provider.api_key()? { + return Ok(CoreAuthProvider { + token: Some(api_key), + account_id: None, + }); + } + + if let Some(token) = provider.experimental_bearer_token.clone() { + return Ok(CoreAuthProvider { + token: Some(token), + account_id: None, + }); + } + + if let Some(auth) = auth { + let token = auth.get_token().await?; + Ok(CoreAuthProvider { + token: Some(token), + account_id: auth.get_account_id(), + }) + } else { + Ok(CoreAuthProvider { + token: None, + account_id: None, + }) + } +} + +#[derive(Debug, Deserialize)] +struct UsageErrorResponse { + error: UsageErrorBody, +} + +#[derive(Debug, Deserialize)] +struct UsageErrorBody { + #[serde(rename = "type")] + error_type: Option, + plan_type: Option, + resets_at: Option, +} + +#[derive(Clone, Default)] +pub(crate) struct CoreAuthProvider { + token: Option, + account_id: Option, +} + +impl ApiAuthProvider for CoreAuthProvider { + fn bearer_token(&self) -> Option { + self.token.clone() + } + + fn account_id(&self) -> Option { + self.account_id.clone() + } +} diff --git a/codex-rs/core/src/bash.rs b/codex-rs/core/src/bash.rs index 0ffb8e785e..cb8248ec1f 100644 --- a/codex-rs/core/src/bash.rs +++ b/codex-rs/core/src/bash.rs @@ -100,7 +100,7 @@ pub fn extract_bash_command(command: &[String]) -> Option<(&str, &str)> { if !matches!(flag.as_str(), "-lc" | "-c") || !matches!( detect_shell_type(&PathBuf::from(shell)), - Some(ShellType::Zsh) | Some(ShellType::Bash) + Some(ShellType::Zsh) | Some(ShellType::Bash) | Some(ShellType::Sh) ) { return None; diff --git a/codex-rs/core/src/chat_completions.rs b/codex-rs/core/src/chat_completions.rs deleted file mode 100644 index 785a4d4ce5..0000000000 --- a/codex-rs/core/src/chat_completions.rs +++ /dev/null @@ -1,981 +0,0 @@ -use std::time::Duration; - -use crate::ModelProviderInfo; -use crate::client_common::Prompt; -use crate::client_common::ResponseEvent; -use crate::client_common::ResponseStream; -use crate::default_client::CodexHttpClient; -use crate::error::CodexErr; -use crate::error::ConnectionFailedError; -use crate::error::ResponseStreamFailed; -use crate::error::Result; -use crate::error::RetryLimitReachedError; -use crate::error::UnexpectedResponseError; -use crate::model_family::ModelFamily; -use crate::tools::spec::create_tools_json_for_chat_completions_api; -use crate::util::backoff; -use bytes::Bytes; -use codex_otel::otel_event_manager::OtelEventManager; -use codex_protocol::models::ContentItem; -use codex_protocol::models::FunctionCallOutputContentItem; -use codex_protocol::models::ReasoningItemContent; -use codex_protocol::models::ResponseItem; -use codex_protocol::protocol::SessionSource; -use codex_protocol::protocol::SubAgentSource; -use eventsource_stream::Eventsource; -use futures::Stream; -use futures::StreamExt; -use futures::TryStreamExt; -use reqwest::StatusCode; -use serde_json::json; -use std::pin::Pin; -use std::task::Context; -use std::task::Poll; -use tokio::sync::mpsc; -use tokio::time::timeout; -use tracing::debug; -use tracing::trace; - -/// Implementation for the classic Chat Completions API. -pub(crate) async fn stream_chat_completions( - prompt: &Prompt, - model_family: &ModelFamily, - client: &CodexHttpClient, - provider: &ModelProviderInfo, - otel_event_manager: &OtelEventManager, - session_source: &SessionSource, -) -> Result { - if prompt.output_schema.is_some() { - return Err(CodexErr::UnsupportedOperation( - "output_schema is not supported for Chat Completions API".to_string(), - )); - } - - // Build messages array - let mut messages = Vec::::new(); - - let full_instructions = prompt.get_full_instructions(model_family); - messages.push(json!({"role": "system", "content": full_instructions})); - - let input = prompt.get_formatted_input(); - - // Pre-scan: map Reasoning blocks to the adjacent assistant anchor after the last user. - // - If the last emitted message is a user message, drop all reasoning. - // - Otherwise, for each Reasoning item after the last user message, attach it - // to the immediate previous assistant message (stop turns) or the immediate - // next assistant anchor (tool-call turns: function/local shell call, or assistant message). - let mut reasoning_by_anchor_index: std::collections::HashMap = - std::collections::HashMap::new(); - - // Determine the last role that would be emitted to Chat Completions. - let mut last_emitted_role: Option<&str> = None; - for item in &input { - match item { - ResponseItem::Message { role, .. } => last_emitted_role = Some(role.as_str()), - ResponseItem::FunctionCall { .. } | ResponseItem::LocalShellCall { .. } => { - last_emitted_role = Some("assistant") - } - ResponseItem::FunctionCallOutput { .. } => last_emitted_role = Some("tool"), - ResponseItem::Reasoning { .. } | ResponseItem::Other => {} - ResponseItem::CustomToolCall { .. } => {} - ResponseItem::CustomToolCallOutput { .. } => {} - ResponseItem::WebSearchCall { .. } => {} - ResponseItem::GhostSnapshot { .. } => {} - ResponseItem::CompactionSummary { .. } => {} - } - } - - // Find the last user message index in the input. - let mut last_user_index: Option = None; - for (idx, item) in input.iter().enumerate() { - if let ResponseItem::Message { role, .. } = item - && role == "user" - { - last_user_index = Some(idx); - } - } - - // Attach reasoning only if the conversation does not end with a user message. - if !matches!(last_emitted_role, Some("user")) { - for (idx, item) in input.iter().enumerate() { - // Only consider reasoning that appears after the last user message. - if let Some(u_idx) = last_user_index - && idx <= u_idx - { - continue; - } - - if let ResponseItem::Reasoning { - content: Some(items), - .. - } = item - { - let mut text = String::new(); - for entry in items { - match entry { - ReasoningItemContent::ReasoningText { text: segment } - | ReasoningItemContent::Text { text: segment } => text.push_str(segment), - } - } - if text.trim().is_empty() { - continue; - } - - // Prefer immediate previous assistant message (stop turns) - let mut attached = false; - if idx > 0 - && let ResponseItem::Message { role, .. } = &input[idx - 1] - && role == "assistant" - { - reasoning_by_anchor_index - .entry(idx - 1) - .and_modify(|v| v.push_str(&text)) - .or_insert(text.clone()); - attached = true; - } - - // Otherwise, attach to immediate next assistant anchor (tool-calls or assistant message) - if !attached && idx + 1 < input.len() { - match &input[idx + 1] { - ResponseItem::FunctionCall { .. } | ResponseItem::LocalShellCall { .. } => { - reasoning_by_anchor_index - .entry(idx + 1) - .and_modify(|v| v.push_str(&text)) - .or_insert(text.clone()); - } - ResponseItem::Message { role, .. } if role == "assistant" => { - reasoning_by_anchor_index - .entry(idx + 1) - .and_modify(|v| v.push_str(&text)) - .or_insert(text.clone()); - } - _ => {} - } - } - } - } - } - - // Track last assistant text we emitted to avoid duplicate assistant messages - // in the outbound Chat Completions payload (can happen if a final - // aggregated assistant message was recorded alongside an earlier partial). - let mut last_assistant_text: Option = None; - - for (idx, item) in input.iter().enumerate() { - match item { - ResponseItem::Message { role, content, .. } => { - // Build content either as a plain string (typical for assistant text) - // or as an array of content items when images are present (user/tool multimodal). - let mut text = String::new(); - let mut items: Vec = Vec::new(); - let mut saw_image = false; - - for c in content { - match c { - ContentItem::InputText { text: t } - | ContentItem::OutputText { text: t } => { - text.push_str(t); - items.push(json!({"type":"text","text": t})); - } - ContentItem::InputImage { image_url } => { - saw_image = true; - items.push(json!({"type":"image_url","image_url": {"url": image_url}})); - } - } - } - - // Skip exact-duplicate assistant messages. - if role == "assistant" { - if let Some(prev) = &last_assistant_text - && prev == &text - { - continue; - } - last_assistant_text = Some(text.clone()); - } - - // For assistant messages, always send a plain string for compatibility. - // For user messages, if an image is present, send an array of content items. - let content_value = if role == "assistant" { - json!(text) - } else if saw_image { - json!(items) - } else { - json!(text) - }; - - let mut msg = json!({"role": role, "content": content_value}); - if role == "assistant" - && let Some(reasoning) = reasoning_by_anchor_index.get(&idx) - && let Some(obj) = msg.as_object_mut() - { - obj.insert("reasoning".to_string(), json!(reasoning)); - } - messages.push(msg); - } - ResponseItem::FunctionCall { - name, - arguments, - call_id, - .. - } => { - let mut msg = json!({ - "role": "assistant", - "content": null, - "tool_calls": [{ - "id": call_id, - "type": "function", - "function": { - "name": name, - "arguments": arguments, - } - }] - }); - if let Some(reasoning) = reasoning_by_anchor_index.get(&idx) - && let Some(obj) = msg.as_object_mut() - { - obj.insert("reasoning".to_string(), json!(reasoning)); - } - messages.push(msg); - } - ResponseItem::LocalShellCall { - id, - call_id: _, - status, - action, - } => { - // Confirm with API team. - let mut msg = json!({ - "role": "assistant", - "content": null, - "tool_calls": [{ - "id": id.clone().unwrap_or_else(|| "".to_string()), - "type": "local_shell_call", - "status": status, - "action": action, - }] - }); - if let Some(reasoning) = reasoning_by_anchor_index.get(&idx) - && let Some(obj) = msg.as_object_mut() - { - obj.insert("reasoning".to_string(), json!(reasoning)); - } - messages.push(msg); - } - ResponseItem::FunctionCallOutput { call_id, output } => { - // Prefer structured content items when available (e.g., images) - // otherwise fall back to the legacy plain-string content. - let content_value = if let Some(items) = &output.content_items { - let mapped: Vec = items - .iter() - .map(|it| match it { - FunctionCallOutputContentItem::InputText { text } => { - json!({"type":"text","text": text}) - } - FunctionCallOutputContentItem::InputImage { image_url } => { - json!({"type":"image_url","image_url": {"url": image_url}}) - } - }) - .collect(); - json!(mapped) - } else { - json!(output.content) - }; - - messages.push(json!({ - "role": "tool", - "tool_call_id": call_id, - "content": content_value, - })); - } - ResponseItem::CustomToolCall { - id, - call_id: _, - name, - input, - status: _, - } => { - messages.push(json!({ - "role": "assistant", - "content": null, - "tool_calls": [{ - "id": id, - "type": "custom", - "custom": { - "name": name, - "input": input, - } - }] - })); - } - ResponseItem::CustomToolCallOutput { call_id, output } => { - messages.push(json!({ - "role": "tool", - "tool_call_id": call_id, - "content": output, - })); - } - ResponseItem::GhostSnapshot { .. } => { - // Ghost snapshots annotate history but are not sent to the model. - continue; - } - ResponseItem::Reasoning { .. } - | ResponseItem::WebSearchCall { .. } - | ResponseItem::Other - | ResponseItem::CompactionSummary { .. } => { - // Omit these items from the conversation history. - continue; - } - } - } - - let tools_json = create_tools_json_for_chat_completions_api(&prompt.tools)?; - let payload = json!({ - "model": model_family.slug, - "messages": messages, - "stream": true, - "tools": tools_json, - }); - - debug!( - "POST to {}: {}", - provider.get_full_url(&None), - payload.to_string() - ); - - let mut attempt = 0; - let max_retries = provider.request_max_retries(); - loop { - attempt += 1; - - let mut req_builder = provider.create_request_builder(client, &None).await?; - - // Include subagent header only for subagent sessions. - if let SessionSource::SubAgent(sub) = session_source.clone() { - let subagent = if let SubAgentSource::Other(label) = sub { - label - } else { - serde_json::to_value(&sub) - .ok() - .and_then(|v| v.as_str().map(std::string::ToString::to_string)) - .unwrap_or_else(|| "other".to_string()) - }; - req_builder = req_builder.header("x-openai-subagent", subagent); - } - - let res = otel_event_manager - .log_request(attempt, || { - req_builder - .header(reqwest::header::ACCEPT, "text/event-stream") - .json(&payload) - .send() - }) - .await; - - match res { - Ok(resp) if resp.status().is_success() => { - let (tx_event, rx_event) = mpsc::channel::>(1600); - let stream = resp.bytes_stream().map_err(|e| { - CodexErr::ResponseStreamFailed(ResponseStreamFailed { - source: e, - request_id: None, - }) - }); - tokio::spawn(process_chat_sse( - stream, - tx_event, - provider.stream_idle_timeout(), - otel_event_manager.clone(), - )); - return Ok(ResponseStream { rx_event }); - } - Ok(res) => { - let status = res.status(); - if !(status == StatusCode::TOO_MANY_REQUESTS || status.is_server_error()) { - let body = (res.text().await).unwrap_or_default(); - return Err(CodexErr::UnexpectedStatus(UnexpectedResponseError { - status, - body, - request_id: None, - })); - } - - if attempt > max_retries { - return Err(CodexErr::RetryLimit(RetryLimitReachedError { - status, - request_id: None, - })); - } - - let retry_after_secs = res - .headers() - .get(reqwest::header::RETRY_AFTER) - .and_then(|v| v.to_str().ok()) - .and_then(|s| s.parse::().ok()); - - let delay = retry_after_secs - .map(|s| Duration::from_millis(s * 1_000)) - .unwrap_or_else(|| backoff(attempt)); - tokio::time::sleep(delay).await; - } - Err(e) => { - if attempt > max_retries { - return Err(CodexErr::ConnectionFailed(ConnectionFailedError { - source: e, - })); - } - let delay = backoff(attempt); - tokio::time::sleep(delay).await; - } - } - } -} - -async fn append_assistant_text( - tx_event: &mpsc::Sender>, - assistant_item: &mut Option, - text: String, -) { - if assistant_item.is_none() { - let item = ResponseItem::Message { - id: None, - role: "assistant".to_string(), - content: vec![], - }; - *assistant_item = Some(item.clone()); - let _ = tx_event - .send(Ok(ResponseEvent::OutputItemAdded(item))) - .await; - } - - if let Some(ResponseItem::Message { content, .. }) = assistant_item { - content.push(ContentItem::OutputText { text: text.clone() }); - let _ = tx_event - .send(Ok(ResponseEvent::OutputTextDelta(text.clone()))) - .await; - } -} - -async fn append_reasoning_text( - tx_event: &mpsc::Sender>, - reasoning_item: &mut Option, - text: String, -) { - if reasoning_item.is_none() { - let item = ResponseItem::Reasoning { - id: String::new(), - summary: Vec::new(), - content: Some(vec![]), - encrypted_content: None, - }; - *reasoning_item = Some(item.clone()); - let _ = tx_event - .send(Ok(ResponseEvent::OutputItemAdded(item))) - .await; - } - - if let Some(ResponseItem::Reasoning { - content: Some(content), - .. - }) = reasoning_item - { - let content_index = content.len() as i64; - content.push(ReasoningItemContent::ReasoningText { text: text.clone() }); - - let _ = tx_event - .send(Ok(ResponseEvent::ReasoningContentDelta { - delta: text.clone(), - content_index, - })) - .await; - } -} -/// Lightweight SSE processor for the Chat Completions streaming format. The -/// output is mapped onto Codex's internal [`ResponseEvent`] so that the rest -/// of the pipeline can stay agnostic of the underlying wire format. -async fn process_chat_sse( - stream: S, - tx_event: mpsc::Sender>, - idle_timeout: Duration, - otel_event_manager: OtelEventManager, -) where - S: Stream> + Unpin, -{ - let mut stream = stream.eventsource(); - - // State to accumulate a function call across streaming chunks. - // OpenAI may split the `arguments` string over multiple `delta` events - // until the chunk whose `finish_reason` is `tool_calls` is emitted. We - // keep collecting the pieces here and forward a single - // `ResponseItem::FunctionCall` once the call is complete. - #[derive(Default)] - struct FunctionCallState { - name: Option, - arguments: String, - call_id: Option, - active: bool, - } - - let mut fn_call_state = FunctionCallState::default(); - let mut assistant_item: Option = None; - let mut reasoning_item: Option = None; - - loop { - let start = std::time::Instant::now(); - let response = timeout(idle_timeout, stream.next()).await; - let duration = start.elapsed(); - otel_event_manager.log_sse_event(&response, duration); - - let sse = match response { - Ok(Some(Ok(ev))) => ev, - Ok(Some(Err(e))) => { - let _ = tx_event - .send(Err(CodexErr::Stream(e.to_string(), None))) - .await; - return; - } - Ok(None) => { - // Stream closed gracefully – emit Completed with dummy id. - let _ = tx_event - .send(Ok(ResponseEvent::Completed { - response_id: String::new(), - token_usage: None, - })) - .await; - return; - } - Err(_) => { - let _ = tx_event - .send(Err(CodexErr::Stream( - "idle timeout waiting for SSE".into(), - None, - ))) - .await; - return; - } - }; - - // OpenAI Chat streaming sends a literal string "[DONE]" when finished. - if sse.data.trim() == "[DONE]" { - // Emit any finalized items before closing so downstream consumers receive - // terminal events for both assistant content and raw reasoning. - if let Some(item) = assistant_item { - let _ = tx_event.send(Ok(ResponseEvent::OutputItemDone(item))).await; - } - - if let Some(item) = reasoning_item { - let _ = tx_event.send(Ok(ResponseEvent::OutputItemDone(item))).await; - } - - let _ = tx_event - .send(Ok(ResponseEvent::Completed { - response_id: String::new(), - token_usage: None, - })) - .await; - return; - } - - // Parse JSON chunk - let chunk: serde_json::Value = match serde_json::from_str(&sse.data) { - Ok(v) => v, - Err(_) => continue, - }; - trace!("chat_completions received SSE chunk: {chunk:?}"); - - let choice_opt = chunk.get("choices").and_then(|c| c.get(0)); - - if let Some(choice) = choice_opt { - // Handle assistant content tokens as streaming deltas. - if let Some(content) = choice - .get("delta") - .and_then(|d| d.get("content")) - .and_then(|c| c.as_str()) - && !content.is_empty() - { - append_assistant_text(&tx_event, &mut assistant_item, content.to_string()).await; - } - - // Forward any reasoning/thinking deltas if present. - // Some providers stream `reasoning` as a plain string while others - // nest the text under an object (e.g. `{ "reasoning": { "text": "…" } }`). - if let Some(reasoning_val) = choice.get("delta").and_then(|d| d.get("reasoning")) { - let mut maybe_text = reasoning_val - .as_str() - .map(str::to_string) - .filter(|s| !s.is_empty()); - - if maybe_text.is_none() && reasoning_val.is_object() { - if let Some(s) = reasoning_val - .get("text") - .and_then(|t| t.as_str()) - .filter(|s| !s.is_empty()) - { - maybe_text = Some(s.to_string()); - } else if let Some(s) = reasoning_val - .get("content") - .and_then(|t| t.as_str()) - .filter(|s| !s.is_empty()) - { - maybe_text = Some(s.to_string()); - } - } - - if let Some(reasoning) = maybe_text { - // Accumulate so we can emit a terminal Reasoning item at the end. - append_reasoning_text(&tx_event, &mut reasoning_item, reasoning).await; - } - } - - // Some providers only include reasoning on the final message object. - if let Some(message_reasoning) = choice.get("message").and_then(|m| m.get("reasoning")) - { - // Accept either a plain string or an object with { text | content } - if let Some(s) = message_reasoning.as_str() { - if !s.is_empty() { - append_reasoning_text(&tx_event, &mut reasoning_item, s.to_string()).await; - } - } else if let Some(obj) = message_reasoning.as_object() - && let Some(s) = obj - .get("text") - .and_then(|v| v.as_str()) - .or_else(|| obj.get("content").and_then(|v| v.as_str())) - && !s.is_empty() - { - append_reasoning_text(&tx_event, &mut reasoning_item, s.to_string()).await; - } - } - - // Handle streaming function / tool calls. - if let Some(tool_calls) = choice - .get("delta") - .and_then(|d| d.get("tool_calls")) - .and_then(|tc| tc.as_array()) - && let Some(tool_call) = tool_calls.first() - { - // Mark that we have an active function call in progress. - fn_call_state.active = true; - - // Extract call_id if present. - if let Some(id) = tool_call.get("id").and_then(|v| v.as_str()) { - fn_call_state.call_id.get_or_insert_with(|| id.to_string()); - } - - // Extract function details if present. - if let Some(function) = tool_call.get("function") { - if let Some(name) = function.get("name").and_then(|n| n.as_str()) { - fn_call_state.name.get_or_insert_with(|| name.to_string()); - } - - if let Some(args_fragment) = function.get("arguments").and_then(|a| a.as_str()) - { - fn_call_state.arguments.push_str(args_fragment); - } - } - } - - // Emit end-of-turn when finish_reason signals completion. - if let Some(finish_reason) = choice.get("finish_reason").and_then(|v| v.as_str()) - && !finish_reason.is_empty() - { - match finish_reason { - "tool_calls" if fn_call_state.active => { - // First, flush the terminal raw reasoning so UIs can finalize - // the reasoning stream before any exec/tool events begin. - if let Some(item) = reasoning_item.take() { - let _ = tx_event.send(Ok(ResponseEvent::OutputItemDone(item))).await; - } - - // Then emit the FunctionCall response item. - let item = ResponseItem::FunctionCall { - id: None, - name: fn_call_state.name.clone().unwrap_or_else(|| "".to_string()), - arguments: fn_call_state.arguments.clone(), - call_id: fn_call_state.call_id.clone().unwrap_or_else(String::new), - }; - - let _ = tx_event.send(Ok(ResponseEvent::OutputItemDone(item))).await; - } - "stop" => { - // Regular turn without tool-call. Emit the final assistant message - // as a single OutputItemDone so non-delta consumers see the result. - if let Some(item) = assistant_item.take() { - let _ = tx_event.send(Ok(ResponseEvent::OutputItemDone(item))).await; - } - // Also emit a terminal Reasoning item so UIs can finalize raw reasoning. - if let Some(item) = reasoning_item.take() { - let _ = tx_event.send(Ok(ResponseEvent::OutputItemDone(item))).await; - } - } - _ => {} - } - - // Emit Completed regardless of reason so the agent can advance. - let _ = tx_event - .send(Ok(ResponseEvent::Completed { - response_id: String::new(), - token_usage: None, - })) - .await; - - // Prepare for potential next turn (should not happen in same stream). - // fn_call_state = FunctionCallState::default(); - - return; // End processing for this SSE stream. - } - } - } -} - -/// Optional client-side aggregation helper -/// -/// Stream adapter that merges the incremental `OutputItemDone` chunks coming from -/// [`process_chat_sse`] into a *running* assistant message, **suppressing the -/// per-token deltas**. The stream stays silent while the model is thinking -/// and only emits two events per turn: -/// -/// 1. `ResponseEvent::OutputItemDone` with the *complete* assistant message -/// (fully concatenated). -/// 2. The original `ResponseEvent::Completed` right after it. -/// -/// This mirrors the behaviour the TypeScript CLI exposes to its higher layers. -/// -/// The adapter is intentionally *lossless*: callers who do **not** opt in via -/// [`AggregateStreamExt::aggregate()`] keep receiving the original unmodified -/// events. -#[derive(Copy, Clone, Eq, PartialEq)] -enum AggregateMode { - AggregatedOnly, - Streaming, -} -pub(crate) struct AggregatedChatStream { - inner: S, - cumulative: String, - cumulative_reasoning: String, - pending: std::collections::VecDeque, - mode: AggregateMode, -} - -impl Stream for AggregatedChatStream -where - S: Stream> + Unpin, -{ - type Item = Result; - - fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - let this = self.get_mut(); - - // First, flush any buffered events from the previous call. - if let Some(ev) = this.pending.pop_front() { - return Poll::Ready(Some(Ok(ev))); - } - - loop { - match Pin::new(&mut this.inner).poll_next(cx) { - Poll::Pending => return Poll::Pending, - Poll::Ready(None) => return Poll::Ready(None), - Poll::Ready(Some(Err(e))) => return Poll::Ready(Some(Err(e))), - Poll::Ready(Some(Ok(ResponseEvent::OutputItemDone(item)))) => { - // If this is an incremental assistant message chunk, accumulate but - // do NOT emit yet. Forward any other item (e.g. FunctionCall) right - // away so downstream consumers see it. - - let is_assistant_message = matches!( - &item, - codex_protocol::models::ResponseItem::Message { role, .. } if role == "assistant" - ); - - if is_assistant_message { - match this.mode { - AggregateMode::AggregatedOnly => { - // Only use the final assistant message if we have not - // seen any deltas; otherwise, deltas already built the - // cumulative text and this would duplicate it. - if this.cumulative.is_empty() - && let codex_protocol::models::ResponseItem::Message { - content, - .. - } = &item - && let Some(text) = content.iter().find_map(|c| match c { - codex_protocol::models::ContentItem::OutputText { - text, - } => Some(text), - _ => None, - }) - { - this.cumulative.push_str(text); - } - // Swallow assistant message here; emit on Completed. - continue; - } - AggregateMode::Streaming => { - // In streaming mode, if we have not seen any deltas, forward - // the final assistant message directly. If deltas were seen, - // suppress the final message to avoid duplication. - if this.cumulative.is_empty() { - return Poll::Ready(Some(Ok(ResponseEvent::OutputItemDone( - item, - )))); - } else { - continue; - } - } - } - } - - // Not an assistant message – forward immediately. - return Poll::Ready(Some(Ok(ResponseEvent::OutputItemDone(item)))); - } - Poll::Ready(Some(Ok(ResponseEvent::RateLimits(snapshot)))) => { - return Poll::Ready(Some(Ok(ResponseEvent::RateLimits(snapshot)))); - } - Poll::Ready(Some(Ok(ResponseEvent::Completed { - response_id, - token_usage, - }))) => { - // Build any aggregated items in the correct order: Reasoning first, then Message. - let mut emitted_any = false; - - if !this.cumulative_reasoning.is_empty() - && matches!(this.mode, AggregateMode::AggregatedOnly) - { - let aggregated_reasoning = - codex_protocol::models::ResponseItem::Reasoning { - id: String::new(), - summary: Vec::new(), - content: Some(vec![ - codex_protocol::models::ReasoningItemContent::ReasoningText { - text: std::mem::take(&mut this.cumulative_reasoning), - }, - ]), - encrypted_content: None, - }; - this.pending - .push_back(ResponseEvent::OutputItemDone(aggregated_reasoning)); - emitted_any = true; - } - - // Always emit the final aggregated assistant message when any - // content deltas have been observed. In AggregatedOnly mode this - // is the sole assistant output; in Streaming mode this finalizes - // the streamed deltas into a terminal OutputItemDone so callers - // can persist/render the message once per turn. - if !this.cumulative.is_empty() { - let aggregated_message = codex_protocol::models::ResponseItem::Message { - id: None, - role: "assistant".to_string(), - content: vec![codex_protocol::models::ContentItem::OutputText { - text: std::mem::take(&mut this.cumulative), - }], - }; - this.pending - .push_back(ResponseEvent::OutputItemDone(aggregated_message)); - emitted_any = true; - } - - // Always emit Completed last when anything was aggregated. - if emitted_any { - this.pending.push_back(ResponseEvent::Completed { - response_id: response_id.clone(), - token_usage: token_usage.clone(), - }); - // Return the first pending event now. - if let Some(ev) = this.pending.pop_front() { - return Poll::Ready(Some(Ok(ev))); - } - } - - // Nothing aggregated – forward Completed directly. - return Poll::Ready(Some(Ok(ResponseEvent::Completed { - response_id, - token_usage, - }))); - } - Poll::Ready(Some(Ok(ResponseEvent::Created))) => { - // These events are exclusive to the Responses API and - // will never appear in a Chat Completions stream. - continue; - } - Poll::Ready(Some(Ok(ResponseEvent::OutputTextDelta(delta)))) => { - // Always accumulate deltas so we can emit a final OutputItemDone at Completed. - this.cumulative.push_str(&delta); - if matches!(this.mode, AggregateMode::Streaming) { - // In streaming mode, also forward the delta immediately. - return Poll::Ready(Some(Ok(ResponseEvent::OutputTextDelta(delta)))); - } else { - continue; - } - } - Poll::Ready(Some(Ok(ResponseEvent::ReasoningContentDelta { - delta, - content_index, - }))) => { - // Always accumulate reasoning deltas so we can emit a final Reasoning item at Completed. - this.cumulative_reasoning.push_str(&delta); - if matches!(this.mode, AggregateMode::Streaming) { - // In streaming mode, also forward the delta immediately. - return Poll::Ready(Some(Ok(ResponseEvent::ReasoningContentDelta { - delta, - content_index, - }))); - } else { - continue; - } - } - Poll::Ready(Some(Ok(ResponseEvent::ReasoningSummaryDelta { .. }))) => { - continue; - } - Poll::Ready(Some(Ok(ResponseEvent::ReasoningSummaryPartAdded { .. }))) => { - continue; - } - Poll::Ready(Some(Ok(ResponseEvent::OutputItemAdded(item)))) => { - return Poll::Ready(Some(Ok(ResponseEvent::OutputItemAdded(item)))); - } - } - } - } -} - -/// Extension trait that activates aggregation on any stream of [`ResponseEvent`]. -pub(crate) trait AggregateStreamExt: Stream> + Sized { - /// Returns a new stream that emits **only** the final assistant message - /// per turn instead of every incremental delta. The produced - /// `ResponseEvent` sequence for a typical text turn looks like: - /// - /// ```ignore - /// OutputItemDone() - /// Completed - /// ``` - /// - /// No other `OutputItemDone` events will be seen by the caller. - /// - /// Usage: - /// - /// ```ignore - /// let agg_stream = client.stream(&prompt).await?.aggregate(); - /// while let Some(event) = agg_stream.next().await { - /// // event now contains cumulative text - /// } - /// ``` - fn aggregate(self) -> AggregatedChatStream { - AggregatedChatStream::new(self, AggregateMode::AggregatedOnly) - } -} - -impl AggregateStreamExt for T where T: Stream> + Sized {} - -impl AggregatedChatStream { - fn new(inner: S, mode: AggregateMode) -> Self { - AggregatedChatStream { - inner, - cumulative: String::new(), - cumulative_reasoning: String::new(), - pending: std::collections::VecDeque::new(), - mode, - } - } - - pub(crate) fn streaming_mode(inner: S) -> Self { - Self::new(inner, AggregateMode::Streaming) - } -} diff --git a/codex-rs/core/src/client.rs b/codex-rs/core/src/client.rs index 68bd30f4e7..82839522c9 100644 --- a/codex-rs/core/src/client.rs +++ b/codex-rs/core/src/client.rs @@ -1,12 +1,22 @@ -use std::io::BufRead; -use std::path::Path; use std::sync::Arc; -use std::sync::OnceLock; -use std::time::Duration; -use bytes::Bytes; -use chrono::DateTime; -use chrono::Utc; +use crate::api_bridge::auth_provider_from_auth; +use crate::api_bridge::map_api_error; +use codex_api::AggregateStreamExt; +use codex_api::ChatClient as ApiChatClient; +use codex_api::CompactClient as ApiCompactClient; +use codex_api::CompactionInput as ApiCompactionInput; +use codex_api::Prompt as ApiPrompt; +use codex_api::RequestTelemetry; +use codex_api::ReqwestTransport; +use codex_api::ResponseStream as ApiResponseStream; +use codex_api::ResponsesClient as ApiResponsesClient; +use codex_api::ResponsesOptions as ApiResponsesOptions; +use codex_api::SseTelemetry; +use codex_api::TransportError; +use codex_api::common::Reasoning; +use codex_api::create_text_param_for_request; +use codex_api::error::ApiError; use codex_app_server_protocol::AuthMode; use codex_otel::otel_event_manager::OtelEventManager; use codex_protocol::ConversationId; @@ -14,90 +24,40 @@ use codex_protocol::config_types::ReasoningEffort as ReasoningEffortConfig; use codex_protocol::config_types::ReasoningSummary as ReasoningSummaryConfig; use codex_protocol::models::ResponseItem; use codex_protocol::protocol::SessionSource; -use eventsource_stream::Eventsource; -use futures::prelude::*; -use regex_lite::Regex; +use eventsource_stream::Event; +use eventsource_stream::EventStreamError; +use futures::StreamExt; +use http::HeaderMap as ApiHeaderMap; +use http::HeaderValue; +use http::StatusCode as HttpStatusCode; use reqwest::StatusCode; -use reqwest::header::HeaderMap; -use serde::Deserialize; -use serde::Serialize; use serde_json::Value; +use std::time::Duration; use tokio::sync::mpsc; -use tokio::time::timeout; -use tokio_util::io::ReaderStream; -use tracing::debug; -use tracing::enabled; -use tracing::trace; use tracing::warn; use crate::AuthManager; -use crate::auth::CodexAuth; use crate::auth::RefreshTokenError; -use crate::chat_completions::AggregateStreamExt; -use crate::chat_completions::stream_chat_completions; use crate::client_common::Prompt; -use crate::client_common::Reasoning; use crate::client_common::ResponseEvent; use crate::client_common::ResponseStream; -use crate::client_common::ResponsesApiRequest; -use crate::client_common::create_text_param_for_request; use crate::config::Config; -use crate::default_client::CodexHttpClient; -use crate::default_client::create_client; +use crate::default_client::build_reqwest_client; use crate::error::CodexErr; -use crate::error::ConnectionFailedError; -use crate::error::ResponseStreamFailed; use crate::error::Result; -use crate::error::RetryLimitReachedError; -use crate::error::UnexpectedResponseError; -use crate::error::UsageLimitReachedError; use crate::flags::CODEX_RS_SSE_FIXTURE; use crate::model_family::ModelFamily; use crate::model_provider_info::ModelProviderInfo; use crate::model_provider_info::WireApi; use crate::openai_model_info::get_model_info; -use crate::protocol::CreditsSnapshot; -use crate::protocol::RateLimitSnapshot; -use crate::protocol::RateLimitWindow; -use crate::protocol::TokenUsage; -use crate::token_data::PlanType; +use crate::tools::spec::create_tools_json_for_chat_completions_api; use crate::tools::spec::create_tools_json_for_responses_api; -use crate::util::backoff; - -#[derive(Debug, Deserialize)] -struct ErrorResponse { - error: Error, -} - -#[derive(Debug, Deserialize)] -struct Error { - r#type: Option, - code: Option, - message: Option, - - // Optional fields available on "usage_limit_reached" and "usage_not_included" errors - plan_type: Option, - resets_at: Option, -} - -#[derive(Debug, Serialize)] -struct CompactHistoryRequest<'a> { - model: &'a str, - input: &'a [ResponseItem], - instructions: &'a str, -} - -#[derive(Debug, Deserialize)] -struct CompactHistoryResponse { - output: Vec, -} #[derive(Debug, Clone)] pub struct ModelClient { config: Arc, auth_manager: Option>, otel_event_manager: OtelEventManager, - client: CodexHttpClient, provider: ModelProviderInfo, conversation_id: ConversationId, effort: Option, @@ -117,13 +77,10 @@ impl ModelClient { conversation_id: ConversationId, session_source: SessionSource, ) -> Self { - let client = create_client(); - Self { config, auth_manager, otel_event_manager, - client, provider, conversation_id, effort, @@ -154,65 +111,102 @@ impl ModelClient { &self.provider } + /// Streams a single model turn using either the Responses or Chat + /// Completions wire API, depending on the configured provider. + /// + /// For Chat providers, the underlying stream is optionally aggregated + /// based on the `show_raw_agent_reasoning` flag in the config. pub async fn stream(&self, prompt: &Prompt) -> Result { match self.provider.wire_api { - WireApi::Responses => self.stream_responses(prompt).await, + WireApi::Responses => self.stream_responses_api(prompt).await, WireApi::Chat => { - // Create the raw streaming connection first. - let response_stream = stream_chat_completions( - prompt, - &self.config.model_family, - &self.client, - &self.provider, - &self.otel_event_manager, - &self.session_source, - ) - .await?; + let api_stream = self.stream_chat_completions(prompt).await?; - // Wrap it with the aggregation adapter so callers see *only* - // the final assistant message per turn (matching the - // behaviour of the Responses API). - let mut aggregated = if self.config.show_raw_agent_reasoning { - crate::chat_completions::AggregatedChatStream::streaming_mode(response_stream) + if self.config.show_raw_agent_reasoning { + Ok(map_response_stream( + api_stream.streaming_mode(), + self.otel_event_manager.clone(), + )) } else { - response_stream.aggregate() - }; - - // Bridge the aggregated stream back into a standard - // `ResponseStream` by forwarding events through a channel. - let (tx, rx) = mpsc::channel::>(16); - - tokio::spawn(async move { - use futures::StreamExt; - while let Some(ev) = aggregated.next().await { - // Exit early if receiver hung up. - if tx.send(ev).await.is_err() { - break; - } - } - }); - - Ok(ResponseStream { rx_event: rx }) + Ok(map_response_stream( + api_stream.aggregate(), + self.otel_event_manager.clone(), + )) + } } } } - /// Implementation for the OpenAI *Responses* experimental API. - async fn stream_responses(&self, prompt: &Prompt) -> Result { - if let Some(path) = &*CODEX_RS_SSE_FIXTURE { - // short circuit for tests - warn!(path, "Streaming from fixture"); - return stream_from_fixture( - path, - self.provider.clone(), - self.otel_event_manager.clone(), - ) - .await; + /// Streams a turn via the OpenAI Chat Completions API. + /// + /// This path is only used when the provider is configured with + /// `WireApi::Chat`; it does not support `output_schema` today. + async fn stream_chat_completions(&self, prompt: &Prompt) -> Result { + if prompt.output_schema.is_some() { + return Err(CodexErr::UnsupportedOperation( + "output_schema is not supported for Chat Completions API".to_string(), + )); } let auth_manager = self.auth_manager.clone(); + let instructions = prompt + .get_full_instructions(&self.config.model_family) + .into_owned(); + let tools_json = create_tools_json_for_chat_completions_api(&prompt.tools)?; + let api_prompt = build_api_prompt(prompt, instructions, tools_json); + let conversation_id = self.conversation_id.to_string(); + let session_source = self.session_source.clone(); - let full_instructions = prompt.get_full_instructions(&self.config.model_family); + let mut refreshed = false; + loop { + let auth = auth_manager.as_ref().and_then(|m| m.auth()); + let api_provider = self + .provider + .to_api_provider(auth.as_ref().map(|a| a.mode))?; + let api_auth = auth_provider_from_auth(auth.clone(), &self.provider).await?; + let transport = ReqwestTransport::new(build_reqwest_client()); + let (request_telemetry, sse_telemetry) = self.build_streaming_telemetry(); + let client = ApiChatClient::new(transport, api_provider, api_auth) + .with_telemetry(Some(request_telemetry), Some(sse_telemetry)); + + let stream_result = client + .stream_prompt( + &self.config.model, + &api_prompt, + Some(conversation_id.clone()), + Some(session_source.clone()), + ) + .await; + + match stream_result { + Ok(stream) => return Ok(stream), + Err(ApiError::Transport(TransportError::Http { status, .. })) + if status == StatusCode::UNAUTHORIZED => + { + handle_unauthorized(status, &mut refreshed, &auth_manager, &auth).await?; + continue; + } + Err(err) => return Err(map_api_error(err)), + } + } + } + + /// Streams a turn via the OpenAI Responses API. + /// + /// Handles SSE fixtures, reasoning summaries, verbosity, and the + /// `text` controls used for output schemas. + async fn stream_responses_api(&self, prompt: &Prompt) -> Result { + if let Some(path) = &*CODEX_RS_SSE_FIXTURE { + warn!(path, "Streaming from fixture"); + let stream = codex_api::stream_from_fixture(path, self.provider.stream_idle_timeout()) + .map_err(map_api_error)?; + return Ok(map_response_stream(stream, self.otel_event_manager.clone())); + } + + let auth_manager = self.auth_manager.clone(); + let instructions = prompt + .get_full_instructions(&self.config.model_family) + .into_owned(); let tools_json: Vec = create_tools_json_for_responses_api(&prompt.tools)?; let reasoning = if self.config.model_family.supports_reasoning_summaries { @@ -232,8 +226,6 @@ impl ModelClient { vec![] }; - let input_with_instructions = prompt.get_formatted_input(); - let verbosity = if self.config.model_family.support_verbosity { self.config .model_verbosity @@ -248,241 +240,49 @@ impl ModelClient { None }; - // Only include `text.verbosity` for GPT-5 family models let text = create_text_param_for_request(verbosity, &prompt.output_schema); + let api_prompt = build_api_prompt(prompt, instructions.clone(), tools_json); + let conversation_id = self.conversation_id.to_string(); + let session_source = self.session_source.clone(); - // In general, we want to explicitly send `store: false` when using the Responses API, - // but in practice, the Azure Responses API rejects `store: false`: - // - // - If store = false and id is sent an error is thrown that ID is not found - // - If store = false and id is not sent an error is thrown that ID is required - // - // For Azure, we send `store: true` and preserve reasoning item IDs. - let azure_workaround = self.provider.is_azure_responses_endpoint(); + let mut refreshed = false; + loop { + let auth = auth_manager.as_ref().and_then(|m| m.auth()); + let api_provider = self + .provider + .to_api_provider(auth.as_ref().map(|a| a.mode))?; + let api_auth = auth_provider_from_auth(auth.clone(), &self.provider).await?; + let transport = ReqwestTransport::new(build_reqwest_client()); + let (request_telemetry, sse_telemetry) = self.build_streaming_telemetry(); + let client = ApiResponsesClient::new(transport, api_provider, api_auth) + .with_telemetry(Some(request_telemetry), Some(sse_telemetry)); - let payload = ResponsesApiRequest { - model: &self.config.model, - instructions: &full_instructions, - input: &input_with_instructions, - tools: &tools_json, - tool_choice: "auto", - parallel_tool_calls: prompt.parallel_tool_calls, - reasoning, - store: azure_workaround, - stream: true, - include, - prompt_cache_key: Some(self.conversation_id.to_string()), - text, - }; - - let mut payload_json = serde_json::to_value(&payload)?; - if azure_workaround { - attach_item_ids(&mut payload_json, &input_with_instructions); - } - - let max_attempts = self.provider.request_max_retries(); - for attempt in 0..=max_attempts { - match self - .attempt_stream_responses(attempt, &payload_json, &auth_manager) - .await - { - Ok(stream) => { - return Ok(stream); - } - Err(StreamAttemptError::Fatal(e)) => { - return Err(e); - } - Err(retryable_attempt_error) => { - if attempt == max_attempts { - return Err(retryable_attempt_error.into_error()); - } - - tokio::time::sleep(retryable_attempt_error.delay(attempt)).await; - } - } - } - - unreachable!("stream_responses_attempt should always return"); - } - - /// Single attempt to start a streaming Responses API call. - async fn attempt_stream_responses( - &self, - attempt: u64, - payload_json: &Value, - auth_manager: &Option>, - ) -> std::result::Result { - // Always fetch the latest auth in case a prior attempt refreshed the token. - let auth = auth_manager.as_ref().and_then(|m| m.auth()); - - trace!( - "POST to {}: {}", - self.provider.get_full_url(&auth), - payload_json.to_string() - ); - - let mut req_builder = self - .provider - .create_request_builder(&self.client, &auth) - .await - .map_err(StreamAttemptError::Fatal)?; - - // Include subagent header only for subagent sessions. - if let SessionSource::SubAgent(sub) = &self.session_source { - let subagent = if let crate::protocol::SubAgentSource::Other(label) = sub { - label.clone() - } else { - serde_json::to_value(sub) - .ok() - .and_then(|v| v.as_str().map(std::string::ToString::to_string)) - .unwrap_or_else(|| "other".to_string()) + let options = ApiResponsesOptions { + reasoning: reasoning.clone(), + include: include.clone(), + prompt_cache_key: Some(conversation_id.clone()), + text: text.clone(), + store_override: None, + conversation_id: Some(conversation_id.clone()), + session_source: Some(session_source.clone()), }; - req_builder = req_builder.header("x-openai-subagent", subagent); - } - req_builder = req_builder - // Send session_id for compatibility. - .header("conversation_id", self.conversation_id.to_string()) - .header("session_id", self.conversation_id.to_string()) - .header(reqwest::header::ACCEPT, "text/event-stream") - .json(payload_json); + let stream_result = client + .stream_prompt(&self.config.model, &api_prompt, options) + .await; - if let Some(auth) = auth.as_ref() - && auth.mode == AuthMode::ChatGPT - && let Some(account_id) = auth.get_account_id() - { - req_builder = req_builder.header("chatgpt-account-id", account_id); - } - - let res = self - .otel_event_manager - .log_request(attempt, || req_builder.send()) - .await; - - let mut request_id = None; - if let Ok(resp) = &res { - request_id = resp - .headers() - .get("cf-ray") - .map(|v| v.to_str().unwrap_or_default().to_string()); - } - - match res { - Ok(resp) if resp.status().is_success() => { - let (tx_event, rx_event) = mpsc::channel::>(1600); - - if let Some(snapshot) = parse_rate_limit_snapshot(resp.headers()) - && tx_event - .send(Ok(ResponseEvent::RateLimits(snapshot))) - .await - .is_err() - { - debug!("receiver dropped rate limit snapshot event"); + match stream_result { + Ok(stream) => { + return Ok(map_response_stream(stream, self.otel_event_manager.clone())); } - - // spawn task to process SSE - let stream = resp.bytes_stream().map_err(move |e| { - CodexErr::ResponseStreamFailed(ResponseStreamFailed { - source: e, - request_id: request_id.clone(), - }) - }); - tokio::spawn(process_sse( - stream, - tx_event, - self.provider.stream_idle_timeout(), - self.otel_event_manager.clone(), - )); - - Ok(ResponseStream { rx_event }) + Err(ApiError::Transport(TransportError::Http { status, .. })) + if status == StatusCode::UNAUTHORIZED => + { + handle_unauthorized(status, &mut refreshed, &auth_manager, &auth).await?; + continue; + } + Err(err) => return Err(map_api_error(err)), } - Ok(res) => { - let status = res.status(); - - // Pull out Retry‑After header if present. - let retry_after_secs = res - .headers() - .get(reqwest::header::RETRY_AFTER) - .and_then(|v| v.to_str().ok()) - .and_then(|s| s.parse::().ok()); - let retry_after = retry_after_secs.map(|s| Duration::from_millis(s * 1_000)); - - if status == StatusCode::UNAUTHORIZED - && let Some(manager) = auth_manager.as_ref() - && let Some(auth) = auth.as_ref() - && auth.mode == AuthMode::ChatGPT - && let Err(err) = manager.refresh_token().await - { - let stream_error = match err { - RefreshTokenError::Permanent(failed) => { - StreamAttemptError::Fatal(CodexErr::RefreshTokenFailed(failed)) - } - RefreshTokenError::Transient(other) => { - StreamAttemptError::RetryableTransportError(CodexErr::Io(other)) - } - }; - return Err(stream_error); - } - - // The OpenAI Responses endpoint returns structured JSON bodies even for 4xx/5xx - // errors. When we bubble early with only the HTTP status the caller sees an opaque - // "unexpected status 400 Bad Request" which makes debugging nearly impossible. - // Instead, read (and include) the response text so higher layers and users see the - // exact error message (e.g. "Unknown parameter: 'input[0].metadata'"). The body is - // small and this branch only runs on error paths so the extra allocation is - // negligible. - if !(status == StatusCode::TOO_MANY_REQUESTS - || status == StatusCode::UNAUTHORIZED - || status.is_server_error()) - { - // Surface the error body to callers. Use `unwrap_or_default` per Clippy. - let body = res.text().await.unwrap_or_default(); - return Err(StreamAttemptError::Fatal(CodexErr::UnexpectedStatus( - UnexpectedResponseError { - status, - body, - request_id: None, - }, - ))); - } - - if status == StatusCode::TOO_MANY_REQUESTS { - let rate_limit_snapshot = parse_rate_limit_snapshot(res.headers()); - let body = res.json::().await.ok(); - if let Some(ErrorResponse { error }) = body { - if error.r#type.as_deref() == Some("usage_limit_reached") { - // Prefer the plan_type provided in the error message if present - // because it's more up to date than the one encoded in the auth - // token. - let plan_type = error - .plan_type - .or_else(|| auth.as_ref().and_then(CodexAuth::get_plan_type)); - let resets_at = error - .resets_at - .and_then(|seconds| DateTime::::from_timestamp(seconds, 0)); - let codex_err = CodexErr::UsageLimitReached(UsageLimitReachedError { - plan_type, - resets_at, - rate_limits: rate_limit_snapshot, - }); - return Err(StreamAttemptError::Fatal(codex_err)); - } else if error.r#type.as_deref() == Some("usage_not_included") { - return Err(StreamAttemptError::Fatal(CodexErr::UsageNotIncluded)); - } else if is_quota_exceeded_error(&error) { - return Err(StreamAttemptError::Fatal(CodexErr::QuotaExceeded)); - } - } - } - - Err(StreamAttemptError::RetryableHttpError { - status, - retry_after, - request_id, - }) - } - Err(e) => Err(StreamAttemptError::RetryableTransportError( - CodexErr::ConnectionFailed(ConnectionFailedError { source: e }), - )), } } @@ -522,16 +322,35 @@ impl ModelClient { self.auth_manager.clone() } + /// Compacts the current conversation history using the Compact endpoint. + /// + /// This is a unary call (no streaming) that returns a new list of + /// `ResponseItem`s representing the compacted transcript. pub async fn compact_conversation_history(&self, prompt: &Prompt) -> Result> { if prompt.input.is_empty() { return Ok(Vec::new()); } let auth_manager = self.auth_manager.clone(); let auth = auth_manager.as_ref().and_then(|m| m.auth()); - let mut req_builder = self + let api_provider = self .provider - .create_compact_request_builder(&self.client, &auth) - .await?; + .to_api_provider(auth.as_ref().map(|a| a.mode))?; + let api_auth = auth_provider_from_auth(auth.clone(), &self.provider).await?; + let transport = ReqwestTransport::new(build_reqwest_client()); + let request_telemetry = self.build_request_telemetry(); + let client = ApiCompactClient::new(transport, api_provider, api_auth) + .with_telemetry(Some(request_telemetry)); + + let instructions = prompt + .get_full_instructions(&self.config.model_family) + .into_owned(); + let payload = ApiCompactionInput { + model: &self.config.model, + input: &prompt.input, + instructions: &instructions, + }; + + let mut extra_headers = ApiHeaderMap::new(); if let SessionSource::SubAgent(sub) = &self.session_source { let subagent = if let crate::protocol::SubAgentSource::Other(label) = sub { label.clone() @@ -541,1109 +360,183 @@ impl ModelClient { .and_then(|v| v.as_str().map(std::string::ToString::to_string)) .unwrap_or_else(|| "other".to_string()) }; - req_builder = req_builder.header("x-openai-subagent", subagent); - } - if let Some(auth) = auth.as_ref() - && auth.mode == AuthMode::ChatGPT - && let Some(account_id) = auth.get_account_id() - { - req_builder = req_builder.header("chatgpt-account-id", account_id); - } - let payload = CompactHistoryRequest { - model: &self.config.model, - input: &prompt.input, - instructions: &prompt.get_full_instructions(&self.config.model_family), - }; - - if enabled!(tracing::Level::TRACE) { - trace!( - "POST to {}: {}", - self.provider - .get_compact_url(&auth) - .unwrap_or("".to_string()), - serde_json::to_value(&payload).unwrap_or_default() - ); + if let Ok(val) = HeaderValue::from_str(&subagent) { + extra_headers.insert("x-openai-subagent", val); + } } - let response = req_builder - .json(&payload) - .send() + client + .compact_input(&payload, extra_headers) .await - .map_err(|source| CodexErr::ConnectionFailed(ConnectionFailedError { source }))?; - let status = response.status(); - let body = response - .text() - .await - .map_err(|source| CodexErr::ConnectionFailed(ConnectionFailedError { source }))?; - if !status.is_success() { - return Err(CodexErr::UnexpectedStatus(UnexpectedResponseError { - status, - body, - request_id: None, - })); - } - let CompactHistoryResponse { output } = serde_json::from_str(&body)?; - Ok(output) + .map_err(map_api_error) } } -enum StreamAttemptError { - RetryableHttpError { - status: StatusCode, - retry_after: Option, - request_id: Option, - }, - RetryableTransportError(CodexErr), - Fatal(CodexErr), -} - -impl StreamAttemptError { - /// attempt is 0-based. - fn delay(&self, attempt: u64) -> Duration { - // backoff() uses 1-based attempts. - let backoff_attempt = attempt + 1; - match self { - Self::RetryableHttpError { retry_after, .. } => { - retry_after.unwrap_or_else(|| backoff(backoff_attempt)) - } - Self::RetryableTransportError { .. } => backoff(backoff_attempt), - Self::Fatal(_) => { - // Should not be called on Fatal errors. - Duration::from_secs(0) - } - } +impl ModelClient { + /// Builds request and SSE telemetry for streaming API calls (Chat/Responses). + fn build_streaming_telemetry(&self) -> (Arc, Arc) { + let telemetry = Arc::new(ApiTelemetry::new(self.otel_event_manager.clone())); + let request_telemetry: Arc = telemetry.clone(); + let sse_telemetry: Arc = telemetry; + (request_telemetry, sse_telemetry) } - fn into_error(self) -> CodexErr { - match self { - Self::RetryableHttpError { - status, request_id, .. - } => { - if status == StatusCode::INTERNAL_SERVER_ERROR { - CodexErr::InternalServerError - } else { - CodexErr::RetryLimit(RetryLimitReachedError { status, request_id }) - } - } - Self::RetryableTransportError(error) => error, - Self::Fatal(error) => error, - } + /// Builds request telemetry for unary API calls (e.g., Compact endpoint). + fn build_request_telemetry(&self) -> Arc { + let telemetry = Arc::new(ApiTelemetry::new(self.otel_event_manager.clone())); + let request_telemetry: Arc = telemetry; + request_telemetry } } -#[derive(Debug, Deserialize, Serialize)] -struct SseEvent { - #[serde(rename = "type")] - kind: String, - response: Option, - item: Option, - delta: Option, - summary_index: Option, - content_index: Option, -} - -#[derive(Debug, Deserialize)] -struct ResponseCompleted { - id: String, - usage: Option, -} - -#[derive(Debug, Deserialize)] -struct ResponseCompletedUsage { - input_tokens: i64, - input_tokens_details: Option, - output_tokens: i64, - output_tokens_details: Option, - total_tokens: i64, -} - -impl From for TokenUsage { - fn from(val: ResponseCompletedUsage) -> Self { - TokenUsage { - input_tokens: val.input_tokens, - cached_input_tokens: val - .input_tokens_details - .map(|d| d.cached_tokens) - .unwrap_or(0), - output_tokens: val.output_tokens, - reasoning_output_tokens: val - .output_tokens_details - .map(|d| d.reasoning_tokens) - .unwrap_or(0), - total_tokens: val.total_tokens, - } +/// Adapts the core `Prompt` type into the `codex-api` payload shape. +fn build_api_prompt(prompt: &Prompt, instructions: String, tools_json: Vec) -> ApiPrompt { + ApiPrompt { + instructions, + input: prompt.get_formatted_input(), + tools: tools_json, + parallel_tool_calls: prompt.parallel_tool_calls, + output_schema: prompt.output_schema.clone(), } } -#[derive(Debug, Deserialize)] -struct ResponseCompletedInputTokensDetails { - cached_tokens: i64, -} - -#[derive(Debug, Deserialize)] -struct ResponseCompletedOutputTokensDetails { - reasoning_tokens: i64, -} - -fn attach_item_ids(payload_json: &mut Value, original_items: &[ResponseItem]) { - let Some(input_value) = payload_json.get_mut("input") else { - return; - }; - let serde_json::Value::Array(items) = input_value else { - return; - }; - - for (value, item) in items.iter_mut().zip(original_items.iter()) { - if let ResponseItem::Reasoning { id, .. } - | ResponseItem::Message { id: Some(id), .. } - | ResponseItem::WebSearchCall { id: Some(id), .. } - | ResponseItem::FunctionCall { id: Some(id), .. } - | ResponseItem::LocalShellCall { id: Some(id), .. } - | ResponseItem::CustomToolCall { id: Some(id), .. } = item - { - if id.is_empty() { - continue; - } - - if let Some(obj) = value.as_object_mut() { - obj.insert("id".to_string(), Value::String(id.clone())); - } - } - } -} - -fn parse_rate_limit_snapshot(headers: &HeaderMap) -> Option { - let primary = parse_rate_limit_window( - headers, - "x-codex-primary-used-percent", - "x-codex-primary-window-minutes", - "x-codex-primary-reset-at", - ); - - let secondary = parse_rate_limit_window( - headers, - "x-codex-secondary-used-percent", - "x-codex-secondary-window-minutes", - "x-codex-secondary-reset-at", - ); - - let credits = parse_credits_snapshot(headers); - - Some(RateLimitSnapshot { - primary, - secondary, - credits, - }) -} - -fn parse_rate_limit_window( - headers: &HeaderMap, - used_percent_header: &str, - window_minutes_header: &str, - resets_at_header: &str, -) -> Option { - let used_percent: Option = parse_header_f64(headers, used_percent_header); - - used_percent.and_then(|used_percent| { - let window_minutes = parse_header_i64(headers, window_minutes_header); - let resets_at = parse_header_i64(headers, resets_at_header); - - let has_data = used_percent != 0.0 - || window_minutes.is_some_and(|minutes| minutes != 0) - || resets_at.is_some(); - - has_data.then_some(RateLimitWindow { - used_percent, - window_minutes, - resets_at, - }) - }) -} - -fn parse_credits_snapshot(headers: &HeaderMap) -> Option { - let has_credits = parse_header_bool(headers, "x-codex-credits-has-credits")?; - let unlimited = parse_header_bool(headers, "x-codex-credits-unlimited")?; - let balance = parse_header_str(headers, "x-codex-credits-balance") - .map(str::trim) - .filter(|value| !value.is_empty()) - .map(std::string::ToString::to_string); - Some(CreditsSnapshot { - has_credits, - unlimited, - balance, - }) -} - -fn parse_header_f64(headers: &HeaderMap, name: &str) -> Option { - parse_header_str(headers, name)? - .parse::() - .ok() - .filter(|v| v.is_finite()) -} - -fn parse_header_i64(headers: &HeaderMap, name: &str) -> Option { - parse_header_str(headers, name)?.parse::().ok() -} - -fn parse_header_bool(headers: &HeaderMap, name: &str) -> Option { - let raw = parse_header_str(headers, name)?; - if raw.eq_ignore_ascii_case("true") || raw == "1" { - Some(true) - } else if raw.eq_ignore_ascii_case("false") || raw == "0" { - Some(false) - } else { - None - } -} - -fn parse_header_str<'a>(headers: &'a HeaderMap, name: &str) -> Option<&'a str> { - headers.get(name)?.to_str().ok() -} - -async fn process_sse( - stream: S, - tx_event: mpsc::Sender>, - idle_timeout: Duration, - otel_event_manager: OtelEventManager, -) where - S: Stream> + Unpin, +fn map_response_stream(api_stream: S, otel_event_manager: OtelEventManager) -> ResponseStream +where + S: futures::Stream> + + Unpin + + Send + + 'static, { - let mut stream = stream.eventsource(); - - // If the stream stays completely silent for an extended period treat it as disconnected. - // The response id returned from the "complete" message. - let mut response_completed: Option = None; - let mut response_error: Option = None; - - loop { - let start = std::time::Instant::now(); - let response = timeout(idle_timeout, stream.next()).await; - let duration = start.elapsed(); - otel_event_manager.log_sse_event(&response, duration); - - let sse = match response { - Ok(Some(Ok(sse))) => sse, - Ok(Some(Err(e))) => { - debug!("SSE Error: {e:#}"); - let event = CodexErr::Stream(e.to_string(), None); - let _ = tx_event.send(Err(event)).await; - return; - } - Ok(None) => { - match response_completed { - Some(ResponseCompleted { - id: response_id, - usage, - }) => { - if let Some(token_usage) = &usage { - otel_event_manager.sse_event_completed( - token_usage.input_tokens, - token_usage.output_tokens, - token_usage - .input_tokens_details - .as_ref() - .map(|d| d.cached_tokens), - token_usage - .output_tokens_details - .as_ref() - .map(|d| d.reasoning_tokens), - token_usage.total_tokens, - ); - } - let event = ResponseEvent::Completed { - response_id, - token_usage: usage.map(Into::into), - }; - let _ = tx_event.send(Ok(event)).await; - } - None => { - let error = response_error.unwrap_or(CodexErr::Stream( - "stream closed before response.completed".into(), - None, - )); - otel_event_manager.see_event_completed_failed(&error); - - let _ = tx_event.send(Err(error)).await; - } - } - return; - } - Err(_) => { - let _ = tx_event - .send(Err(CodexErr::Stream( - "idle timeout waiting for SSE".into(), - None, - ))) - .await; - return; - } - }; - - let raw = sse.data.clone(); - trace!("SSE event: {}", raw); - - let event: SseEvent = match serde_json::from_str(&sse.data) { - Ok(event) => event, - Err(e) => { - debug!("Failed to parse SSE event: {e}, data: {}", &sse.data); - continue; - } - }; - - match event.kind.as_str() { - // Individual output item finalised. Forward immediately so the - // rest of the agent can stream assistant text/functions *live* - // instead of waiting for the final `response.completed` envelope. - // - // IMPORTANT: We used to ignore these events and forward the - // duplicated `output` array embedded in the `response.completed` - // payload. That produced two concrete issues: - // 1. No real‑time streaming – the user only saw output after the - // entire turn had finished, which broke the "typing" UX and - // made long‑running turns look stalled. - // 2. Duplicate `function_call_output` items – both the - // individual *and* the completed array were forwarded, which - // confused the backend and triggered 400 - // "previous_response_not_found" errors because the duplicated - // IDs did not match the incremental turn chain. - // - // The fix is to forward the incremental events *as they come* and - // drop the duplicated list inside `response.completed`. - "response.output_item.done" => { - let Some(item_val) = event.item else { continue }; - let Ok(item) = serde_json::from_value::(item_val) else { - debug!("failed to parse ResponseItem from output_item.done"); - continue; - }; - - let event = ResponseEvent::OutputItemDone(item); - if tx_event.send(Ok(event)).await.is_err() { - return; - } - } - "response.output_text.delta" => { - if let Some(delta) = event.delta { - let event = ResponseEvent::OutputTextDelta(delta); - if tx_event.send(Ok(event)).await.is_err() { - return; - } - } - } - "response.reasoning_summary_text.delta" => { - if let (Some(delta), Some(summary_index)) = (event.delta, event.summary_index) { - let event = ResponseEvent::ReasoningSummaryDelta { - delta, - summary_index, - }; - if tx_event.send(Ok(event)).await.is_err() { - return; - } - } - } - "response.reasoning_text.delta" => { - if let (Some(delta), Some(content_index)) = (event.delta, event.content_index) { - let event = ResponseEvent::ReasoningContentDelta { - delta, - content_index, - }; - if tx_event.send(Ok(event)).await.is_err() { - return; - } - } - } - "response.created" => { - if event.response.is_some() { - let _ = tx_event.send(Ok(ResponseEvent::Created {})).await; - } - } - "response.failed" => { - if let Some(resp_val) = event.response { - response_error = Some(CodexErr::Stream( - "response.failed event received".to_string(), - None, - )); - - let error = resp_val.get("error"); - - if let Some(error) = error { - match serde_json::from_value::(error.clone()) { - Ok(error) => { - if is_context_window_error(&error) { - response_error = Some(CodexErr::ContextWindowExceeded); - } else if is_quota_exceeded_error(&error) { - response_error = Some(CodexErr::QuotaExceeded); - } else { - let delay = try_parse_retry_after(&error); - let message = error.message.clone().unwrap_or_default(); - response_error = Some(CodexErr::Stream(message, delay)); - } - } - Err(e) => { - let error = format!("failed to parse ErrorResponse: {e}"); - debug!(error); - response_error = Some(CodexErr::Stream(error, None)) - } - } - } - } - } - // Final response completed – includes array of output items & id - "response.completed" => { - if let Some(resp_val) = event.response { - match serde_json::from_value::(resp_val) { - Ok(r) => { - response_completed = Some(r); - } - Err(e) => { - let error = format!("failed to parse ResponseCompleted: {e}"); - debug!(error); - response_error = Some(CodexErr::Stream(error, None)); - continue; - } - }; - }; - } - "response.content_part.done" - | "response.function_call_arguments.delta" - | "response.custom_tool_call_input.delta" - | "response.custom_tool_call_input.done" // also emitted as response.output_item.done - | "response.in_progress" - | "response.output_text.done" => {} - "response.output_item.added" => { - let Some(item_val) = event.item else { continue }; - let Ok(item) = serde_json::from_value::(item_val) else { - debug!("failed to parse ResponseItem from output_item.done"); - continue; - }; - - let event = ResponseEvent::OutputItemAdded(item); - if tx_event.send(Ok(event)).await.is_err() { - return; - } - } - "response.reasoning_summary_part.added" => { - if let Some(summary_index) = event.summary_index { - // Boundary between reasoning summary sections (e.g., titles). - let event = ResponseEvent::ReasoningSummaryPartAdded { summary_index }; - if tx_event.send(Ok(event)).await.is_err() { - return; - } - } - } - "response.reasoning_summary_text.done" => {} - _ => {} - } - } -} - -/// used in tests to stream from a text SSE file -async fn stream_from_fixture( - path: impl AsRef, - provider: ModelProviderInfo, - otel_event_manager: OtelEventManager, -) -> Result { let (tx_event, rx_event) = mpsc::channel::>(1600); - let f = std::fs::File::open(path.as_ref())?; - let lines = std::io::BufReader::new(f).lines(); - - // insert \n\n after each line for proper SSE parsing - let mut content = String::new(); - for line in lines { - content.push_str(&line?); - content.push_str("\n\n"); - } - - let rdr = std::io::Cursor::new(content); - let stream = ReaderStream::new(rdr).map_err(CodexErr::Io); - tokio::spawn(process_sse( - stream, - tx_event, - provider.stream_idle_timeout(), - otel_event_manager, - )); - Ok(ResponseStream { rx_event }) -} - -fn rate_limit_regex() -> &'static Regex { - static RE: OnceLock = OnceLock::new(); - - // Match both OpenAI-style messages like "Please try again in 1.898s" - // and Azure OpenAI-style messages like "Try again in 35 seconds". - #[expect(clippy::unwrap_used)] - RE.get_or_init(|| Regex::new(r"(?i)try again in\s*(\d+(?:\.\d+)?)\s*(s|ms|seconds?)").unwrap()) -} - -fn try_parse_retry_after(err: &Error) -> Option { - if err.code != Some("rate_limit_exceeded".to_string()) { - return None; - } - - // parse retry hints like "try again in 1.898s" or - // "Try again in 35 seconds" using regex - let re = rate_limit_regex(); - if let Some(message) = &err.message - && let Some(captures) = re.captures(message) - { - let seconds = captures.get(1); - let unit = captures.get(2); - - if let (Some(value), Some(unit)) = (seconds, unit) { - let value = value.as_str().parse::().ok()?; - let unit = unit.as_str().to_ascii_lowercase(); - - if unit == "s" || unit.starts_with("second") { - return Some(Duration::from_secs_f64(value)); - } else if unit == "ms" { - return Some(Duration::from_millis(value as u64)); - } - } - } - None -} - -fn is_context_window_error(error: &Error) -> bool { - error.code.as_deref() == Some("context_length_exceeded") -} - -fn is_quota_exceeded_error(error: &Error) -> bool { - error.code.as_deref() == Some("insufficient_quota") -} - -#[cfg(test)] -mod tests { - use super::*; - use assert_matches::assert_matches; - use serde_json::json; - use tokio::sync::mpsc; - use tokio_test::io::Builder as IoBuilder; - use tokio_util::io::ReaderStream; - - // ──────────────────────────── - // Helpers - // ──────────────────────────── - - /// Runs the SSE parser on pre-chunked byte slices and returns every event - /// (including any final `Err` from a stream-closure check). - async fn collect_events( - chunks: &[&[u8]], - provider: ModelProviderInfo, - otel_event_manager: OtelEventManager, - ) -> Vec> { - let mut builder = IoBuilder::new(); - for chunk in chunks { - builder.read(chunk); - } - - let reader = builder.build(); - let stream = ReaderStream::new(reader).map_err(CodexErr::Io); - let (tx, mut rx) = mpsc::channel::>(16); - tokio::spawn(process_sse( - stream, - tx, - provider.stream_idle_timeout(), - otel_event_manager, - )); - - let mut events = Vec::new(); - while let Some(ev) = rx.recv().await { - events.push(ev); - } - events - } - - /// Builds an in-memory SSE stream from JSON fixtures and returns only the - /// successfully parsed events (panics on internal channel errors). - async fn run_sse( - events: Vec, - provider: ModelProviderInfo, - otel_event_manager: OtelEventManager, - ) -> Vec { - let mut body = String::new(); - for e in events { - let kind = e - .get("type") - .and_then(|v| v.as_str()) - .expect("fixture event missing type"); - if e.as_object().map(|o| o.len() == 1).unwrap_or(false) { - body.push_str(&format!("event: {kind}\n\n")); - } else { - body.push_str(&format!("event: {kind}\ndata: {e}\n\n")); - } - } - - let (tx, mut rx) = mpsc::channel::>(8); - let stream = ReaderStream::new(std::io::Cursor::new(body)).map_err(CodexErr::Io); - tokio::spawn(process_sse( - stream, - tx, - provider.stream_idle_timeout(), - otel_event_manager, - )); - - let mut out = Vec::new(); - while let Some(ev) = rx.recv().await { - out.push(ev.expect("channel closed")); - } - out - } - - fn otel_event_manager() -> OtelEventManager { - OtelEventManager::new( - ConversationId::new(), - "test", - "test", - None, - Some("test@test.com".to_string()), - Some(AuthMode::ChatGPT), - false, - "test".to_string(), - ) - } - - // ──────────────────────────── - // Tests from `implement-test-for-responses-api-sse-parser` - // ──────────────────────────── - - #[tokio::test] - async fn parses_items_and_completed() { - let item1 = json!({ - "type": "response.output_item.done", - "item": { - "type": "message", - "role": "assistant", - "content": [{"type": "output_text", "text": "Hello"}] - } - }) - .to_string(); - - let item2 = json!({ - "type": "response.output_item.done", - "item": { - "type": "message", - "role": "assistant", - "content": [{"type": "output_text", "text": "World"}] - } - }) - .to_string(); - - let completed = json!({ - "type": "response.completed", - "response": { "id": "resp1" } - }) - .to_string(); - - let sse1 = format!("event: response.output_item.done\ndata: {item1}\n\n"); - let sse2 = format!("event: response.output_item.done\ndata: {item2}\n\n"); - let sse3 = format!("event: response.completed\ndata: {completed}\n\n"); - - let provider = ModelProviderInfo { - name: "test".to_string(), - base_url: Some("https://test.com".to_string()), - env_key: Some("TEST_API_KEY".to_string()), - env_key_instructions: None, - experimental_bearer_token: None, - wire_api: WireApi::Responses, - query_params: None, - http_headers: None, - env_http_headers: None, - request_max_retries: Some(0), - stream_max_retries: Some(0), - stream_idle_timeout_ms: Some(1000), - requires_openai_auth: false, - }; - - let otel_event_manager = otel_event_manager(); - - let events = collect_events( - &[sse1.as_bytes(), sse2.as_bytes(), sse3.as_bytes()], - provider, - otel_event_manager, - ) - .await; - - assert_eq!(events.len(), 3); - - matches!( - &events[0], - Ok(ResponseEvent::OutputItemDone(ResponseItem::Message { role, .. })) - if role == "assistant" - ); - - matches!( - &events[1], - Ok(ResponseEvent::OutputItemDone(ResponseItem::Message { role, .. })) - if role == "assistant" - ); - - match &events[2] { - Ok(ResponseEvent::Completed { - response_id, - token_usage, - }) => { - assert_eq!(response_id, "resp1"); - assert!(token_usage.is_none()); - } - other => panic!("unexpected third event: {other:?}"), - } - } - - #[tokio::test] - async fn error_when_missing_completed() { - let item1 = json!({ - "type": "response.output_item.done", - "item": { - "type": "message", - "role": "assistant", - "content": [{"type": "output_text", "text": "Hello"}] - } - }) - .to_string(); - - let sse1 = format!("event: response.output_item.done\ndata: {item1}\n\n"); - let provider = ModelProviderInfo { - name: "test".to_string(), - base_url: Some("https://test.com".to_string()), - env_key: Some("TEST_API_KEY".to_string()), - env_key_instructions: None, - experimental_bearer_token: None, - wire_api: WireApi::Responses, - query_params: None, - http_headers: None, - env_http_headers: None, - request_max_retries: Some(0), - stream_max_retries: Some(0), - stream_idle_timeout_ms: Some(1000), - requires_openai_auth: false, - }; - - let otel_event_manager = otel_event_manager(); - - let events = collect_events(&[sse1.as_bytes()], provider, otel_event_manager).await; - - assert_eq!(events.len(), 2); - - matches!(events[0], Ok(ResponseEvent::OutputItemDone(_))); - - match &events[1] { - Err(CodexErr::Stream(msg, _)) => { - assert_eq!(msg, "stream closed before response.completed") - } - other => panic!("unexpected second event: {other:?}"), - } - } - - #[tokio::test] - async fn error_when_error_event() { - let raw_error = r#"{"type":"response.failed","sequence_number":3,"response":{"id":"resp_689bcf18d7f08194bf3440ba62fe05d803fee0cdac429894","object":"response","created_at":1755041560,"status":"failed","background":false,"error":{"code":"rate_limit_exceeded","message":"Rate limit reached for gpt-5.1 in organization org-AAA on tokens per min (TPM): Limit 30000, Used 22999, Requested 12528. Please try again in 11.054s. Visit https://platform.openai.com/account/rate-limits to learn more."}, "usage":null,"user":null,"metadata":{}}}"#; - - let sse1 = format!("event: response.failed\ndata: {raw_error}\n\n"); - let provider = ModelProviderInfo { - name: "test".to_string(), - base_url: Some("https://test.com".to_string()), - env_key: Some("TEST_API_KEY".to_string()), - env_key_instructions: None, - experimental_bearer_token: None, - wire_api: WireApi::Responses, - query_params: None, - http_headers: None, - env_http_headers: None, - request_max_retries: Some(0), - stream_max_retries: Some(0), - stream_idle_timeout_ms: Some(1000), - requires_openai_auth: false, - }; - - let otel_event_manager = otel_event_manager(); - - let events = collect_events(&[sse1.as_bytes()], provider, otel_event_manager).await; - - assert_eq!(events.len(), 1); - - match &events[0] { - Err(CodexErr::Stream(msg, delay)) => { - assert_eq!( - msg, - "Rate limit reached for gpt-5.1 in organization org-AAA on tokens per min (TPM): Limit 30000, Used 22999, Requested 12528. Please try again in 11.054s. Visit https://platform.openai.com/account/rate-limits to learn more." - ); - assert_eq!(*delay, Some(Duration::from_secs_f64(11.054))); - } - other => panic!("unexpected second event: {other:?}"), - } - } - - #[tokio::test] - async fn context_window_error_is_fatal() { - let raw_error = r#"{"type":"response.failed","sequence_number":3,"response":{"id":"resp_5c66275b97b9baef1ed95550adb3b7ec13b17aafd1d2f11b","object":"response","created_at":1759510079,"status":"failed","background":false,"error":{"code":"context_length_exceeded","message":"Your input exceeds the context window of this model. Please adjust your input and try again."},"usage":null,"user":null,"metadata":{}}}"#; - - let sse1 = format!("event: response.failed\ndata: {raw_error}\n\n"); - let provider = ModelProviderInfo { - name: "test".to_string(), - base_url: Some("https://test.com".to_string()), - env_key: Some("TEST_API_KEY".to_string()), - env_key_instructions: None, - experimental_bearer_token: None, - wire_api: WireApi::Responses, - query_params: None, - http_headers: None, - env_http_headers: None, - request_max_retries: Some(0), - stream_max_retries: Some(0), - stream_idle_timeout_ms: Some(1000), - requires_openai_auth: false, - }; - - let otel_event_manager = otel_event_manager(); - - let events = collect_events(&[sse1.as_bytes()], provider, otel_event_manager).await; - - assert_eq!(events.len(), 1); - - match &events[0] { - Err(err @ CodexErr::ContextWindowExceeded) => { - assert_eq!(err.to_string(), CodexErr::ContextWindowExceeded.to_string()); - } - other => panic!("unexpected context window event: {other:?}"), - } - } - - #[tokio::test] - async fn context_window_error_with_newline_is_fatal() { - let raw_error = r#"{"type":"response.failed","sequence_number":4,"response":{"id":"resp_fatal_newline","object":"response","created_at":1759510080,"status":"failed","background":false,"error":{"code":"context_length_exceeded","message":"Your input exceeds the context window of this model. Please adjust your input and try\nagain."},"usage":null,"user":null,"metadata":{}}}"#; - - let sse1 = format!("event: response.failed\ndata: {raw_error}\n\n"); - let provider = ModelProviderInfo { - name: "test".to_string(), - base_url: Some("https://test.com".to_string()), - env_key: Some("TEST_API_KEY".to_string()), - env_key_instructions: None, - experimental_bearer_token: None, - wire_api: WireApi::Responses, - query_params: None, - http_headers: None, - env_http_headers: None, - request_max_retries: Some(0), - stream_max_retries: Some(0), - stream_idle_timeout_ms: Some(1000), - requires_openai_auth: false, - }; - - let otel_event_manager = otel_event_manager(); - - let events = collect_events(&[sse1.as_bytes()], provider, otel_event_manager).await; - - assert_eq!(events.len(), 1); - - match &events[0] { - Err(err @ CodexErr::ContextWindowExceeded) => { - assert_eq!(err.to_string(), CodexErr::ContextWindowExceeded.to_string()); - } - other => panic!("unexpected context window event: {other:?}"), - } - } - - #[tokio::test] - async fn quota_exceeded_error_is_fatal() { - let raw_error = r#"{"type":"response.failed","sequence_number":3,"response":{"id":"resp_fatal_quota","object":"response","created_at":1759771626,"status":"failed","background":false,"error":{"code":"insufficient_quota","message":"You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."},"incomplete_details":null}}"#; - - let sse1 = format!("event: response.failed\ndata: {raw_error}\n\n"); - let provider = ModelProviderInfo { - name: "test".to_string(), - base_url: Some("https://test.com".to_string()), - env_key: Some("TEST_API_KEY".to_string()), - env_key_instructions: None, - experimental_bearer_token: None, - wire_api: WireApi::Responses, - query_params: None, - http_headers: None, - env_http_headers: None, - request_max_retries: Some(0), - stream_max_retries: Some(0), - stream_idle_timeout_ms: Some(1000), - requires_openai_auth: false, - }; - - let otel_event_manager = otel_event_manager(); - - let events = collect_events(&[sse1.as_bytes()], provider, otel_event_manager).await; - - assert_eq!(events.len(), 1); - - match &events[0] { - Err(err @ CodexErr::QuotaExceeded) => { - assert_eq!(err.to_string(), CodexErr::QuotaExceeded.to_string()); - } - other => panic!("unexpected quota exceeded event: {other:?}"), - } - } - - // ──────────────────────────── - // Table-driven test from `main` - // ──────────────────────────── - - /// Verifies that the adapter produces the right `ResponseEvent` for a - /// variety of incoming `type` values. - #[tokio::test] - async fn table_driven_event_kinds() { - struct TestCase { - name: &'static str, - event: serde_json::Value, - expect_first: fn(&ResponseEvent) -> bool, - expected_len: usize, - } - - fn is_created(ev: &ResponseEvent) -> bool { - matches!(ev, ResponseEvent::Created) - } - fn is_output(ev: &ResponseEvent) -> bool { - matches!(ev, ResponseEvent::OutputItemDone(_)) - } - fn is_completed(ev: &ResponseEvent) -> bool { - matches!(ev, ResponseEvent::Completed { .. }) - } - - let completed = json!({ - "type": "response.completed", - "response": { - "id": "c", - "usage": { - "input_tokens": 0, - "input_tokens_details": null, - "output_tokens": 0, - "output_tokens_details": null, - "total_tokens": 0 - }, - "output": [] - } - }); - - let cases = vec![ - TestCase { - name: "created", - event: json!({"type": "response.created", "response": {}}), - expect_first: is_created, - expected_len: 2, - }, - TestCase { - name: "output_item.done", - event: json!({ - "type": "response.output_item.done", - "item": { - "type": "message", - "role": "assistant", - "content": [ - {"type": "output_text", "text": "hi"} - ] + let manager = otel_event_manager; + + tokio::spawn(async move { + let mut logged_error = false; + let mut api_stream = api_stream; + while let Some(event) = api_stream.next().await { + match event { + Ok(ResponseEvent::Completed { + response_id, + token_usage, + }) => { + if let Some(usage) = &token_usage { + manager.sse_event_completed( + usage.input_tokens, + usage.output_tokens, + Some(usage.cached_input_tokens), + Some(usage.reasoning_output_tokens), + usage.total_tokens, + ); } - }), - expect_first: is_output, - expected_len: 2, - }, - TestCase { - name: "unknown", - event: json!({"type": "response.new_tool_event"}), - expect_first: is_completed, - expected_len: 1, - }, - ]; - - for case in cases { - let mut evs = vec![case.event]; - evs.push(completed.clone()); - - let provider = ModelProviderInfo { - name: "test".to_string(), - base_url: Some("https://test.com".to_string()), - env_key: Some("TEST_API_KEY".to_string()), - env_key_instructions: None, - experimental_bearer_token: None, - wire_api: WireApi::Responses, - query_params: None, - http_headers: None, - env_http_headers: None, - request_max_retries: Some(0), - stream_max_retries: Some(0), - stream_idle_timeout_ms: Some(1000), - requires_openai_auth: false, - }; - - let otel_event_manager = otel_event_manager(); - - let out = run_sse(evs, provider, otel_event_manager).await; - assert_eq!(out.len(), case.expected_len, "case {}", case.name); - assert!( - (case.expect_first)(&out[0]), - "first event mismatch in case {}", - case.name - ); + if tx_event + .send(Ok(ResponseEvent::Completed { + response_id, + token_usage, + })) + .await + .is_err() + { + return; + } + } + Ok(event) => { + if tx_event.send(Ok(event)).await.is_err() { + return; + } + } + Err(err) => { + let mapped = map_api_error(err); + if !logged_error { + manager.see_event_completed_failed(&mapped); + logged_error = true; + } + if tx_event.send(Err(mapped)).await.is_err() { + return; + } + } + } } + }); + + ResponseStream { rx_event } +} + +/// Handles a 401 response by optionally refreshing ChatGPT tokens once. +/// +/// When refresh succeeds, the caller should retry the API call; otherwise +/// the mapped `CodexErr` is returned to the caller. +async fn handle_unauthorized( + status: StatusCode, + refreshed: &mut bool, + auth_manager: &Option>, + auth: &Option, +) -> Result<()> { + if *refreshed { + return Err(map_unauthorized_status(status)); } - #[test] - fn test_try_parse_retry_after() { - let err = Error { - r#type: None, - message: Some("Rate limit reached for gpt-5.1 in organization org- on tokens per min (TPM): Limit 1, Used 1, Requested 19304. Please try again in 28ms. Visit https://platform.openai.com/account/rate-limits to learn more.".to_string()), - code: Some("rate_limit_exceeded".to_string()), - plan_type: None, - resets_at: None - }; - - let delay = try_parse_retry_after(&err); - assert_eq!(delay, Some(Duration::from_millis(28))); - } - - #[test] - fn test_try_parse_retry_after_no_delay() { - let err = Error { - r#type: None, - message: Some("Rate limit reached for gpt-5.1 in organization on tokens per min (TPM): Limit 30000, Used 6899, Requested 24050. Please try again in 1.898s. Visit https://platform.openai.com/account/rate-limits to learn more.".to_string()), - code: Some("rate_limit_exceeded".to_string()), - plan_type: None, - resets_at: None - }; - let delay = try_parse_retry_after(&err); - assert_eq!(delay, Some(Duration::from_secs_f64(1.898))); - } - - #[test] - fn test_try_parse_retry_after_azure() { - let err = Error { - r#type: None, - message: Some("Rate limit exceeded. Try again in 35 seconds.".to_string()), - code: Some("rate_limit_exceeded".to_string()), - plan_type: None, - resets_at: None, - }; - let delay = try_parse_retry_after(&err); - assert_eq!(delay, Some(Duration::from_secs(35))); - } - - #[test] - fn error_response_deserializes_schema_known_plan_type_and_serializes_back() { - use crate::token_data::KnownPlan; - use crate::token_data::PlanType; - - let json = - r#"{"error":{"type":"usage_limit_reached","plan_type":"pro","resets_at":1704067200}}"#; - let resp: ErrorResponse = serde_json::from_str(json).expect("should deserialize schema"); - - assert_matches!(resp.error.plan_type, Some(PlanType::Known(KnownPlan::Pro))); - - let plan_json = serde_json::to_string(&resp.error.plan_type).expect("serialize plan_type"); - assert_eq!(plan_json, "\"pro\""); - } - - #[test] - fn error_response_deserializes_schema_unknown_plan_type_and_serializes_back() { - use crate::token_data::PlanType; - - let json = - r#"{"error":{"type":"usage_limit_reached","plan_type":"vip","resets_at":1704067260}}"#; - let resp: ErrorResponse = serde_json::from_str(json).expect("should deserialize schema"); - - assert_matches!(resp.error.plan_type, Some(PlanType::Unknown(ref s)) if s == "vip"); - - let plan_json = serde_json::to_string(&resp.error.plan_type).expect("serialize plan_type"); - assert_eq!(plan_json, "\"vip\""); + if let Some(manager) = auth_manager.as_ref() + && let Some(auth) = auth.as_ref() + && auth.mode == AuthMode::ChatGPT + { + match manager.refresh_token().await { + Ok(_) => { + *refreshed = true; + Ok(()) + } + Err(RefreshTokenError::Permanent(failed)) => Err(CodexErr::RefreshTokenFailed(failed)), + Err(RefreshTokenError::Transient(other)) => Err(CodexErr::Io(other)), + } + } else { + Err(map_unauthorized_status(status)) + } +} + +fn map_unauthorized_status(status: StatusCode) -> CodexErr { + map_api_error(ApiError::Transport(TransportError::Http { + status, + headers: None, + body: None, + })) +} + +struct ApiTelemetry { + otel_event_manager: OtelEventManager, +} + +impl ApiTelemetry { + fn new(otel_event_manager: OtelEventManager) -> Self { + Self { otel_event_manager } + } +} + +impl RequestTelemetry for ApiTelemetry { + fn on_request( + &self, + attempt: u64, + status: Option, + error: Option<&TransportError>, + duration: Duration, + ) { + let error_message = error.map(std::string::ToString::to_string); + self.otel_event_manager.record_api_request( + attempt, + status.map(|s| s.as_u16()), + error_message.as_deref(), + duration, + ); + } +} + +impl SseTelemetry for ApiTelemetry { + fn on_sse_poll( + &self, + result: &std::result::Result< + Option>>, + tokio::time::error::Elapsed, + >, + duration: Duration, + ) { + self.otel_event_manager.log_sse_event(result, duration); } } diff --git a/codex-rs/core/src/client_common.rs b/codex-rs/core/src/client_common.rs index 21f6fc6576..a249ca6fcc 100644 --- a/codex-rs/core/src/client_common.rs +++ b/codex-rs/core/src/client_common.rs @@ -1,16 +1,11 @@ use crate::client_common::tools::ToolSpec; use crate::error::Result; use crate::model_family::ModelFamily; -use crate::protocol::RateLimitSnapshot; -use crate::protocol::TokenUsage; +pub use codex_api::common::ResponseEvent; use codex_apply_patch::APPLY_PATCH_TOOL_INSTRUCTIONS; -use codex_protocol::config_types::ReasoningEffort as ReasoningEffortConfig; -use codex_protocol::config_types::ReasoningSummary as ReasoningSummaryConfig; -use codex_protocol::config_types::Verbosity as VerbosityConfig; use codex_protocol::models::ResponseItem; use futures::Stream; use serde::Deserialize; -use serde::Serialize; use serde_json::Value; use std::borrow::Cow; use std::collections::HashSet; @@ -184,104 +179,6 @@ fn strip_total_output_header(output: &str) -> Option<(&str, u32)> { Some((remainder, total_lines)) } -#[derive(Debug)] -pub enum ResponseEvent { - Created, - OutputItemDone(ResponseItem), - OutputItemAdded(ResponseItem), - Completed { - response_id: String, - token_usage: Option, - }, - OutputTextDelta(String), - ReasoningSummaryDelta { - delta: String, - summary_index: i64, - }, - ReasoningContentDelta { - delta: String, - content_index: i64, - }, - ReasoningSummaryPartAdded { - summary_index: i64, - }, - RateLimits(RateLimitSnapshot), -} - -#[derive(Debug, Serialize)] -pub(crate) struct Reasoning { - #[serde(skip_serializing_if = "Option::is_none")] - pub(crate) effort: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub(crate) summary: Option, -} - -#[derive(Debug, Serialize, Default, Clone)] -#[serde(rename_all = "snake_case")] -pub(crate) enum TextFormatType { - #[default] - JsonSchema, -} - -#[derive(Debug, Serialize, Default, Clone)] -pub(crate) struct TextFormat { - pub(crate) r#type: TextFormatType, - pub(crate) strict: bool, - pub(crate) schema: Value, - pub(crate) name: String, -} - -/// Controls under the `text` field in the Responses API for GPT-5. -#[derive(Debug, Serialize, Default, Clone)] -pub(crate) struct TextControls { - #[serde(skip_serializing_if = "Option::is_none")] - pub(crate) verbosity: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub(crate) format: Option, -} - -#[derive(Debug, Serialize, Default, Clone)] -#[serde(rename_all = "lowercase")] -pub(crate) enum OpenAiVerbosity { - Low, - #[default] - Medium, - High, -} - -impl From for OpenAiVerbosity { - fn from(v: VerbosityConfig) -> Self { - match v { - VerbosityConfig::Low => OpenAiVerbosity::Low, - VerbosityConfig::Medium => OpenAiVerbosity::Medium, - VerbosityConfig::High => OpenAiVerbosity::High, - } - } -} - -/// Request object that is serialized as JSON and POST'ed when using the -/// Responses API. -#[derive(Debug, Serialize)] -pub(crate) struct ResponsesApiRequest<'a> { - pub(crate) model: &'a str, - pub(crate) instructions: &'a str, - // TODO(mbolin): ResponseItem::Other should not be serialized. Currently, - // we code defensively to avoid this case, but perhaps we should use a - // separate enum for serialization. - pub(crate) input: &'a Vec, - pub(crate) tools: &'a [serde_json::Value], - pub(crate) tool_choice: &'static str, - pub(crate) parallel_tool_calls: bool, - pub(crate) reasoning: Option, - pub(crate) store: bool, - pub(crate) stream: bool, - pub(crate) include: Vec, - #[serde(skip_serializing_if = "Option::is_none")] - pub(crate) prompt_cache_key: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub(crate) text: Option, -} - pub(crate) mod tools { use crate::tools::spec::JsonSchema; use serde::Deserialize; @@ -341,25 +238,6 @@ pub(crate) mod tools { } } -pub(crate) fn create_text_param_for_request( - verbosity: Option, - output_schema: &Option, -) -> Option { - if verbosity.is_none() && output_schema.is_none() { - return None; - } - - Some(TextControls { - verbosity: verbosity.map(std::convert::Into::into), - format: output_schema.as_ref().map(|schema| TextFormat { - r#type: TextFormatType::JsonSchema, - strict: true, - schema: schema.clone(), - name: "codex_output_schema".to_string(), - }), - }) -} - pub struct ResponseStream { pub(crate) rx_event: mpsc::Receiver>, } @@ -375,6 +253,10 @@ impl Stream for ResponseStream { #[cfg(test)] mod tests { use crate::model_family::find_family_for_model; + use codex_api::ResponsesApiRequest; + use codex_api::common::OpenAiVerbosity; + use codex_api::common::TextControls; + use codex_api::create_text_param_for_request; use pretty_assertions::assert_eq; use super::*; diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index ad8283169d..79789014cb 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -5,6 +5,7 @@ use std::sync::Arc; use std::sync::atomic::AtomicU64; use crate::AuthManager; +use crate::SandboxState; use crate::client_common::REVIEW_PROMPT; use crate::compact; use crate::compact::run_inline_auto_compact_task; @@ -34,6 +35,7 @@ use codex_protocol::protocol::SessionSource; use codex_protocol::protocol::TaskStartedEvent; use codex_protocol::protocol::TurnAbortReason; use codex_protocol::protocol::TurnContextItem; +use codex_rmcp_client::ElicitationResponse; use futures::future::BoxFuture; use futures::prelude::*; use futures::stream::FuturesOrdered; @@ -44,6 +46,7 @@ use mcp_types::ListResourcesRequestParams; use mcp_types::ListResourcesResult; use mcp_types::ReadResourceRequestParams; use mcp_types::ReadResourceResult; +use mcp_types::RequestId; use serde_json; use serde_json::Value; use tokio::sync::Mutex; @@ -79,7 +82,6 @@ use crate::protocol::ApplyPatchApprovalRequestEvent; use crate::protocol::AskForApproval; use crate::protocol::BackgroundEventEvent; use crate::protocol::DeprecationNoticeEvent; -use crate::protocol::ErrorEvent; use crate::protocol::Event; use crate::protocol::EventMsg; use crate::protocol::ExecApprovalRequestEvent; @@ -100,6 +102,7 @@ use crate::protocol::TurnDiffEvent; use crate::protocol::WarningEvent; use crate::rollout::RolloutRecorder; use crate::rollout::RolloutRecorderParams; +use crate::rollout::map_session_init_error; use crate::saved_sessions::build_saved_session_entry; use crate::saved_sessions::upsert_saved_session; use crate::shell; @@ -131,11 +134,11 @@ use codex_protocol::models::ContentItem; use codex_protocol::models::FunctionCallOutputPayload; use codex_protocol::models::ResponseInputItem; use codex_protocol::models::ResponseItem; +use codex_protocol::protocol::CodexErrorInfo; use codex_protocol::protocol::InitialHistory; use codex_protocol::user_input::UserInput; use codex_utils_readiness::Readiness; use codex_utils_readiness::ReadinessFlag; -use codex_utils_tokenizer::warm_model_cache; use reqwest::StatusCode; use std::path::Path; @@ -209,7 +212,7 @@ impl Codex { .await .map_err(|e| { error!("Failed to create session: {e:#}"); - CodexErr::InternalAgentDied + map_session_init_error(&e, &config.codex_home) })?; let conversation_id = session.conversation_id; @@ -500,7 +503,7 @@ impl Session { // - load history metadata let rollout_fut = RolloutRecorder::new(&config, rollout_params); - let default_shell_fut = shell::default_user_shell(); + let default_shell = shell::default_user_shell(); let history_meta_fut = crate::message_history::history_metadata(&config); let auth_statuses_fut = compute_auth_statuses( config.mcp_servers.iter(), @@ -508,16 +511,12 @@ impl Session { ); // Join all independent futures. - let (rollout_recorder, default_shell, (history_log_id, history_entry_count), auth_statuses) = tokio::join!( - rollout_fut, - default_shell_fut, - history_meta_fut, - auth_statuses_fut - ); + let (rollout_recorder, (history_log_id, history_entry_count), auth_statuses) = + tokio::join!(rollout_fut, history_meta_fut, auth_statuses_fut); let rollout_recorder = rollout_recorder.map_err(|e| { error!("failed to initialize rollout recorder: {e:#}"); - anyhow::anyhow!("failed to initialize rollout recorder: {e:#}") + anyhow::Error::from(e) })?; let rollout_path = rollout_recorder.rollout_path.clone(); @@ -555,7 +554,6 @@ impl Session { config.model_reasoning_effort, config.model_reasoning_summary, config.model_context_window, - config.model_max_output_tokens, config.model_auto_compact_token_limit, config.approval_policy, config.sandbox_policy.clone(), @@ -566,9 +564,6 @@ impl Session { // Create the mutable state for the Session. let state = SessionState::new(session_configuration.clone()); - // Warm the tokenizer cache for the session model without blocking startup. - warm_model_cache(&session_configuration.model); - let services = SessionServices { mcp_connection_manager: Arc::new(RwLock::new(McpConnectionManager::default())), mcp_startup_cancellation_token: CancellationToken::new(), @@ -628,6 +623,22 @@ impl Session { ) .await; + let sandbox_state = SandboxState { + sandbox_policy: session_configuration.sandbox_policy.clone(), + codex_linux_sandbox_exe: config.codex_linux_sandbox_exe.clone(), + sandbox_cwd: session_configuration.cwd.clone(), + }; + if let Err(e) = sess + .services + .mcp_connection_manager + .read() + .await + .notify_sandbox_state_change(&sandbox_state) + .await + { + tracing::error!("Failed to notify sandbox state change: {e}"); + } + // record_initial_history can emit events. We record only after the SessionConfiguredEvent is emitted. sess.record_initial_history(initial_history).await; @@ -708,6 +719,11 @@ impl Session { format!("auto-compact-{id}") } + async fn get_total_token_usage(&self) -> i64 { + let state = self.state.lock().await; + state.get_total_token_usage() + } + async fn record_initial_history(&self, conversation_history: InitialHistory) { let turn_context = self.new_turn(SessionSettingsUpdate::default()).await; match conversation_history { @@ -1016,6 +1032,20 @@ impl Session { } } + pub async fn resolve_elicitation( + &self, + server_name: String, + id: RequestId, + response: ElicitationResponse, + ) -> anyhow::Result<()> { + self.services + .mcp_connection_manager + .read() + .await + .resolve_elicitation(server_name, id, response) + .await + } + /// Records input items: always append to conversation history and /// persist these response items to rollout. pub(crate) async fn record_conversation_items( @@ -1073,6 +1103,22 @@ impl Session { state.record_items(items.iter(), turn_context.truncation_policy); } + pub(crate) async fn record_model_warning(&self, message: impl Into, ctx: &TurnContext) { + if !self.enabled(Feature::ModelWarnings).await { + return; + } + + let item = ResponseItem::Message { + id: None, + role: "user".to_string(), + content: vec![ContentItem::InputText { + text: format!("Warning: {}", message.into()), + }], + }; + + self.record_conversation_items(ctx, &[item]).await; + } + pub(crate) async fn replace_history(&self, items: Vec) { let mut state = self.state.lock().await; state.replace_history(items); @@ -1124,7 +1170,7 @@ impl Session { Some(turn_context.cwd.clone()), Some(turn_context.approval_policy), Some(turn_context.sandbox_policy.clone()), - Some(self.user_shell().clone()), + self.user_shell().clone(), ))); items } @@ -1228,22 +1274,17 @@ impl Session { } } - /// Record a user input item to conversation history and also persist a - /// corresponding UserMessage EventMsg to rollout. - async fn record_input_and_rollout_usermsg( + pub(crate) async fn record_response_item_and_emit_turn_item( &self, turn_context: &TurnContext, - response_input: &ResponseInputItem, + response_item: ResponseItem, ) { - let response_item: ResponseItem = response_input.clone().into(); - // Add to conversation history and persist response item to rollout + // Add to conversation history and persist response item to rollout. self.record_conversation_items(turn_context, std::slice::from_ref(&response_item)) .await; - // Derive user message events and persist only UserMessage to rollout - let turn_item = parse_turn_item(&response_item); - - if let Some(item @ TurnItem::UserMessage(_)) = turn_item { + // Derive a turn item and emit lifecycle events if applicable. + if let Some(item) = parse_turn_item(&response_item) { self.emit_turn_item_started(turn_context, &item).await; self.emit_turn_item_completed(turn_context, item).await; } @@ -1264,9 +1305,14 @@ impl Session { &self, turn_context: &TurnContext, message: impl Into, + codex_error: CodexErr, ) { + let codex_error_info = CodexErrorInfo::ResponseStreamDisconnected { + http_status_code: codex_error.http_status_code_value(), + }; let event = EventMsg::StreamError(StreamErrorEvent { message: message.into(), + codex_error_info: Some(codex_error_info), }); self.send_event(turn_context, event).await; } @@ -1484,6 +1530,13 @@ async fn submission_loop(sess: Arc, config: Arc, rx_sub: Receiv ) .await; } + Op::ResolveElicitation { + server_name, + request_id, + decision, + } => { + handlers::resolve_elicitation(&sess, server_name, request_id, decision).await; + } Op::Shutdown => { if handlers::shutdown(&sess, sub.id.clone()).await { break; @@ -1510,11 +1563,13 @@ mod handlers { use crate::codex::spawn_review_thread; use crate::config::Config; use crate::mcp::auth::compute_auth_statuses; + use crate::review_prompts::resolve_review_request; use crate::tasks::CompactTask; use crate::tasks::RegularTask; use crate::tasks::UndoTask; use crate::tasks::UserShellCommandTask; use codex_protocol::custom_prompts::CustomPrompt; + use codex_protocol::protocol::CodexErrorInfo; use codex_protocol::protocol::ErrorEvent; use codex_protocol::protocol::Event; use codex_protocol::protocol::EventMsg; @@ -1526,6 +1581,9 @@ mod handlers { use codex_protocol::protocol::TurnAbortReason; use codex_protocol::user_input::UserInput; + use codex_rmcp_client::ElicitationAction; + use codex_rmcp_client::ElicitationResponse; + use mcp_types::RequestId; use std::sync::Arc; use tracing::info; use tracing::warn; @@ -1609,6 +1667,32 @@ mod handlers { *previous_context = Some(turn_context); } + pub async fn resolve_elicitation( + sess: &Arc, + server_name: String, + request_id: RequestId, + decision: codex_protocol::approvals::ElicitationAction, + ) { + let action = match decision { + codex_protocol::approvals::ElicitationAction::Accept => ElicitationAction::Accept, + codex_protocol::approvals::ElicitationAction::Decline => ElicitationAction::Decline, + codex_protocol::approvals::ElicitationAction::Cancel => ElicitationAction::Cancel, + }; + let response = ElicitationResponse { + action, + content: None, + }; + if let Err(err) = sess + .resolve_elicitation(server_name, request_id, response) + .await + { + warn!( + error = %err, + "failed to resolve elicitation request in session" + ); + } + } + pub async fn exec_approval(sess: &Arc, id: String, decision: ReviewDecision) { match decision { ReviewDecision::Abort => { @@ -1778,6 +1862,10 @@ mod handlers { pub async fn shutdown(sess: &Arc, sub_id: String) -> bool { sess.abort_all_tasks(TurnAbortReason::Interrupted).await; + sess.services + .unified_exec_manager + .terminate_all_sessions() + .await; info!("Shutting down Codex instance"); // Gracefully flush and shutdown rollout recorder on session end so tests @@ -1794,6 +1882,7 @@ mod handlers { id: sub_id.clone(), msg: EventMsg::Error(ErrorEvent { message: "Failed to shutdown rollout recorder".to_string(), + codex_error_info: Some(CodexErrorInfo::Other), }), }; sess.send_event_raw(event).await; @@ -1816,14 +1905,28 @@ mod handlers { let turn_context = sess .new_turn_with_sub_id(sub_id.clone(), SessionSettingsUpdate::default()) .await; - spawn_review_thread( - Arc::clone(sess), - Arc::clone(config), - turn_context.clone(), - sub_id, - review_request, - ) - .await; + match resolve_review_request(review_request, config.cwd.as_path()) { + Ok(resolved) => { + spawn_review_thread( + Arc::clone(sess), + Arc::clone(config), + turn_context.clone(), + sub_id, + resolved, + ) + .await; + } + Err(err) => { + let event = Event { + id: sub_id, + msg: EventMsg::Error(ErrorEvent { + message: err.to_string(), + codex_error_info: Some(CodexErrorInfo::Other), + }), + }; + sess.send_event(&turn_context, event.msg).await; + } + } } } @@ -1833,7 +1936,7 @@ async fn spawn_review_thread( config: Arc, parent_turn_context: Arc, sub_id: String, - review_request: ReviewRequest, + resolved: crate::review_prompts::ResolvedReviewRequest, ) { let model = config.review_model.clone(); let review_model_family = find_family_for_model(&model) @@ -1849,7 +1952,7 @@ async fn spawn_review_thread( }); let base_instructions = REVIEW_PROMPT.to_string(); - let review_prompt = review_request.prompt.clone(); + let review_prompt = resolved.prompt.clone(); let provider = parent_turn_context.client.get_provider(); let auth_manager = parent_turn_context.client.get_auth_manager(); let model_family = review_model_family.clone(); @@ -1908,14 +2011,13 @@ async fn spawn_review_thread( text: review_prompt, }]; let tc = Arc::new(review_turn_context); - sess.spawn_task( - tc.clone(), - input, - ReviewTask::new(review_request.append_to_original_thread), - ) - .await; + sess.spawn_task(tc.clone(), input, ReviewTask::new()).await; // Announce entering review mode so UIs can switch modes. + let review_request = ReviewRequest { + target: resolved.target, + user_facing_hint: Some(resolved.user_facing_hint), + }; sess.send_event(&tc, EventMsg::EnteredReviewMode(review_request)) .await; } @@ -1949,7 +2051,8 @@ pub(crate) async fn run_task( sess.send_event(&turn_context, event).await; let initial_input_for_turn: ResponseInputItem = ResponseInputItem::from(input); - sess.record_input_and_rollout_usermsg(turn_context.as_ref(), &initial_input_for_turn) + let response_item: ResponseItem = initial_input_for_turn.clone().into(); + sess.record_response_item_and_emit_turn_item(turn_context.as_ref(), response_item) .await; sess.maybe_start_ghost_snapshot(Arc::clone(&turn_context), cancellation_token.child_token()) @@ -1995,20 +2098,13 @@ pub(crate) async fn run_task( .await { Ok(turn_output) => { - let TurnRunResult { - processed_items, - total_token_usage, - } = turn_output; + let processed_items = turn_output; let limit = turn_context .client .get_auto_compact_token_limit() .unwrap_or(i64::MAX); - let total_usage_tokens = total_token_usage - .as_ref() - .map(TokenUsage::tokens_in_context_window); - let token_limit_reached = total_usage_tokens - .map(|tokens| tokens >= limit) - .unwrap_or(false); + let total_usage_tokens = sess.get_total_token_usage().await; + let token_limit_reached = total_usage_tokens >= limit; let (responses, items_to_record_in_conversation_history) = process_items(processed_items, &sess, &turn_context).await; @@ -2046,11 +2142,16 @@ pub(crate) async fn run_task( // Aborted turn is reported via a different event. break; } + Err(CodexErr::InvalidImageRequest()) => { + let mut state = sess.state.lock().await; + error_or_panic( + "Invalid image detected, replacing it in the last turn to prevent poisoning", + ); + state.history.replace_last_turn_images("Invalid image"); + } Err(e) => { info!("Turn error: {e:#}"); - let event = EventMsg::Error(ErrorEvent { - message: e.to_string(), - }); + let event = EventMsg::Error(e.to_error_event(None)); sess.send_event(&turn_context, event).await; // let the user continue the conversation break; @@ -2067,7 +2168,7 @@ async fn run_turn( turn_diff_tracker: SharedTurnDiffTracker, input: Vec, cancellation_token: CancellationToken, -) -> CodexResult { +) -> CodexResult> { let mcp_tools = sess .services .mcp_connection_manager @@ -2155,6 +2256,8 @@ async fn run_turn( } Err(CodexErr::UsageNotIncluded) => return Err(CodexErr::UsageNotIncluded), Err(e @ CodexErr::QuotaExceeded) => return Err(e), + Err(e @ CodexErr::InvalidImageRequest()) => return Err(e), + Err(e @ CodexErr::InvalidRequest(_)) => return Err(e), Err(e @ CodexErr::RefreshTokenFailed(_)) => return Err(e), Err(e) => { // Use the configured provider-specific stream retry budget. @@ -2175,6 +2278,7 @@ async fn run_turn( sess.notify_stream_error( &turn_context, format!("Reconnecting... {retries}/{max_retries}"), + e, ) .await; @@ -2197,12 +2301,6 @@ pub struct ProcessedResponseItem { pub response: Option, } -#[derive(Debug)] -struct TurnRunResult { - processed_items: Vec, - total_token_usage: Option, -} - #[allow(clippy::too_many_arguments)] async fn try_run_turn( router: Arc, @@ -2211,7 +2309,7 @@ async fn try_run_turn( turn_diff_tracker: SharedTurnDiffTracker, prompt: &Prompt, cancellation_token: CancellationToken, -) -> CodexResult { +) -> CodexResult> { let rollout_item = RolloutItem::TurnContext(TurnContextItem { cwd: turn_context.cwd.clone(), approval_policy: turn_context.approval_policy, @@ -2373,12 +2471,7 @@ async fn try_run_turn( sess.send_event(&turn_context, msg).await; } - let result = TurnRunResult { - processed_items, - total_token_usage: token_usage.clone(), - }; - - return Ok(result); + return Ok(processed_items); } ResponseEvent::OutputTextDelta(delta) => { // In review child threads, suppress assistant text deltas; the @@ -2393,7 +2486,7 @@ async fn try_run_turn( sess.send_event(&turn_context, EventMsg::AgentMessageContentDelta(event)) .await; } else { - error_or_panic("ReasoningSummaryDelta without active item".to_string()); + error_or_panic("OutputTextDelta without active item".to_string()); } } ResponseEvent::ReasoningSummaryDelta { @@ -2487,16 +2580,23 @@ use crate::features::Features; #[cfg(test)] pub(crate) use tests::make_session_and_context; +#[cfg(test)] +pub(crate) use tests::make_session_and_context_with_rx; + #[cfg(test)] mod tests { use super::*; use crate::config::ConfigOverrides; use crate::config::ConfigToml; use crate::exec::ExecToolCallOutput; + use crate::shell::default_user_shell; use crate::tools::format_exec_output_str; use crate::protocol::CompactedItem; + use crate::protocol::CreditsSnapshot; use crate::protocol::InitialHistory; + use crate::protocol::RateLimitSnapshot; + use crate::protocol::RateLimitWindow; use crate::protocol::ResumedHistory; use crate::state::TaskKind; use crate::tasks::SessionTask; @@ -2566,6 +2666,75 @@ mod tests { assert_eq!(expected, actual); } + #[test] + fn set_rate_limits_retains_previous_credits() { + let codex_home = tempfile::tempdir().expect("create temp dir"); + let config = Config::load_from_base_config_with_overrides( + ConfigToml::default(), + ConfigOverrides::default(), + codex_home.path().to_path_buf(), + ) + .expect("load default test config"); + let config = Arc::new(config); + let session_configuration = SessionConfiguration { + provider: config.model_provider.clone(), + model: config.model.clone(), + model_reasoning_effort: config.model_reasoning_effort, + model_reasoning_summary: config.model_reasoning_summary, + developer_instructions: config.developer_instructions.clone(), + user_instructions: config.user_instructions.clone(), + base_instructions: config.base_instructions.clone(), + compact_prompt: config.compact_prompt.clone(), + approval_policy: config.approval_policy, + sandbox_policy: config.sandbox_policy.clone(), + cwd: config.cwd.clone(), + original_config_do_not_use: Arc::clone(&config), + features: Features::default(), + exec_policy: Arc::new(ExecPolicy::empty()), + session_source: SessionSource::Exec, + }; + + let mut state = SessionState::new(session_configuration); + let initial = RateLimitSnapshot { + primary: Some(RateLimitWindow { + used_percent: 10.0, + window_minutes: Some(15), + resets_at: Some(1_700), + }), + secondary: None, + credits: Some(CreditsSnapshot { + has_credits: true, + unlimited: false, + balance: Some("10.00".to_string()), + }), + }; + state.set_rate_limits(initial.clone()); + + let update = RateLimitSnapshot { + primary: Some(RateLimitWindow { + used_percent: 40.0, + window_minutes: Some(30), + resets_at: Some(1_800), + }), + secondary: Some(RateLimitWindow { + used_percent: 5.0, + window_minutes: Some(60), + resets_at: Some(1_900), + }), + credits: None, + }; + state.set_rate_limits(update.clone()); + + assert_eq!( + state.latest_rate_limits, + Some(RateLimitSnapshot { + primary: update.primary.clone(), + secondary: update.secondary, + credits: initial.credits, + }) + ); + } + #[test] fn prefers_structured_content_when_present() { let ctr = CallToolResult { @@ -2732,7 +2901,7 @@ mod tests { unified_exec_manager: UnifiedExecSessionManager::default(), notifier: UserNotifier::new(None), rollout: Mutex::new(None), - user_shell: shell::Shell::Unknown, + user_shell: default_user_shell(), show_raw_agent_reasoning: config.show_raw_agent_reasoning, auth_manager: Arc::clone(&auth_manager), otel_event_manager: otel_event_manager.clone(), @@ -2762,7 +2931,7 @@ mod tests { // Like make_session_and_context, but returns Arc and the event receiver // so tests can assert on emitted events. - fn make_session_and_context_with_rx() -> ( + pub(crate) fn make_session_and_context_with_rx() -> ( Arc, Arc, async_channel::Receiver, @@ -2810,7 +2979,7 @@ mod tests { unified_exec_manager: UnifiedExecSessionManager::default(), notifier: UserNotifier::new(None), rollout: Mutex::new(None), - user_shell: shell::Shell::Unknown, + user_shell: default_user_shell(), show_raw_agent_reasoning: config.show_raw_agent_reasoning, auth_manager: Arc::clone(&auth_manager), otel_event_manager: otel_event_manager.clone(), @@ -2838,6 +3007,40 @@ mod tests { (session, turn_context, rx_event) } + #[tokio::test] + async fn record_model_warning_appends_user_message() { + let (session, turn_context) = make_session_and_context(); + + session + .state + .lock() + .await + .session_configuration + .features + .enable(Feature::ModelWarnings); + + session + .record_model_warning("too many unified exec sessions", &turn_context) + .await; + + let mut history = session.clone_history().await; + let history_items = history.get_history(); + let last = history_items.last().expect("warning recorded"); + + match last { + ResponseItem::Message { role, content, .. } => { + assert_eq!(role, "user"); + assert_eq!( + content, + &vec![ContentItem::InputText { + text: "Warning: too many unified exec sessions".to_string(), + }] + ); + } + other => panic!("expected user message, got {other:?}"), + } + } + #[derive(Clone, Copy)] struct NeverEndingTask { kind: TaskKind, @@ -2929,7 +3132,7 @@ mod tests { let input = vec![UserInput::Text { text: "start review".to_string(), }]; - sess.spawn_task(Arc::clone(&tc), input, ReviewTask::new(true)) + sess.spawn_task(Arc::clone(&tc), input, ReviewTask::new()) .await; sess.abort_all_tasks(TurnAbortReason::Interrupted).await; @@ -2957,6 +3160,8 @@ mod tests { .expect("event"); match evt.msg { EventMsg::RawResponseItem(_) => continue, + EventMsg::ItemStarted(_) | EventMsg::ItemCompleted(_) => continue, + EventMsg::AgentMessage(_) => continue, EventMsg::TurnAborted(e) => { assert_eq!(TurnAbortReason::Interrupted, e.reason); break; @@ -2966,23 +3171,7 @@ mod tests { } let history = sess.clone_history().await.get_history(); - let found = history.iter().any(|item| match item { - ResponseItem::Message { role, content, .. } if role == "user" => { - content.iter().any(|ci| match ci { - ContentItem::InputText { text } => { - text.contains("") - && text.contains("review") - && text.contains("interrupted") - } - _ => false, - }) - } - _ => false, - }); - assert!( - found, - "synthetic review interruption not recorded in history" - ); + let _ = history; } #[tokio::test] @@ -3155,6 +3344,7 @@ mod tests { let session = Arc::new(session); let mut turn_context = Arc::new(turn_context_raw); + let timeout_ms = 1000; let params = ExecParams { command: if cfg!(windows) { vec![ @@ -3170,7 +3360,7 @@ mod tests { ] }, cwd: turn_context.cwd.clone(), - timeout_ms: Some(1000), + expiration: timeout_ms.into(), env: HashMap::new(), with_escalated_permissions: Some(true), justification: Some("test".to_string()), @@ -3179,7 +3369,12 @@ mod tests { let params2 = ExecParams { with_escalated_permissions: Some(false), - ..params.clone() + command: params.command.clone(), + cwd: params.cwd.clone(), + expiration: timeout_ms.into(), + env: HashMap::new(), + justification: params.justification.clone(), + arg0: None, }; let turn_diff_tracker = Arc::new(tokio::sync::Mutex::new(TurnDiffTracker::new())); @@ -3199,7 +3394,7 @@ mod tests { arguments: serde_json::json!({ "command": params.command.clone(), "workdir": Some(turn_context.cwd.to_string_lossy().to_string()), - "timeout_ms": params.timeout_ms, + "timeout_ms": params.expiration.timeout_ms(), "with_escalated_permissions": params.with_escalated_permissions, "justification": params.justification.clone(), }) @@ -3236,7 +3431,7 @@ mod tests { arguments: serde_json::json!({ "command": params2.command.clone(), "workdir": Some(turn_context.cwd.to_string_lossy().to_string()), - "timeout_ms": params2.timeout_ms, + "timeout_ms": params2.expiration.timeout_ms(), "with_escalated_permissions": params2.with_escalated_permissions, "justification": params2.justification.clone(), }) diff --git a/codex-rs/core/src/codex_delegate.rs b/codex-rs/core/src/codex_delegate.rs index 4cb4d4a06a..796331d1e8 100644 --- a/codex-rs/core/src/codex_delegate.rs +++ b/codex-rs/core/src/codex_delegate.rs @@ -13,6 +13,8 @@ use codex_protocol::protocol::SessionSource; use codex_protocol::protocol::SubAgentSource; use codex_protocol::protocol::Submission; use codex_protocol::user_input::UserInput; +use std::time::Duration; +use tokio::time::timeout; use tokio_util::sync::CancellationToken; use crate::AuthManager; @@ -60,14 +62,13 @@ pub(crate) async fn run_codex_conversation_interactive( let parent_ctx_clone = Arc::clone(&parent_ctx); let codex_for_events = Arc::clone(&codex); tokio::spawn(async move { - let _ = forward_events( + forward_events( codex_for_events, tx_sub, parent_session_clone, parent_ctx_clone, - cancel_token_events.clone(), + cancel_token_events, ) - .or_cancel(&cancel_token_events) .await; }); @@ -156,53 +157,92 @@ async fn forward_events( parent_ctx: Arc, cancel_token: CancellationToken, ) { - while let Ok(event) = codex.next_event().await { - match event { - // ignore all legacy delta events - Event { - id: _, - msg: EventMsg::AgentMessageDelta(_) | EventMsg::AgentReasoningDelta(_), - } => continue, - Event { - id: _, - msg: EventMsg::SessionConfigured(_), - } => continue, - Event { - id, - msg: EventMsg::ExecApprovalRequest(event), - } => { - // Initiate approval via parent session; do not surface to consumer. - handle_exec_approval( - &codex, - id, - &parent_session, - &parent_ctx, - event, - &cancel_token, - ) - .await; + let cancelled = cancel_token.cancelled(); + tokio::pin!(cancelled); + + loop { + tokio::select! { + _ = &mut cancelled => { + shutdown_delegate(&codex).await; + break; } - Event { - id, - msg: EventMsg::ApplyPatchApprovalRequest(event), - } => { - handle_patch_approval( - &codex, - id, - &parent_session, - &parent_ctx, - event, - &cancel_token, - ) - .await; - } - other => { - let _ = tx_sub.send(other).await; + event = codex.next_event() => { + let event = match event { + Ok(event) => event, + Err(_) => break, + }; + match event { + // ignore all legacy delta events + Event { + id: _, + msg: EventMsg::AgentMessageDelta(_) | EventMsg::AgentReasoningDelta(_), + } => {} + Event { + id: _, + msg: EventMsg::SessionConfigured(_), + } => {} + Event { + id, + msg: EventMsg::ExecApprovalRequest(event), + } => { + // Initiate approval via parent session; do not surface to consumer. + handle_exec_approval( + &codex, + id, + &parent_session, + &parent_ctx, + event, + &cancel_token, + ) + .await; + } + Event { + id, + msg: EventMsg::ApplyPatchApprovalRequest(event), + } => { + handle_patch_approval( + &codex, + id, + &parent_session, + &parent_ctx, + event, + &cancel_token, + ) + .await; + } + other => { + match tx_sub.send(other).or_cancel(&cancel_token).await { + Ok(Ok(())) => {} + _ => { + shutdown_delegate(&codex).await; + break; + } + } + } + } } } } } +/// Ask the delegate to stop and drain its events so background sends do not hit a closed channel. +async fn shutdown_delegate(codex: &Codex) { + let _ = codex.submit(Op::Interrupt).await; + let _ = codex.submit(Op::Shutdown {}).await; + + let _ = timeout(Duration::from_millis(500), async { + while let Ok(event) = codex.next_event().await { + if matches!( + event.msg, + EventMsg::TurnAborted(_) | EventMsg::TaskComplete(_) + ) { + break; + } + } + }) + .await; +} + /// Forward ops from a caller to a sub-agent, respecting cancellation. async fn forward_ops( codex: Arc, @@ -298,3 +338,85 @@ where } } } + +#[cfg(test)] +mod tests { + use super::*; + use async_channel::bounded; + use codex_protocol::models::ResponseItem; + use codex_protocol::protocol::RawResponseItemEvent; + use codex_protocol::protocol::TurnAbortReason; + use codex_protocol::protocol::TurnAbortedEvent; + use pretty_assertions::assert_eq; + + #[tokio::test] + async fn forward_events_cancelled_while_send_blocked_shuts_down_delegate() { + let (tx_events, rx_events) = bounded(1); + let (tx_sub, rx_sub) = bounded(SUBMISSION_CHANNEL_CAPACITY); + let codex = Arc::new(Codex { + next_id: AtomicU64::new(0), + tx_sub, + rx_event: rx_events, + }); + + let (session, ctx, _rx_evt) = crate::codex::make_session_and_context_with_rx(); + + let (tx_out, rx_out) = bounded(1); + tx_out + .send(Event { + id: "full".to_string(), + msg: EventMsg::TurnAborted(TurnAbortedEvent { + reason: TurnAbortReason::Interrupted, + }), + }) + .await + .unwrap(); + + let cancel = CancellationToken::new(); + let forward = tokio::spawn(forward_events( + Arc::clone(&codex), + tx_out.clone(), + session, + ctx, + cancel.clone(), + )); + + tx_events + .send(Event { + id: "evt".to_string(), + msg: EventMsg::RawResponseItem(RawResponseItemEvent { + item: ResponseItem::CustomToolCall { + id: None, + status: None, + call_id: "call-1".to_string(), + name: "tool".to_string(), + input: "{}".to_string(), + }, + }), + }) + .await + .unwrap(); + + drop(tx_events); + cancel.cancel(); + timeout(std::time::Duration::from_millis(1000), forward) + .await + .expect("forward_events hung") + .expect("forward_events join error"); + + let received = rx_out.recv().await.expect("prefilled event missing"); + assert_eq!("full", received.id); + let mut ops = Vec::new(); + while let Ok(sub) = rx_sub.try_recv() { + ops.push(sub.op); + } + assert!( + ops.iter().any(|op| matches!(op, Op::Interrupt)), + "expected Interrupt op after cancellation" + ); + assert!( + ops.iter().any(|op| matches!(op, Op::Shutdown)), + "expected Shutdown op after cancellation" + ); + } +} diff --git a/codex-rs/core/src/command_safety/is_dangerous_command.rs b/codex-rs/core/src/command_safety/is_dangerous_command.rs index 5df2023f06..96f73f3e8f 100644 --- a/codex-rs/core/src/command_safety/is_dangerous_command.rs +++ b/codex-rs/core/src/command_safety/is_dangerous_command.rs @@ -5,6 +5,9 @@ use crate::sandboxing::SandboxPermissions; use crate::bash::parse_shell_lc_plain_commands; use crate::is_safe_command::is_known_safe_command; +#[cfg(windows)] +#[path = "windows_dangerous_commands.rs"] +mod windows_dangerous_commands; pub fn requires_initial_appoval( policy: AskForApproval, @@ -36,6 +39,13 @@ pub fn requires_initial_appoval( } pub fn command_might_be_dangerous(command: &[String]) -> bool { + #[cfg(windows)] + { + if windows_dangerous_commands::is_dangerous_command_windows(command) { + return true; + } + } + if is_dangerous_to_call_with_exec(command) { return true; } diff --git a/codex-rs/core/src/command_safety/is_safe_command.rs b/codex-rs/core/src/command_safety/is_safe_command.rs index ab084c191b..5e7a7034d8 100644 --- a/codex-rs/core/src/command_safety/is_safe_command.rs +++ b/codex-rs/core/src/command_safety/is_safe_command.rs @@ -267,6 +267,20 @@ mod tests { } } + #[test] + fn windows_powershell_full_path_is_safe() { + if !cfg!(windows) { + // Windows only because on Linux path splitting doesn't handle `/` separators properly + return; + } + + assert!(is_known_safe_command(&vec_str(&[ + r"C:\Program Files\PowerShell\7\pwsh.exe", + "-Command", + "Get-Location", + ]))); + } + #[test] fn bash_lc_safe_examples() { assert!(is_known_safe_command(&vec_str(&["bash", "-lc", "ls"]))); diff --git a/codex-rs/core/src/command_safety/windows_dangerous_commands.rs b/codex-rs/core/src/command_safety/windows_dangerous_commands.rs new file mode 100644 index 0000000000..d4b418d93a --- /dev/null +++ b/codex-rs/core/src/command_safety/windows_dangerous_commands.rs @@ -0,0 +1,316 @@ +use std::path::Path; + +use once_cell::sync::Lazy; +use regex::Regex; +use shlex::split as shlex_split; +use url::Url; + +pub fn is_dangerous_command_windows(command: &[String]) -> bool { + // Prefer structured parsing for PowerShell/CMD so we can spot URL-bearing + // invocations of ShellExecute-style entry points before falling back to + // simple argv heuristics. + if is_dangerous_powershell(command) { + return true; + } + + if is_dangerous_cmd(command) { + return true; + } + + is_direct_gui_launch(command) +} + +fn is_dangerous_powershell(command: &[String]) -> bool { + let Some((exe, rest)) = command.split_first() else { + return false; + }; + if !is_powershell_executable(exe) { + return false; + } + // Parse the PowerShell invocation to get a flat token list we can scan for + // dangerous cmdlets/COM calls plus any URL-looking arguments. This is a + // best-effort shlex split of the script text, not a full PS parser. + let Some(parsed) = parse_powershell_invocation(rest) else { + return false; + }; + + let tokens_lc: Vec = parsed + .tokens + .iter() + .map(|t| t.trim_matches('\'').trim_matches('"').to_ascii_lowercase()) + .collect(); + let has_url = args_have_url(&parsed.tokens); + + if has_url + && tokens_lc.iter().any(|t| { + matches!( + t.as_str(), + "start-process" | "start" | "saps" | "invoke-item" | "ii" + ) || t.contains("start-process") + || t.contains("invoke-item") + }) + { + return true; + } + + if has_url + && tokens_lc + .iter() + .any(|t| t.contains("shellexecute") || t.contains("shell.application")) + { + return true; + } + + if let Some(first) = tokens_lc.first() { + // Legacy ShellExecute path via url.dll + if first == "rundll32" + && tokens_lc + .iter() + .any(|t| t.contains("url.dll,fileprotocolhandler")) + && has_url + { + return true; + } + if first == "mshta" && has_url { + return true; + } + if is_browser_executable(first) && has_url { + return true; + } + if matches!(first.as_str(), "explorer" | "explorer.exe") && has_url { + return true; + } + } + + false +} + +fn is_dangerous_cmd(command: &[String]) -> bool { + let Some((exe, rest)) = command.split_first() else { + return false; + }; + let Some(base) = executable_basename(exe) else { + return false; + }; + if base != "cmd" && base != "cmd.exe" { + return false; + } + + let mut iter = rest.iter(); + for arg in iter.by_ref() { + let lower = arg.to_ascii_lowercase(); + match lower.as_str() { + "/c" | "/r" | "-c" => break, + _ if lower.starts_with('/') => continue, + // Unknown tokens before the command body => bail. + _ => return false, + } + } + + let Some(first_cmd) = iter.next() else { + return false; + }; + // Classic `cmd /c start https://...` ShellExecute path. + if !first_cmd.eq_ignore_ascii_case("start") { + return false; + } + let remaining: Vec = iter.cloned().collect(); + args_have_url(&remaining) +} + +fn is_direct_gui_launch(command: &[String]) -> bool { + let Some((exe, rest)) = command.split_first() else { + return false; + }; + let Some(base) = executable_basename(exe) else { + return false; + }; + + // Explorer/rundll32/mshta or direct browser exe with a URL anywhere in args. + if matches!(base.as_str(), "explorer" | "explorer.exe") && args_have_url(rest) { + return true; + } + if matches!(base.as_str(), "mshta" | "mshta.exe") && args_have_url(rest) { + return true; + } + if (base == "rundll32" || base == "rundll32.exe") + && rest.iter().any(|t| { + t.to_ascii_lowercase() + .contains("url.dll,fileprotocolhandler") + }) + && args_have_url(rest) + { + return true; + } + if is_browser_executable(&base) && args_have_url(rest) { + return true; + } + + false +} + +fn args_have_url(args: &[String]) -> bool { + args.iter().any(|arg| looks_like_url(arg)) +} + +fn looks_like_url(token: &str) -> bool { + // Strip common PowerShell punctuation around inline URLs (quotes, parens, trailing semicolons). + // Capture the middle token after trimming leading quotes/parens/whitespace and trailing semicolons/closing parens. + static RE: Lazy> = + Lazy::new(|| Regex::new(r#"^[ "'\(\s]*([^\s"'\);]+)[\s;\)]*$"#).ok()); + // If the token embeds a URL alongside other text (e.g., Start-Process('https://...')) + // as a single shlex token, grab the substring starting at the first URL prefix. + let urlish = token + .find("https://") + .or_else(|| token.find("http://")) + .map(|idx| &token[idx..]) + .unwrap_or(token); + + let candidate = RE + .as_ref() + .and_then(|re| re.captures(urlish)) + .and_then(|caps| caps.get(1)) + .map(|m| m.as_str()) + .unwrap_or(urlish); + let Ok(url) = Url::parse(candidate) else { + return false; + }; + matches!(url.scheme(), "http" | "https") +} + +fn executable_basename(exe: &str) -> Option { + Path::new(exe) + .file_name() + .and_then(|osstr| osstr.to_str()) + .map(str::to_ascii_lowercase) +} + +fn is_powershell_executable(exe: &str) -> bool { + matches!( + executable_basename(exe).as_deref(), + Some("powershell") | Some("powershell.exe") | Some("pwsh") | Some("pwsh.exe") + ) +} + +fn is_browser_executable(name: &str) -> bool { + matches!( + name, + "chrome" + | "chrome.exe" + | "msedge" + | "msedge.exe" + | "firefox" + | "firefox.exe" + | "iexplore" + | "iexplore.exe" + ) +} + +struct ParsedPowershell { + tokens: Vec, +} + +fn parse_powershell_invocation(args: &[String]) -> Option { + if args.is_empty() { + return None; + } + + let mut idx = 0; + while idx < args.len() { + let arg = &args[idx]; + let lower = arg.to_ascii_lowercase(); + match lower.as_str() { + "-command" | "/command" | "-c" => { + let script = args.get(idx + 1)?; + if idx + 2 != args.len() { + return None; + } + let tokens = shlex_split(script)?; + return Some(ParsedPowershell { tokens }); + } + _ if lower.starts_with("-command:") || lower.starts_with("/command:") => { + if idx + 1 != args.len() { + return None; + } + let (_, script) = arg.split_once(':')?; + let tokens = shlex_split(script)?; + return Some(ParsedPowershell { tokens }); + } + "-nologo" | "-noprofile" | "-noninteractive" | "-mta" | "-sta" => { + idx += 1; + } + _ if lower.starts_with('-') => { + idx += 1; + } + _ => { + let rest = args[idx..].to_vec(); + return Some(ParsedPowershell { tokens: rest }); + } + } + } + + None +} + +#[cfg(test)] +mod tests { + use super::is_dangerous_command_windows; + + fn vec_str(items: &[&str]) -> Vec { + items.iter().map(std::string::ToString::to_string).collect() + } + + #[test] + fn powershell_start_process_url_is_dangerous() { + assert!(is_dangerous_command_windows(&vec_str(&[ + "powershell", + "-NoLogo", + "-Command", + "Start-Process 'https://example.com'" + ]))); + } + + #[test] + fn powershell_start_process_url_with_trailing_semicolon_is_dangerous() { + assert!(is_dangerous_command_windows(&vec_str(&[ + "powershell", + "-Command", + "Start-Process('https://example.com');" + ]))); + } + + #[test] + fn powershell_start_process_local_is_not_flagged() { + assert!(!is_dangerous_command_windows(&vec_str(&[ + "powershell", + "-Command", + "Start-Process notepad.exe" + ]))); + } + + #[test] + fn cmd_start_with_url_is_dangerous() { + assert!(is_dangerous_command_windows(&vec_str(&[ + "cmd", + "/c", + "start", + "https://example.com" + ]))); + } + + #[test] + fn msedge_with_url_is_dangerous() { + assert!(is_dangerous_command_windows(&vec_str(&[ + "msedge.exe", + "https://example.com" + ]))); + } + + #[test] + fn explorer_with_directory_is_not_flagged() { + assert!(!is_dangerous_command_windows(&vec_str(&[ + "explorer.exe", + "." + ]))); + } +} diff --git a/codex-rs/core/src/command_safety/windows_safe_commands.rs b/codex-rs/core/src/command_safety/windows_safe_commands.rs index ff0a3d2e7b..a1d3b297f9 100644 --- a/codex-rs/core/src/command_safety/windows_safe_commands.rs +++ b/codex-rs/core/src/command_safety/windows_safe_commands.rs @@ -1,4 +1,5 @@ use shlex::split as shlex_split; +use std::path::Path; /// On Windows, we conservatively allow only clearly read-only PowerShell invocations /// that match a small safelist. Anything else (including direct CMD commands) is unsafe. @@ -131,8 +132,14 @@ fn split_into_commands(tokens: Vec) -> Option>> { /// Returns true when the executable name is one of the supported PowerShell binaries. fn is_powershell_executable(exe: &str) -> bool { + let executable_name = Path::new(exe) + .file_name() + .and_then(|osstr| osstr.to_str()) + .unwrap_or(exe) + .to_ascii_lowercase(); + matches!( - exe.to_ascii_lowercase().as_str(), + executable_name.as_str(), "powershell" | "powershell.exe" | "pwsh" | "pwsh.exe" ) } @@ -313,6 +320,27 @@ mod tests { ]))); } + #[test] + fn accepts_full_path_powershell_invocations() { + if !cfg!(windows) { + // Windows only because on Linux path splitting doesn't handle `/` separators properly + return; + } + + assert!(is_safe_command_windows(&vec_str(&[ + r"C:\Program Files\PowerShell\7\pwsh.exe", + "-NoProfile", + "-Command", + "Get-ChildItem -Path .", + ]))); + + assert!(is_safe_command_windows(&vec_str(&[ + r"C:\Windows\System32\WindowsPowerShell\v1.0\powershell.exe", + "-Command", + "Get-Content Cargo.toml", + ]))); + } + #[test] fn allows_read_only_pipelines_and_git_usage() { assert!(is_safe_command_windows(&vec_str(&[ diff --git a/codex-rs/core/src/compact.rs b/codex-rs/core/src/compact.rs index 8c38f93937..fb5c187b7f 100644 --- a/codex-rs/core/src/compact.rs +++ b/codex-rs/core/src/compact.rs @@ -8,9 +8,8 @@ use crate::codex::get_last_assistant_message_from_turn; use crate::error::CodexErr; use crate::error::Result as CodexResult; use crate::features::Feature; -use crate::protocol::AgentMessageEvent; use crate::protocol::CompactedItem; -use crate::protocol::ErrorEvent; +use crate::protocol::ContextCompactedEvent; use crate::protocol::EventMsg; use crate::protocol::TaskStartedEvent; use crate::protocol::TurnContextItem; @@ -128,9 +127,7 @@ async fn run_compact_task_inner( continue; } sess.set_total_tokens_full(turn_context.as_ref()).await; - let event = EventMsg::Error(ErrorEvent { - message: e.to_string(), - }); + let event = EventMsg::Error(e.to_error_event(None)); sess.send_event(&turn_context, event).await; return; } @@ -141,14 +138,13 @@ async fn run_compact_task_inner( sess.notify_stream_error( turn_context.as_ref(), format!("Reconnecting... {retries}/{max_retries}"), + e, ) .await; tokio::time::sleep(delay).await; continue; } else { - let event = EventMsg::Error(ErrorEvent { - message: e.to_string(), - }); + let event = EventMsg::Error(e.to_error_event(None)); sess.send_event(&turn_context, event).await; return; } @@ -179,9 +175,7 @@ async fn run_compact_task_inner( }); sess.persist_rollout_items(&[rollout_item]).await; - let event = EventMsg::AgentMessage(AgentMessageEvent { - message: "Compact task completed".to_string(), - }); + let event = EventMsg::ContextCompacted(ContextCompactedEvent {}); sess.send_event(&turn_context, event).await; let warning = EventMsg::Warning(WarningEvent { diff --git a/codex-rs/core/src/compact_remote.rs b/codex-rs/core/src/compact_remote.rs index 0d2e0f138d..b855f28d39 100644 --- a/codex-rs/core/src/compact_remote.rs +++ b/codex-rs/core/src/compact_remote.rs @@ -4,9 +4,8 @@ use crate::Prompt; use crate::codex::Session; use crate::codex::TurnContext; use crate::error::Result as CodexResult; -use crate::protocol::AgentMessageEvent; use crate::protocol::CompactedItem; -use crate::protocol::ErrorEvent; +use crate::protocol::ContextCompactedEvent; use crate::protocol::EventMsg; use crate::protocol::RolloutItem; use crate::protocol::TaskStartedEvent; @@ -30,9 +29,9 @@ pub(crate) async fn run_remote_compact_task(sess: Arc, turn_context: Ar async fn run_remote_compact_task_inner(sess: &Arc, turn_context: &Arc) { if let Err(err) = run_remote_compact_task_inner_impl(sess, turn_context).await { - let event = EventMsg::Error(ErrorEvent { - message: format!("Error running remote compact task: {err}"), - }); + let event = EventMsg::Error( + err.to_error_event(Some("Error running remote compact task".to_string())), + ); sess.send_event(turn_context, event).await; } } @@ -75,9 +74,7 @@ async fn run_remote_compact_task_inner_impl( sess.persist_rollout_items(&[RolloutItem::Compacted(compacted_item)]) .await; - let event = EventMsg::AgentMessage(AgentMessageEvent { - message: "Compact task completed".to_string(), - }); + let event = EventMsg::ContextCompacted(ContextCompactedEvent {}); sess.send_event(turn_context, event).await; Ok(()) diff --git a/codex-rs/core/src/config/edit.rs b/codex-rs/core/src/config/edit.rs index bd7c16a0ae..b8862fa5c5 100644 --- a/codex-rs/core/src/config/edit.rs +++ b/codex-rs/core/src/config/edit.rs @@ -4,7 +4,6 @@ use crate::config::types::Notice; use anyhow::Context; use codex_protocol::config_types::ReasoningEffort; use codex_protocol::config_types::TrustLevel; -use codex_utils_tokenizer::warm_model_cache; use std::collections::BTreeMap; use std::path::Path; use std::path::PathBuf; @@ -231,9 +230,6 @@ impl ConfigDocument { fn apply(&mut self, edit: &ConfigEdit) -> anyhow::Result { match edit { ConfigEdit::SetModel { model, effort } => Ok({ - if let Some(model) = &model { - warm_model_cache(model) - } let mut mutated = false; mutated |= self.write_profile_value( &["model"], diff --git a/codex-rs/core/src/config/mod.rs b/codex-rs/core/src/config/mod.rs index 4cdb1b0b03..6276d3b6e3 100644 --- a/codex-rs/core/src/config/mod.rs +++ b/codex-rs/core/src/config/mod.rs @@ -34,6 +34,7 @@ use crate::project_doc::DEFAULT_PROJECT_DOC_FILENAME; use crate::project_doc::LOCAL_PROJECT_DOC_FILENAME; use crate::protocol::AskForApproval; use crate::protocol::SandboxPolicy; +use crate::util::resolve_path; use codex_app_server_protocol::Tools; use codex_app_server_protocol::UserSavedConfig; use codex_protocol::config_types::ForcedLoginMethod; @@ -70,7 +71,7 @@ pub const GPT_5_CODEX_MEDIUM_MODEL: &str = "gpt-5.1-codex"; /// the context window. pub(crate) const PROJECT_DOC_MAX_BYTES: usize = 32 * 1024; // 32 KiB -pub(crate) const CONFIG_TOML_FILE: &str = "config.toml"; +pub const CONFIG_TOML_FILE: &str = "config.toml"; /// Application configuration loaded from disk and merged with overrides. #[derive(Debug, Clone, PartialEq)] @@ -86,9 +87,6 @@ pub struct Config { /// Size of the context window for the model, in tokens. pub model_context_window: Option, - /// Maximum number of output tokens. - pub model_max_output_tokens: Option, - /// Token usage threshold triggering auto-compaction of conversation history. pub model_auto_compact_token_limit: Option, @@ -160,6 +158,12 @@ pub struct Config { /// and turn completions when not focused. pub tui_notifications: Notifications, + /// Enable ASCII animations and shimmer effects in the TUI. + pub animations: bool, + + /// Show startup tooltips in the TUI welcome screen. + pub show_tooltips: bool, + /// The directory that should be treated as the current working directory /// for the session. All relative paths inside the business-logic layer are /// resolved against this path. @@ -267,6 +271,11 @@ pub struct Config { /// Collection of various notices we show the user pub notices: Notice, + /// When `true`, checks for Codex updates on startup and surfaces update prompts. + /// Set to `false` only if your Codex updates are centrally managed. + /// Defaults to `true`. + pub check_for_update_on_startup: bool, + /// When true, disables burst-paste detection for typed input entirely. /// All characters are inserted as they are received, and no buffering /// or placeholder replacement will occur for fast keypress bursts. @@ -567,9 +576,6 @@ pub struct ConfigToml { /// Size of the context window for the model, in tokens. pub model_context_window: Option, - /// Maximum number of output tokens. - pub model_max_output_tokens: Option, - /// Token usage threshold triggering auto-compaction of conversation history. pub model_auto_compact_token_limit: Option, @@ -688,6 +694,11 @@ pub struct ConfigToml { #[serde(default)] pub features: Option, + /// When `true`, checks for Codex updates on startup and surfaces update prompts. + /// Set to `false` only if your Codex updates are centrally managed. + /// Defaults to `true`. + pub check_for_update_on_startup: Option, + /// When true, disables burst-paste detection for typed input entirely. /// All characters are inserted as they are received, and no buffering /// or placeholder replacement will occur for fast keypress bursts. @@ -1009,15 +1020,8 @@ impl Config { let additional_writable_roots: Vec = additional_writable_roots .into_iter() .map(|path| { - let absolute = if path.is_absolute() { - path - } else { - resolved_cwd.join(path) - }; - match canonicalize(&absolute) { - Ok(canonical) => canonical, - Err(_) => absolute, - } + let absolute = resolve_path(&resolved_cwd, &path); + canonicalize(&absolute).unwrap_or(absolute) }) .collect(); let active_project = cfg @@ -1119,11 +1123,6 @@ impl Config { let model_context_window = cfg .model_context_window .or_else(|| openai_model_info.as_ref().map(|info| info.context_window)); - let model_max_output_tokens = cfg.model_max_output_tokens.or_else(|| { - openai_model_info - .as_ref() - .map(|info| info.max_output_tokens) - }); let model_auto_compact_token_limit = cfg.model_auto_compact_token_limit.or_else(|| { openai_model_info .as_ref() @@ -1170,12 +1169,13 @@ impl Config { .or(cfg.review_model) .unwrap_or_else(default_review_model); + let check_for_update_on_startup = cfg.check_for_update_on_startup.unwrap_or(true); + let config = Self { model, review_model, model_family, model_context_window, - model_max_output_tokens, model_auto_compact_token_limit, model_provider_id, model_provider, @@ -1247,12 +1247,15 @@ impl Config { active_project, windows_wsl_setup_acknowledged: cfg.windows_wsl_setup_acknowledged.unwrap_or(false), notices: cfg.notice.unwrap_or_default(), + check_for_update_on_startup, disable_paste_burst: cfg.disable_paste_burst.unwrap_or(false), tui_notifications: cfg .tui .as_ref() .map(|t| t.notifications.clone()) .unwrap_or_default(), + animations: cfg.tui.as_ref().map(|t| t.animations).unwrap_or(true), + show_tooltips: cfg.tui.as_ref().map(|t| t.show_tooltips).unwrap_or(true), otel: { let t: OtelConfigToml = cfg.otel.unwrap_or_default(); let log_user_prompt = t.log_user_prompt.unwrap_or(false); @@ -1294,11 +1297,7 @@ impl Config { return Ok(None); }; - let full_path = if p.is_relative() { - cwd.join(p) - } else { - p.to_path_buf() - }; + let full_path = resolve_path(cwd, p); let contents = std::fs::read_to_string(&full_path).map_err(|e| { std::io::Error::new( @@ -1431,6 +1430,7 @@ persistence = "none" let tui = parsed.tui.expect("config should include tui section"); assert_eq!(tui.notifications, Notifications::Enabled(true)); + assert!(tui.show_tooltips); } #[test] @@ -2957,7 +2957,6 @@ model_verbosity = "high" review_model: OPENAI_DEFAULT_REVIEW_MODEL.to_string(), model_family: find_family_for_model("o3").expect("known model slug"), model_context_window: Some(200_000), - model_max_output_tokens: Some(100_000), model_auto_compact_token_limit: Some(180_000), model_provider_id: "openai".to_string(), model_provider: fixture.openai_provider.clone(), @@ -3001,8 +3000,11 @@ model_verbosity = "high" active_project: ProjectConfig { trust_level: None }, windows_wsl_setup_acknowledged: false, notices: Default::default(), + check_for_update_on_startup: true, disable_paste_burst: false, tui_notifications: Default::default(), + animations: true, + show_tooltips: true, otel: OtelConfig::default(), }, o3_profile_config @@ -3029,7 +3031,6 @@ model_verbosity = "high" review_model: OPENAI_DEFAULT_REVIEW_MODEL.to_string(), model_family: find_family_for_model("gpt-3.5-turbo").expect("known model slug"), model_context_window: Some(16_385), - model_max_output_tokens: Some(4_096), model_auto_compact_token_limit: Some(14_746), model_provider_id: "openai-chat-completions".to_string(), model_provider: fixture.openai_chat_completions_provider.clone(), @@ -3073,8 +3074,11 @@ model_verbosity = "high" active_project: ProjectConfig { trust_level: None }, windows_wsl_setup_acknowledged: false, notices: Default::default(), + check_for_update_on_startup: true, disable_paste_burst: false, tui_notifications: Default::default(), + animations: true, + show_tooltips: true, otel: OtelConfig::default(), }; @@ -3116,7 +3120,6 @@ model_verbosity = "high" review_model: OPENAI_DEFAULT_REVIEW_MODEL.to_string(), model_family: find_family_for_model("o3").expect("known model slug"), model_context_window: Some(200_000), - model_max_output_tokens: Some(100_000), model_auto_compact_token_limit: Some(180_000), model_provider_id: "openai".to_string(), model_provider: fixture.openai_provider.clone(), @@ -3160,8 +3163,11 @@ model_verbosity = "high" active_project: ProjectConfig { trust_level: None }, windows_wsl_setup_acknowledged: false, notices: Default::default(), + check_for_update_on_startup: true, disable_paste_burst: false, tui_notifications: Default::default(), + animations: true, + show_tooltips: true, otel: OtelConfig::default(), }; @@ -3189,7 +3195,6 @@ model_verbosity = "high" review_model: OPENAI_DEFAULT_REVIEW_MODEL.to_string(), model_family: find_family_for_model("gpt-5.1").expect("known model slug"), model_context_window: Some(272_000), - model_max_output_tokens: Some(128_000), model_auto_compact_token_limit: Some(244_800), model_provider_id: "openai".to_string(), model_provider: fixture.openai_provider.clone(), @@ -3233,8 +3238,11 @@ model_verbosity = "high" active_project: ProjectConfig { trust_level: None }, windows_wsl_setup_acknowledged: false, notices: Default::default(), + check_for_update_on_startup: true, disable_paste_burst: false, tui_notifications: Default::default(), + animations: true, + show_tooltips: true, otel: OtelConfig::default(), }; diff --git a/codex-rs/core/src/config/types.rs b/codex-rs/core/src/config/types.rs index 8f5077410e..5e1b78aa7b 100644 --- a/codex-rs/core/src/config/types.rs +++ b/codex-rs/core/src/config/types.rs @@ -256,8 +256,8 @@ pub struct History { /// If true, history entries will not be written to disk. pub persistence: HistoryPersistence, - /// If set, the maximum size of the history file in bytes. - /// TODO(mbolin): Not currently honored. + /// If set, the maximum size of the history file in bytes. The oldest entries + /// are dropped once the file exceeds this limit. pub max_bytes: Option, } @@ -363,6 +363,20 @@ pub struct Tui { /// Defaults to `true`. #[serde(default)] pub notifications: Notifications, + + /// Enable animations (welcome screen, shimmer effects, spinners). + /// Defaults to `true`. + #[serde(default = "default_true")] + pub animations: bool, + + /// Show startup tooltips in the TUI welcome screen. + /// Defaults to `true`. + #[serde(default = "default_true")] + pub show_tooltips: bool, +} + +const fn default_true() -> bool { + true } /// Settings for notices we display to users via the tui and app-server clients diff --git a/codex-rs/core/src/config_loader/mod.rs b/codex-rs/core/src/config_loader/mod.rs index 6b55b015a3..4cd541d3e5 100644 --- a/codex-rs/core/src/config_loader/mod.rs +++ b/codex-rs/core/src/config_loader/mod.rs @@ -11,15 +11,15 @@ use toml::Value as TomlValue; #[cfg(unix)] const CODEX_MANAGED_CONFIG_SYSTEM_PATH: &str = "/etc/codex/managed_config.toml"; -#[derive(Debug)] -pub(crate) struct LoadedConfigLayers { +#[derive(Debug, Clone)] +pub struct LoadedConfigLayers { pub base: TomlValue, pub managed_config: Option, pub managed_preferences: Option, } -#[derive(Debug, Default)] -pub(crate) struct LoaderOverrides { +#[derive(Debug, Default, Clone)] +pub struct LoaderOverrides { pub managed_config_path: Option, #[cfg(target_os = "macos")] pub managed_preferences_base64: Option, @@ -47,11 +47,15 @@ pub async fn load_config_as_toml(codex_home: &Path) -> io::Result { load_config_as_toml_with_overrides(codex_home, LoaderOverrides::default()).await } +pub async fn load_config_layers(codex_home: &Path) -> io::Result { + load_config_layers_with_overrides(codex_home, LoaderOverrides::default()).await +} + fn default_empty_table() -> TomlValue { TomlValue::Table(Default::default()) } -pub(crate) async fn load_config_layers_with_overrides( +pub async fn load_config_layers_with_overrides( codex_home: &Path, overrides: LoaderOverrides, ) -> io::Result { @@ -130,7 +134,7 @@ async fn read_config_from_path( } /// Merge config `overlay` into `base`, giving `overlay` precedence. -pub(crate) fn merge_toml_values(base: &mut TomlValue, overlay: &TomlValue) { +pub fn merge_toml_values(base: &mut TomlValue, overlay: &TomlValue) { if let TomlValue::Table(overlay_table) = overlay && let TomlValue::Table(base_table) = base { @@ -147,6 +151,10 @@ pub(crate) fn merge_toml_values(base: &mut TomlValue, overlay: &TomlValue) { } fn managed_config_default_path(codex_home: &Path) -> PathBuf { + if let Ok(path) = std::env::var("CODEX_MANAGED_CONFIG_PATH") { + return PathBuf::from(path); + } + #[cfg(unix)] { let _ = codex_home; diff --git a/codex-rs/core/src/context_manager/history.rs b/codex-rs/core/src/context_manager/history.rs index 8575fe9f6c..b9a9c58f63 100644 --- a/codex-rs/core/src/context_manager/history.rs +++ b/codex-rs/core/src/context_manager/history.rs @@ -1,13 +1,16 @@ use crate::codex::TurnContext; use crate::context_manager::normalize; use crate::truncate::TruncationPolicy; +use crate::truncate::approx_token_count; +use crate::truncate::approx_tokens_from_byte_count; use crate::truncate::truncate_function_output_items_with_policy; use crate::truncate::truncate_text; +use codex_protocol::models::ContentItem; +use codex_protocol::models::FunctionCallOutputContentItem; use codex_protocol::models::FunctionCallOutputPayload; use codex_protocol::models::ResponseItem; use codex_protocol::protocol::TokenUsage; use codex_protocol::protocol::TokenUsageInfo; -use codex_utils_tokenizer::Tokenizer; use std::ops::Deref; /// Transcript of conversation history @@ -74,26 +77,31 @@ impl ContextManager { history } - // Estimate the number of tokens in the history. Return None if no tokenizer - // is available. This does not consider the reasoning traces. - // /!\ The value is a lower bound estimate and does not represent the exact - // context length. + // Estimate token usage using byte-based heuristics from the truncation helpers. + // This is a coarse lower bound, not a tokenizer-accurate count. pub(crate) fn estimate_token_count(&self, turn_context: &TurnContext) -> Option { - let model = turn_context.client.get_model(); - let tokenizer = Tokenizer::for_model(model.as_str()).ok()?; let model_family = turn_context.client.get_model_family(); + let base_tokens = + i64::try_from(approx_token_count(model_family.base_instructions.as_str())) + .unwrap_or(i64::MAX); - Some( - self.items - .iter() - .map(|item| { - serde_json::to_string(&item) - .map(|item| tokenizer.count(&item)) - .unwrap_or_default() - }) - .sum::() - + tokenizer.count(model_family.base_instructions.as_str()), - ) + let items_tokens = self.items.iter().fold(0i64, |acc, item| { + acc + match item { + ResponseItem::Reasoning { + encrypted_content: Some(content), + .. + } + | ResponseItem::CompactionSummary { + encrypted_content: content, + } => estimate_reasoning_length(content.len()) as i64, + item => { + let serialized = serde_json::to_string(item).unwrap_or_default(); + i64::try_from(approx_token_count(&serialized)).unwrap_or(i64::MAX) + } + } + }); + + Some(base_tokens.saturating_add(items_tokens)) } pub(crate) fn remove_first_item(&mut self) { @@ -112,6 +120,37 @@ impl ContextManager { self.items = items; } + pub(crate) fn replace_last_turn_images(&mut self, placeholder: &str) { + let Some(last_item) = self.items.last_mut() else { + return; + }; + + match last_item { + ResponseItem::Message { role, content, .. } if role == "user" => { + for item in content.iter_mut() { + if matches!(item, ContentItem::InputImage { .. }) { + *item = ContentItem::InputText { + text: placeholder.to_string(), + }; + } + } + } + ResponseItem::FunctionCallOutput { output, .. } => { + let Some(content_items) = output.content_items.as_mut() else { + return; + }; + for item in content_items.iter_mut() { + if matches!(item, FunctionCallOutputContentItem::InputImage { .. }) { + *item = FunctionCallOutputContentItem::InputText { + text: placeholder.to_string(), + }; + } + } + } + _ => {} + } + } + pub(crate) fn update_token_info( &mut self, usage: &TokenUsage, @@ -124,6 +163,46 @@ impl ContextManager { ); } + fn get_non_last_reasoning_items_tokens(&self) -> usize { + // get reasoning items excluding all the ones after the last user message + let Some(last_user_index) = self + .items + .iter() + .rposition(|item| matches!(item, ResponseItem::Message { role, .. } if role == "user")) + else { + return 0usize; + }; + + let total_reasoning_bytes = self + .items + .iter() + .take(last_user_index) + .filter_map(|item| { + if let ResponseItem::Reasoning { + encrypted_content: Some(content), + .. + } = item + { + Some(content.len()) + } else { + None + } + }) + .map(estimate_reasoning_length) + .fold(0usize, usize::saturating_add); + + let token_estimate = approx_tokens_from_byte_count(total_reasoning_bytes); + token_estimate as usize + } + + pub(crate) fn get_total_token_usage(&self) -> i64 { + self.token_info + .as_ref() + .map(|info| info.last_token_usage.total_tokens) + .unwrap_or(0) + .saturating_add(self.get_non_last_reasoning_items_tokens() as i64) + } + /// This function enforces a couple of invariants on the in-memory history: /// 1. every call (function/custom) has a corresponding output entry /// 2. every output has a corresponding call entry @@ -203,6 +282,14 @@ fn is_api_message(message: &ResponseItem) -> bool { } } +fn estimate_reasoning_length(encoded_len: usize) -> usize { + encoded_len + .saturating_mul(3) + .checked_div(4) + .unwrap_or(0) + .saturating_sub(650) +} + #[cfg(test)] #[path = "history_tests.rs"] mod tests; diff --git a/codex-rs/core/src/context_manager/history_tests.rs b/codex-rs/core/src/context_manager/history_tests.rs index 1a01604a70..cb59c97231 100644 --- a/codex-rs/core/src/context_manager/history_tests.rs +++ b/codex-rs/core/src/context_manager/history_tests.rs @@ -56,6 +56,17 @@ fn reasoning_msg(text: &str) -> ResponseItem { } } +fn reasoning_with_encrypted_content(len: usize) -> ResponseItem { + ResponseItem::Reasoning { + id: String::new(), + summary: vec![ReasoningItemReasoningSummary::SummaryText { + text: "summary".to_string(), + }], + content: None, + encrypted_content: Some("a".repeat(len)), + } +} + fn truncate_exec_output(content: &str) -> String { truncate::truncate_text(content, TruncationPolicy::Tokens(EXEC_FORMAT_MAX_TOKENS)) } @@ -112,6 +123,28 @@ fn filters_non_api_messages() { ); } +#[test] +fn non_last_reasoning_tokens_return_zero_when_no_user_messages() { + let history = create_history_with_items(vec![reasoning_with_encrypted_content(800)]); + + assert_eq!(history.get_non_last_reasoning_items_tokens(), 0); +} + +#[test] +fn non_last_reasoning_tokens_ignore_entries_after_last_user() { + let history = create_history_with_items(vec![ + reasoning_with_encrypted_content(900), + user_msg("first"), + reasoning_with_encrypted_content(1_000), + user_msg("second"), + reasoning_with_encrypted_content(2_000), + ]); + // first: (900 * 0.75 - 650) / 4 = 6.25 tokens + // second: (1000 * 0.75 - 650) / 4 = 25 tokens + // first + second = 62.5 + assert_eq!(history.get_non_last_reasoning_items_tokens(), 32); +} + #[test] fn get_history_for_prompt_drops_ghost_commits() { let items = vec![ResponseItem::GhostSnapshot { diff --git a/codex-rs/core/src/default_client.rs b/codex-rs/core/src/default_client.rs index 8e4635460c..29986c401d 100644 --- a/codex-rs/core/src/default_client.rs +++ b/codex-rs/core/src/default_client.rs @@ -258,6 +258,11 @@ fn sanitize_user_agent(candidate: String, fallback: &str) -> String { /// Create an HTTP client with default `originator` and `User-Agent` headers set. pub fn create_client() -> CodexHttpClient { + let inner = build_reqwest_client(); + CodexHttpClient::new(inner) +} + +pub fn build_reqwest_client() -> reqwest::Client { use reqwest::header::HeaderMap; let mut headers = HeaderMap::new(); @@ -272,8 +277,7 @@ pub fn create_client() -> CodexHttpClient { builder = builder.no_proxy(); } - let inner = builder.build().unwrap_or_else(|_| reqwest::Client::new()); - CodexHttpClient::new(inner) + builder.build().unwrap_or_else(|_| reqwest::Client::new()) } fn is_sandboxed() -> bool { diff --git a/codex-rs/core/src/environment_context.rs b/codex-rs/core/src/environment_context.rs index 6db2bb8f52..56e7f6cadb 100644 --- a/codex-rs/core/src/environment_context.rs +++ b/codex-rs/core/src/environment_context.rs @@ -6,6 +6,7 @@ use crate::codex::TurnContext; use crate::protocol::AskForApproval; use crate::protocol::SandboxPolicy; use crate::shell::Shell; +use crate::shell::default_user_shell; use codex_protocol::config_types::SandboxMode; use codex_protocol::models::ContentItem; use codex_protocol::models::ResponseItem; @@ -28,7 +29,7 @@ pub(crate) struct EnvironmentContext { pub sandbox_mode: Option, pub network_access: Option, pub writable_roots: Option>, - pub shell: Option, + pub shell: Shell, } impl EnvironmentContext { @@ -36,7 +37,7 @@ impl EnvironmentContext { cwd: Option, approval_policy: Option, sandbox_policy: Option, - shell: Option, + shell: Shell, ) -> Self { Self { cwd, @@ -110,7 +111,7 @@ impl EnvironmentContext { } else { None }; - EnvironmentContext::new(cwd, approval_policy, sandbox_policy, None) + EnvironmentContext::new(cwd, approval_policy, sandbox_policy, default_user_shell()) } } @@ -121,7 +122,7 @@ impl From<&TurnContext> for EnvironmentContext { Some(turn_context.approval_policy), Some(turn_context.sandbox_policy.clone()), // Shell is not configurable from turn to turn - None, + default_user_shell(), ) } } @@ -169,11 +170,9 @@ impl EnvironmentContext { } lines.push(" ".to_string()); } - if let Some(shell) = self.shell - && let Some(shell_name) = shell.name() - { - lines.push(format!(" {shell_name}")); - } + + let shell_name = self.shell.name(); + lines.push(format!(" {shell_name}")); lines.push(ENVIRONMENT_CONTEXT_CLOSE_TAG.to_string()); lines.join("\n") } @@ -193,12 +192,18 @@ impl From for ResponseItem { #[cfg(test)] mod tests { - use crate::shell::BashShell; - use crate::shell::ZshShell; + use crate::shell::ShellType; use super::*; use pretty_assertions::assert_eq; + fn fake_shell() -> Shell { + Shell { + shell_type: ShellType::Bash, + shell_path: PathBuf::from("/bin/bash"), + } + } + fn workspace_write_policy(writable_roots: Vec<&str>, network_access: bool) -> SandboxPolicy { SandboxPolicy::WorkspaceWrite { writable_roots: writable_roots.into_iter().map(PathBuf::from).collect(), @@ -214,7 +219,7 @@ mod tests { Some(PathBuf::from("/repo")), Some(AskForApproval::OnRequest), Some(workspace_write_policy(vec!["/repo", "/tmp"], false)), - None, + fake_shell(), ); let expected = r#" @@ -226,6 +231,7 @@ mod tests { /repo /tmp + bash "#; assert_eq!(context.serialize_to_xml(), expected); @@ -237,13 +243,14 @@ mod tests { None, Some(AskForApproval::Never), Some(SandboxPolicy::ReadOnly), - None, + fake_shell(), ); let expected = r#" never read-only restricted + bash "#; assert_eq!(context.serialize_to_xml(), expected); @@ -255,13 +262,14 @@ mod tests { None, Some(AskForApproval::OnFailure), Some(SandboxPolicy::DangerFullAccess), - None, + fake_shell(), ); let expected = r#" on-failure danger-full-access enabled + bash "#; assert_eq!(context.serialize_to_xml(), expected); @@ -274,13 +282,13 @@ mod tests { Some(PathBuf::from("/repo")), Some(AskForApproval::OnRequest), Some(workspace_write_policy(vec!["/repo"], false)), - None, + fake_shell(), ); let context2 = EnvironmentContext::new( Some(PathBuf::from("/repo")), Some(AskForApproval::Never), Some(workspace_write_policy(vec!["/repo"], true)), - None, + fake_shell(), ); assert!(!context1.equals_except_shell(&context2)); } @@ -291,13 +299,13 @@ mod tests { Some(PathBuf::from("/repo")), Some(AskForApproval::OnRequest), Some(SandboxPolicy::new_read_only_policy()), - None, + fake_shell(), ); let context2 = EnvironmentContext::new( Some(PathBuf::from("/repo")), Some(AskForApproval::OnRequest), Some(SandboxPolicy::new_workspace_write_policy()), - None, + fake_shell(), ); assert!(!context1.equals_except_shell(&context2)); @@ -309,13 +317,13 @@ mod tests { Some(PathBuf::from("/repo")), Some(AskForApproval::OnRequest), Some(workspace_write_policy(vec!["/repo", "/tmp", "/var"], false)), - None, + fake_shell(), ); let context2 = EnvironmentContext::new( Some(PathBuf::from("/repo")), Some(AskForApproval::OnRequest), Some(workspace_write_policy(vec!["/repo", "/tmp"], true)), - None, + fake_shell(), ); assert!(!context1.equals_except_shell(&context2)); @@ -327,17 +335,19 @@ mod tests { Some(PathBuf::from("/repo")), Some(AskForApproval::OnRequest), Some(workspace_write_policy(vec!["/repo"], false)), - Some(Shell::Bash(BashShell { + Shell { + shell_type: ShellType::Bash, shell_path: "/bin/bash".into(), - })), + }, ); let context2 = EnvironmentContext::new( Some(PathBuf::from("/repo")), Some(AskForApproval::OnRequest), Some(workspace_write_policy(vec!["/repo"], false)), - Some(Shell::Zsh(ZshShell { + Shell { + shell_type: ShellType::Zsh, shell_path: "/bin/zsh".into(), - })), + }, ); assert!(context1.equals_except_shell(&context2)); diff --git a/codex-rs/core/src/error.rs b/codex-rs/core/src/error.rs index b2027dc940..a25261d649 100644 --- a/codex-rs/core/src/error.rs +++ b/codex-rs/core/src/error.rs @@ -10,6 +10,8 @@ use chrono::Local; use chrono::Utc; use codex_async_utils::CancelErr; use codex_protocol::ConversationId; +use codex_protocol::protocol::CodexErrorInfo; +use codex_protocol::protocol::ErrorEvent; use codex_protocol::protocol::RateLimitSnapshot; use reqwest::StatusCode; use serde_json; @@ -101,6 +103,14 @@ pub enum CodexErr { #[error("{0}")] UnexpectedStatus(UnexpectedResponseError), + /// Invalid request. + #[error("{0}")] + InvalidRequest(String), + + /// Invalid image. + #[error("Image poisoning")] + InvalidImageRequest(), + #[error("{0}")] UsageLimitReached(UsageLimitReachedError), @@ -430,6 +440,57 @@ impl CodexErr { pub fn downcast_ref(&self) -> Option<&T> { (self as &dyn std::any::Any).downcast_ref::() } + + /// Translate core error to client-facing protocol error. + pub fn to_codex_protocol_error(&self) -> CodexErrorInfo { + match self { + CodexErr::ContextWindowExceeded => CodexErrorInfo::ContextWindowExceeded, + CodexErr::UsageLimitReached(_) + | CodexErr::QuotaExceeded + | CodexErr::UsageNotIncluded => CodexErrorInfo::UsageLimitExceeded, + CodexErr::RetryLimit(_) => CodexErrorInfo::ResponseTooManyFailedAttempts { + http_status_code: self.http_status_code_value(), + }, + CodexErr::ConnectionFailed(_) => CodexErrorInfo::HttpConnectionFailed { + http_status_code: self.http_status_code_value(), + }, + CodexErr::ResponseStreamFailed(_) => CodexErrorInfo::ResponseStreamConnectionFailed { + http_status_code: self.http_status_code_value(), + }, + CodexErr::RefreshTokenFailed(_) => CodexErrorInfo::Unauthorized, + CodexErr::SessionConfiguredNotFirstEvent + | CodexErr::InternalServerError + | CodexErr::InternalAgentDied => CodexErrorInfo::InternalServerError, + CodexErr::UnsupportedOperation(_) | CodexErr::ConversationNotFound(_) => { + CodexErrorInfo::BadRequest + } + CodexErr::Sandbox(_) => CodexErrorInfo::SandboxError, + _ => CodexErrorInfo::Other, + } + } + + pub fn to_error_event(&self, message_prefix: Option) -> ErrorEvent { + let error_message = self.to_string(); + let message: String = match message_prefix { + Some(prefix) => format!("{prefix}: {error_message}"), + None => error_message, + }; + ErrorEvent { + message, + codex_error_info: Some(self.to_codex_protocol_error()), + } + } + + pub fn http_status_code_value(&self) -> Option { + let http_status_code = match self { + CodexErr::RetryLimit(err) => Some(err.status), + CodexErr::UnexpectedStatus(err) => Some(err.status), + CodexErr::ConnectionFailed(err) => err.source.status(), + CodexErr::ResponseStreamFailed(err) => err.source.status(), + _ => None, + }; + http_status_code.as_ref().map(StatusCode::as_u16) + } } pub fn get_error_message_ui(e: &CodexErr) -> String { @@ -478,6 +539,10 @@ mod tests { use chrono::Utc; use codex_protocol::protocol::RateLimitWindow; use pretty_assertions::assert_eq; + use reqwest::Response; + use reqwest::ResponseBuilderExt; + use reqwest::StatusCode; + use reqwest::Url; fn rate_limit_snapshot() -> RateLimitSnapshot { let primary_reset_at = Utc @@ -573,6 +638,33 @@ mod tests { assert_eq!(get_error_message_ui(&err), "stdout only"); } + #[test] + fn to_error_event_handles_response_stream_failed() { + let response = http::Response::builder() + .status(StatusCode::TOO_MANY_REQUESTS) + .url(Url::parse("http://example.com").unwrap()) + .body("") + .unwrap(); + let source = Response::from(response).error_for_status_ref().unwrap_err(); + let err = CodexErr::ResponseStreamFailed(ResponseStreamFailed { + source, + request_id: Some("req-123".to_string()), + }); + + let event = err.to_error_event(Some("prefix".to_string())); + + assert_eq!( + event.message, + "prefix: Error while reading the server response: HTTP status client error (429 Too Many Requests) for url (http://example.com/), request id: req-123" + ); + assert_eq!( + event.codex_error_info, + Some(CodexErrorInfo::ResponseStreamConnectionFailed { + http_status_code: Some(429) + }) + ); + } + #[test] fn sandbox_denied_reports_exit_code_when_no_output_available() { let output = ExecToolCallOutput { diff --git a/codex-rs/core/src/event_mapping.rs b/codex-rs/core/src/event_mapping.rs index d0aa1d8188..6b4bed4db3 100644 --- a/codex-rs/core/src/event_mapping.rs +++ b/codex-rs/core/src/event_mapping.rs @@ -117,7 +117,7 @@ pub fn parse_turn_item(item: &ResponseItem) -> Option { .. } => Some(TurnItem::WebSearch(WebSearchItem { id: id.clone().unwrap_or_default(), - query: query.clone(), + query: query.clone().unwrap_or_default(), })), _ => None, } @@ -306,7 +306,7 @@ mod tests { id: Some("ws_1".to_string()), status: Some("completed".to_string()), action: WebSearchAction::Search { - query: "weather".to_string(), + query: Some("weather".to_string()), }, }; diff --git a/codex-rs/core/src/exec.rs b/codex-rs/core/src/exec.rs index 583c2233a2..f46444675f 100644 --- a/codex-rs/core/src/exec.rs +++ b/codex-rs/core/src/exec.rs @@ -14,10 +14,12 @@ use tokio::io::AsyncRead; use tokio::io::AsyncReadExt; use tokio::io::BufReader; use tokio::process::Child; +use tokio_util::sync::CancellationToken; use crate::error::CodexErr; use crate::error::Result; use crate::error::SandboxErr; +use crate::get_platform_sandbox; use crate::protocol::Event; use crate::protocol::EventMsg; use crate::protocol::ExecCommandOutputDeltaEvent; @@ -28,8 +30,9 @@ use crate::sandboxing::ExecEnv; use crate::sandboxing::SandboxManager; use crate::spawn::StdioPolicy; use crate::spawn::spawn_child_async; +use crate::text_encoding::bytes_to_string_smart; -const DEFAULT_TIMEOUT_MS: u64 = 10_000; +pub const DEFAULT_EXEC_COMMAND_TIMEOUT_MS: u64 = 10_000; // Hardcode these since it does not seem worth including the libc crate just // for these. @@ -46,20 +49,59 @@ const AGGREGATE_BUFFER_INITIAL_CAPACITY: usize = 8 * 1024; // 8 KiB /// Aggregation still collects full output; only the live event stream is capped. pub(crate) const MAX_EXEC_OUTPUT_DELTAS_PER_CALL: usize = 10_000; -#[derive(Clone, Debug)] +#[derive(Debug)] pub struct ExecParams { pub command: Vec, pub cwd: PathBuf, - pub timeout_ms: Option, + pub expiration: ExecExpiration, pub env: HashMap, pub with_escalated_permissions: Option, pub justification: Option, pub arg0: Option, } -impl ExecParams { - pub fn timeout_duration(&self) -> Duration { - Duration::from_millis(self.timeout_ms.unwrap_or(DEFAULT_TIMEOUT_MS)) +/// Mechanism to terminate an exec invocation before it finishes naturally. +#[derive(Debug)] +pub enum ExecExpiration { + Timeout(Duration), + DefaultTimeout, + Cancellation(CancellationToken), +} + +impl From> for ExecExpiration { + fn from(timeout_ms: Option) -> Self { + timeout_ms.map_or(ExecExpiration::DefaultTimeout, |timeout_ms| { + ExecExpiration::Timeout(Duration::from_millis(timeout_ms)) + }) + } +} + +impl From for ExecExpiration { + fn from(timeout_ms: u64) -> Self { + ExecExpiration::Timeout(Duration::from_millis(timeout_ms)) + } +} + +impl ExecExpiration { + async fn wait(self) { + match self { + ExecExpiration::Timeout(duration) => tokio::time::sleep(duration).await, + ExecExpiration::DefaultTimeout => { + tokio::time::sleep(Duration::from_millis(DEFAULT_EXEC_COMMAND_TIMEOUT_MS)).await + } + ExecExpiration::Cancellation(cancel) => { + cancel.cancelled().await; + } + } + } + + /// If ExecExpiration is a timeout, returns the timeout in milliseconds. + pub(crate) fn timeout_ms(&self) -> Option { + match self { + ExecExpiration::Timeout(duration) => Some(duration.as_millis() as u64), + ExecExpiration::DefaultTimeout => Some(DEFAULT_EXEC_COMMAND_TIMEOUT_MS), + ExecExpiration::Cancellation(_) => None, + } } } @@ -86,16 +128,21 @@ pub struct StdoutStream { pub async fn process_exec_tool_call( params: ExecParams, - sandbox_type: SandboxType, sandbox_policy: &SandboxPolicy, sandbox_cwd: &Path, codex_linux_sandbox_exe: &Option, stdout_stream: Option, ) -> Result { + let sandbox_type = match &sandbox_policy { + SandboxPolicy::DangerFullAccess => SandboxType::None, + _ => get_platform_sandbox().unwrap_or(SandboxType::None), + }; + tracing::debug!("Sandbox type: {sandbox_type:?}"); + let ExecParams { command, cwd, - timeout_ms, + expiration, env, with_escalated_permissions, justification, @@ -114,7 +161,7 @@ pub async fn process_exec_tool_call( args: args.to_vec(), cwd, env, - timeout_ms, + expiration, with_escalated_permissions, justification, }; @@ -122,7 +169,7 @@ pub async fn process_exec_tool_call( let manager = SandboxManager::new(); let exec_env = manager .transform( - &spec, + spec, sandbox_policy, sandbox_type, sandbox_cwd, @@ -131,7 +178,7 @@ pub async fn process_exec_tool_call( .map_err(CodexErr::from)?; // Route through the sandboxing module for a single, unified execution path. - crate::sandboxing::execute_env(&exec_env, sandbox_policy, stdout_stream).await + crate::sandboxing::execute_env(exec_env, sandbox_policy, stdout_stream).await } pub(crate) async fn execute_exec_env( @@ -143,7 +190,7 @@ pub(crate) async fn execute_exec_env( command, cwd, env, - timeout_ms, + expiration, sandbox, with_escalated_permissions, justification, @@ -153,7 +200,7 @@ pub(crate) async fn execute_exec_env( let params = ExecParams { command, cwd, - timeout_ms, + expiration, env, with_escalated_permissions, justification, @@ -178,9 +225,12 @@ async fn exec_windows_sandbox( command, cwd, env, - timeout_ms, + expiration, .. } = params; + // TODO(iceweasel-oai): run_windows_sandbox_capture should support all + // variants of ExecExpiration, not just timeout. + let timeout_ms = expiration.timeout_ms(); let policy_str = serde_json::to_string(sandbox_policy).map_err(|err| { CodexErr::Io(io::Error::other(format!( @@ -414,7 +464,7 @@ impl StreamOutput { impl StreamOutput> { pub fn from_utf8_lossy(&self) -> StreamOutput { StreamOutput { - text: String::from_utf8_lossy(&self.text).to_string(), + text: bytes_to_string_smart(&self.text), truncated_after_lines: self.truncated_after_lines, } } @@ -448,12 +498,12 @@ async fn exec( { return exec_windows_sandbox(params, sandbox_policy).await; } - let timeout = params.timeout_duration(); let ExecParams { command, cwd, env, arg0, + expiration, .. } = params; @@ -474,14 +524,14 @@ async fn exec( env, ) .await?; - consume_truncated_output(child, timeout, stdout_stream).await + consume_truncated_output(child, expiration, stdout_stream).await } /// Consumes the output of a child process, truncating it so it is suitable for /// use as the output of a `shell` tool call. Also enforces specified timeout. async fn consume_truncated_output( mut child: Child, - timeout: Duration, + expiration: ExecExpiration, stdout_stream: Option, ) -> Result { // Both stdout and stderr were configured with `Stdio::piped()` @@ -515,20 +565,14 @@ async fn consume_truncated_output( )); let (exit_status, timed_out) = tokio::select! { - result = tokio::time::timeout(timeout, child.wait()) => { - match result { - Ok(status_result) => { - let exit_status = status_result?; - (exit_status, false) - } - Err(_) => { - // timeout - kill_child_process_group(&mut child)?; - child.start_kill()?; - // Debatable whether `child.wait().await` should be called here. - (synthetic_exit_status(EXIT_CODE_SIGNAL_BASE + TIMEOUT_CODE), true) - } - } + status_result = child.wait() => { + let exit_status = status_result?; + (exit_status, false) + } + _ = expiration.wait() => { + kill_child_process_group(&mut child)?; + child.start_kill()?; + (synthetic_exit_status(EXIT_CODE_SIGNAL_BASE + TIMEOUT_CODE), true) } _ = tokio::signal::ctrl_c() => { kill_child_process_group(&mut child)?; @@ -780,6 +824,15 @@ mod tests { #[cfg(unix)] #[tokio::test] async fn kill_child_process_group_kills_grandchildren_on_timeout() -> Result<()> { + // On Linux/macOS, /bin/bash is typically present; on FreeBSD/OpenBSD, + // prefer /bin/sh to avoid NotFound errors. + #[cfg(any(target_os = "freebsd", target_os = "openbsd"))] + let command = vec![ + "/bin/sh".to_string(), + "-c".to_string(), + "sleep 60 & echo $!; sleep 60".to_string(), + ]; + #[cfg(all(unix, not(any(target_os = "freebsd", target_os = "openbsd"))))] let command = vec![ "/bin/bash".to_string(), "-c".to_string(), @@ -789,7 +842,7 @@ mod tests { let params = ExecParams { command, cwd: std::env::current_dir()?, - timeout_ms: Some(500), + expiration: 500.into(), env, with_escalated_permissions: None, justification: None, @@ -823,4 +876,61 @@ mod tests { assert!(killed, "grandchild process with pid {pid} is still alive"); Ok(()) } + + #[tokio::test] + async fn process_exec_tool_call_respects_cancellation_token() -> Result<()> { + let command = long_running_command(); + let cwd = std::env::current_dir()?; + let env: HashMap = std::env::vars().collect(); + let cancel_token = CancellationToken::new(); + let cancel_tx = cancel_token.clone(); + let params = ExecParams { + command, + cwd: cwd.clone(), + expiration: ExecExpiration::Cancellation(cancel_token), + env, + with_escalated_permissions: None, + justification: None, + arg0: None, + }; + tokio::spawn(async move { + tokio::time::sleep(Duration::from_millis(1_000)).await; + cancel_tx.cancel(); + }); + let result = process_exec_tool_call( + params, + &SandboxPolicy::DangerFullAccess, + cwd.as_path(), + &None, + None, + ) + .await; + let output = match result { + Err(CodexErr::Sandbox(SandboxErr::Timeout { output })) => output, + other => panic!("expected timeout error, got {other:?}"), + }; + assert!(output.timed_out); + assert_eq!(output.exit_code, EXEC_TIMEOUT_EXIT_CODE); + Ok(()) + } + + #[cfg(unix)] + fn long_running_command() -> Vec { + vec![ + "/bin/sh".to_string(), + "-c".to_string(), + "sleep 30".to_string(), + ] + } + + #[cfg(windows)] + fn long_running_command() -> Vec { + vec![ + "powershell.exe".to_string(), + "-NonInteractive".to_string(), + "-NoLogo".to_string(), + "-Command".to_string(), + "Start-Sleep -Seconds 30".to_string(), + ] + } } diff --git a/codex-rs/core/src/exec_policy.rs b/codex-rs/core/src/exec_policy.rs index 9525cc1acf..602e5d679e 100644 --- a/codex-rs/core/src/exec_policy.rs +++ b/codex-rs/core/src/exec_policy.rs @@ -107,13 +107,15 @@ fn evaluate_with_policy( }) } } - Decision::Allow => Some(ApprovalRequirement::Skip), + Decision::Allow => Some(ApprovalRequirement::Skip { + bypass_sandbox: true, + }), }, - Evaluation::NoMatch => None, + Evaluation::NoMatch { .. } => None, } } -pub(crate) fn create_approval_requirement_for_command( +pub(crate) async fn create_approval_requirement_for_command( policy: &Policy, command: &[String], approval_policy: AskForApproval, @@ -132,7 +134,9 @@ pub(crate) fn create_approval_requirement_for_command( ) { ApprovalRequirement::NeedsApproval { reason: None } } else { - ApprovalRequirement::Skip + ApprovalRequirement::Skip { + bypass_sandbox: false, + } } } @@ -206,7 +210,7 @@ mod tests { let commands = [vec!["rm".to_string()]]; assert!(matches!( policy.check_multiple(commands.iter()), - Evaluation::NoMatch + Evaluation::NoMatch { .. } )); assert!(!temp_dir.path().join(POLICY_DIR_NAME).exists()); } @@ -259,7 +263,7 @@ mod tests { let command = [vec!["ls".to_string()]]; assert!(matches!( policy.check_multiple(command.iter()), - Evaluation::NoMatch + Evaluation::NoMatch { .. } )); } @@ -292,8 +296,8 @@ prefix_rule(pattern=["rm"], decision="forbidden") ); } - #[test] - fn approval_requirement_prefers_execpolicy_match() { + #[tokio::test] + async fn approval_requirement_prefers_execpolicy_match() { let policy_src = r#"prefix_rule(pattern=["rm"], decision="prompt")"#; let mut parser = PolicyParser::new(); parser @@ -308,7 +312,8 @@ prefix_rule(pattern=["rm"], decision="forbidden") AskForApproval::OnRequest, &SandboxPolicy::DangerFullAccess, SandboxPermissions::UseDefault, - ); + ) + .await; assert_eq!( requirement, @@ -318,8 +323,8 @@ prefix_rule(pattern=["rm"], decision="forbidden") ); } - #[test] - fn approval_requirement_respects_approval_policy() { + #[tokio::test] + async fn approval_requirement_respects_approval_policy() { let policy_src = r#"prefix_rule(pattern=["rm"], decision="prompt")"#; let mut parser = PolicyParser::new(); parser @@ -334,7 +339,8 @@ prefix_rule(pattern=["rm"], decision="forbidden") AskForApproval::Never, &SandboxPolicy::DangerFullAccess, SandboxPermissions::UseDefault, - ); + ) + .await; assert_eq!( requirement, @@ -344,8 +350,8 @@ prefix_rule(pattern=["rm"], decision="forbidden") ); } - #[test] - fn approval_requirement_falls_back_to_heuristics() { + #[tokio::test] + async fn approval_requirement_falls_back_to_heuristics() { let command = vec!["python".to_string()]; let empty_policy = Policy::empty(); @@ -355,7 +361,8 @@ prefix_rule(pattern=["rm"], decision="forbidden") AskForApproval::UnlessTrusted, &SandboxPolicy::ReadOnly, SandboxPermissions::UseDefault, - ); + ) + .await; assert_eq!( requirement, diff --git a/codex-rs/core/src/features.rs b/codex-rs/core/src/features.rs index 75622e5254..687e8b1627 100644 --- a/codex-rs/core/src/features.rs +++ b/codex-rs/core/src/features.rs @@ -31,9 +31,6 @@ pub enum Feature { GhostCommit, /// Use the single unified PTY-backed exec tool. UnifiedExec, - /// Use the shell command tool that takes `command` as a single string of - /// shell instead of an array of args passed to `execvp(3)`. - ShellCommandTool, /// Enable experimental RMCP features such as OAuth login. RmcpClient, /// Include the freeform apply_patch tool. @@ -54,6 +51,10 @@ pub enum Feature { ShellTool, /// Allow model to call multiple tools in parallel (only for models supporting it). ParallelToolCalls, + /// Experimental skills injection (CLI flag-driven). + Skills, + /// Send warnings to the model to correct it on the tool usage. + ModelWarnings, } impl Feature { @@ -268,6 +269,12 @@ pub const FEATURES: &[FeatureSpec] = &[ stage: Stage::Stable, default_enabled: true, }, + FeatureSpec { + id: Feature::ShellTool, + key: "shell_tool", + stage: Stage::Stable, + default_enabled: true, + }, // Unstable features. FeatureSpec { id: Feature::UnifiedExec, @@ -275,12 +282,6 @@ pub const FEATURES: &[FeatureSpec] = &[ stage: Stage::Experimental, default_enabled: false, }, - FeatureSpec { - id: Feature::ShellCommandTool, - key: "shell_command_tool", - stage: Stage::Experimental, - default_enabled: false, - }, FeatureSpec { id: Feature::RmcpClient, key: "rmcp_client", @@ -330,9 +331,15 @@ pub const FEATURES: &[FeatureSpec] = &[ default_enabled: false, }, FeatureSpec { - id: Feature::ShellTool, - key: "shell_tool", - stage: Stage::Stable, - default_enabled: true, + id: Feature::ModelWarnings, + key: "warnings", + stage: Stage::Experimental, + default_enabled: false, + }, + FeatureSpec { + id: Feature::Skills, + key: "skills", + stage: Stage::Experimental, + default_enabled: false, }, ]; diff --git a/codex-rs/core/src/git_info.rs b/codex-rs/core/src/git_info.rs index 387e9a682e..065f1280a4 100644 --- a/codex-rs/core/src/git_info.rs +++ b/codex-rs/core/src/git_info.rs @@ -2,6 +2,7 @@ use std::collections::HashSet; use std::path::Path; use std::path::PathBuf; +use crate::util::resolve_path; use codex_app_server_protocol::GitSha; use codex_protocol::protocol::GitInfo; use futures::future::join_all; @@ -131,11 +132,15 @@ pub async fn recent_commits(cwd: &Path, limit: usize) -> Vec { } let fmt = "%H%x1f%ct%x1f%s"; // - let n = limit.max(1).to_string(); - let Some(log_out) = - run_git_command_with_timeout(&["log", "-n", &n, &format!("--pretty=format:{fmt}")], cwd) - .await - else { + let limit_arg = (limit > 0).then(|| limit.to_string()); + let mut args: Vec = vec!["log".to_string()]; + if let Some(n) = &limit_arg { + args.push("-n".to_string()); + args.push(n.clone()); + } + args.push(format!("--pretty=format:{fmt}")); + let arg_refs: Vec<&str> = args.iter().map(String::as_str).collect(); + let Some(log_out) = run_git_command_with_timeout(&arg_refs, cwd).await else { return Vec::new(); }; if !log_out.status.success() { @@ -544,11 +549,7 @@ pub fn resolve_root_git_project_for_trust(cwd: &Path) -> Option { .trim() .to_string(); - let git_dir_path_raw = if Path::new(&git_dir_s).is_absolute() { - PathBuf::from(&git_dir_s) - } else { - base.join(&git_dir_s) - }; + let git_dir_path_raw = resolve_path(base, &PathBuf::from(&git_dir_s)); // Normalize to handle macOS /var vs /private/var and resolve ".." segments. let git_dir_path = std::fs::canonicalize(&git_dir_path_raw).unwrap_or(git_dir_path_raw); @@ -825,11 +826,21 @@ mod tests { .await .expect("Should collect git info from repo"); + let remote_url_output = Command::new("git") + .args(["remote", "get-url", "origin"]) + .current_dir(&repo_path) + .output() + .await + .expect("Failed to read remote url"); + // Some dev environments rewrite remotes (e.g., force SSH), so compare against + // whatever URL Git reports instead of a fixed placeholder. + let expected_remote = String::from_utf8(remote_url_output.stdout) + .unwrap() + .trim() + .to_string(); + // Should have repository URL - assert_eq!( - git_info.repository_url, - Some("https://github.com/example/repo.git".to_string()) - ); + assert_eq!(git_info.repository_url, Some(expected_remote)); } #[tokio::test] diff --git a/codex-rs/core/src/lib.rs b/codex-rs/core/src/lib.rs index 27d7cdaea3..f7179d9bbb 100644 --- a/codex-rs/core/src/lib.rs +++ b/codex-rs/core/src/lib.rs @@ -5,10 +5,10 @@ // the TUI or the tracing stack). #![deny(clippy::print_stdout, clippy::print_stderr)] +pub mod api_bridge; mod apply_patch; pub mod auth; pub mod bash; -mod chat_completions; mod client; mod client_common; pub mod codex; @@ -32,6 +32,9 @@ pub mod git_info; pub mod landlock; pub mod mcp; mod mcp_connection_manager; +pub use mcp_connection_manager::MCP_SANDBOX_STATE_CAPABILITY; +pub use mcp_connection_manager::MCP_SANDBOX_STATE_NOTIFICATION; +pub use mcp_connection_manager::SandboxState; mod mcp_tool_call; mod message_history; mod model_provider_info; @@ -39,6 +42,7 @@ pub mod parse_command; pub mod powershell; mod response_processing; pub mod sandboxing; +mod text_encoding; pub mod token_data; mod truncate; mod unified_exec; @@ -54,6 +58,7 @@ pub use model_provider_info::create_oss_provider_with_base_url; mod conversation_manager; mod event_mapping; pub mod review_format; +pub mod review_prompts; pub use codex_protocol::protocol::InitialHistory; pub use conversation_manager::ConversationManager; pub use conversation_manager::NewConversation; @@ -69,6 +74,7 @@ pub(crate) mod safety; pub mod saved_sessions; pub mod seatbelt; pub mod shell; +pub mod skills; pub mod spawn; pub mod terminal; mod tools; diff --git a/codex-rs/core/src/mcp_connection_manager.rs b/codex-rs/core/src/mcp_connection_manager.rs index d8869e5e9f..11a90f77a8 100644 --- a/codex-rs/core/src/mcp_connection_manager.rs +++ b/codex-rs/core/src/mcp_connection_manager.rs @@ -10,6 +10,7 @@ use std::collections::HashMap; use std::collections::HashSet; use std::env; use std::ffi::OsString; +use std::path::PathBuf; use std::sync::Arc; use std::time::Duration; @@ -20,14 +21,18 @@ use anyhow::anyhow; use async_channel::Sender; use codex_async_utils::CancelErr; use codex_async_utils::OrCancelExt; +use codex_protocol::approvals::ElicitationRequestEvent; use codex_protocol::protocol::Event; use codex_protocol::protocol::EventMsg; use codex_protocol::protocol::McpStartupCompleteEvent; use codex_protocol::protocol::McpStartupFailure; use codex_protocol::protocol::McpStartupStatus; use codex_protocol::protocol::McpStartupUpdateEvent; +use codex_protocol::protocol::SandboxPolicy; +use codex_rmcp_client::ElicitationResponse; use codex_rmcp_client::OAuthCredentialsStoreMode; use codex_rmcp_client::RmcpClient; +use codex_rmcp_client::SendElicitation; use futures::future::BoxFuture; use futures::future::FutureExt; use futures::future::Shared; @@ -39,13 +44,18 @@ use mcp_types::ListResourcesRequestParams; use mcp_types::ListResourcesResult; use mcp_types::ReadResourceRequestParams; use mcp_types::ReadResourceResult; +use mcp_types::RequestId; use mcp_types::Resource; use mcp_types::ResourceTemplate; use mcp_types::Tool; +use serde::Deserialize; +use serde::Serialize; use serde_json::json; use sha1::Digest; use sha1::Sha1; +use tokio::sync::Mutex; +use tokio::sync::oneshot; use tokio::task::JoinSet; use tokio_util::sync::CancellationToken; use tracing::warn; @@ -110,12 +120,66 @@ pub(crate) struct ToolInfo { pub(crate) tool: Tool, } +type ResponderMap = HashMap<(String, RequestId), oneshot::Sender>; + +#[derive(Clone, Default)] +struct ElicitationRequestManager { + requests: Arc>, +} + +impl ElicitationRequestManager { + async fn resolve( + &self, + server_name: String, + id: RequestId, + response: ElicitationResponse, + ) -> Result<()> { + self.requests + .lock() + .await + .remove(&(server_name, id)) + .ok_or_else(|| anyhow!("elicitation request not found"))? + .send(response) + .map_err(|e| anyhow!("failed to send elicitation response: {e:?}")) + } + + fn make_sender(&self, server_name: String, tx_event: Sender) -> SendElicitation { + let elicitation_requests = self.requests.clone(); + Box::new(move |id, elicitation| { + let elicitation_requests = elicitation_requests.clone(); + let tx_event = tx_event.clone(); + let server_name = server_name.clone(); + async move { + let (tx, rx) = oneshot::channel(); + { + let mut lock = elicitation_requests.lock().await; + lock.insert((server_name.clone(), id.clone()), tx); + } + let _ = tx_event + .send(Event { + id: "mcp_elicitation_request".to_string(), + msg: EventMsg::ElicitationRequest(ElicitationRequestEvent { + server_name, + id, + message: elicitation.message, + }), + }) + .await; + rx.await + .context("elicitation request channel closed unexpectedly") + } + .boxed() + }) + } +} + #[derive(Clone)] struct ManagedClient { client: Arc, tools: Vec, tool_filter: ToolFilter, tool_timeout: Option, + server_supports_sandbox_state_capability: bool, } #[derive(Clone)] @@ -129,19 +193,33 @@ impl AsyncManagedClient { config: McpServerConfig, store_mode: OAuthCredentialsStoreMode, cancel_token: CancellationToken, + tx_event: Sender, + elicitation_requests: ElicitationRequestManager, ) -> Self { let tool_filter = ToolFilter::from_config(&config); - let fut = start_server_task( - server_name, - config.transport, - store_mode, - config - .startup_timeout_sec - .unwrap_or(DEFAULT_STARTUP_TIMEOUT), - config.tool_timeout_sec.unwrap_or(DEFAULT_TOOL_TIMEOUT), - tool_filter, - cancel_token, - ); + let fut = async move { + if let Err(error) = validate_mcp_server_name(&server_name) { + return Err(error.into()); + } + + let client = + Arc::new(make_rmcp_client(&server_name, config.transport, store_mode).await?); + match start_server_task( + server_name, + client, + config.startup_timeout_sec.or(Some(DEFAULT_STARTUP_TIMEOUT)), + config.tool_timeout_sec.unwrap_or(DEFAULT_TOOL_TIMEOUT), + tool_filter, + tx_event, + elicitation_requests, + ) + .or_cancel(&cancel_token) + .await + { + Ok(result) => result, + Err(CancelErr::Cancelled) => Err(StartupOutcomeError::Cancelled), + } + }; Self { client: fut.boxed().shared(), } @@ -150,12 +228,42 @@ impl AsyncManagedClient { async fn client(&self) -> Result { self.client.clone().await } + + async fn notify_sandbox_state_change(&self, sandbox_state: &SandboxState) -> Result<()> { + let managed = self.client().await?; + if !managed.server_supports_sandbox_state_capability { + return Ok(()); + } + + managed + .client + .send_custom_notification( + MCP_SANDBOX_STATE_NOTIFICATION, + Some(serde_json::to_value(sandbox_state)?), + ) + .await + } +} + +pub const MCP_SANDBOX_STATE_CAPABILITY: &str = "codex/sandbox-state"; + +/// Custom MCP notification for sandbox state updates. +/// When used, the `params` field of the notification is [`SandboxState`]. +pub const MCP_SANDBOX_STATE_NOTIFICATION: &str = "codex/sandbox-state/update"; + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SandboxState { + pub sandbox_policy: SandboxPolicy, + pub codex_linux_sandbox_exe: Option, + pub sandbox_cwd: PathBuf, } /// A thin wrapper around a set of running [`RmcpClient`] instances. #[derive(Default)] pub(crate) struct McpConnectionManager { clients: HashMap, + elicitation_requests: ElicitationRequestManager, } impl McpConnectionManager { @@ -172,6 +280,7 @@ impl McpConnectionManager { } let mut clients = HashMap::new(); let mut join_set = JoinSet::new(); + let elicitation_requests = ElicitationRequestManager::default(); for (server_name, cfg) in mcp_servers.into_iter().filter(|(_, cfg)| cfg.enabled) { let cancel_token = cancel_token.child_token(); let _ = emit_update( @@ -182,8 +291,14 @@ impl McpConnectionManager { }, ) .await; - let async_managed_client = - AsyncManagedClient::new(server_name.clone(), cfg, store_mode, cancel_token.clone()); + let async_managed_client = AsyncManagedClient::new( + server_name.clone(), + cfg, + store_mode, + cancel_token.clone(), + tx_event.clone(), + elicitation_requests.clone(), + ); clients.insert(server_name.clone(), async_managed_client.clone()); let tx_event = tx_event.clone(); let auth_entry = auth_entries.get(&server_name).cloned(); @@ -217,6 +332,7 @@ impl McpConnectionManager { }); } self.clients = clients; + self.elicitation_requests = elicitation_requests.clone(); tokio::spawn(async move { let outcomes = join_set.join_all().await; let mut summary = McpStartupCompleteEvent::default(); @@ -250,6 +366,17 @@ impl McpConnectionManager { .context("failed to get client") } + pub async fn resolve_elicitation( + &self, + server_name: String, + id: RequestId, + response: ElicitationResponse, + ) -> Result<()> { + self.elicitation_requests + .resolve(server_name, id, response) + .await + } + /// Returns a single map that contains all tools. Each key is the /// fully-qualified name for the tool. pub async fn list_all_tools(&self) -> HashMap { @@ -477,6 +604,34 @@ impl McpConnectionManager { .get(tool_name) .map(|tool| (tool.server_name.clone(), tool.tool_name.clone())) } + + pub async fn notify_sandbox_state_change(&self, sandbox_state: &SandboxState) -> Result<()> { + let mut join_set = JoinSet::new(); + + for async_managed_client in self.clients.values() { + let sandbox_state = sandbox_state.clone(); + let async_managed_client = async_managed_client.clone(); + join_set.spawn(async move { + async_managed_client + .notify_sandbox_state_change(&sandbox_state) + .await + }); + } + + while let Some(join_res) = join_set.join_next().await { + match join_res { + Ok(Ok(())) => {} + Ok(Err(err)) => { + warn!("Failed to notify sandbox state change to MCP server: {err:#}"); + } + Err(err) => { + warn!("Task panic when notifying sandbox state change to MCP server: {err:#}"); + } + } + } + + Ok(()) + } } async fn emit_update( @@ -580,43 +735,12 @@ impl From for StartupOutcomeError { async fn start_server_task( server_name: String, - transport: McpServerTransportConfig, - store_mode: OAuthCredentialsStoreMode, - startup_timeout: Duration, // TODO: cancel_token should handle this. - tool_timeout: Duration, - tool_filter: ToolFilter, - cancel_token: CancellationToken, -) -> Result { - if cancel_token.is_cancelled() { - return Err(StartupOutcomeError::Cancelled); - } - if let Err(error) = validate_mcp_server_name(&server_name) { - return Err(error.into()); - } - - match start_server_work( - server_name, - transport, - store_mode, - startup_timeout, - tool_timeout, - tool_filter, - ) - .or_cancel(&cancel_token) - .await - { - Ok(result) => result, - Err(CancelErr::Cancelled) => Err(StartupOutcomeError::Cancelled), - } -} - -async fn start_server_work( - server_name: String, - transport: McpServerTransportConfig, - store_mode: OAuthCredentialsStoreMode, - startup_timeout: Duration, + client: Arc, + startup_timeout: Option, // TODO: cancel_token should handle this. tool_timeout: Duration, tool_filter: ToolFilter, + tx_event: Sender, + elicitation_requests: ElicitationRequestManager, ) -> Result { let params = mcp_types::InitializeRequestParams { capabilities: ClientCapabilities { @@ -639,7 +763,41 @@ async fn start_server_work( protocol_version: mcp_types::MCP_SCHEMA_VERSION.to_owned(), }; - let client_result = match transport { + let send_elicitation = elicitation_requests.make_sender(server_name.clone(), tx_event); + + let initialize_result = client + .initialize(params, startup_timeout, send_elicitation) + .await + .map_err(StartupOutcomeError::from)?; + + let tools = list_tools_for_client(&server_name, &client, startup_timeout) + .await + .map_err(StartupOutcomeError::from)?; + + let server_supports_sandbox_state_capability = initialize_result + .capabilities + .experimental + .as_ref() + .and_then(|exp| exp.get(MCP_SANDBOX_STATE_CAPABILITY)) + .is_some(); + + let managed = ManagedClient { + client: Arc::clone(&client), + tools, + tool_timeout: Some(tool_timeout), + tool_filter, + server_supports_sandbox_state_capability, + }; + + Ok(managed) +} + +async fn make_rmcp_client( + server_name: &str, + transport: McpServerTransportConfig, + store_mode: OAuthCredentialsStoreMode, +) -> Result { + match transport { McpServerTransportConfig::Stdio { command, args, @@ -649,16 +807,9 @@ async fn start_server_work( } => { let command_os: OsString = command.into(); let args_os: Vec = args.into_iter().map(Into::into).collect(); - match RmcpClient::new_stdio_client(command_os, args_os, env, &env_vars, cwd).await { - Ok(client) => { - let client = Arc::new(client); - client - .initialize(params.clone(), Some(startup_timeout)) - .await - .map(|_| client) - } - Err(err) => Err(err.into()), - } + RmcpClient::new_stdio_client(command_os, args_os, env, &env_vars, cwd) + .await + .map_err(|err| StartupOutcomeError::from(anyhow!(err))) } McpServerTransportConfig::StreamableHttp { url, @@ -667,12 +818,12 @@ async fn start_server_work( bearer_token_env_var, } => { let resolved_bearer_token = - match resolve_bearer_token(&server_name, bearer_token_env_var.as_deref()) { + match resolve_bearer_token(server_name, bearer_token_env_var.as_deref()) { Ok(token) => token, Err(error) => return Err(error.into()), }; - match RmcpClient::new_streamable_http_client( - &server_name, + RmcpClient::new_streamable_http_client( + server_name, &url, resolved_bearer_token, http_headers, @@ -680,49 +831,17 @@ async fn start_server_work( store_mode, ) .await - { - Ok(client) => { - let client = Arc::new(client); - client - .initialize(params.clone(), Some(startup_timeout)) - .await - .map(|_| client) - } - Err(err) => Err(err), - } + .map_err(StartupOutcomeError::from) } - }; - - let client = match client_result { - Ok(client) => client, - Err(error) => { - return Err(error.into()); - } - }; - - let tools = match list_tools_for_client(&server_name, &client, startup_timeout).await { - Ok(tools) => tools, - Err(error) => { - return Err(error.into()); - } - }; - - let managed = ManagedClient { - client: Arc::clone(&client), - tools, - tool_timeout: Some(tool_timeout), - tool_filter, - }; - - Ok(managed) + } } async fn list_tools_for_client( server_name: &str, client: &Arc, - timeout: Duration, + timeout: Option, ) -> Result> { - let resp = client.list_tools(None, Some(timeout)).await?; + let resp = client.list_tools(None, timeout).await?; Ok(resp .tools .into_iter() diff --git a/codex-rs/core/src/message_history.rs b/codex-rs/core/src/message_history.rs index e96d8d5026..e46dd93067 100644 --- a/codex-rs/core/src/message_history.rs +++ b/codex-rs/core/src/message_history.rs @@ -16,8 +16,14 @@ use std::fs::File; use std::fs::OpenOptions; +use std::io::BufRead; +use std::io::BufReader; +use std::io::Read; use std::io::Result; +use std::io::Seek; +use std::io::SeekFrom; use std::io::Write; +use std::path::Path; use std::path::PathBuf; use serde::Deserialize; @@ -39,10 +45,13 @@ use std::os::unix::fs::PermissionsExt; /// Filename that stores the message history inside `~/.codex`. const HISTORY_FILENAME: &str = "history.jsonl"; +/// When history exceeds the hard cap, trim it down to this fraction of `max_bytes`. +const HISTORY_SOFT_CAP_RATIO: f64 = 0.8; + const MAX_RETRIES: usize = 10; const RETRY_SLEEP: Duration = Duration::from_millis(100); -#[derive(Serialize, Deserialize, Debug, Clone)] +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] pub struct HistoryEntry { pub session_id: String, pub ts: u64, @@ -97,11 +106,12 @@ pub(crate) async fn append_entry( .map_err(|e| std::io::Error::other(format!("failed to serialise history entry: {e}")))?; line.push('\n'); - // Open in append-only mode. + // Open the history file for read/write access (append-only on Unix). let mut options = OpenOptions::new(); - options.append(true).read(true).create(true); + options.read(true).write(true).create(true); #[cfg(unix)] { + options.append(true); options.mode(0o600); } @@ -110,6 +120,8 @@ pub(crate) async fn append_entry( // Ensure permissions. ensure_owner_only_permissions(&history_file).await?; + let history_max_bytes = config.history.max_bytes; + // Perform a blocking write under an advisory write lock using std::fs. tokio::task::spawn_blocking(move || -> Result<()> { // Retry a few times to avoid indefinite blocking when contended. @@ -117,8 +129,12 @@ pub(crate) async fn append_entry( match history_file.try_lock() { Ok(()) => { // While holding the exclusive lock, write the full line. + // We do not open the file with `append(true)` on Windows, so ensure the + // cursor is positioned at the end before writing. + history_file.seek(SeekFrom::End(0))?; history_file.write_all(line.as_bytes())?; history_file.flush()?; + enforce_history_limit(&mut history_file, history_max_bytes)?; return Ok(()); } Err(std::fs::TryLockError::WouldBlock) => { @@ -138,27 +154,144 @@ pub(crate) async fn append_entry( Ok(()) } +/// Trim the history file to honor `max_bytes`, dropping the oldest lines while holding +/// the write lock so the newest entry is always retained. When the file exceeds the +/// hard cap, it rewrites the remaining tail to a soft cap to avoid trimming again +/// immediately on the next write. +fn enforce_history_limit(file: &mut File, max_bytes: Option) -> Result<()> { + let Some(max_bytes) = max_bytes else { + return Ok(()); + }; + + if max_bytes == 0 { + return Ok(()); + } + + let max_bytes = match u64::try_from(max_bytes) { + Ok(value) => value, + Err(_) => return Ok(()), + }; + + let mut current_len = file.metadata()?.len(); + + if current_len <= max_bytes { + return Ok(()); + } + + let mut reader_file = file.try_clone()?; + reader_file.seek(SeekFrom::Start(0))?; + + let mut buf_reader = BufReader::new(reader_file); + let mut line_lengths = Vec::new(); + let mut line_buf = String::new(); + + loop { + line_buf.clear(); + + let bytes = buf_reader.read_line(&mut line_buf)?; + + if bytes == 0 { + break; + } + + line_lengths.push(bytes as u64); + } + + if line_lengths.is_empty() { + return Ok(()); + } + + let last_index = line_lengths.len() - 1; + let trim_target = trim_target_bytes(max_bytes, line_lengths[last_index]); + + let mut drop_bytes = 0u64; + let mut idx = 0usize; + + while current_len > trim_target && idx < last_index { + current_len = current_len.saturating_sub(line_lengths[idx]); + drop_bytes += line_lengths[idx]; + idx += 1; + } + + if drop_bytes == 0 { + return Ok(()); + } + + let mut reader = buf_reader.into_inner(); + reader.seek(SeekFrom::Start(drop_bytes))?; + + let capacity = usize::try_from(current_len).unwrap_or(0); + let mut tail = Vec::with_capacity(capacity); + + reader.read_to_end(&mut tail)?; + + file.set_len(0)?; + file.seek(SeekFrom::Start(0))?; + file.write_all(&tail)?; + file.flush()?; + + Ok(()) +} + +fn trim_target_bytes(max_bytes: u64, newest_entry_len: u64) -> u64 { + let soft_cap_bytes = ((max_bytes as f64) * HISTORY_SOFT_CAP_RATIO) + .floor() + .clamp(1.0, max_bytes as f64) as u64; + + soft_cap_bytes.max(newest_entry_len) +} + /// Asynchronously fetch the history file's *identifier* (inode on Unix) and /// the current number of entries by counting newline characters. pub(crate) async fn history_metadata(config: &Config) -> (u64, usize) { let path = history_filepath(config); + history_metadata_for_file(&path).await +} - #[cfg(unix)] - let log_id = { - use std::os::unix::fs::MetadataExt; - // Obtain metadata (async) to get the identifier. - let meta = match fs::metadata(&path).await { - Ok(m) => m, - Err(e) if e.kind() == std::io::ErrorKind::NotFound => return (0, 0), - Err(_) => return (0, 0), - }; - meta.ino() +/// Given a `log_id` (on Unix this is the file's inode number, +/// on Windows this is the file's creation time) and a zero-based +/// `offset`, return the corresponding `HistoryEntry` if the identifier matches +/// the current history file **and** the requested offset exists. Any I/O or +/// parsing errors are logged and result in `None`. +/// +/// Note this function is not async because it uses a sync advisory file +/// locking API. +pub(crate) fn lookup(log_id: u64, offset: usize, config: &Config) -> Option { + let path = history_filepath(config); + lookup_history_entry(&path, log_id, offset) +} + +/// On Unix systems, ensure the file permissions are `0o600` (rw-------). If the +/// permissions cannot be changed the error is propagated to the caller. +#[cfg(unix)] +async fn ensure_owner_only_permissions(file: &File) -> Result<()> { + let metadata = file.metadata()?; + let current_mode = metadata.permissions().mode() & 0o777; + if current_mode != 0o600 { + let mut perms = metadata.permissions(); + perms.set_mode(0o600); + let perms_clone = perms.clone(); + let file_clone = file.try_clone()?; + tokio::task::spawn_blocking(move || file_clone.set_permissions(perms_clone)).await??; + } + Ok(()) +} + +#[cfg(windows)] +// On Windows, simply succeed. +async fn ensure_owner_only_permissions(_file: &File) -> Result<()> { + Ok(()) +} + +async fn history_metadata_for_file(path: &Path) -> (u64, usize) { + let log_id = match fs::metadata(path).await { + Ok(metadata) => history_log_id(&metadata).unwrap_or(0), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return (0, 0), + Err(_) => return (0, 0), }; - #[cfg(not(unix))] - let log_id = 0u64; // Open the file. - let mut file = match fs::File::open(&path).await { + let mut file = match fs::File::open(path).await { Ok(f) => f, Err(_) => return (log_id, 0), }; @@ -179,21 +312,11 @@ pub(crate) async fn history_metadata(config: &Config) -> (u64, usize) { (log_id, count) } -/// Given a `log_id` (on Unix this is the file's inode number) and a zero-based -/// `offset`, return the corresponding `HistoryEntry` if the identifier matches -/// the current history file **and** the requested offset exists. Any I/O or -/// parsing errors are logged and result in `None`. -/// -/// Note this function is not async because it uses a sync advisory file -/// locking API. -#[cfg(unix)] -pub(crate) fn lookup(log_id: u64, offset: usize, config: &Config) -> Option { +fn lookup_history_entry(path: &Path, log_id: u64, offset: usize) -> Option { use std::io::BufRead; use std::io::BufReader; - use std::os::unix::fs::MetadataExt; - let path = history_filepath(config); - let file: File = match OpenOptions::new().read(true).open(&path) { + let file: File = match OpenOptions::new().read(true).open(path) { Ok(f) => f, Err(e) => { tracing::warn!(error = %e, "failed to open history file"); @@ -209,7 +332,9 @@ pub(crate) fn lookup(log_id: u64, offset: usize, config: &Config) -> Option Option Option { - let _ = (log_id, offset, config); +#[cfg(unix)] +fn history_log_id(metadata: &std::fs::Metadata) -> Option { + use std::os::unix::fs::MetadataExt; + Some(metadata.ino()) +} + +#[cfg(windows)] +fn history_log_id(metadata: &std::fs::Metadata) -> Option { + use std::os::windows::fs::MetadataExt; + Some(metadata.creation_time()) +} + +#[cfg(not(any(unix, windows)))] +fn history_log_id(_metadata: &std::fs::Metadata) -> Option { None } -/// On Unix systems ensure the file permissions are `0o600` (rw-------). If the -/// permissions cannot be changed the error is propagated to the caller. -#[cfg(unix)] -async fn ensure_owner_only_permissions(file: &File) -> Result<()> { - let metadata = file.metadata()?; - let current_mode = metadata.permissions().mode() & 0o777; - if current_mode != 0o600 { - let mut perms = metadata.permissions(); - perms.set_mode(0o600); - let perms_clone = perms.clone(); - let file_clone = file.try_clone()?; - tokio::task::spawn_blocking(move || file_clone.set_permissions(perms_clone)).await??; - } - Ok(()) -} +#[cfg(test)] +mod tests { + use super::*; + use crate::config::Config; + use crate::config::ConfigOverrides; + use crate::config::ConfigToml; + use codex_protocol::ConversationId; + use pretty_assertions::assert_eq; + use std::fs::File; + use std::io::Write; + use tempfile::TempDir; -#[cfg(not(unix))] -async fn ensure_owner_only_permissions(_file: &File) -> Result<()> { - // For now, on non-Unix, simply succeed. - Ok(()) + #[tokio::test] + async fn lookup_reads_history_entries() { + let temp_dir = TempDir::new().expect("create temp dir"); + let history_path = temp_dir.path().join(HISTORY_FILENAME); + + let entries = vec![ + HistoryEntry { + session_id: "first-session".to_string(), + ts: 1, + text: "first".to_string(), + }, + HistoryEntry { + session_id: "second-session".to_string(), + ts: 2, + text: "second".to_string(), + }, + ]; + + let mut file = File::create(&history_path).expect("create history file"); + for entry in &entries { + writeln!( + file, + "{}", + serde_json::to_string(entry).expect("serialize history entry") + ) + .expect("write history entry"); + } + + let (log_id, count) = history_metadata_for_file(&history_path).await; + assert_eq!(count, entries.len()); + + let second_entry = + lookup_history_entry(&history_path, log_id, 1).expect("fetch second history entry"); + assert_eq!(second_entry, entries[1]); + } + + #[tokio::test] + async fn lookup_uses_stable_log_id_after_appends() { + let temp_dir = TempDir::new().expect("create temp dir"); + let history_path = temp_dir.path().join(HISTORY_FILENAME); + + let initial = HistoryEntry { + session_id: "first-session".to_string(), + ts: 1, + text: "first".to_string(), + }; + let appended = HistoryEntry { + session_id: "second-session".to_string(), + ts: 2, + text: "second".to_string(), + }; + + let mut file = File::create(&history_path).expect("create history file"); + writeln!( + file, + "{}", + serde_json::to_string(&initial).expect("serialize initial entry") + ) + .expect("write initial entry"); + + let (log_id, count) = history_metadata_for_file(&history_path).await; + assert_eq!(count, 1); + + let mut append = std::fs::OpenOptions::new() + .append(true) + .open(&history_path) + .expect("open history file for append"); + writeln!( + append, + "{}", + serde_json::to_string(&appended).expect("serialize appended entry") + ) + .expect("append history entry"); + + let fetched = + lookup_history_entry(&history_path, log_id, 1).expect("lookup appended history entry"); + assert_eq!(fetched, appended); + } + + #[tokio::test] + async fn append_entry_trims_history_when_beyond_max_bytes() { + let codex_home = TempDir::new().expect("create temp dir"); + + let mut config = Config::load_from_base_config_with_overrides( + ConfigToml::default(), + ConfigOverrides::default(), + codex_home.path().to_path_buf(), + ) + .expect("load config"); + + let conversation_id = ConversationId::new(); + + let entry_one = "a".repeat(200); + let entry_two = "b".repeat(200); + + let history_path = codex_home.path().join("history.jsonl"); + + append_entry(&entry_one, &conversation_id, &config) + .await + .expect("write first entry"); + + let first_len = std::fs::metadata(&history_path).expect("metadata").len(); + let limit_bytes = first_len + 10; + + config.history.max_bytes = + Some(usize::try_from(limit_bytes).expect("limit should fit into usize")); + + append_entry(&entry_two, &conversation_id, &config) + .await + .expect("write second entry"); + + let contents = std::fs::read_to_string(&history_path).expect("read history"); + + let entries = contents + .lines() + .map(|line| serde_json::from_str::(line).expect("parse entry")) + .collect::>(); + + assert_eq!( + entries.len(), + 1, + "only one entry left because entry_one should be evicted" + ); + assert_eq!(entries[0].text, entry_two); + assert!(std::fs::metadata(&history_path).expect("metadata").len() <= limit_bytes); + } + + #[tokio::test] + async fn append_entry_trims_history_to_soft_cap() { + let codex_home = TempDir::new().expect("create temp dir"); + + let mut config = Config::load_from_base_config_with_overrides( + ConfigToml::default(), + ConfigOverrides::default(), + codex_home.path().to_path_buf(), + ) + .expect("load config"); + + let conversation_id = ConversationId::new(); + + let short_entry = "a".repeat(200); + let long_entry = "b".repeat(400); + + let history_path = codex_home.path().join("history.jsonl"); + + append_entry(&short_entry, &conversation_id, &config) + .await + .expect("write first entry"); + + let short_entry_len = std::fs::metadata(&history_path).expect("metadata").len(); + + append_entry(&long_entry, &conversation_id, &config) + .await + .expect("write second entry"); + + let two_entry_len = std::fs::metadata(&history_path).expect("metadata").len(); + + let long_entry_len = two_entry_len + .checked_sub(short_entry_len) + .expect("second entry length should be larger than first entry length"); + + config.history.max_bytes = Some( + usize::try_from((2 * long_entry_len) + (short_entry_len / 2)) + .expect("max bytes should fit into usize"), + ); + + append_entry(&long_entry, &conversation_id, &config) + .await + .expect("write third entry"); + + let contents = std::fs::read_to_string(&history_path).expect("read history"); + + let entries = contents + .lines() + .map(|line| serde_json::from_str::(line).expect("parse entry")) + .collect::>(); + + assert_eq!(entries.len(), 1); + assert_eq!(entries[0].text, long_entry); + + let pruned_len = std::fs::metadata(&history_path).expect("metadata").len() as u64; + let max_bytes = config + .history + .max_bytes + .expect("max bytes should be configured") as u64; + + assert!(pruned_len <= max_bytes); + + let soft_cap_bytes = ((max_bytes as f64) * HISTORY_SOFT_CAP_RATIO) + .floor() + .clamp(1.0, max_bytes as f64) as u64; + let len_without_first = 2 * long_entry_len; + + assert!( + len_without_first <= max_bytes, + "dropping only the first entry would satisfy the hard cap" + ); + assert!( + len_without_first > soft_cap_bytes, + "soft cap should require more aggressive trimming than the hard cap" + ); + + assert_eq!(pruned_len, long_entry_len); + assert!(pruned_len <= soft_cap_bytes.max(long_entry_len)); + } } diff --git a/codex-rs/core/src/model_family.rs b/codex-rs/core/src/model_family.rs index ef54a95841..5dea1c0168 100644 --- a/codex-rs/core/src/model_family.rs +++ b/codex-rs/core/src/model_family.rs @@ -173,6 +173,19 @@ pub fn find_family_for_model(slug: &str) -> Option { support_verbosity: true, truncation_policy: TruncationPolicy::Tokens(10_000), ) + } else if slug.starts_with("exp-") { + model_family!( + slug, slug, + supports_reasoning_summaries: true, + apply_patch_tool_type: Some(ApplyPatchToolType::Freeform), + support_verbosity: true, + default_verbosity: Some(Verbosity::Low), + base_instructions: BASE_INSTRUCTIONS.to_string(), + default_reasoning_effort: Some(ReasoningEffort::Medium), + truncation_policy: TruncationPolicy::Bytes(10_000), + shell_type: ConfigShellToolType::UnifiedExec, + supports_parallel_tool_calls: true, + ) // Production models. } else if slug.starts_with("gpt-5.1-codex-max") { diff --git a/codex-rs/core/src/model_provider_info.rs b/codex-rs/core/src/model_provider_info.rs index 3ab341eea5..4912a64694 100644 --- a/codex-rs/core/src/model_provider_info.rs +++ b/codex-rs/core/src/model_provider_info.rs @@ -5,11 +5,13 @@ //! 2. User-defined entries inside `~/.codex/config.toml` under the `model_providers` //! key. These override or extend the defaults at runtime. -use crate::CodexAuth; -use crate::default_client::CodexHttpClient; -use crate::default_client::CodexRequestBuilder; -use crate::error::CodexErr; +use codex_api::Provider as ApiProvider; +use codex_api::WireApi as ApiWireApi; +use codex_api::provider::RetryConfig as ApiRetryConfig; use codex_app_server_protocol::AuthMode; +use http::HeaderMap; +use http::header::HeaderName; +use http::header::HeaderValue; use serde::Deserialize; use serde::Serialize; use std::collections::HashMap; @@ -97,148 +99,14 @@ pub struct ModelProviderInfo { } impl ModelProviderInfo { - /// Construct a `POST` RequestBuilder for the given URL using the provided - /// [`CodexHttpClient`] applying: - /// • provider-specific headers (static + env based) - /// • Bearer auth header when an API key is available. - /// • Auth token for OAuth. - /// - /// If the provider declares an `env_key` but the variable is missing/empty, returns an [`Err`] identical to the - /// one produced by [`ModelProviderInfo::api_key`]. - pub async fn create_request_builder<'a>( - &'a self, - client: &'a CodexHttpClient, - auth: &Option, - ) -> crate::error::Result { - let effective_auth = self.effective_auth(auth)?; - - let url = self.get_full_url(&effective_auth); - - let mut builder = client.post(url); - - if let Some(auth) = effective_auth.as_ref() { - builder = builder.bearer_auth(auth.get_token().await?); - } - - Ok(self.apply_http_headers(builder)) - } - - pub async fn create_compact_request_builder<'a>( - &'a self, - client: &'a CodexHttpClient, - auth: &Option, - ) -> crate::error::Result { - if self.wire_api != WireApi::Responses { - return Err(CodexErr::UnsupportedOperation( - "Compaction endpoint requires Responses API providers".to_string(), - )); - } - let effective_auth = self.effective_auth(auth)?; - - let url = self.get_compact_url(&effective_auth).ok_or_else(|| { - CodexErr::UnsupportedOperation( - "Compaction endpoint requires Responses API providers".to_string(), - ) - })?; - - let mut builder = client.post(url); - - if let Some(auth) = effective_auth.as_ref() { - builder = builder.bearer_auth(auth.get_token().await?); - } - - Ok(self.apply_http_headers(builder)) - } - - fn effective_auth(&self, auth: &Option) -> crate::error::Result> { - if let Some(secret_key) = &self.experimental_bearer_token { - return Ok(Some(CodexAuth::from_api_key(secret_key))); - } - - match self.api_key() { - Ok(Some(key)) => Ok(Some(CodexAuth::from_api_key(&key))), - Ok(None) => Ok(auth.clone()), - Err(err) => { - if auth.is_some() { - Ok(auth.clone()) - } else { - Err(err) - } - } - } - } - - fn get_query_string(&self) -> String { - self.query_params - .as_ref() - .map_or_else(String::new, |params| { - let full_params = params - .iter() - .map(|(k, v)| format!("{k}={v}")) - .collect::>() - .join("&"); - format!("?{full_params}") - }) - } - - pub(crate) fn get_full_url(&self, auth: &Option) -> String { - let default_base_url = if matches!( - auth, - Some(CodexAuth { - mode: AuthMode::ChatGPT, - .. - }) - ) { - "https://chatgpt.com/backend-api/codex" - } else { - "https://api.openai.com/v1" - }; - let query_string = self.get_query_string(); - let base_url = self - .base_url - .clone() - .unwrap_or(default_base_url.to_string()); - - match self.wire_api { - WireApi::Responses => format!("{base_url}/responses{query_string}"), - WireApi::Chat => format!("{base_url}/chat/completions{query_string}"), - } - } - - pub(crate) fn get_compact_url(&self, auth: &Option) -> Option { - if self.wire_api != WireApi::Responses { - return None; - } - let full = self.get_full_url(auth); - if let Some((path, query)) = full.split_once('?') { - Some(format!("{path}/compact?{query}")) - } else { - Some(format!("{full}/compact")) - } - } - - pub(crate) fn is_azure_responses_endpoint(&self) -> bool { - if self.wire_api != WireApi::Responses { - return false; - } - - if self.name.eq_ignore_ascii_case("azure") { - return true; - } - - self.base_url - .as_ref() - .map(|base| matches_azure_responses_base_url(base)) - .unwrap_or(false) - } - - /// Apply provider-specific HTTP headers (both static and environment-based) - /// onto an existing [`CodexRequestBuilder`] and return the updated - /// builder. - fn apply_http_headers(&self, mut builder: CodexRequestBuilder) -> CodexRequestBuilder { + #[allow(dead_code)] + fn build_header_map(&self) -> crate::error::Result { + let mut headers = HeaderMap::new(); if let Some(extra) = &self.http_headers { for (k, v) in extra { - builder = builder.header(k, v); + if let (Ok(name), Ok(value)) = (HeaderName::try_from(k), HeaderValue::try_from(v)) { + headers.insert(name, value); + } } } @@ -246,12 +114,52 @@ impl ModelProviderInfo { for (header, env_var) in env_headers { if let Ok(val) = std::env::var(env_var) && !val.trim().is_empty() + && let (Ok(name), Ok(value)) = + (HeaderName::try_from(header), HeaderValue::try_from(val)) { - builder = builder.header(header, val); + headers.insert(name, value); } } } - builder + + Ok(headers) + } + + pub(crate) fn to_api_provider( + &self, + auth_mode: Option, + ) -> crate::error::Result { + let default_base_url = if matches!(auth_mode, Some(AuthMode::ChatGPT)) { + "https://chatgpt.com/backend-api/codex" + } else { + "https://api.openai.com/v1" + }; + let base_url = self + .base_url + .clone() + .unwrap_or_else(|| default_base_url.to_string()); + + let headers = self.build_header_map()?; + let retry = ApiRetryConfig { + max_attempts: self.request_max_retries(), + base_delay: Duration::from_millis(200), + retry_429: false, + retry_5xx: true, + retry_transport: true, + }; + + Ok(ApiProvider { + name: self.name.clone(), + base_url, + query_params: self.query_params.clone(), + wire: match self.wire_api { + WireApi::Responses => ApiWireApi::Responses, + WireApi::Chat => ApiWireApi::Chat, + }, + headers, + retry, + stream_idle_timeout: self.stream_idle_timeout(), + }) } /// If `env_key` is Some, returns the API key for this provider if present @@ -409,18 +317,6 @@ pub fn create_oss_provider_with_base_url(base_url: &str, wire_api: WireApi) -> M } } -fn matches_azure_responses_base_url(base_url: &str) -> bool { - let base = base_url.to_ascii_lowercase(); - const AZURE_MARKERS: [&str; 5] = [ - "openai.azure.", - "cognitiveservices.azure.", - "aoai.azure.", - "azure-api.", - "azurefd.", - ]; - AZURE_MARKERS.iter().any(|marker| base.contains(marker)) -} - #[cfg(test)] mod tests { use super::*; @@ -517,8 +413,16 @@ env_http_headers = { "X-Example-Env-Header" = "EXAMPLE_ENV_VAR" } #[test] fn detects_azure_responses_base_urls() { - fn provider_for(base_url: &str) -> ModelProviderInfo { - ModelProviderInfo { + let positive_cases = [ + "https://foo.openai.azure.com/openai", + "https://foo.openai.azure.us/openai/deployments/bar", + "https://foo.cognitiveservices.azure.cn/openai", + "https://foo.aoai.azure.com/openai", + "https://foo.openai.azure-api.net/openai", + "https://foo.z01.azurefd.net/", + ]; + for base_url in positive_cases { + let provider = ModelProviderInfo { name: "test".into(), base_url: Some(base_url.into()), env_key: None, @@ -532,21 +436,10 @@ env_http_headers = { "X-Example-Env-Header" = "EXAMPLE_ENV_VAR" } stream_max_retries: None, stream_idle_timeout_ms: None, requires_openai_auth: false, - } - } - - let positive_cases = [ - "https://foo.openai.azure.com/openai", - "https://foo.openai.azure.us/openai/deployments/bar", - "https://foo.cognitiveservices.azure.cn/openai", - "https://foo.aoai.azure.com/openai", - "https://foo.openai.azure-api.net/openai", - "https://foo.z01.azurefd.net/", - ]; - for base_url in positive_cases { - let provider = provider_for(base_url); + }; + let api = provider.to_api_provider(None).expect("api provider"); assert!( - provider.is_azure_responses_endpoint(), + api.is_azure_responses_endpoint(), "expected {base_url} to be detected as Azure" ); } @@ -566,7 +459,8 @@ env_http_headers = { "X-Example-Env-Header" = "EXAMPLE_ENV_VAR" } stream_idle_timeout_ms: None, requires_openai_auth: false, }; - assert!(named_provider.is_azure_responses_endpoint()); + let named_api = named_provider.to_api_provider(None).expect("api provider"); + assert!(named_api.is_azure_responses_endpoint()); let negative_cases = [ "https://api.openai.com/v1", @@ -574,9 +468,24 @@ env_http_headers = { "X-Example-Env-Header" = "EXAMPLE_ENV_VAR" } "https://myproxy.azurewebsites.net/openai", ]; for base_url in negative_cases { - let provider = provider_for(base_url); + let provider = ModelProviderInfo { + name: "test".into(), + base_url: Some(base_url.into()), + env_key: None, + env_key_instructions: None, + experimental_bearer_token: None, + wire_api: WireApi::Responses, + query_params: None, + http_headers: None, + env_http_headers: None, + request_max_retries: None, + stream_max_retries: None, + stream_idle_timeout_ms: None, + requires_openai_auth: false, + }; + let api = provider.to_api_provider(None).expect("api provider"); assert!( - !provider.is_azure_responses_endpoint(), + !api.is_azure_responses_endpoint(), "expected {base_url} not to be detected as Azure" ); } diff --git a/codex-rs/core/src/openai_model_info.rs b/codex-rs/core/src/openai_model_info.rs index bd0ca9d31d..96f3ed77cb 100644 --- a/codex-rs/core/src/openai_model_info.rs +++ b/codex-rs/core/src/openai_model_info.rs @@ -2,7 +2,6 @@ use crate::model_family::ModelFamily; // Shared constants for commonly used window/token sizes. pub(crate) const CONTEXT_WINDOW_272K: i64 = 272_000; -pub(crate) const MAX_OUTPUT_TOKENS_128K: i64 = 128_000; /// Metadata about a model, particularly OpenAI models. /// We may want to consider including details like the pricing for @@ -14,19 +13,15 @@ pub(crate) struct ModelInfo { /// Size of the context window in tokens. This is the maximum size of the input context. pub(crate) context_window: i64, - /// Maximum number of output tokens that can be generated for the model. - pub(crate) max_output_tokens: i64, - /// Token threshold where we should automatically compact conversation history. This considers /// input tokens + output tokens of this turn. pub(crate) auto_compact_token_limit: Option, } impl ModelInfo { - const fn new(context_window: i64, max_output_tokens: i64) -> Self { + const fn new(context_window: i64) -> Self { Self { context_window, - max_output_tokens, auto_compact_token_limit: Some(Self::default_auto_compact_limit(context_window)), } } @@ -42,48 +37,46 @@ pub(crate) fn get_model_info(model_family: &ModelFamily) -> Option { // OSS models have a 128k shared token pool. // Arbitrarily splitting it: 3/4 input context, 1/4 output. // https://openai.com/index/gpt-oss-model-card/ - "gpt-oss-20b" => Some(ModelInfo::new(96_000, 32_000)), - "gpt-oss-120b" => Some(ModelInfo::new(96_000, 32_000)), + "gpt-oss-20b" => Some(ModelInfo::new(96_000)), + "gpt-oss-120b" => Some(ModelInfo::new(96_000)), // https://platform.openai.com/docs/models/o3 - "o3" => Some(ModelInfo::new(200_000, 100_000)), + "o3" => Some(ModelInfo::new(200_000)), // https://platform.openai.com/docs/models/o4-mini - "o4-mini" => Some(ModelInfo::new(200_000, 100_000)), + "o4-mini" => Some(ModelInfo::new(200_000)), // https://platform.openai.com/docs/models/codex-mini-latest - "codex-mini-latest" => Some(ModelInfo::new(200_000, 100_000)), + "codex-mini-latest" => Some(ModelInfo::new(200_000)), // As of Jun 25, 2025, gpt-4.1 defaults to gpt-4.1-2025-04-14. // https://platform.openai.com/docs/models/gpt-4.1 - "gpt-4.1" | "gpt-4.1-2025-04-14" => Some(ModelInfo::new(1_047_576, 32_768)), + "gpt-4.1" | "gpt-4.1-2025-04-14" => Some(ModelInfo::new(1_047_576)), // As of Jun 25, 2025, gpt-4o defaults to gpt-4o-2024-08-06. // https://platform.openai.com/docs/models/gpt-4o - "gpt-4o" | "gpt-4o-2024-08-06" => Some(ModelInfo::new(128_000, 16_384)), + "gpt-4o" | "gpt-4o-2024-08-06" => Some(ModelInfo::new(128_000)), // https://platform.openai.com/docs/models/gpt-4o?snapshot=gpt-4o-2024-05-13 - "gpt-4o-2024-05-13" => Some(ModelInfo::new(128_000, 4_096)), + "gpt-4o-2024-05-13" => Some(ModelInfo::new(128_000)), // https://platform.openai.com/docs/models/gpt-4o?snapshot=gpt-4o-2024-11-20 - "gpt-4o-2024-11-20" => Some(ModelInfo::new(128_000, 16_384)), + "gpt-4o-2024-11-20" => Some(ModelInfo::new(128_000)), // https://platform.openai.com/docs/models/gpt-3.5-turbo - "gpt-3.5-turbo" => Some(ModelInfo::new(16_385, 4_096)), + "gpt-3.5-turbo" => Some(ModelInfo::new(16_385)), _ if slug.starts_with("gpt-5-codex") || slug.starts_with("gpt-5.1-codex") || slug.starts_with("gpt-5.1-codex-max") => { - Some(ModelInfo::new(CONTEXT_WINDOW_272K, MAX_OUTPUT_TOKENS_128K)) + Some(ModelInfo::new(CONTEXT_WINDOW_272K)) } - _ if slug.starts_with("gpt-5") => { - Some(ModelInfo::new(CONTEXT_WINDOW_272K, MAX_OUTPUT_TOKENS_128K)) - } + _ if slug.starts_with("gpt-5") => Some(ModelInfo::new(CONTEXT_WINDOW_272K)), - _ if slug.starts_with("codex-") => { - Some(ModelInfo::new(CONTEXT_WINDOW_272K, MAX_OUTPUT_TOKENS_128K)) - } + _ if slug.starts_with("codex-") => Some(ModelInfo::new(CONTEXT_WINDOW_272K)), + + _ if slug.starts_with("exp-") => Some(ModelInfo::new(CONTEXT_WINDOW_272K)), _ => None, } diff --git a/codex-rs/core/src/project_doc.rs b/codex-rs/core/src/project_doc.rs index c8076aa822..ee3148e7e7 100644 --- a/codex-rs/core/src/project_doc.rs +++ b/codex-rs/core/src/project_doc.rs @@ -14,6 +14,9 @@ //! 3. We do **not** walk past the Git root. use crate::config::Config; +use crate::features::Feature; +use crate::skills::load_skills; +use crate::skills::render_skills_section; use dunce::canonicalize as normalize_path; use std::path::PathBuf; use tokio::io::AsyncReadExt; @@ -31,18 +34,47 @@ const PROJECT_DOC_SEPARATOR: &str = "\n\n--- project-doc ---\n\n"; /// Combines `Config::instructions` and `AGENTS.md` (if present) into a single /// string of instructions. pub(crate) async fn get_user_instructions(config: &Config) -> Option { - match read_project_docs(config).await { - Ok(Some(project_doc)) => match &config.user_instructions { - Some(original_instructions) => Some(format!( - "{original_instructions}{PROJECT_DOC_SEPARATOR}{project_doc}" - )), - None => Some(project_doc), - }, - Ok(None) => config.user_instructions.clone(), + let skills_section = if config.features.enabled(Feature::Skills) { + let skills_outcome = load_skills(config); + for err in &skills_outcome.errors { + error!( + "failed to load skill {}: {}", + err.path.display(), + err.message + ); + } + render_skills_section(&skills_outcome.skills) + } else { + None + }; + + let project_docs = match read_project_docs(config).await { + Ok(docs) => docs, Err(e) => { error!("error trying to find project doc: {e:#}"); - config.user_instructions.clone() + return config.user_instructions.clone(); } + }; + + let combined_project_docs = merge_project_docs_with_skills(project_docs, skills_section); + + let mut parts: Vec = Vec::new(); + + if let Some(instructions) = config.user_instructions.clone() { + parts.push(instructions); + } + + if let Some(project_doc) = combined_project_docs { + if !parts.is_empty() { + parts.push(PROJECT_DOC_SEPARATOR.to_string()); + } + parts.push(project_doc); + } + + if parts.is_empty() { + None + } else { + Some(parts.concat()) } } @@ -195,12 +227,25 @@ fn candidate_filenames<'a>(config: &'a Config) -> Vec<&'a str> { names } +fn merge_project_docs_with_skills( + project_doc: Option, + skills_section: Option, +) -> Option { + match (project_doc, skills_section) { + (Some(doc), Some(skills)) => Some(format!("{doc}\n\n{skills}")), + (Some(doc), None) => Some(doc), + (None, Some(skills)) => Some(skills), + (None, None) => None, + } +} + #[cfg(test)] mod tests { use super::*; use crate::config::ConfigOverrides; use crate::config::ConfigToml; use std::fs; + use std::path::PathBuf; use tempfile::TempDir; /// Helper that returns a `Config` pointing at `root` and using `limit` as @@ -219,6 +264,7 @@ mod tests { config.cwd = root.path().to_path_buf(); config.project_doc_max_bytes = limit; + config.features.enable(Feature::Skills); config.user_instructions = instructions.map(ToOwned::to_owned); config @@ -447,4 +493,58 @@ mod tests { .eq(DEFAULT_PROJECT_DOC_FILENAME) ); } + + #[tokio::test] + async fn skills_are_appended_to_project_doc() { + let tmp = tempfile::tempdir().expect("tempdir"); + fs::write(tmp.path().join("AGENTS.md"), "base doc").unwrap(); + + let cfg = make_config(&tmp, 4096, None); + create_skill( + cfg.codex_home.clone(), + "pdf-processing", + "extract from pdfs", + ); + + let res = get_user_instructions(&cfg) + .await + .expect("instructions expected"); + let expected_path = dunce::canonicalize( + cfg.codex_home + .join("skills/pdf-processing/SKILL.md") + .as_path(), + ) + .unwrap_or_else(|_| cfg.codex_home.join("skills/pdf-processing/SKILL.md")); + let expected_path_str = expected_path.to_string_lossy().replace('\\', "/"); + let expected = format!( + "base doc\n\n## Skills\nThese skills are discovered at startup from ~/.codex/skills; each entry shows name, description, and file path so you can open the source for full instructions. Content is not inlined to keep context lean.\n- pdf-processing: extract from pdfs (file: {expected_path_str})" + ); + assert_eq!(res, expected); + } + + #[tokio::test] + async fn skills_render_without_project_doc() { + let tmp = tempfile::tempdir().expect("tempdir"); + let cfg = make_config(&tmp, 4096, None); + create_skill(cfg.codex_home.clone(), "linting", "run clippy"); + + let res = get_user_instructions(&cfg) + .await + .expect("instructions expected"); + let expected_path = + dunce::canonicalize(cfg.codex_home.join("skills/linting/SKILL.md").as_path()) + .unwrap_or_else(|_| cfg.codex_home.join("skills/linting/SKILL.md")); + let expected_path_str = expected_path.to_string_lossy().replace('\\', "/"); + let expected = format!( + "## Skills\nThese skills are discovered at startup from ~/.codex/skills; each entry shows name, description, and file path so you can open the source for full instructions. Content is not inlined to keep context lean.\n- linting: run clippy (file: {expected_path_str})" + ); + assert_eq!(res, expected); + } + + fn create_skill(codex_home: PathBuf, name: &str, description: &str) { + let skill_dir = codex_home.join(format!("skills/{name}")); + fs::create_dir_all(&skill_dir).unwrap(); + let content = format!("---\nname: {name}\ndescription: {description}\n---\n\n# Body\n"); + fs::write(skill_dir.join("SKILL.md"), content).unwrap(); + } } diff --git a/codex-rs/core/src/review_format.rs b/codex-rs/core/src/review_format.rs index 6a64d06fa1..10f53066a7 100644 --- a/codex-rs/core/src/review_format.rs +++ b/codex-rs/core/src/review_format.rs @@ -1,4 +1,5 @@ use crate::protocol::ReviewFinding; +use crate::protocol::ReviewOutputEvent; // Note: We keep this module UI-agnostic. It returns plain strings that // higher layers (e.g., TUI) may style as needed. @@ -10,6 +11,8 @@ fn format_location(item: &ReviewFinding) -> String { format!("{path}:{start}-{end}") } +const REVIEW_FALLBACK_MESSAGE: &str = "Reviewer failed to output a response."; + /// Format a full review findings block as plain text lines. /// /// - When `selection` is `Some`, each item line includes a checkbox marker: @@ -53,3 +56,27 @@ pub fn format_review_findings_block( lines.join("\n") } + +/// Render a human-readable review summary suitable for a user-facing message. +/// +/// Returns either the explanation, the formatted findings block, or both +/// separated by a blank line. If neither is present, emits a fallback message. +pub fn render_review_output_text(output: &ReviewOutputEvent) -> String { + let mut sections = Vec::new(); + let explanation = output.overall_explanation.trim(); + if !explanation.is_empty() { + sections.push(explanation.to_string()); + } + if !output.findings.is_empty() { + let findings = format_review_findings_block(&output.findings, None); + let trimmed = findings.trim(); + if !trimmed.is_empty() { + sections.push(trimmed.to_string()); + } + } + if sections.is_empty() { + REVIEW_FALLBACK_MESSAGE.to_string() + } else { + sections.join("\n\n") + } +} diff --git a/codex-rs/core/src/review_prompts.rs b/codex-rs/core/src/review_prompts.rs new file mode 100644 index 0000000000..cfcb7e0b13 --- /dev/null +++ b/codex-rs/core/src/review_prompts.rs @@ -0,0 +1,93 @@ +use codex_git::merge_base_with_head; +use codex_protocol::protocol::ReviewRequest; +use codex_protocol::protocol::ReviewTarget; +use std::path::Path; + +#[derive(Clone, Debug, PartialEq)] +pub struct ResolvedReviewRequest { + pub target: ReviewTarget, + pub prompt: String, + pub user_facing_hint: String, +} + +const UNCOMMITTED_PROMPT: &str = "Review the current code changes (staged, unstaged, and untracked files) and provide prioritized findings."; + +const BASE_BRANCH_PROMPT_BACKUP: &str = "Review the code changes against the base branch '{branch}'. Start by finding the merge diff between the current branch and {branch}'s upstream e.g. (`git merge-base HEAD \"$(git rev-parse --abbrev-ref \"{branch}@{upstream}\")\"`), then run `git diff` against that SHA to see what changes we would merge into the {branch} branch. Provide prioritized, actionable findings."; +const BASE_BRANCH_PROMPT: &str = "Review the code changes against the base branch '{baseBranch}'. The merge base commit for this comparison is {mergeBaseSha}. Run `git diff {mergeBaseSha}` to inspect the changes relative to {baseBranch}. Provide prioritized, actionable findings."; + +const COMMIT_PROMPT_WITH_TITLE: &str = "Review the code changes introduced by commit {sha} (\"{title}\"). Provide prioritized, actionable findings."; +const COMMIT_PROMPT: &str = + "Review the code changes introduced by commit {sha}. Provide prioritized, actionable findings."; + +pub fn resolve_review_request( + request: ReviewRequest, + cwd: &Path, +) -> anyhow::Result { + let target = request.target; + let prompt = review_prompt(&target, cwd)?; + let user_facing_hint = request + .user_facing_hint + .unwrap_or_else(|| user_facing_hint(&target)); + + Ok(ResolvedReviewRequest { + target, + prompt, + user_facing_hint, + }) +} + +pub fn review_prompt(target: &ReviewTarget, cwd: &Path) -> anyhow::Result { + match target { + ReviewTarget::UncommittedChanges => Ok(UNCOMMITTED_PROMPT.to_string()), + ReviewTarget::BaseBranch { branch } => { + if let Some(commit) = merge_base_with_head(cwd, branch)? { + Ok(BASE_BRANCH_PROMPT + .replace("{baseBranch}", branch) + .replace("{mergeBaseSha}", &commit)) + } else { + Ok(BASE_BRANCH_PROMPT_BACKUP.replace("{branch}", branch)) + } + } + ReviewTarget::Commit { sha, title } => { + if let Some(title) = title { + Ok(COMMIT_PROMPT_WITH_TITLE + .replace("{sha}", sha) + .replace("{title}", title)) + } else { + Ok(COMMIT_PROMPT.replace("{sha}", sha)) + } + } + ReviewTarget::Custom { instructions } => { + let prompt = instructions.trim(); + if prompt.is_empty() { + anyhow::bail!("Review prompt cannot be empty"); + } + Ok(prompt.to_string()) + } + } +} + +pub fn user_facing_hint(target: &ReviewTarget) -> String { + match target { + ReviewTarget::UncommittedChanges => "current changes".to_string(), + ReviewTarget::BaseBranch { branch } => format!("changes against '{branch}'"), + ReviewTarget::Commit { sha, title } => { + let short_sha: String = sha.chars().take(7).collect(); + if let Some(title) = title { + format!("commit {short_sha}: {title}") + } else { + format!("commit {short_sha}") + } + } + ReviewTarget::Custom { instructions } => instructions.trim().to_string(), + } +} + +impl From for ReviewRequest { + fn from(resolved: ResolvedReviewRequest) -> Self { + ReviewRequest { + target: resolved.target, + user_facing_hint: Some(resolved.user_facing_hint), + } + } +} diff --git a/codex-rs/core/src/rollout/error.rs b/codex-rs/core/src/rollout/error.rs new file mode 100644 index 0000000000..e924dd2d28 --- /dev/null +++ b/codex-rs/core/src/rollout/error.rs @@ -0,0 +1,49 @@ +use std::io::ErrorKind; +use std::path::Path; + +use crate::error::CodexErr; +use crate::rollout::SESSIONS_SUBDIR; + +pub(crate) fn map_session_init_error(err: &anyhow::Error, codex_home: &Path) -> CodexErr { + if let Some(mapped) = err + .chain() + .filter_map(|cause| cause.downcast_ref::()) + .find_map(|io_err| map_rollout_io_error(io_err, codex_home)) + { + return mapped; + } + + CodexErr::Fatal(format!("Failed to initialize session: {err:#}")) +} + +fn map_rollout_io_error(io_err: &std::io::Error, codex_home: &Path) -> Option { + let sessions_dir = codex_home.join(SESSIONS_SUBDIR); + let hint = match io_err.kind() { + ErrorKind::PermissionDenied => format!( + "Codex cannot access session files at {} (permission denied). If sessions were created using sudo, fix ownership: sudo chown -R $(whoami) {}", + sessions_dir.display(), + codex_home.display() + ), + ErrorKind::NotFound => format!( + "Session storage missing at {}. Create the directory or choose a different Codex home.", + sessions_dir.display() + ), + ErrorKind::AlreadyExists => format!( + "Session storage path {} is blocked by an existing file. Remove or rename it so Codex can create sessions.", + sessions_dir.display() + ), + ErrorKind::InvalidData | ErrorKind::InvalidInput => format!( + "Session data under {} looks corrupt or unreadable. Clearing the sessions directory may help (this will remove saved conversations).", + sessions_dir.display() + ), + ErrorKind::IsADirectory | ErrorKind::NotADirectory => format!( + "Session storage path {} has an unexpected type. Ensure it is a directory Codex can use for session files.", + sessions_dir.display() + ), + _ => return None, + }; + + Some(CodexErr::Fatal(format!( + "{hint} (underlying error: {io_err})" + ))) +} diff --git a/codex-rs/core/src/rollout/list.rs b/codex-rs/core/src/rollout/list.rs index 781e3fe372..e2ef0e883c 100644 --- a/codex-rs/core/src/rollout/list.rs +++ b/codex-rs/core/src/rollout/list.rs @@ -9,6 +9,7 @@ use std::sync::atomic::AtomicBool; use time::OffsetDateTime; use time::PrimitiveDateTime; use time::format_description::FormatItem; +use time::format_description::well_known::Rfc3339; use time::macros::format_description; use uuid::Uuid; @@ -39,18 +40,15 @@ pub struct ConversationItem { pub path: PathBuf, /// First up to `HEAD_RECORD_LIMIT` JSONL records parsed as JSON (includes meta line). pub head: Vec, - /// Last up to `TAIL_RECORD_LIMIT` JSONL response records parsed as JSON. - pub tail: Vec, /// RFC3339 timestamp string for when the session was created, if available. pub created_at: Option, - /// RFC3339 timestamp string for the most recent response in the tail, if available. + /// RFC3339 timestamp string for the most recent update (from file mtime). pub updated_at: Option, } #[derive(Default)] struct HeadTailSummary { head: Vec, - tail: Vec, saw_session_meta: bool, saw_user_event: bool, source: Option, @@ -62,7 +60,6 @@ struct HeadTailSummary { /// Hard cap to bound worst‑case work per request. const MAX_SCAN_FILES: usize = 10000; const HEAD_RECORD_LIMIT: usize = 10; -const TAIL_RECORD_LIMIT: usize = 10; /// Pagination cursor identifying a file by timestamp and UUID. #[derive(Debug, Clone, PartialEq, Eq)] @@ -141,13 +138,6 @@ pub(crate) async fn get_conversations( Ok(result) } -/// Load the full contents of a single conversation session file at `path`. -/// Returns the entire file contents as a String. -#[allow(dead_code)] -pub(crate) async fn get_conversation(path: &Path) -> io::Result { - tokio::fs::read_to_string(path).await -} - /// Load conversation file paths from disk using directory traversal. /// /// Directory layout: `~/.codex/sessions/YYYY/MM/DD/rollout-YYYY-MM-DDThh-mm-ss-.jsonl` @@ -212,9 +202,8 @@ async fn traverse_directories_for_paths( more_matches_available = true; break 'outer; } - // Read head and simultaneously detect message events within the same - // first N JSONL records to avoid a second file read. - let summary = read_head_and_tail(&path, HEAD_RECORD_LIMIT, TAIL_RECORD_LIMIT) + // Read head and detect message events; stop once meta + user are found. + let summary = read_head_summary(&path, HEAD_RECORD_LIMIT) .await .unwrap_or_default(); if !allowed_sources.is_empty() @@ -233,16 +222,19 @@ async fn traverse_directories_for_paths( if summary.saw_session_meta && summary.saw_user_event { let HeadTailSummary { head, - tail, created_at, mut updated_at, .. } = summary; - updated_at = updated_at.or_else(|| created_at.clone()); + if updated_at.is_none() { + updated_at = file_modified_rfc3339(&path) + .await + .unwrap_or(None) + .or_else(|| created_at.clone()); + } items.push(ConversationItem { path, head, - tail, created_at, updated_at, }); @@ -384,11 +376,7 @@ impl<'a> ProviderMatcher<'a> { } } -async fn read_head_and_tail( - path: &Path, - head_limit: usize, - tail_limit: usize, -) -> io::Result { +async fn read_head_summary(path: &Path, head_limit: usize) -> io::Result { use tokio::io::AsyncBufReadExt; let file = tokio::fs::File::open(path).await?; @@ -441,107 +429,30 @@ async fn read_head_and_tail( } } } + + if summary.saw_session_meta && summary.saw_user_event { + break; + } } - if tail_limit != 0 { - let (tail, updated_at) = read_tail_records(path, tail_limit).await?; - summary.tail = tail; - summary.updated_at = updated_at; - } Ok(summary) } /// Read up to `HEAD_RECORD_LIMIT` records from the start of the rollout file at `path`. /// This should be enough to produce a summary including the session meta line. pub async fn read_head_for_summary(path: &Path) -> io::Result> { - let summary = read_head_and_tail(path, HEAD_RECORD_LIMIT, 0).await?; + let summary = read_head_summary(path, HEAD_RECORD_LIMIT).await?; Ok(summary.head) } -async fn read_tail_records( - path: &Path, - max_records: usize, -) -> io::Result<(Vec, Option)> { - use std::io::SeekFrom; - use tokio::io::AsyncReadExt; - use tokio::io::AsyncSeekExt; - - if max_records == 0 { - return Ok((Vec::new(), None)); - } - - const CHUNK_SIZE: usize = 8192; - - let mut file = tokio::fs::File::open(path).await?; - let mut pos = file.seek(SeekFrom::End(0)).await?; - if pos == 0 { - return Ok((Vec::new(), None)); - } - - let mut buffer: Vec = Vec::new(); - let mut latest_timestamp: Option = None; - - loop { - let slice_start = match (pos > 0, buffer.iter().position(|&b| b == b'\n')) { - (true, Some(idx)) => idx + 1, - _ => 0, - }; - let (tail, newest_ts) = collect_last_response_values(&buffer[slice_start..], max_records); - if latest_timestamp.is_none() { - latest_timestamp = newest_ts.clone(); - } - if tail.len() >= max_records || pos == 0 { - return Ok((tail, latest_timestamp.or(newest_ts))); - } - - let read_size = CHUNK_SIZE.min(pos as usize); - if read_size == 0 { - return Ok((tail, latest_timestamp.or(newest_ts))); - } - pos -= read_size as u64; - file.seek(SeekFrom::Start(pos)).await?; - let mut chunk = vec![0; read_size]; - file.read_exact(&mut chunk).await?; - chunk.extend_from_slice(&buffer); - buffer = chunk; - } -} - -fn collect_last_response_values( - buffer: &[u8], - max_records: usize, -) -> (Vec, Option) { - use std::borrow::Cow; - - if buffer.is_empty() || max_records == 0 { - return (Vec::new(), None); - } - - let text: Cow<'_, str> = String::from_utf8_lossy(buffer); - let mut collected_rev: Vec = Vec::new(); - let mut latest_timestamp: Option = None; - for line in text.lines().rev() { - let trimmed = line.trim(); - if trimmed.is_empty() { - continue; - } - let parsed: serde_json::Result = serde_json::from_str(trimmed); - let Ok(rollout_line) = parsed else { continue }; - let RolloutLine { timestamp, item } = rollout_line; - if let RolloutItem::ResponseItem(item) = item - && let Ok(val) = serde_json::to_value(&item) - { - if latest_timestamp.is_none() { - latest_timestamp = Some(timestamp.clone()); - } - collected_rev.push(val); - if collected_rev.len() == max_records { - break; - } - } - } - collected_rev.reverse(); - (collected_rev, latest_timestamp) +async fn file_modified_rfc3339(path: &Path) -> io::Result> { + let meta = tokio::fs::metadata(path).await?; + let modified = meta.modified().ok(); + let Some(modified) = modified else { + return Ok(None); + }; + let dt = OffsetDateTime::from(modified); + Ok(dt.format(&Rfc3339).ok()) } /// Locate a recorded conversation rollout file by its UUID string using the existing diff --git a/codex-rs/core/src/rollout/mod.rs b/codex-rs/core/src/rollout/mod.rs index 23410bd4d0..540d204be3 100644 --- a/codex-rs/core/src/rollout/mod.rs +++ b/codex-rs/core/src/rollout/mod.rs @@ -7,11 +7,13 @@ pub const ARCHIVED_SESSIONS_SUBDIR: &str = "archived_sessions"; pub const INTERACTIVE_SESSION_SOURCES: &[SessionSource] = &[SessionSource::Cli, SessionSource::VSCode]; +pub(crate) mod error; pub mod list; pub(crate) mod policy; pub mod recorder; pub use codex_protocol::protocol::SessionMeta; +pub(crate) use error::map_session_init_error; pub use list::find_conversation_path_by_id_str; pub use recorder::RolloutRecorder; pub use recorder::RolloutRecorderParams; diff --git a/codex-rs/core/src/rollout/policy.rs b/codex-rs/core/src/rollout/policy.rs index 6bbe20bdad..bb29551a2f 100644 --- a/codex-rs/core/src/rollout/policy.rs +++ b/codex-rs/core/src/rollout/policy.rs @@ -42,6 +42,7 @@ pub(crate) fn should_persist_event_msg(ev: &EventMsg) -> bool { | EventMsg::AgentReasoning(_) | EventMsg::AgentReasoningRawContent(_) | EventMsg::TokenCount(_) + | EventMsg::ContextCompacted(_) | EventMsg::EnteredReviewMode(_) | EventMsg::ExitedReviewMode(_) | EventMsg::UndoCompleted(_) @@ -64,6 +65,7 @@ pub(crate) fn should_persist_event_msg(ev: &EventMsg) -> bool { | EventMsg::ExecCommandOutputDelta(_) | EventMsg::ExecCommandEnd(_) | EventMsg::ExecApprovalRequest(_) + | EventMsg::ElicitationRequest(_) | EventMsg::ApplyPatchApprovalRequest(_) | EventMsg::BackgroundEvent(_) | EventMsg::SaveSessionResponse(_) diff --git a/codex-rs/core/src/rollout/tests.rs b/codex-rs/core/src/rollout/tests.rs index 1471f12370..625c1ae254 100644 --- a/codex-rs/core/src/rollout/tests.rs +++ b/codex-rs/core/src/rollout/tests.rs @@ -16,13 +16,11 @@ use crate::rollout::INTERACTIVE_SESSION_SOURCES; use crate::rollout::list::ConversationItem; use crate::rollout::list::ConversationsPage; use crate::rollout::list::Cursor; -use crate::rollout::list::get_conversation; use crate::rollout::list::get_conversations; use anyhow::Result; use codex_protocol::ConversationId; use codex_protocol::models::ContentItem; use codex_protocol::models::ResponseItem; -use codex_protocol::protocol::CompactedItem; use codex_protocol::protocol::EventMsg; use codex_protocol::protocol::RolloutItem; use codex_protocol::protocol::RolloutLine; @@ -226,28 +224,28 @@ async fn test_list_conversations_latest_first() { "model_provider": "test-provider", })]; + let updated_times: Vec> = + page.items.iter().map(|i| i.updated_at.clone()).collect(); + let expected = ConversationsPage { items: vec![ ConversationItem { path: p1, head: head_3, - tail: Vec::new(), created_at: Some("2025-01-03T12-00-00".into()), - updated_at: Some("2025-01-03T12-00-00".into()), + updated_at: updated_times.first().cloned().flatten(), }, ConversationItem { path: p2, head: head_2, - tail: Vec::new(), created_at: Some("2025-01-02T12-00-00".into()), - updated_at: Some("2025-01-02T12-00-00".into()), + updated_at: updated_times.get(1).cloned().flatten(), }, ConversationItem { path: p3, head: head_1, - tail: Vec::new(), created_at: Some("2025-01-01T12-00-00".into()), - updated_at: Some("2025-01-01T12-00-00".into()), + updated_at: updated_times.get(2).cloned().flatten(), }, ], next_cursor: None, @@ -355,6 +353,8 @@ async fn test_pagination_cursor() { "source": "vscode", "model_provider": "test-provider", })]; + let updated_page1: Vec> = + page1.items.iter().map(|i| i.updated_at.clone()).collect(); let expected_cursor1: Cursor = serde_json::from_str(&format!("\"2025-03-04T09-00-00|{u4}\"")).unwrap(); let expected_page1 = ConversationsPage { @@ -362,16 +362,14 @@ async fn test_pagination_cursor() { ConversationItem { path: p5, head: head_5, - tail: Vec::new(), created_at: Some("2025-03-05T09-00-00".into()), - updated_at: Some("2025-03-05T09-00-00".into()), + updated_at: updated_page1.first().cloned().flatten(), }, ConversationItem { path: p4, head: head_4, - tail: Vec::new(), created_at: Some("2025-03-04T09-00-00".into()), - updated_at: Some("2025-03-04T09-00-00".into()), + updated_at: updated_page1.get(1).cloned().flatten(), }, ], next_cursor: Some(expected_cursor1.clone()), @@ -422,6 +420,8 @@ async fn test_pagination_cursor() { "source": "vscode", "model_provider": "test-provider", })]; + let updated_page2: Vec> = + page2.items.iter().map(|i| i.updated_at.clone()).collect(); let expected_cursor2: Cursor = serde_json::from_str(&format!("\"2025-03-02T09-00-00|{u2}\"")).unwrap(); let expected_page2 = ConversationsPage { @@ -429,16 +429,14 @@ async fn test_pagination_cursor() { ConversationItem { path: p3, head: head_3, - tail: Vec::new(), created_at: Some("2025-03-03T09-00-00".into()), - updated_at: Some("2025-03-03T09-00-00".into()), + updated_at: updated_page2.first().cloned().flatten(), }, ConversationItem { path: p2, head: head_2, - tail: Vec::new(), created_at: Some("2025-03-02T09-00-00".into()), - updated_at: Some("2025-03-02T09-00-00".into()), + updated_at: updated_page2.get(1).cloned().flatten(), }, ], next_cursor: Some(expected_cursor2.clone()), @@ -473,13 +471,14 @@ async fn test_pagination_cursor() { "source": "vscode", "model_provider": "test-provider", })]; + let updated_page3: Vec> = + page3.items.iter().map(|i| i.updated_at.clone()).collect(); let expected_page3 = ConversationsPage { items: vec![ConversationItem { path: p1, head: head_1, - tail: Vec::new(), created_at: Some("2025-03-01T09-00-00".into()), - updated_at: Some("2025-03-01T09-00-00".into()), + updated_at: updated_page3.first().cloned().flatten(), }], next_cursor: None, num_scanned_files: 5, // scanned 05, 04 (anchor), 03, 02 (anchor), 01 @@ -510,7 +509,7 @@ async fn test_get_conversation_contents() { .unwrap(); let path = &page.items[0].path; - let content = get_conversation(path).await.unwrap(); + let content = tokio::fs::read_to_string(path).await.unwrap(); // Page equality (single item) let expected_path = home @@ -533,9 +532,8 @@ async fn test_get_conversation_contents() { items: vec![ConversationItem { path: expected_path, head: expected_head, - tail: Vec::new(), created_at: Some(ts.into()), - updated_at: Some(ts.into()), + updated_at: page.items[0].updated_at.clone(), }], next_cursor: None, num_scanned_files: 1, @@ -570,7 +568,7 @@ async fn test_get_conversation_contents() { } #[tokio::test] -async fn test_tail_includes_last_response_items() -> Result<()> { +async fn test_updated_at_uses_file_mtime() -> Result<()> { let temp = TempDir::new().unwrap(); let home = temp.path(); @@ -637,231 +635,16 @@ async fn test_tail_includes_last_response_items() -> Result<()> { ) .await?; let item = page.items.first().expect("conversation item"); - let tail_len = item.tail.len(); - assert_eq!(tail_len, 10usize.min(total_messages)); - - let expected: Vec = (total_messages - tail_len..total_messages) - .map(|idx| { - serde_json::json!({ - "type": "message", - "role": "assistant", - "content": [ - { - "type": "output_text", - "text": format!("reply-{idx}"), - } - ], - }) - }) - .collect(); - - assert_eq!(item.tail, expected); assert_eq!(item.created_at.as_deref(), Some(ts)); - let expected_updated = format!("{ts}-{last:02}", last = total_messages - 1); - assert_eq!(item.updated_at.as_deref(), Some(expected_updated.as_str())); - - Ok(()) -} - -#[tokio::test] -async fn test_tail_handles_short_sessions() -> Result<()> { - let temp = TempDir::new().unwrap(); - let home = temp.path(); - - let ts = "2025-06-02T08-30-00"; - let uuid = Uuid::from_u128(7); - let day_dir = home.join("sessions").join("2025").join("06").join("02"); - fs::create_dir_all(&day_dir)?; - let file_path = day_dir.join(format!("rollout-{ts}-{uuid}.jsonl")); - let mut file = File::create(&file_path)?; - - let conversation_id = ConversationId::from_string(&uuid.to_string())?; - let meta_line = RolloutLine { - timestamp: ts.to_string(), - item: RolloutItem::SessionMeta(SessionMetaLine { - meta: SessionMeta { - id: conversation_id, - timestamp: ts.to_string(), - instructions: None, - cwd: ".".into(), - originator: "test_originator".into(), - cli_version: "test_version".into(), - source: SessionSource::VSCode, - model_provider: Some("test-provider".into()), - name: None, - }, - git: None, - }), - }; - writeln!(file, "{}", serde_json::to_string(&meta_line)?)?; - - let user_event_line = RolloutLine { - timestamp: ts.to_string(), - item: RolloutItem::EventMsg(EventMsg::UserMessage(UserMessageEvent { - message: "hi".into(), - images: None, - })), - }; - writeln!(file, "{}", serde_json::to_string(&user_event_line)?)?; - - for idx in 0..3 { - let response_line = RolloutLine { - timestamp: format!("{ts}-{idx:02}"), - item: RolloutItem::ResponseItem(ResponseItem::Message { - id: None, - role: "assistant".into(), - content: vec![ContentItem::OutputText { - text: format!("short-{idx}"), - }], - }), - }; - writeln!(file, "{}", serde_json::to_string(&response_line)?)?; - } - drop(file); - - let provider_filter = provider_vec(&[TEST_PROVIDER]); - let page = get_conversations( - home, - 1, - None, - INTERACTIVE_SESSION_SOURCES, - Some(provider_filter.as_slice()), - TEST_PROVIDER, - ) - .await?; - let tail = &page.items.first().expect("conversation item").tail; - - assert_eq!(tail.len(), 3); - - let expected: Vec = (0..3) - .map(|idx| { - serde_json::json!({ - "type": "message", - "role": "assistant", - "content": [ - { - "type": "output_text", - "text": format!("short-{idx}"), - } - ], - }) - }) - .collect(); - - assert_eq!(tail, &expected); - let expected_updated = format!("{ts}-{last:02}", last = 2); - assert_eq!( - page.items[0].updated_at.as_deref(), - Some(expected_updated.as_str()) - ); - - Ok(()) -} - -#[tokio::test] -async fn test_tail_skips_trailing_non_responses() -> Result<()> { - let temp = TempDir::new().unwrap(); - let home = temp.path(); - - let ts = "2025-06-03T10-00-00"; - let uuid = Uuid::from_u128(11); - let day_dir = home.join("sessions").join("2025").join("06").join("03"); - fs::create_dir_all(&day_dir)?; - let file_path = day_dir.join(format!("rollout-{ts}-{uuid}.jsonl")); - let mut file = File::create(&file_path)?; - - let conversation_id = ConversationId::from_string(&uuid.to_string())?; - let meta_line = RolloutLine { - timestamp: ts.to_string(), - item: RolloutItem::SessionMeta(SessionMetaLine { - meta: SessionMeta { - id: conversation_id, - timestamp: ts.to_string(), - instructions: None, - cwd: ".".into(), - originator: "test_originator".into(), - cli_version: "test_version".into(), - source: SessionSource::VSCode, - model_provider: Some("test-provider".into()), - name: None, - }, - git: None, - }), - }; - writeln!(file, "{}", serde_json::to_string(&meta_line)?)?; - - let user_event_line = RolloutLine { - timestamp: ts.to_string(), - item: RolloutItem::EventMsg(EventMsg::UserMessage(UserMessageEvent { - message: "hello".into(), - images: None, - })), - }; - writeln!(file, "{}", serde_json::to_string(&user_event_line)?)?; - - for idx in 0..4 { - let response_line = RolloutLine { - timestamp: format!("{ts}-{idx:02}"), - item: RolloutItem::ResponseItem(ResponseItem::Message { - id: None, - role: "assistant".into(), - content: vec![ContentItem::OutputText { - text: format!("response-{idx}"), - }], - }), - }; - writeln!(file, "{}", serde_json::to_string(&response_line)?)?; - } - - let compacted_line = RolloutLine { - timestamp: format!("{ts}-compacted"), - item: RolloutItem::Compacted(CompactedItem { - message: "compacted".into(), - replacement_history: None, - }), - }; - writeln!(file, "{}", serde_json::to_string(&compacted_line)?)?; - - let shutdown_event = RolloutLine { - timestamp: format!("{ts}-shutdown"), - item: RolloutItem::EventMsg(EventMsg::ShutdownComplete), - }; - writeln!(file, "{}", serde_json::to_string(&shutdown_event)?)?; - drop(file); - - let provider_filter = provider_vec(&[TEST_PROVIDER]); - let page = get_conversations( - home, - 1, - None, - INTERACTIVE_SESSION_SOURCES, - Some(provider_filter.as_slice()), - TEST_PROVIDER, - ) - .await?; - let tail = &page.items.first().expect("conversation item").tail; - - let expected: Vec = (0..4) - .map(|idx| { - serde_json::json!({ - "type": "message", - "role": "assistant", - "content": [ - { - "type": "output_text", - "text": format!("response-{idx}"), - } - ], - }) - }) - .collect(); - - assert_eq!(tail, &expected); - let expected_updated = format!("{ts}-{last:02}", last = 3); - assert_eq!( - page.items[0].updated_at.as_deref(), - Some(expected_updated.as_str()) - ); + let updated = item + .updated_at + .as_deref() + .and_then(|s| chrono::DateTime::parse_from_rfc3339(s).ok()) + .map(|dt| dt.with_timezone(&chrono::Utc)) + .expect("updated_at set from file mtime"); + let now = chrono::Utc::now(); + let age = now - updated; + assert!(age.num_seconds().abs() < 30); Ok(()) } @@ -916,22 +699,22 @@ async fn test_stable_ordering_same_second_pagination() { "model_provider": "test-provider", })] }; + let updated_page1: Vec> = + page1.items.iter().map(|i| i.updated_at.clone()).collect(); let expected_cursor1: Cursor = serde_json::from_str(&format!("\"{ts}|{u2}\"")).unwrap(); let expected_page1 = ConversationsPage { items: vec![ ConversationItem { path: p3, head: head(u3), - tail: Vec::new(), created_at: Some(ts.to_string()), - updated_at: Some(ts.to_string()), + updated_at: updated_page1.first().cloned().flatten(), }, ConversationItem { path: p2, head: head(u2), - tail: Vec::new(), created_at: Some(ts.to_string()), - updated_at: Some(ts.to_string()), + updated_at: updated_page1.get(1).cloned().flatten(), }, ], next_cursor: Some(expected_cursor1.clone()), @@ -956,13 +739,14 @@ async fn test_stable_ordering_same_second_pagination() { .join("07") .join("01") .join(format!("rollout-2025-07-01T00-00-00-{u1}.jsonl")); + let updated_page2: Vec> = + page2.items.iter().map(|i| i.updated_at.clone()).collect(); let expected_page2 = ConversationsPage { items: vec![ConversationItem { path: p1, head: head(u1), - tail: Vec::new(), created_at: Some(ts.to_string()), - updated_at: Some(ts.to_string()), + updated_at: updated_page2.first().cloned().flatten(), }], next_cursor: None, num_scanned_files: 3, // scanned u3, u2 (anchor), u1 diff --git a/codex-rs/core/src/safety.rs b/codex-rs/core/src/safety.rs index 1e81981fd1..e82e107453 100644 --- a/codex-rs/core/src/safety.rs +++ b/codex-rs/core/src/safety.rs @@ -6,6 +6,7 @@ use codex_apply_patch::ApplyPatchAction; use codex_apply_patch::ApplyPatchFileChange; use crate::exec::SandboxType; +use crate::util::resolve_path; use crate::protocol::AskForApproval; use crate::protocol::SandboxPolicy; @@ -150,11 +151,7 @@ fn is_write_patch_constrained_to_writable_paths( // and roots are converted to absolute, normalized forms before the // prefix check. let is_path_writable = |p: &PathBuf| { - let abs = if p.is_absolute() { - p.clone() - } else { - cwd.join(p) - }; + let abs = resolve_path(cwd, p); let abs = match normalize(&abs) { Some(v) => v, None => return false, diff --git a/codex-rs/core/src/sandboxing/assessment.rs b/codex-rs/core/src/sandboxing/assessment.rs index 31e76777bb..719e3be1f0 100644 --- a/codex-rs/core/src/sandboxing/assessment.rs +++ b/codex-rs/core/src/sandboxing/assessment.rs @@ -14,6 +14,7 @@ use crate::protocol::SandboxPolicy; use askama::Template; use codex_otel::otel_event_manager::OtelEventManager; use codex_protocol::ConversationId; +use codex_protocol::config_types::ReasoningEffort as ReasoningEffortConfig; use codex_protocol::models::ContentItem; use codex_protocol::models::ResponseItem; use codex_protocol::protocol::SandboxCommandAssessment; @@ -23,7 +24,8 @@ use serde_json::json; use tokio::time::timeout; use tracing::warn; -const SANDBOX_ASSESSMENT_TIMEOUT: Duration = Duration::from_secs(5); +const SANDBOX_ASSESSMENT_TIMEOUT: Duration = Duration::from_secs(15); +const SANDBOX_ASSESSMENT_REASONING_EFFORT: ReasoningEffortConfig = ReasoningEffortConfig::Medium; #[derive(Template)] #[template(path = "sandboxing/assessment_prompt.md", escape = "none")] @@ -130,7 +132,7 @@ pub(crate) async fn assess_command( Some(auth_manager), child_otel, provider, - config.model_reasoning_effort, + Some(SANDBOX_ASSESSMENT_REASONING_EFFORT), config.model_reasoning_summary, conversation_id, session_source, diff --git a/codex-rs/core/src/sandboxing/mod.rs b/codex-rs/core/src/sandboxing/mod.rs index 4ecb2a8c12..d43646021e 100644 --- a/codex-rs/core/src/sandboxing/mod.rs +++ b/codex-rs/core/src/sandboxing/mod.rs @@ -8,6 +8,7 @@ ready‑to‑spawn environment. pub mod assessment; +use crate::exec::ExecExpiration; use crate::exec::ExecToolCallOutput; use crate::exec::SandboxType; use crate::exec::StdoutStream; @@ -48,23 +49,23 @@ impl From for SandboxPermissions { } } -#[derive(Clone, Debug)] +#[derive(Debug)] pub struct CommandSpec { pub program: String, pub args: Vec, pub cwd: PathBuf, pub env: HashMap, - pub timeout_ms: Option, + pub expiration: ExecExpiration, pub with_escalated_permissions: Option, pub justification: Option, } -#[derive(Clone, Debug)] +#[derive(Debug)] pub struct ExecEnv { pub command: Vec, pub cwd: PathBuf, pub env: HashMap, - pub timeout_ms: Option, + pub expiration: ExecExpiration, pub sandbox: SandboxType, pub with_escalated_permissions: Option, pub justification: Option, @@ -115,13 +116,13 @@ impl SandboxManager { pub(crate) fn transform( &self, - spec: &CommandSpec, + mut spec: CommandSpec, policy: &SandboxPolicy, sandbox: SandboxType, sandbox_policy_cwd: &Path, codex_linux_sandbox_exe: Option<&PathBuf>, ) -> Result { - let mut env = spec.env.clone(); + let mut env = spec.env; if !policy.has_full_network_access() { env.insert( CODEX_SANDBOX_NETWORK_DISABLED_ENV_VAR.to_string(), @@ -130,8 +131,8 @@ impl SandboxManager { } let mut command = Vec::with_capacity(1 + spec.args.len()); - command.push(spec.program.clone()); - command.extend(spec.args.iter().cloned()); + command.push(spec.program); + command.append(&mut spec.args); let (command, sandbox_env, arg0_override) = match sandbox { SandboxType::None => (command, HashMap::new(), None), @@ -176,12 +177,12 @@ impl SandboxManager { Ok(ExecEnv { command, - cwd: spec.cwd.clone(), + cwd: spec.cwd, env, - timeout_ms: spec.timeout_ms, + expiration: spec.expiration, sandbox, with_escalated_permissions: spec.with_escalated_permissions, - justification: spec.justification.clone(), + justification: spec.justification, arg0: arg0_override, }) } @@ -192,9 +193,9 @@ impl SandboxManager { } pub async fn execute_env( - env: &ExecEnv, + env: ExecEnv, policy: &SandboxPolicy, stdout_stream: Option, ) -> crate::error::Result { - execute_exec_env(env.clone(), policy, stdout_stream).await + execute_exec_env(env, policy, stdout_stream).await } diff --git a/codex-rs/core/src/shell.rs b/codex-rs/core/src/shell.rs index 1fb2b97dd7..ac115facb6 100644 --- a/codex-rs/core/src/shell.rs +++ b/codex-rs/core/src/shell.rs @@ -7,61 +7,41 @@ pub enum ShellType { Zsh, Bash, PowerShell, + Sh, + Cmd, } #[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)] -pub struct ZshShell { +pub struct Shell { + pub(crate) shell_type: ShellType, pub(crate) shell_path: PathBuf, } -#[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)] -pub struct BashShell { - pub(crate) shell_path: PathBuf, -} - -#[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)] -pub struct PowerShellConfig { - pub(crate) shell_path: PathBuf, // Executable name or path, e.g. "pwsh" or "powershell.exe". -} - -#[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)] -pub enum Shell { - Zsh(ZshShell), - Bash(BashShell), - PowerShell(PowerShellConfig), - Unknown, -} - impl Shell { - pub fn name(&self) -> Option { - match self { - Shell::Zsh(ZshShell { shell_path, .. }) | Shell::Bash(BashShell { shell_path, .. }) => { - std::path::Path::new(shell_path) - .file_name() - .map(|s| s.to_string_lossy().to_string()) - } - Shell::PowerShell(ps) => ps - .shell_path - .file_stem() - .map(|s| s.to_string_lossy().to_string()), - Shell::Unknown => None, + pub fn name(&self) -> &'static str { + match self.shell_type { + ShellType::Zsh => "zsh", + ShellType::Bash => "bash", + ShellType::PowerShell => "powershell", + ShellType::Sh => "sh", + ShellType::Cmd => "cmd", } } /// Takes a string of shell and returns the full list of command args to /// use with `exec()` to run the shell command. pub fn derive_exec_args(&self, command: &str, use_login_shell: bool) -> Vec { - match self { - Shell::Zsh(ZshShell { shell_path, .. }) | Shell::Bash(BashShell { shell_path, .. }) => { + match self.shell_type { + ShellType::Zsh | ShellType::Bash | ShellType::Sh => { let arg = if use_login_shell { "-lc" } else { "-c" }; vec![ - shell_path.to_string_lossy().to_string(), + self.shell_path.to_string_lossy().to_string(), arg.to_string(), command.to_string(), ] } - Shell::PowerShell(ps) => { - let mut args = vec![ps.shell_path.to_string_lossy().to_string()]; + ShellType::PowerShell => { + let mut args = vec![self.shell_path.to_string_lossy().to_string()]; if !use_login_shell { args.push("-NoProfile".to_string()); } @@ -70,7 +50,12 @@ impl Shell { args.push(command.to_string()); args } - Shell::Unknown => shlex::split(command).unwrap_or_else(|| vec![command.to_string()]), + ShellType::Cmd => { + let mut args = vec![self.shell_path.to_string_lossy().to_string()]; + args.push("/c".to_string()); + args.push(command.to_string()); + args + } } } } @@ -143,19 +128,34 @@ fn get_shell_path( None } -fn get_zsh_shell(path: Option<&PathBuf>) -> Option { +fn get_zsh_shell(path: Option<&PathBuf>) -> Option { let shell_path = get_shell_path(ShellType::Zsh, path, "zsh", vec!["/bin/zsh"]); - shell_path.map(|shell_path| ZshShell { shell_path }) + shell_path.map(|shell_path| Shell { + shell_type: ShellType::Zsh, + shell_path, + }) } -fn get_bash_shell(path: Option<&PathBuf>) -> Option { +fn get_bash_shell(path: Option<&PathBuf>) -> Option { let shell_path = get_shell_path(ShellType::Bash, path, "bash", vec!["/bin/bash"]); - shell_path.map(|shell_path| BashShell { shell_path }) + shell_path.map(|shell_path| Shell { + shell_type: ShellType::Bash, + shell_path, + }) } -fn get_powershell_shell(path: Option<&PathBuf>) -> Option { +fn get_sh_shell(path: Option<&PathBuf>) -> Option { + let shell_path = get_shell_path(ShellType::Sh, path, "sh", vec!["/bin/sh"]); + + shell_path.map(|shell_path| Shell { + shell_type: ShellType::Sh, + shell_path, + }) +} + +fn get_powershell_shell(path: Option<&PathBuf>) -> Option { let shell_path = get_shell_path( ShellType::PowerShell, path, @@ -164,26 +164,56 @@ fn get_powershell_shell(path: Option<&PathBuf>) -> Option { ) .or_else(|| get_shell_path(ShellType::PowerShell, path, "powershell", vec![])); - shell_path.map(|shell_path| PowerShellConfig { shell_path }) + shell_path.map(|shell_path| Shell { + shell_type: ShellType::PowerShell, + shell_path, + }) +} + +fn get_cmd_shell(path: Option<&PathBuf>) -> Option { + let shell_path = get_shell_path(ShellType::Cmd, path, "cmd", vec![]); + + shell_path.map(|shell_path| Shell { + shell_type: ShellType::Cmd, + shell_path, + }) +} + +fn ultimate_fallback_shell() -> Shell { + if cfg!(windows) { + Shell { + shell_type: ShellType::Cmd, + shell_path: PathBuf::from("cmd.exe"), + } + } else { + Shell { + shell_type: ShellType::Sh, + shell_path: PathBuf::from("/bin/sh"), + } + } } pub fn get_shell_by_model_provided_path(shell_path: &PathBuf) -> Shell { detect_shell_type(shell_path) .and_then(|shell_type| get_shell(shell_type, Some(shell_path))) - .unwrap_or(Shell::Unknown) + .unwrap_or(ultimate_fallback_shell()) } pub fn get_shell(shell_type: ShellType, path: Option<&PathBuf>) -> Option { match shell_type { - ShellType::Zsh => get_zsh_shell(path).map(Shell::Zsh), - ShellType::Bash => get_bash_shell(path).map(Shell::Bash), - ShellType::PowerShell => get_powershell_shell(path).map(Shell::PowerShell), + ShellType::Zsh => get_zsh_shell(path), + ShellType::Bash => get_bash_shell(path), + ShellType::PowerShell => get_powershell_shell(path), + ShellType::Sh => get_sh_shell(path), + ShellType::Cmd => get_cmd_shell(path), } } pub fn detect_shell_type(shell_path: &PathBuf) -> Option { match shell_path.as_os_str().to_str() { Some("zsh") => Some(ShellType::Zsh), + Some("sh") => Some(ShellType::Sh), + Some("cmd") => Some(ShellType::Cmd), Some("bash") => Some(ShellType::Bash), Some("pwsh") => Some(ShellType::PowerShell), Some("powershell") => Some(ShellType::PowerShell), @@ -200,11 +230,15 @@ pub fn detect_shell_type(shell_path: &PathBuf) -> Option { } } -pub async fn default_user_shell() -> Shell { +pub fn default_user_shell() -> Shell { + default_user_shell_from_path(get_user_shell_path()) +} + +fn default_user_shell_from_path(user_shell_path: Option) -> Shell { if cfg!(windows) { - get_shell(ShellType::PowerShell, None).unwrap_or(Shell::Unknown) + get_shell(ShellType::PowerShell, None).unwrap_or(ultimate_fallback_shell()) } else { - let user_default_shell = get_user_shell_path() + let user_default_shell = user_shell_path .and_then(|shell| detect_shell_type(&shell)) .and_then(|shell_type| get_shell(shell_type, None)); @@ -218,7 +252,7 @@ pub async fn default_user_shell() -> Shell { .or_else(|| get_shell(ShellType::Zsh, None)) }; - shell_with_fallback.unwrap_or(Shell::Unknown) + shell_with_fallback.unwrap_or(ultimate_fallback_shell()) } } @@ -274,6 +308,19 @@ mod detect_shell_type_tests { detect_shell_type(&PathBuf::from("/usr/local/bin/pwsh")), Some(ShellType::PowerShell) ); + assert_eq!( + detect_shell_type(&PathBuf::from("/bin/sh")), + Some(ShellType::Sh) + ); + assert_eq!(detect_shell_type(&PathBuf::from("sh")), Some(ShellType::Sh)); + assert_eq!( + detect_shell_type(&PathBuf::from("cmd")), + Some(ShellType::Cmd) + ); + assert_eq!( + detect_shell_type(&PathBuf::from("cmd.exe")), + Some(ShellType::Cmd) + ); } } @@ -289,10 +336,17 @@ mod tests { fn detects_zsh() { let zsh_shell = get_shell(ShellType::Zsh, None).unwrap(); - let ZshShell { shell_path } = match zsh_shell { - Shell::Zsh(zsh_shell) => zsh_shell, - _ => panic!("expected zsh shell"), - }; + let shell_path = zsh_shell.shell_path; + + assert_eq!(shell_path, PathBuf::from("/bin/zsh")); + } + + #[test] + #[cfg(target_os = "macos")] + fn fish_fallback_to_zsh() { + let zsh_shell = default_user_shell_from_path(Some(PathBuf::from("/bin/fish"))); + + let shell_path = zsh_shell.shell_path; assert_eq!(shell_path, PathBuf::from("/bin/zsh")); } @@ -300,18 +354,60 @@ mod tests { #[test] fn detects_bash() { let bash_shell = get_shell(ShellType::Bash, None).unwrap(); - let BashShell { shell_path } = match bash_shell { - Shell::Bash(bash_shell) => bash_shell, - _ => panic!("expected bash shell"), - }; + let shell_path = bash_shell.shell_path; assert!( shell_path == PathBuf::from("/bin/bash") - || shell_path == PathBuf::from("/usr/bin/bash"), + || shell_path == PathBuf::from("/usr/bin/bash") + || shell_path == PathBuf::from("/usr/local/bin/bash"), "shell path: {shell_path:?}", ); } + #[test] + fn detects_sh() { + let sh_shell = get_shell(ShellType::Sh, None).unwrap(); + let shell_path = sh_shell.shell_path; + assert!( + shell_path == PathBuf::from("/bin/sh") || shell_path == PathBuf::from("/usr/bin/sh"), + "shell path: {shell_path:?}", + ); + } + + #[test] + fn can_run_on_shell_test() { + let cmd = "echo \"Works\""; + if cfg!(windows) { + assert!(shell_works( + get_shell(ShellType::PowerShell, None), + "Out-String 'Works'", + true, + )); + assert!(shell_works(get_shell(ShellType::Cmd, None), cmd, true,)); + assert!(shell_works(Some(ultimate_fallback_shell()), cmd, true)); + } else { + assert!(shell_works(Some(ultimate_fallback_shell()), cmd, true)); + assert!(shell_works(get_shell(ShellType::Zsh, None), cmd, false)); + assert!(shell_works(get_shell(ShellType::Bash, None), cmd, true)); + assert!(shell_works(get_shell(ShellType::Sh, None), cmd, true)); + } + } + + fn shell_works(shell: Option, command: &str, required: bool) -> bool { + if let Some(shell) = shell { + let args = shell.derive_exec_args(command, false); + let output = Command::new(args[0].clone()) + .args(&args[1..]) + .output() + .unwrap(); + assert!(output.status.success()); + assert!(String::from_utf8_lossy(&output.stdout).contains("Works")); + true + } else { + !required + } + } + #[tokio::test] async fn test_current_shell_detects_zsh() { let shell = Command::new("sh") @@ -323,10 +419,11 @@ mod tests { let shell_path = String::from_utf8_lossy(&shell.stdout).trim().to_string(); if shell_path.ends_with("/zsh") { assert_eq!( - default_user_shell().await, - Shell::Zsh(ZshShell { + default_user_shell(), + Shell { + shell_type: ShellType::Zsh, shell_path: PathBuf::from(shell_path), - }) + } ); } } @@ -337,11 +434,8 @@ mod tests { return; } - let powershell_shell = default_user_shell().await; - let PowerShellConfig { shell_path } = match powershell_shell { - Shell::PowerShell(powershell_shell) => powershell_shell, - _ => panic!("expected powershell shell"), - }; + let powershell_shell = default_user_shell(); + let shell_path = powershell_shell.shell_path; assert!(shell_path.ends_with("pwsh.exe") || shell_path.ends_with("powershell.exe")); } @@ -353,10 +447,7 @@ mod tests { } let powershell_shell = get_shell(ShellType::PowerShell, None).unwrap(); - let PowerShellConfig { shell_path } = match powershell_shell { - Shell::PowerShell(powershell_shell) => powershell_shell, - _ => panic!("expected powershell shell"), - }; + let shell_path = powershell_shell.shell_path; assert!(shell_path.ends_with("pwsh.exe") || shell_path.ends_with("powershell.exe")); } diff --git a/codex-rs/core/src/skills/loader.rs b/codex-rs/core/src/skills/loader.rs new file mode 100644 index 0000000000..a9ea156f02 --- /dev/null +++ b/codex-rs/core/src/skills/loader.rs @@ -0,0 +1,291 @@ +use crate::config::Config; +use crate::skills::model::SkillError; +use crate::skills::model::SkillLoadOutcome; +use crate::skills::model::SkillMetadata; +use dunce::canonicalize as normalize_path; +use serde::Deserialize; +use std::collections::VecDeque; +use std::error::Error; +use std::fmt; +use std::fs; +use std::path::Path; +use std::path::PathBuf; +use tracing::error; + +#[derive(Debug, Deserialize)] +struct SkillFrontmatter { + name: String, + description: String, +} + +const SKILLS_FILENAME: &str = "SKILL.md"; +const SKILLS_DIR_NAME: &str = "skills"; +const MAX_NAME_LEN: usize = 100; +const MAX_DESCRIPTION_LEN: usize = 500; + +#[derive(Debug)] +enum SkillParseError { + Read(std::io::Error), + MissingFrontmatter, + InvalidYaml(serde_yaml::Error), + MissingField(&'static str), + InvalidField { field: &'static str, reason: String }, +} + +impl fmt::Display for SkillParseError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + SkillParseError::Read(e) => write!(f, "failed to read file: {e}"), + SkillParseError::MissingFrontmatter => { + write!(f, "missing YAML frontmatter delimited by ---") + } + SkillParseError::InvalidYaml(e) => write!(f, "invalid YAML: {e}"), + SkillParseError::MissingField(field) => write!(f, "missing field `{field}`"), + SkillParseError::InvalidField { field, reason } => { + write!(f, "invalid {field}: {reason}") + } + } + } +} + +impl Error for SkillParseError {} + +pub fn load_skills(config: &Config) -> SkillLoadOutcome { + let mut outcome = SkillLoadOutcome::default(); + let roots = skill_roots(config); + for root in roots { + discover_skills_under_root(&root, &mut outcome); + } + + outcome + .skills + .sort_by(|a, b| a.name.cmp(&b.name).then_with(|| a.path.cmp(&b.path))); + + outcome +} + +fn skill_roots(config: &Config) -> Vec { + vec![config.codex_home.join(SKILLS_DIR_NAME)] +} + +fn discover_skills_under_root(root: &Path, outcome: &mut SkillLoadOutcome) { + let Ok(root) = normalize_path(root) else { + return; + }; + + if !root.is_dir() { + return; + } + + let mut queue: VecDeque = VecDeque::from([root]); + while let Some(dir) = queue.pop_front() { + let entries = match fs::read_dir(&dir) { + Ok(entries) => entries, + Err(e) => { + error!("failed to read skills dir {}: {e:#}", dir.display()); + continue; + } + }; + + for entry in entries.flatten() { + let path = entry.path(); + let file_name = match path.file_name().and_then(|f| f.to_str()) { + Some(name) => name, + None => continue, + }; + + if file_name.starts_with('.') { + continue; + } + + let Ok(file_type) = entry.file_type() else { + continue; + }; + + if file_type.is_symlink() { + continue; + } + + if file_type.is_dir() { + queue.push_back(path); + continue; + } + + if file_type.is_file() && file_name == SKILLS_FILENAME { + match parse_skill_file(&path) { + Ok(skill) => outcome.skills.push(skill), + Err(err) => outcome.errors.push(SkillError { + path, + message: err.to_string(), + }), + } + } + } + } +} + +fn parse_skill_file(path: &Path) -> Result { + let contents = fs::read_to_string(path).map_err(SkillParseError::Read)?; + + let frontmatter = extract_frontmatter(&contents).ok_or(SkillParseError::MissingFrontmatter)?; + + let parsed: SkillFrontmatter = + serde_yaml::from_str(&frontmatter).map_err(SkillParseError::InvalidYaml)?; + + let name = sanitize_single_line(&parsed.name); + let description = sanitize_single_line(&parsed.description); + + validate_field(&name, MAX_NAME_LEN, "name")?; + validate_field(&description, MAX_DESCRIPTION_LEN, "description")?; + + let resolved_path = normalize_path(path).unwrap_or_else(|_| path.to_path_buf()); + + Ok(SkillMetadata { + name, + description, + path: resolved_path, + }) +} + +fn sanitize_single_line(raw: &str) -> String { + raw.split_whitespace().collect::>().join(" ") +} + +fn validate_field( + value: &str, + max_len: usize, + field_name: &'static str, +) -> Result<(), SkillParseError> { + if value.is_empty() { + return Err(SkillParseError::MissingField(field_name)); + } + if value.len() > max_len { + return Err(SkillParseError::InvalidField { + field: field_name, + reason: format!("exceeds maximum length of {max_len} characters"), + }); + } + Ok(()) +} + +fn extract_frontmatter(contents: &str) -> Option { + let mut lines = contents.lines(); + if !matches!(lines.next(), Some(line) if line.trim() == "---") { + return None; + } + + let mut frontmatter_lines: Vec<&str> = Vec::new(); + let mut found_closing = false; + for line in lines.by_ref() { + if line.trim() == "---" { + found_closing = true; + break; + } + frontmatter_lines.push(line); + } + + if frontmatter_lines.is_empty() || !found_closing { + return None; + } + + Some(frontmatter_lines.join("\n")) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::config::ConfigOverrides; + use crate::config::ConfigToml; + use tempfile::TempDir; + + fn make_config(codex_home: &TempDir) -> Config { + let mut config = Config::load_from_base_config_with_overrides( + ConfigToml::default(), + ConfigOverrides::default(), + codex_home.path().to_path_buf(), + ) + .expect("defaults for test should always succeed"); + + config.cwd = codex_home.path().to_path_buf(); + config + } + + fn write_skill(codex_home: &TempDir, dir: &str, name: &str, description: &str) -> PathBuf { + let skill_dir = codex_home.path().join(format!("skills/{dir}")); + fs::create_dir_all(&skill_dir).unwrap(); + let indented_description = description.replace('\n', "\n "); + let content = format!( + "---\nname: {name}\ndescription: |-\n {indented_description}\n---\n\n# Body\n" + ); + let path = skill_dir.join(SKILLS_FILENAME); + fs::write(&path, content).unwrap(); + path + } + + #[test] + fn loads_valid_skill() { + let codex_home = tempfile::tempdir().expect("tempdir"); + write_skill(&codex_home, "demo", "demo-skill", "does things\ncarefully"); + let cfg = make_config(&codex_home); + + let outcome = load_skills(&cfg); + assert!( + outcome.errors.is_empty(), + "unexpected errors: {:?}", + outcome.errors + ); + assert_eq!(outcome.skills.len(), 1); + let skill = &outcome.skills[0]; + assert_eq!(skill.name, "demo-skill"); + assert_eq!(skill.description, "does things carefully"); + let path_str = skill.path.to_string_lossy().replace('\\', "/"); + assert!( + path_str.ends_with("skills/demo/SKILL.md"), + "unexpected path {path_str}" + ); + } + + #[test] + fn skips_hidden_and_invalid() { + let codex_home = tempfile::tempdir().expect("tempdir"); + let hidden_dir = codex_home.path().join("skills/.hidden"); + fs::create_dir_all(&hidden_dir).unwrap(); + fs::write( + hidden_dir.join(SKILLS_FILENAME), + "---\nname: hidden\ndescription: hidden\n---\n", + ) + .unwrap(); + + // Invalid because missing closing frontmatter. + let invalid_dir = codex_home.path().join("skills/invalid"); + fs::create_dir_all(&invalid_dir).unwrap(); + fs::write(invalid_dir.join(SKILLS_FILENAME), "---\nname: bad").unwrap(); + + let cfg = make_config(&codex_home); + let outcome = load_skills(&cfg); + assert_eq!(outcome.skills.len(), 0); + assert_eq!(outcome.errors.len(), 1); + assert!( + outcome.errors[0] + .message + .contains("missing YAML frontmatter"), + "expected frontmatter error" + ); + } + + #[test] + fn enforces_length_limits() { + let codex_home = tempfile::tempdir().expect("tempdir"); + let long_desc = "a".repeat(MAX_DESCRIPTION_LEN + 1); + write_skill(&codex_home, "too-long", "toolong", &long_desc); + let cfg = make_config(&codex_home); + + let outcome = load_skills(&cfg); + assert_eq!(outcome.skills.len(), 0); + assert_eq!(outcome.errors.len(), 1); + assert!( + outcome.errors[0].message.contains("invalid description"), + "expected length error" + ); + } +} diff --git a/codex-rs/core/src/skills/mod.rs b/codex-rs/core/src/skills/mod.rs new file mode 100644 index 0000000000..ebb1490c99 --- /dev/null +++ b/codex-rs/core/src/skills/mod.rs @@ -0,0 +1,9 @@ +pub mod loader; +pub mod model; +pub mod render; + +pub use loader::load_skills; +pub use model::SkillError; +pub use model::SkillLoadOutcome; +pub use model::SkillMetadata; +pub use render::render_skills_section; diff --git a/codex-rs/core/src/skills/model.rs b/codex-rs/core/src/skills/model.rs new file mode 100644 index 0000000000..83a3317d57 --- /dev/null +++ b/codex-rs/core/src/skills/model.rs @@ -0,0 +1,20 @@ +use std::path::PathBuf; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SkillMetadata { + pub name: String, + pub description: String, + pub path: PathBuf, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SkillError { + pub path: PathBuf, + pub message: String, +} + +#[derive(Debug, Clone, Default)] +pub struct SkillLoadOutcome { + pub skills: Vec, + pub errors: Vec, +} diff --git a/codex-rs/core/src/skills/render.rs b/codex-rs/core/src/skills/render.rs new file mode 100644 index 0000000000..d547e21c28 --- /dev/null +++ b/codex-rs/core/src/skills/render.rs @@ -0,0 +1,21 @@ +use crate::skills::model::SkillMetadata; + +pub fn render_skills_section(skills: &[SkillMetadata]) -> Option { + if skills.is_empty() { + return None; + } + + let mut lines: Vec = Vec::new(); + lines.push("## Skills".to_string()); + lines.push("These skills are discovered at startup from ~/.codex/skills; each entry shows name, description, and file path so you can open the source for full instructions. Content is not inlined to keep context lean.".to_string()); + + for skill in skills { + let path_str = skill.path.to_string_lossy().replace('\\', "/"); + lines.push(format!( + "- {}: {} (file: {})", + skill.name, skill.description, path_str + )); + } + + Some(lines.join("\n")) +} diff --git a/codex-rs/core/src/state/session.rs b/codex-rs/core/src/state/session.rs index 2dfa5199f1..8c739c9243 100644 --- a/codex-rs/core/src/state/session.rs +++ b/codex-rs/core/src/state/session.rs @@ -62,7 +62,10 @@ impl SessionState { } pub(crate) fn set_rate_limits(&mut self, snapshot: RateLimitSnapshot) { - self.latest_rate_limits = Some(snapshot); + self.latest_rate_limits = Some(merge_rate_limit_credits( + self.latest_rate_limits.as_ref(), + snapshot, + )); } pub(crate) fn token_info_and_rate_limits( @@ -74,4 +77,19 @@ impl SessionState { pub(crate) fn set_token_usage_full(&mut self, context_window: i64) { self.history.set_token_usage_full(context_window); } + + pub(crate) fn get_total_token_usage(&self) -> i64 { + self.history.get_total_token_usage() + } +} + +// Sometimes new snapshots don't include credits +fn merge_rate_limit_credits( + previous: Option<&RateLimitSnapshot>, + mut snapshot: RateLimitSnapshot, +) -> RateLimitSnapshot { + if snapshot.credits.is_none() { + snapshot.credits = previous.and_then(|prior| prior.credits.clone()); + } + snapshot } diff --git a/codex-rs/core/src/tasks/review.rs b/codex-rs/core/src/tasks/review.rs index 14a95dba5c..a6ec840a84 100644 --- a/codex-rs/core/src/tasks/review.rs +++ b/codex-rs/core/src/tasks/review.rs @@ -17,6 +17,7 @@ use crate::codex::Session; use crate::codex::TurnContext; use crate::codex_delegate::run_codex_conversation_one_shot; use crate::review_format::format_review_findings_block; +use crate::review_format::render_review_output_text; use crate::state::TaskKind; use codex_protocol::user_input::UserInput; @@ -24,15 +25,11 @@ use super::SessionTask; use super::SessionTaskContext; #[derive(Clone, Copy)] -pub(crate) struct ReviewTask { - append_to_original_thread: bool, -} +pub(crate) struct ReviewTask; impl ReviewTask { - pub(crate) fn new(append_to_original_thread: bool) -> Self { - Self { - append_to_original_thread, - } + pub(crate) fn new() -> Self { + Self } } @@ -62,25 +59,13 @@ impl SessionTask for ReviewTask { None => None, }; if !cancellation_token.is_cancelled() { - exit_review_mode( - session.clone_session(), - output.clone(), - ctx.clone(), - self.append_to_original_thread, - ) - .await; + exit_review_mode(session.clone_session(), output.clone(), ctx.clone()).await; } None } async fn abort(&self, session: Arc, ctx: Arc) { - exit_review_mode( - session.clone_session(), - None, - ctx, - self.append_to_original_thread, - ) - .await; + exit_review_mode(session.clone_session(), None, ctx).await; } } @@ -197,39 +182,57 @@ pub(crate) async fn exit_review_mode( session: Arc, review_output: Option, ctx: Arc, - append_to_original_thread: bool, ) { - if append_to_original_thread { - let user_message = if let Some(out) = review_output.clone() { - let mut findings_str = String::new(); - let text = out.overall_explanation.trim(); - if !text.is_empty() { - findings_str.push_str(text); - } - if !out.findings.is_empty() { - let block = format_review_findings_block(&out.findings, None); - findings_str.push_str(&format!("\n{block}")); - } - crate::client_common::REVIEW_EXIT_SUCCESS_TMPL.replace("{results}", &findings_str) - } else { - crate::client_common::REVIEW_EXIT_INTERRUPTED_TMPL.to_string() - }; + const REVIEW_USER_MESSAGE_ID: &str = "review:rollout:user"; + const REVIEW_ASSISTANT_MESSAGE_ID: &str = "review:rollout:assistant"; + let (user_message, assistant_message) = if let Some(out) = review_output.clone() { + let mut findings_str = String::new(); + let text = out.overall_explanation.trim(); + if !text.is_empty() { + findings_str.push_str(text); + } + if !out.findings.is_empty() { + let block = format_review_findings_block(&out.findings, None); + findings_str.push_str(&format!("\n{block}")); + } + let rendered = + crate::client_common::REVIEW_EXIT_SUCCESS_TMPL.replace("{results}", &findings_str); + let assistant_message = render_review_output_text(&out); + (rendered, assistant_message) + } else { + let rendered = crate::client_common::REVIEW_EXIT_INTERRUPTED_TMPL.to_string(); + let assistant_message = + "Review was interrupted. Please re-run /review and wait for it to complete." + .to_string(); + (rendered, assistant_message) + }; - session - .record_conversation_items( - &ctx, - &[ResponseItem::Message { - id: None, - role: "user".to_string(), - content: vec![ContentItem::InputText { text: user_message }], - }], - ) - .await; - } + session + .record_conversation_items( + &ctx, + &[ResponseItem::Message { + id: Some(REVIEW_USER_MESSAGE_ID.to_string()), + role: "user".to_string(), + content: vec![ContentItem::InputText { text: user_message }], + }], + ) + .await; session .send_event( ctx.as_ref(), EventMsg::ExitedReviewMode(ExitedReviewModeEvent { review_output }), ) .await; + session + .record_response_item_and_emit_turn_item( + ctx.as_ref(), + ResponseItem::Message { + id: Some(REVIEW_ASSISTANT_MESSAGE_ID.to_string()), + role: "assistant".to_string(), + content: vec![ContentItem::OutputText { + text: assistant_message, + }], + }, + ) + .await; } diff --git a/codex-rs/core/src/tasks/user_shell.rs b/codex-rs/core/src/tasks/user_shell.rs index 9644d3678e..ca5243241a 100644 --- a/codex-rs/core/src/tasks/user_shell.rs +++ b/codex-rs/core/src/tasks/user_shell.rs @@ -31,6 +31,8 @@ use crate::user_shell_command::user_shell_command_record_item; use super::SessionTask; use super::SessionTaskContext; +const USER_SHELL_TIMEOUT_MS: u64 = 60 * 60 * 1000; // 1 hour + #[derive(Clone)] pub(crate) struct UserShellCommandTask { command: String, @@ -79,6 +81,7 @@ impl SessionTask for UserShellCommandTask { turn_context.as_ref(), EventMsg::ExecCommandBegin(ExecCommandBeginEvent { call_id: call_id.clone(), + process_id: None, turn_id: turn_context.sub_id.clone(), command: command.clone(), cwd: cwd.clone(), @@ -93,7 +96,9 @@ impl SessionTask for UserShellCommandTask { command: command.clone(), cwd: cwd.clone(), env: create_env(&turn_context.shell_environment_policy), - timeout_ms: None, + // TODO(zhao-oai): Now that we have ExecExpiration::Cancellation, we + // should use that instead of an "arbitrarily large" timeout here. + expiration: USER_SHELL_TIMEOUT_MS.into(), sandbox: SandboxType::None, with_escalated_permissions: None, justification: None, @@ -135,6 +140,7 @@ impl SessionTask for UserShellCommandTask { turn_context.as_ref(), EventMsg::ExecCommandEnd(ExecCommandEndEvent { call_id, + process_id: None, turn_id: turn_context.sub_id.clone(), command: command.clone(), cwd: cwd.clone(), @@ -157,6 +163,7 @@ impl SessionTask for UserShellCommandTask { turn_context.as_ref(), EventMsg::ExecCommandEnd(ExecCommandEndEvent { call_id: call_id.clone(), + process_id: None, turn_id: turn_context.sub_id.clone(), command: command.clone(), cwd: cwd.clone(), @@ -201,6 +208,7 @@ impl SessionTask for UserShellCommandTask { turn_context.as_ref(), EventMsg::ExecCommandEnd(ExecCommandEndEvent { call_id, + process_id: None, turn_id: turn_context.sub_id.clone(), command, cwd, diff --git a/codex-rs/core/src/text_encoding.rs b/codex-rs/core/src/text_encoding.rs new file mode 100644 index 0000000000..fde44c4195 --- /dev/null +++ b/codex-rs/core/src/text_encoding.rs @@ -0,0 +1,461 @@ +//! Text encoding detection and conversion utilities for shell output. +//! +//! Windows users frequently run into code pages such as CP1251 or CP866 when invoking commands +//! through VS Code. Those bytes show up as invalid UTF-8 and used to be replaced with the standard +//! Unicode replacement character. We now lean on `chardetng` and `encoding_rs` so we can +//! automatically detect and decode the vast majority of legacy encodings before falling back to +//! lossy UTF-8 decoding. + +use chardetng::EncodingDetector; +use encoding_rs::Encoding; +use encoding_rs::IBM866; +use encoding_rs::WINDOWS_1252; + +/// Attempts to convert arbitrary bytes to UTF-8 with best-effort encoding detection. +pub fn bytes_to_string_smart(bytes: &[u8]) -> String { + if bytes.is_empty() { + return String::new(); + } + + if let Ok(utf8_str) = std::str::from_utf8(bytes) { + return utf8_str.to_owned(); + } + + let encoding = detect_encoding(bytes); + decode_bytes(bytes, encoding) +} + +// Windows-1252 reassigns a handful of 0x80-0x9F slots to smart punctuation (curly quotes, dashes, +// ™). CP866 uses those *same byte values* for uppercase Cyrillic letters. When chardetng sees shell +// snippets that mix these bytes with ASCII it sometimes guesses IBM866, so “smart quotes” render as +// Cyrillic garbage (“УФЦ”) in VS Code. However, CP866 uppercase tokens are perfectly valid output +// (e.g., `ПРИ test`) so we cannot flip every 0x80-0x9F byte to Windows-1252 either. The compromise +// is to only coerce IBM866 to Windows-1252 when (a) the high bytes are exclusively the punctuation +// values listed below and (b) we spot adjacent ASCII. This targets the real failure case without +// clobbering legitimate Cyrillic text. If another code page has a similar collision, introduce a +// dedicated allowlist (like this one) plus unit tests that capture the actual shell output we want +// to preserve. Windows-1252 byte values for smart punctuation. +const WINDOWS_1252_PUNCT_BYTES: [u8; 8] = [ + 0x91, // ‘ (left single quotation mark) + 0x92, // ’ (right single quotation mark) + 0x93, // “ (left double quotation mark) + 0x94, // ” (right double quotation mark) + 0x95, // • (bullet) + 0x96, // – (en dash) + 0x97, // — (em dash) + 0x99, // ™ (trade mark sign) +]; + +fn detect_encoding(bytes: &[u8]) -> &'static Encoding { + let mut detector = EncodingDetector::new(); + detector.feed(bytes, true); + let (encoding, _is_confident) = detector.guess_assess(None, true); + + // chardetng occasionally reports IBM866 for short strings that only contain Windows-1252 “smart + // punctuation” bytes (0x80-0x9F) because that range maps to Cyrillic letters in IBM866. When + // those bytes show up alongside an ASCII word (typical shell output: `"“`test), we know the + // intent was likely CP1252 quotes/dashes. Prefer WINDOWS_1252 in that specific situation so we + // render the characters users expect instead of Cyrillic junk. References: + // - Windows-1252 reserving 0x80-0x9F for curly quotes/dashes: + // https://en.wikipedia.org/wiki/Windows-1252 + // - CP866 mapping 0x93/0x94/0x96 to Cyrillic letters, so the same bytes show up as “УФЦ” when + // mis-decoded: https://www.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/PC/CP866.TXT + if encoding == IBM866 && looks_like_windows_1252_punctuation(bytes) { + return WINDOWS_1252; + } + + encoding +} + +fn decode_bytes(bytes: &[u8], encoding: &'static Encoding) -> String { + let (decoded, _, had_errors) = encoding.decode(bytes); + + if had_errors { + return String::from_utf8_lossy(bytes).into_owned(); + } + + decoded.into_owned() +} + +/// Detect whether the byte stream looks like Windows-1252 “smart punctuation” wrapped around +/// otherwise-ASCII text. +/// +/// Context: IBM866 and Windows-1252 share the 0x80-0x9F slot range. In IBM866 these bytes decode to +/// Cyrillic letters, whereas Windows-1252 maps them to curly quotes and dashes. chardetng can guess +/// IBM866 for short snippets that only contain those bytes, which turns shell output such as +/// `“test”` into unreadable Cyrillic. To avoid that, we treat inputs comprising a handful of bytes +/// from the problematic range plus ASCII letters as CP1252 punctuation. We deliberately do *not* +/// cap how many of those punctuation bytes we accept: VS Code frequently prints several quoted +/// phrases (e.g., `"foo" – "bar"`), and truncating the count would once again mis-decode those as +/// Cyrillic. If we discover additional encodings with overlapping byte ranges, prefer adding +/// encoding-specific byte allowlists like `WINDOWS_1252_PUNCT` and tests that exercise real-world +/// shell snippets. +fn looks_like_windows_1252_punctuation(bytes: &[u8]) -> bool { + let mut saw_extended_punctuation = false; + let mut saw_ascii_word = false; + + for &byte in bytes { + if byte >= 0xA0 { + return false; + } + if (0x80..=0x9F).contains(&byte) { + if !is_windows_1252_punct(byte) { + return false; + } + saw_extended_punctuation = true; + } + if byte.is_ascii_alphabetic() { + saw_ascii_word = true; + } + } + + saw_extended_punctuation && saw_ascii_word +} + +fn is_windows_1252_punct(byte: u8) -> bool { + WINDOWS_1252_PUNCT_BYTES.contains(&byte) +} + +#[cfg(test)] +mod tests { + use super::*; + use encoding_rs::BIG5; + use encoding_rs::EUC_KR; + use encoding_rs::GBK; + use encoding_rs::ISO_8859_2; + use encoding_rs::ISO_8859_3; + use encoding_rs::ISO_8859_4; + use encoding_rs::ISO_8859_5; + use encoding_rs::ISO_8859_6; + use encoding_rs::ISO_8859_7; + use encoding_rs::ISO_8859_8; + use encoding_rs::ISO_8859_10; + use encoding_rs::ISO_8859_13; + use encoding_rs::SHIFT_JIS; + use encoding_rs::WINDOWS_874; + use encoding_rs::WINDOWS_1250; + use encoding_rs::WINDOWS_1251; + use encoding_rs::WINDOWS_1253; + use encoding_rs::WINDOWS_1254; + use encoding_rs::WINDOWS_1255; + use encoding_rs::WINDOWS_1256; + use encoding_rs::WINDOWS_1257; + use encoding_rs::WINDOWS_1258; + use pretty_assertions::assert_eq; + + #[test] + fn test_utf8_passthrough() { + // Fast path: when UTF-8 is valid we should avoid copies and return as-is. + let utf8_text = "Hello, мир! 世界"; + let bytes = utf8_text.as_bytes(); + assert_eq!(bytes_to_string_smart(bytes), utf8_text); + } + + #[test] + fn test_cp1251_russian_text() { + // Cyrillic text emitted by PowerShell/WSL in CP1251 should decode cleanly. + let bytes = b"\xEF\xF0\xE8\xEC\xE5\xF0"; // "пример" encoded with Windows-1251 + assert_eq!(bytes_to_string_smart(bytes), "пример"); + } + + #[test] + fn test_cp1251_privet_word() { + // Regression: CP1251 words like "Привет" must not be mis-identified as Windows-1252. + let bytes = b"\xCF\xF0\xE8\xE2\xE5\xF2"; // "Привет" encoded with Windows-1251 + assert_eq!(bytes_to_string_smart(bytes), "Привет"); + } + + #[test] + fn test_koi8_r_privet_word() { + // KOI8-R output should decode to the original Cyrillic as well. + let bytes = b"\xF0\xD2\xC9\xD7\xC5\xD4"; // "Привет" encoded with KOI8-R + assert_eq!(bytes_to_string_smart(bytes), "Привет"); + } + + #[test] + fn test_cp866_russian_text() { + // Legacy consoles (cmd.exe) commonly emit CP866 bytes for Cyrillic content. + let bytes = b"\xAF\xE0\xA8\xAC\xA5\xE0"; // "пример" encoded with CP866 + assert_eq!(bytes_to_string_smart(bytes), "пример"); + } + + #[test] + fn test_cp866_uppercase_text() { + // Ensure the IBM866 heuristic still returns IBM866 for uppercase-only words. + let bytes = b"\x8F\x90\x88"; // "ПРИ" encoded with CP866 uppercase letters + assert_eq!(bytes_to_string_smart(bytes), "ПРИ"); + } + + #[test] + fn test_cp866_uppercase_followed_by_ascii() { + // Regression test: uppercase CP866 tokens next to ASCII text should not be treated as + // CP1252. + let bytes = b"\x8F\x90\x88 test"; // "ПРИ test" encoded with CP866 uppercase letters followed by ASCII + assert_eq!(bytes_to_string_smart(bytes), "ПРИ test"); + } + + #[test] + fn test_windows_1252_quotes() { + // Smart detection should map Windows-1252 punctuation into proper Unicode. + let bytes = b"\x93\x94test"; + assert_eq!(bytes_to_string_smart(bytes), "\u{201C}\u{201D}test"); + } + + #[test] + fn test_windows_1252_multiple_quotes() { + // Longer snippets of punctuation (e.g., “foo” – “bar”) should still flip to CP1252. + let bytes = b"\x93foo\x94 \x96 \x93bar\x94"; + assert_eq!( + bytes_to_string_smart(bytes), + "\u{201C}foo\u{201D} \u{2013} \u{201C}bar\u{201D}" + ); + } + + #[test] + fn test_windows_1252_privet_gibberish_is_preserved() { + // Windows-1252 cannot encode Cyrillic; if the input literally contains "ПÑ..." we should not "fix" it. + let bytes = "Привет".as_bytes(); + assert_eq!(bytes_to_string_smart(bytes), "Привет"); + } + + #[test] + fn test_iso8859_1_latin_text() { + // ISO-8859-1 (code page 28591) is the Latin segment used by LatArCyrHeb. + // encoding_rs unifies ISO-8859-1 with Windows-1252, so reuse that constant here. + let (encoded, _, had_errors) = WINDOWS_1252.encode("Hello"); + assert!(!had_errors, "failed to encode Latin sample"); + assert_eq!(bytes_to_string_smart(encoded.as_ref()), "Hello"); + } + + #[test] + fn test_iso8859_2_central_european_text() { + // ISO-8859-2 (code page 28592) covers additional Central European glyphs. + let (encoded, _, had_errors) = ISO_8859_2.encode("Příliš žluťoučký kůň"); + assert!(!had_errors, "failed to encode ISO-8859-2 sample"); + assert_eq!( + bytes_to_string_smart(encoded.as_ref()), + "Příliš žluťoučký kůň" + ); + } + + #[test] + fn test_iso8859_3_south_europe_text() { + // ISO-8859-3 (code page 28593) adds support for Maltese/Esperanto letters. + // chardetng rarely distinguishes ISO-8859-3 from neighboring Latin code pages, so we rely on + // an ASCII-only sample to ensure round-tripping still succeeds. + let (encoded, _, had_errors) = ISO_8859_3.encode("Esperanto and Maltese"); + assert!(!had_errors, "failed to encode ISO-8859-3 sample"); + assert_eq!( + bytes_to_string_smart(encoded.as_ref()), + "Esperanto and Maltese" + ); + } + + #[test] + fn test_iso8859_4_baltic_text() { + // ISO-8859-4 (code page 28594) targets the Baltic/Nordic repertoire. + let sample = "Šis ir rakstzīmju kodēšanas tests. Dažās valodās, kurās tiek \ + izmantotas latīņu valodas burti, lēmuma pieņemšanai mums ir nepieciešams \ + vairāk ieguldījuma."; + let (encoded, _, had_errors) = ISO_8859_4.encode(sample); + assert!(!had_errors, "failed to encode ISO-8859-4 sample"); + assert_eq!(bytes_to_string_smart(encoded.as_ref()), sample); + } + + #[test] + fn test_iso8859_5_cyrillic_text() { + // ISO-8859-5 (code page 28595) covers the Cyrillic portion. + let (encoded, _, had_errors) = ISO_8859_5.encode("Привет"); + assert!(!had_errors, "failed to encode Cyrillic sample"); + assert_eq!(bytes_to_string_smart(encoded.as_ref()), "Привет"); + } + + #[test] + fn test_iso8859_6_arabic_text() { + // ISO-8859-6 (code page 28596) covers the Arabic glyphs. + let (encoded, _, had_errors) = ISO_8859_6.encode("مرحبا"); + assert!(!had_errors, "failed to encode Arabic sample"); + assert_eq!(bytes_to_string_smart(encoded.as_ref()), "مرحبا"); + } + + #[test] + fn test_iso8859_7_greek_text() { + // ISO-8859-7 (code page 28597) is used for Greek locales. + let (encoded, _, had_errors) = ISO_8859_7.encode("Καλημέρα"); + assert!(!had_errors, "failed to encode ISO-8859-7 sample"); + assert_eq!(bytes_to_string_smart(encoded.as_ref()), "Καλημέρα"); + } + + #[test] + fn test_iso8859_8_hebrew_text() { + // ISO-8859-8 (code page 28598) covers the Hebrew glyphs. + let (encoded, _, had_errors) = ISO_8859_8.encode("שלום"); + assert!(!had_errors, "failed to encode Hebrew sample"); + assert_eq!(bytes_to_string_smart(encoded.as_ref()), "שלום"); + } + + #[test] + fn test_iso8859_9_turkish_text() { + // ISO-8859-9 (code page 28599) mirrors Latin-1 but inserts Turkish letters. + // encoding_rs exposes the equivalent Windows-1254 mapping. + let (encoded, _, had_errors) = WINDOWS_1254.encode("İstanbul"); + assert!(!had_errors, "failed to encode ISO-8859-9 sample"); + assert_eq!(bytes_to_string_smart(encoded.as_ref()), "İstanbul"); + } + + #[test] + fn test_iso8859_10_nordic_text() { + // ISO-8859-10 (code page 28600) adds additional Nordic letters. + let sample = "Þetta er prófun fyrir Ægir og Øystein."; + let (encoded, _, had_errors) = ISO_8859_10.encode(sample); + assert!(!had_errors, "failed to encode ISO-8859-10 sample"); + assert_eq!(bytes_to_string_smart(encoded.as_ref()), sample); + } + + #[test] + fn test_iso8859_11_thai_text() { + // ISO-8859-11 (code page 28601) mirrors TIS-620 / Windows-874 for Thai. + let sample = "ภาษาไทยสำหรับการทดสอบ ISO-8859-11"; + // encoding_rs exposes the equivalent Windows-874 encoding, so use that constant. + let (encoded, _, had_errors) = WINDOWS_874.encode(sample); + assert!(!had_errors, "failed to encode ISO-8859-11 sample"); + assert_eq!(bytes_to_string_smart(encoded.as_ref()), sample); + } + + // ISO-8859-12 was never standardized, and encodings 14–16 cannot be distinguished reliably + // without the heuristics we removed (chardetng generally reports neighboring Latin pages), so + // we intentionally omit coverage for those slots until the detector can identify them. + + #[test] + fn test_iso8859_13_baltic_text() { + // ISO-8859-13 (code page 28603) is common across Baltic languages. + let (encoded, _, had_errors) = ISO_8859_13.encode("Sveiki"); + assert!(!had_errors, "failed to encode ISO-8859-13 sample"); + assert_eq!(bytes_to_string_smart(encoded.as_ref()), "Sveiki"); + } + + #[test] + fn test_windows_1250_central_european_text() { + let (encoded, _, had_errors) = WINDOWS_1250.encode("Příliš žluťoučký kůň"); + assert!(!had_errors, "failed to encode Central European sample"); + assert_eq!( + bytes_to_string_smart(encoded.as_ref()), + "Příliš žluťoučký kůň" + ); + } + + #[test] + fn test_windows_1251_encoded_text() { + let (encoded, _, had_errors) = WINDOWS_1251.encode("Привет из Windows-1251"); + assert!(!had_errors, "failed to encode Windows-1251 sample"); + assert_eq!( + bytes_to_string_smart(encoded.as_ref()), + "Привет из Windows-1251" + ); + } + + #[test] + fn test_windows_1253_greek_text() { + let (encoded, _, had_errors) = WINDOWS_1253.encode("Γειά σου"); + assert!(!had_errors, "failed to encode Greek sample"); + assert_eq!(bytes_to_string_smart(encoded.as_ref()), "Γειά σου"); + } + + #[test] + fn test_windows_1254_turkish_text() { + let (encoded, _, had_errors) = WINDOWS_1254.encode("İstanbul"); + assert!(!had_errors, "failed to encode Turkish sample"); + assert_eq!(bytes_to_string_smart(encoded.as_ref()), "İstanbul"); + } + + #[test] + fn test_windows_1255_hebrew_text() { + let (encoded, _, had_errors) = WINDOWS_1255.encode("שלום"); + assert!(!had_errors, "failed to encode Windows-1255 Hebrew sample"); + assert_eq!(bytes_to_string_smart(encoded.as_ref()), "שלום"); + } + + #[test] + fn test_windows_1256_arabic_text() { + let (encoded, _, had_errors) = WINDOWS_1256.encode("مرحبا"); + assert!(!had_errors, "failed to encode Windows-1256 Arabic sample"); + assert_eq!(bytes_to_string_smart(encoded.as_ref()), "مرحبا"); + } + + #[test] + fn test_windows_1257_baltic_text() { + let (encoded, _, had_errors) = WINDOWS_1257.encode("Pērkons"); + assert!(!had_errors, "failed to encode Baltic sample"); + assert_eq!(bytes_to_string_smart(encoded.as_ref()), "Pērkons"); + } + + #[test] + fn test_windows_1258_vietnamese_text() { + let (encoded, _, had_errors) = WINDOWS_1258.encode("Xin chào"); + assert!(!had_errors, "failed to encode Vietnamese sample"); + assert_eq!(bytes_to_string_smart(encoded.as_ref()), "Xin chào"); + } + + #[test] + fn test_windows_874_thai_text() { + let (encoded, _, had_errors) = WINDOWS_874.encode("สวัสดีครับ นี่คือการทดสอบภาษาไทย"); + assert!(!had_errors, "failed to encode Thai sample"); + assert_eq!( + bytes_to_string_smart(encoded.as_ref()), + "สวัสดีครับ นี่คือการทดสอบภาษาไทย" + ); + } + + #[test] + fn test_windows_932_shift_jis_text() { + let (encoded, _, had_errors) = SHIFT_JIS.encode("こんにちは"); + assert!(!had_errors, "failed to encode Shift-JIS sample"); + assert_eq!(bytes_to_string_smart(encoded.as_ref()), "こんにちは"); + } + + #[test] + fn test_windows_936_gbk_text() { + let (encoded, _, had_errors) = GBK.encode("你好，世界，这是一个测试"); + assert!(!had_errors, "failed to encode GBK sample"); + assert_eq!( + bytes_to_string_smart(encoded.as_ref()), + "你好，世界，这是一个测试" + ); + } + + #[test] + fn test_windows_949_korean_text() { + let (encoded, _, had_errors) = EUC_KR.encode("안녕하세요"); + assert!(!had_errors, "failed to encode Korean sample"); + assert_eq!(bytes_to_string_smart(encoded.as_ref()), "안녕하세요"); + } + + #[test] + fn test_windows_950_big5_text() { + let (encoded, _, had_errors) = BIG5.encode("繁體"); + assert!(!had_errors, "failed to encode Big5 sample"); + assert_eq!(bytes_to_string_smart(encoded.as_ref()), "繁體"); + } + + #[test] + fn test_latin1_cafe() { + // Latin-1 bytes remain common in Western-European locales; decode them directly. + let bytes = b"caf\xE9"; // codespell:ignore caf + assert_eq!(bytes_to_string_smart(bytes), "café"); + } + + #[test] + fn test_preserves_ansi_sequences() { + // ANSI escape sequences should survive regardless of the detected encoding. + let bytes = b"\x1b[31mred\x1b[0m"; + assert_eq!(bytes_to_string_smart(bytes), "\x1b[31mred\x1b[0m"); + } + + #[test] + fn test_fallback_to_lossy() { + // Completely invalid sequences fall back to the old lossy behavior. + let invalid_bytes = [0xFF, 0xFE, 0xFD]; + let result = bytes_to_string_smart(&invalid_bytes); + assert_eq!(result, String::from_utf8_lossy(&invalid_bytes)); + } +} diff --git a/codex-rs/core/src/tools/events.rs b/codex-rs/core/src/tools/events.rs index 37df12d4b0..93bce60489 100644 --- a/codex-rs/core/src/tools/events.rs +++ b/codex-rs/core/src/tools/events.rs @@ -65,12 +65,14 @@ pub(crate) async fn emit_exec_command_begin( parsed_cmd: &[ParsedCommand], source: ExecCommandSource, interaction_input: Option, + process_id: Option<&str>, ) { ctx.session .send_event( ctx.turn, EventMsg::ExecCommandBegin(ExecCommandBeginEvent { call_id: ctx.call_id.to_string(), + process_id: process_id.map(str::to_owned), turn_id: ctx.turn.sub_id.clone(), command: command.to_vec(), cwd: cwd.to_path_buf(), @@ -100,6 +102,7 @@ pub(crate) enum ToolEmitter { source: ExecCommandSource, interaction_input: Option, parsed_cmd: Vec, + process_id: Option, }, } @@ -132,6 +135,7 @@ impl ToolEmitter { cwd: PathBuf, source: ExecCommandSource, interaction_input: Option, + process_id: Option, ) -> Self { let parsed_cmd = parse_command(command); Self::UnifiedExec { @@ -140,6 +144,7 @@ impl ToolEmitter { source, interaction_input, parsed_cmd, + process_id, } } @@ -157,7 +162,7 @@ impl ToolEmitter { ) => { emit_exec_stage( ctx, - ExecCommandInput::new(command, cwd.as_path(), parsed_cmd, *source, None), + ExecCommandInput::new(command, cwd.as_path(), parsed_cmd, *source, None, None), stage, ) .await; @@ -229,6 +234,7 @@ impl ToolEmitter { source, interaction_input, parsed_cmd, + process_id, }, stage, ) => { @@ -240,6 +246,7 @@ impl ToolEmitter { parsed_cmd, *source, interaction_input.as_deref(), + process_id.as_deref(), ), stage, ) @@ -319,6 +326,7 @@ struct ExecCommandInput<'a> { parsed_cmd: &'a [ParsedCommand], source: ExecCommandSource, interaction_input: Option<&'a str>, + process_id: Option<&'a str>, } impl<'a> ExecCommandInput<'a> { @@ -328,6 +336,7 @@ impl<'a> ExecCommandInput<'a> { parsed_cmd: &'a [ParsedCommand], source: ExecCommandSource, interaction_input: Option<&'a str>, + process_id: Option<&'a str>, ) -> Self { Self { command, @@ -335,6 +344,7 @@ impl<'a> ExecCommandInput<'a> { parsed_cmd, source, interaction_input, + process_id, } } } @@ -362,6 +372,7 @@ async fn emit_exec_stage( exec_input.parsed_cmd, exec_input.source, exec_input.interaction_input.map(str::to_owned), + exec_input.process_id, ) .await; } @@ -402,6 +413,7 @@ async fn emit_exec_end( ctx.turn, EventMsg::ExecCommandEnd(ExecCommandEndEvent { call_id: ctx.call_id.to_string(), + process_id: exec_input.process_id.map(str::to_owned), turn_id: ctx.turn.sub_id.clone(), command: exec_input.command.to_vec(), cwd: exec_input.cwd.to_path_buf(), diff --git a/codex-rs/core/src/tools/handlers/apply_patch.rs b/codex-rs/core/src/tools/handlers/apply_patch.rs index 2109f1d2c8..4a28619c76 100644 --- a/codex-rs/core/src/tools/handlers/apply_patch.rs +++ b/codex-rs/core/src/tools/handlers/apply_patch.rs @@ -1,4 +1,5 @@ use std::collections::BTreeMap; +use std::path::Path; use crate::apply_patch; use crate::apply_patch::InternalApplyPatchInvocation; @@ -7,7 +8,10 @@ use crate::client_common::tools::FreeformTool; use crate::client_common::tools::FreeformToolFormat; use crate::client_common::tools::ResponsesApiTool; use crate::client_common::tools::ToolSpec; +use crate::codex::Session; +use crate::codex::TurnContext; use crate::function_tool::FunctionCallError; +use crate::tools::context::SharedTurnDiffTracker; use crate::tools::context::ToolInvocation; use crate::tools::context::ToolOutput; use crate::tools::context::ToolPayload; @@ -164,6 +168,86 @@ pub enum ApplyPatchToolType { Function, } +#[allow(clippy::too_many_arguments)] +pub(crate) async fn intercept_apply_patch( + command: &[String], + cwd: &Path, + timeout_ms: Option, + session: &Session, + turn: &TurnContext, + tracker: Option<&SharedTurnDiffTracker>, + call_id: &str, + tool_name: &str, +) -> Result, FunctionCallError> { + match codex_apply_patch::maybe_parse_apply_patch_verified(command, cwd) { + codex_apply_patch::MaybeApplyPatchVerified::Body(changes) => { + session + .record_model_warning( + format!("apply_patch was requested via {tool_name}. Use the apply_patch tool instead of exec_command."), + turn, + ) + .await; + match apply_patch::apply_patch(session, turn, call_id, changes).await { + InternalApplyPatchInvocation::Output(item) => { + let content = item?; + Ok(Some(ToolOutput::Function { + content, + content_items: None, + success: Some(true), + })) + } + InternalApplyPatchInvocation::DelegateToExec(apply) => { + let emitter = ToolEmitter::apply_patch( + convert_apply_patch_to_protocol(&apply.action), + !apply.user_explicitly_approved_this_action, + ); + let event_ctx = + ToolEventCtx::new(session, turn, call_id, tracker.as_ref().copied()); + emitter.begin(event_ctx).await; + + let req = ApplyPatchRequest { + patch: apply.action.patch.clone(), + cwd: apply.action.cwd.clone(), + timeout_ms, + user_explicitly_approved: apply.user_explicitly_approved_this_action, + codex_exe: turn.codex_linux_sandbox_exe.clone(), + }; + + let mut orchestrator = ToolOrchestrator::new(); + let mut runtime = ApplyPatchRuntime::new(); + let tool_ctx = ToolCtx { + session, + turn, + call_id: call_id.to_string(), + tool_name: tool_name.to_string(), + }; + let out = orchestrator + .run(&mut runtime, &req, &tool_ctx, turn, turn.approval_policy) + .await; + let event_ctx = + ToolEventCtx::new(session, turn, call_id, tracker.as_ref().copied()); + let content = emitter.finish(event_ctx, out).await?; + Ok(Some(ToolOutput::Function { + content, + content_items: None, + success: Some(true), + })) + } + } + } + codex_apply_patch::MaybeApplyPatchVerified::CorrectnessError(parse_error) => { + Err(FunctionCallError::RespondToModel(format!( + "apply_patch verification failed: {parse_error}" + ))) + } + codex_apply_patch::MaybeApplyPatchVerified::ShellParseError(error) => { + tracing::trace!("Failed to parse apply_patch input, {error:?}"); + Ok(None) + } + codex_apply_patch::MaybeApplyPatchVerified::NotApplyPatch => Ok(None), + } +} + /// Returns a custom tool that can be used to edit files. Well-suited for GPT-5 models /// https://platform.openai.com/docs/guides/function-calling#custom-tools pub(crate) fn create_apply_patch_freeform_tool() -> ToolSpec { diff --git a/codex-rs/core/src/tools/handlers/shell.rs b/codex-rs/core/src/tools/handlers/shell.rs index fcd2f5b0c9..d1b7d3144c 100644 --- a/codex-rs/core/src/tools/handlers/shell.rs +++ b/codex-rs/core/src/tools/handlers/shell.rs @@ -3,9 +3,6 @@ use codex_protocol::models::ShellCommandToolCallParams; use codex_protocol::models::ShellToolCallParams; use std::sync::Arc; -use crate::apply_patch; -use crate::apply_patch::InternalApplyPatchInvocation; -use crate::apply_patch::convert_apply_patch_to_protocol; use crate::codex::TurnContext; use crate::exec::ExecParams; use crate::exec_env::create_env; @@ -19,11 +16,10 @@ use crate::tools::context::ToolOutput; use crate::tools::context::ToolPayload; use crate::tools::events::ToolEmitter; use crate::tools::events::ToolEventCtx; +use crate::tools::handlers::apply_patch::intercept_apply_patch; use crate::tools::orchestrator::ToolOrchestrator; use crate::tools::registry::ToolHandler; use crate::tools::registry::ToolKind; -use crate::tools::runtimes::apply_patch::ApplyPatchRequest; -use crate::tools::runtimes::apply_patch::ApplyPatchRuntime; use crate::tools::runtimes::shell::ShellRequest; use crate::tools::runtimes::shell::ShellRuntime; use crate::tools::sandboxing::ToolCtx; @@ -37,7 +33,7 @@ impl ShellHandler { ExecParams { command: params.command, cwd: turn_context.resolve_path(params.workdir.clone()), - timeout_ms: params.timeout_ms, + expiration: params.timeout_ms.into(), env: create_env(&turn_context.shell_environment_policy), with_escalated_permissions: params.with_escalated_permissions, justification: params.justification, @@ -59,7 +55,7 @@ impl ShellCommandHandler { ExecParams { command, cwd: turn_context.resolve_path(params.workdir.clone()), - timeout_ms: params.timeout_ms, + expiration: params.timeout_ms.into(), env: create_env(&turn_context.shell_environment_policy), with_escalated_permissions: params.with_escalated_permissions, justification: params.justification, @@ -210,81 +206,19 @@ impl ShellHandler { } // Intercept apply_patch if present. - match codex_apply_patch::maybe_parse_apply_patch_verified( + if let Some(output) = intercept_apply_patch( &exec_params.command, &exec_params.cwd, - ) { - codex_apply_patch::MaybeApplyPatchVerified::Body(changes) => { - match apply_patch::apply_patch(session.as_ref(), turn.as_ref(), &call_id, changes) - .await - { - InternalApplyPatchInvocation::Output(item) => { - // Programmatic apply_patch path; return its result. - let content = item?; - return Ok(ToolOutput::Function { - content, - content_items: None, - success: Some(true), - }); - } - InternalApplyPatchInvocation::DelegateToExec(apply) => { - let emitter = ToolEmitter::apply_patch( - convert_apply_patch_to_protocol(&apply.action), - !apply.user_explicitly_approved_this_action, - ); - let event_ctx = ToolEventCtx::new( - session.as_ref(), - turn.as_ref(), - &call_id, - Some(&tracker), - ); - emitter.begin(event_ctx).await; - - let req = ApplyPatchRequest { - patch: apply.action.patch.clone(), - cwd: apply.action.cwd.clone(), - timeout_ms: exec_params.timeout_ms, - user_explicitly_approved: apply.user_explicitly_approved_this_action, - codex_exe: turn.codex_linux_sandbox_exe.clone(), - }; - let mut orchestrator = ToolOrchestrator::new(); - let mut runtime = ApplyPatchRuntime::new(); - let tool_ctx = ToolCtx { - session: session.as_ref(), - turn: turn.as_ref(), - call_id: call_id.clone(), - tool_name: tool_name.to_string(), - }; - let out = orchestrator - .run(&mut runtime, &req, &tool_ctx, &turn, turn.approval_policy) - .await; - let event_ctx = ToolEventCtx::new( - session.as_ref(), - turn.as_ref(), - &call_id, - Some(&tracker), - ); - let content = emitter.finish(event_ctx, out).await?; - return Ok(ToolOutput::Function { - content, - content_items: None, - success: Some(true), - }); - } - } - } - codex_apply_patch::MaybeApplyPatchVerified::CorrectnessError(parse_error) => { - return Err(FunctionCallError::RespondToModel(format!( - "apply_patch verification failed: {parse_error}" - ))); - } - codex_apply_patch::MaybeApplyPatchVerified::ShellParseError(error) => { - tracing::trace!("Failed to parse shell command, {error:?}"); - // Fall through to regular shell execution. - } - codex_apply_patch::MaybeApplyPatchVerified::NotApplyPatch => { - // Fall through to regular shell execution. - } + exec_params.expiration.timeout_ms(), + session.as_ref(), + turn.as_ref(), + Some(&tracker), + &call_id, + tool_name, + ) + .await? + { + return Ok(output); } let source = ExecCommandSource::Agent; @@ -297,20 +231,23 @@ impl ShellHandler { let event_ctx = ToolEventCtx::new(session.as_ref(), turn.as_ref(), &call_id, None); emitter.begin(event_ctx).await; + let approval_requirement = create_approval_requirement_for_command( + &turn.exec_policy, + &exec_params.command, + turn.approval_policy, + &turn.sandbox_policy, + SandboxPermissions::from(exec_params.with_escalated_permissions.unwrap_or(false)), + ) + .await; + let req = ShellRequest { command: exec_params.command.clone(), cwd: exec_params.cwd.clone(), - timeout_ms: exec_params.timeout_ms, + timeout_ms: exec_params.expiration.timeout_ms(), env: exec_params.env.clone(), with_escalated_permissions: exec_params.with_escalated_permissions, justification: exec_params.justification.clone(), - approval_requirement: create_approval_requirement_for_command( - &turn.exec_policy, - &exec_params.command, - turn.approval_policy, - &turn.sandbox_policy, - SandboxPermissions::from(exec_params.with_escalated_permissions.unwrap_or(false)), - ), + approval_requirement, }; let mut orchestrator = ToolOrchestrator::new(); let mut runtime = ShellRuntime::new(); @@ -338,29 +275,30 @@ mod tests { use std::path::PathBuf; use crate::is_safe_command::is_known_safe_command; - use crate::shell::BashShell; - use crate::shell::PowerShellConfig; use crate::shell::Shell; - use crate::shell::ZshShell; + use crate::shell::ShellType; /// The logic for is_known_safe_command() has heuristics for known shells, /// so we must ensure the commands generated by [ShellCommandHandler] can be /// recognized as safe if the `command` is safe. #[test] fn commands_generated_by_shell_command_handler_can_be_matched_by_is_known_safe_command() { - let bash_shell = Shell::Bash(BashShell { + let bash_shell = Shell { + shell_type: ShellType::Bash, shell_path: PathBuf::from("/bin/bash"), - }); + }; assert_safe(&bash_shell, "ls -la"); - let zsh_shell = Shell::Zsh(ZshShell { + let zsh_shell = Shell { + shell_type: ShellType::Zsh, shell_path: PathBuf::from("/bin/zsh"), - }); + }; assert_safe(&zsh_shell, "ls -la"); - let powershell = Shell::PowerShell(PowerShellConfig { + let powershell = Shell { + shell_type: ShellType::PowerShell, shell_path: PathBuf::from("pwsh.exe"), - }); + }; assert_safe(&powershell, "ls -Name"); } diff --git a/codex-rs/core/src/tools/handlers/unified_exec.rs b/codex-rs/core/src/tools/handlers/unified_exec.rs index 6790b73599..4c943c6285 100644 --- a/codex-rs/core/src/tools/handlers/unified_exec.rs +++ b/codex-rs/core/src/tools/handlers/unified_exec.rs @@ -6,6 +6,7 @@ use crate::protocol::EventMsg; use crate::protocol::ExecCommandOutputDeltaEvent; use crate::protocol::ExecCommandSource; use crate::protocol::ExecOutputStream; +use crate::shell::default_user_shell; use crate::shell::get_shell_by_model_provided_path; use crate::tools::context::ToolInvocation; use crate::tools::context::ToolOutput; @@ -13,6 +14,7 @@ use crate::tools::context::ToolPayload; use crate::tools::events::ToolEmitter; use crate::tools::events::ToolEventCtx; use crate::tools::events::ToolEventStage; +use crate::tools::handlers::apply_patch::intercept_apply_patch; use crate::tools::registry::ToolHandler; use crate::tools::registry::ToolKind; use crate::unified_exec::ExecCommandRequest; @@ -30,8 +32,8 @@ struct ExecCommandArgs { cmd: String, #[serde(default)] workdir: Option, - #[serde(default = "default_shell")] - shell: String, + #[serde(default)] + shell: Option, #[serde(default = "default_login")] login: bool, #[serde(default = "default_exec_yield_time_ms")] @@ -46,6 +48,7 @@ struct ExecCommandArgs { #[derive(Debug, Deserialize)] struct WriteStdinArgs { + // The model is trained on `session_id`. session_id: i32, #[serde(default)] chars: String, @@ -63,10 +66,6 @@ fn default_write_stdin_yield_time_ms() -> u64 { 250 } -fn default_shell() -> String { - "/bin/bash".to_string() -} - fn default_login() -> bool { true } @@ -102,6 +101,7 @@ impl ToolHandler for UnifiedExecHandler { let ToolInvocation { session, turn, + tracker, call_id, tool_name, payload, @@ -128,6 +128,7 @@ impl ToolHandler for UnifiedExecHandler { "failed to parse exec_command arguments: {err:?}" )) })?; + let process_id = manager.allocate_process_id().await; let command = get_command(&args); let ExecCommandArgs { @@ -151,12 +152,26 @@ impl ToolHandler for UnifiedExecHandler { ))); } - let workdir = workdir - .as_deref() - .filter(|value| !value.is_empty()) - .map(PathBuf::from); + let workdir = workdir.filter(|value| !value.is_empty()); + + let workdir = workdir.map(|dir| context.turn.resolve_path(Some(dir))); let cwd = workdir.clone().unwrap_or_else(|| context.turn.cwd.clone()); + if let Some(output) = intercept_apply_patch( + &command, + &cwd, + Some(yield_time_ms), + context.session.as_ref(), + context.turn.as_ref(), + Some(&tracker), + &context.call_id, + tool_name.as_str(), + ) + .await? + { + return Ok(output); + } + let event_ctx = ToolEventCtx::new( context.session.as_ref(), context.turn.as_ref(), @@ -168,6 +183,7 @@ impl ToolHandler for UnifiedExecHandler { cwd.clone(), ExecCommandSource::UnifiedExecStartup, None, + Some(process_id.clone()), ); emitter.emit(event_ctx, ToolEventStage::Begin).await; @@ -175,6 +191,7 @@ impl ToolHandler for UnifiedExecHandler { .exec_command( ExecCommandRequest { command, + process_id, yield_time_ms, max_output_tokens, workdir, @@ -197,7 +214,7 @@ impl ToolHandler for UnifiedExecHandler { manager .write_stdin(WriteStdinRequest { call_id: &call_id, - session_id: args.session_id, + process_id: &args.session_id.to_string(), input: &args.chars, yield_time_ms: args.yield_time_ms, max_output_tokens: args.max_output_tokens, @@ -237,7 +254,12 @@ impl ToolHandler for UnifiedExecHandler { } fn get_command(args: &ExecCommandArgs) -> Vec { - let shell = get_shell_by_model_provided_path(&PathBuf::from(args.shell.clone())); + let shell = if let Some(shell_str) = &args.shell { + get_shell_by_model_provided_path(&PathBuf::from(shell_str)) + } else { + default_user_shell() + }; + shell.derive_exec_args(&args.cmd, args.login) } @@ -255,8 +277,9 @@ fn format_response(response: &UnifiedExecResponse) -> String { sections.push(format!("Process exited with code {exit_code}")); } - if let Some(session_id) = response.session_id { - sections.push(format!("Process running with session ID {session_id}")); + if let Some(process_id) = &response.process_id { + // Training still uses "session ID". + sections.push(format!("Process running with session ID {process_id}")); } if let Some(original_token_count) = response.original_token_count { @@ -268,3 +291,65 @@ fn format_response(response: &UnifiedExecResponse) -> String { sections.join("\n") } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_get_command_uses_default_shell_when_unspecified() { + let json = r#"{"cmd": "echo hello"}"#; + + let args: ExecCommandArgs = + serde_json::from_str(json).expect("deserialize ExecCommandArgs"); + + assert!(args.shell.is_none()); + + let command = get_command(&args); + + assert_eq!(command.len(), 3); + assert_eq!(command[2], "echo hello"); + } + + #[test] + fn test_get_command_respects_explicit_bash_shell() { + let json = r#"{"cmd": "echo hello", "shell": "/bin/bash"}"#; + + let args: ExecCommandArgs = + serde_json::from_str(json).expect("deserialize ExecCommandArgs"); + + assert_eq!(args.shell.as_deref(), Some("/bin/bash")); + + let command = get_command(&args); + + assert_eq!(command[2], "echo hello"); + } + + #[test] + fn test_get_command_respects_explicit_powershell_shell() { + let json = r#"{"cmd": "echo hello", "shell": "powershell"}"#; + + let args: ExecCommandArgs = + serde_json::from_str(json).expect("deserialize ExecCommandArgs"); + + assert_eq!(args.shell.as_deref(), Some("powershell")); + + let command = get_command(&args); + + assert_eq!(command[2], "echo hello"); + } + + #[test] + fn test_get_command_respects_explicit_cmd_shell() { + let json = r#"{"cmd": "echo hello", "shell": "cmd"}"#; + + let args: ExecCommandArgs = + serde_json::from_str(json).expect("deserialize ExecCommandArgs"); + + assert_eq!(args.shell.as_deref(), Some("cmd")); + + let command = get_command(&args); + + assert_eq!(command[2], "echo hello"); + } +} diff --git a/codex-rs/core/src/tools/orchestrator.rs b/codex-rs/core/src/tools/orchestrator.rs index 7e8e152f67..de23d510bf 100644 --- a/codex-rs/core/src/tools/orchestrator.rs +++ b/codex-rs/core/src/tools/orchestrator.rs @@ -14,6 +14,7 @@ use crate::tools::sandboxing::ApprovalCtx; use crate::tools::sandboxing::ApprovalRequirement; use crate::tools::sandboxing::ProvidesSandboxRetryData; use crate::tools::sandboxing::SandboxAttempt; +use crate::tools::sandboxing::SandboxOverride; use crate::tools::sandboxing::ToolCtx; use crate::tools::sandboxing::ToolError; use crate::tools::sandboxing::ToolRuntime; @@ -57,7 +58,7 @@ impl ToolOrchestrator { default_approval_requirement(approval_policy, &turn_ctx.sandbox_policy) }); match requirement { - ApprovalRequirement::Skip => { + ApprovalRequirement::Skip { .. } => { otel.tool_decision(otel_tn, otel_ci, ReviewDecision::Approved, otel_cfg); } ApprovalRequirement::Forbidden { reason } => { @@ -100,12 +101,13 @@ impl ToolOrchestrator { } // 2) First attempt under the selected sandbox. - let mut initial_sandbox = self - .sandbox - .select_initial(&turn_ctx.sandbox_policy, tool.sandbox_preference()); - if tool.wants_escalated_first_attempt(req) { - initial_sandbox = crate::exec::SandboxType::None; - } + let initial_sandbox = match tool.sandbox_mode_for_first_attempt(req) { + SandboxOverride::BypassSandboxFirstAttempt => crate::exec::SandboxType::None, + SandboxOverride::NoOverride => self + .sandbox + .select_initial(&turn_ctx.sandbox_policy, tool.sandbox_preference()), + }; + // Platform-specific flag gating is handled by SandboxManager::select_initial // via crate::safety::get_platform_sandbox(). let initial_attempt = SandboxAttempt { diff --git a/codex-rs/core/src/tools/runtimes/apply_patch.rs b/codex-rs/core/src/tools/runtimes/apply_patch.rs index 0cdddd5087..2334f1e712 100644 --- a/codex-rs/core/src/tools/runtimes/apply_patch.rs +++ b/codex-rs/core/src/tools/runtimes/apply_patch.rs @@ -67,7 +67,7 @@ impl ApplyPatchRuntime { program, args: vec![CODEX_APPLY_PATCH_ARG1.to_string(), req.patch.clone()], cwd: req.cwd.clone(), - timeout_ms: req.timeout_ms, + expiration: req.timeout_ms.into(), // Run apply_patch with a minimal environment for determinism and to avoid leaks. env: HashMap::new(), with_escalated_permissions: None, @@ -153,9 +153,9 @@ impl ToolRuntime for ApplyPatchRuntime { ) -> Result { let spec = Self::build_command_spec(req)?; let env = attempt - .env_for(&spec) + .env_for(spec) .map_err(|err| ToolError::Codex(err.into()))?; - let out = execute_env(&env, attempt.policy, Self::stdout_stream(ctx)) + let out = execute_env(env, attempt.policy, Self::stdout_stream(ctx)) .await .map_err(ToolError::Codex)?; Ok(out) diff --git a/codex-rs/core/src/tools/runtimes/mod.rs b/codex-rs/core/src/tools/runtimes/mod.rs index 212163d72c..437f4af428 100644 --- a/codex-rs/core/src/tools/runtimes/mod.rs +++ b/codex-rs/core/src/tools/runtimes/mod.rs @@ -4,6 +4,7 @@ Module: runtimes Concrete ToolRuntime implementations for specific tools. Each runtime stays small and focused and reuses the orchestrator for approvals + sandbox + retry. */ +use crate::exec::ExecExpiration; use crate::sandboxing::CommandSpec; use crate::tools::sandboxing::ToolError; use std::collections::HashMap; @@ -19,7 +20,7 @@ pub(crate) fn build_command_spec( command: &[String], cwd: &Path, env: &HashMap, - timeout_ms: Option, + expiration: ExecExpiration, with_escalated_permissions: Option, justification: Option, ) -> Result { @@ -31,7 +32,7 @@ pub(crate) fn build_command_spec( args: args.to_vec(), cwd: cwd.to_path_buf(), env: env.clone(), - timeout_ms, + expiration, with_escalated_permissions, justification, }) diff --git a/codex-rs/core/src/tools/runtimes/shell.rs b/codex-rs/core/src/tools/runtimes/shell.rs index d71c4498e6..56c72a8278 100644 --- a/codex-rs/core/src/tools/runtimes/shell.rs +++ b/codex-rs/core/src/tools/runtimes/shell.rs @@ -12,6 +12,7 @@ use crate::tools::sandboxing::ApprovalCtx; use crate::tools::sandboxing::ApprovalRequirement; use crate::tools::sandboxing::ProvidesSandboxRetryData; use crate::tools::sandboxing::SandboxAttempt; +use crate::tools::sandboxing::SandboxOverride; use crate::tools::sandboxing::SandboxRetryData; use crate::tools::sandboxing::Sandboxable; use crate::tools::sandboxing::SandboxablePreference; @@ -117,8 +118,19 @@ impl Approvable for ShellRuntime { Some(req.approval_requirement.clone()) } - fn wants_escalated_first_attempt(&self, req: &ShellRequest) -> bool { - req.with_escalated_permissions.unwrap_or(false) + fn sandbox_mode_for_first_attempt(&self, req: &ShellRequest) -> SandboxOverride { + if req.with_escalated_permissions.unwrap_or(false) + || matches!( + req.approval_requirement, + ApprovalRequirement::Skip { + bypass_sandbox: true + } + ) + { + SandboxOverride::BypassSandboxFirstAttempt + } else { + SandboxOverride::NoOverride + } } } @@ -133,14 +145,14 @@ impl ToolRuntime for ShellRuntime { &req.command, &req.cwd, &req.env, - req.timeout_ms, + req.timeout_ms.into(), req.with_escalated_permissions, req.justification.clone(), )?; let env = attempt - .env_for(&spec) + .env_for(spec) .map_err(|err| ToolError::Codex(err.into()))?; - let out = execute_env(&env, attempt.policy, Self::stdout_stream(ctx)) + let out = execute_env(env, attempt.policy, Self::stdout_stream(ctx)) .await .map_err(ToolError::Codex)?; Ok(out) diff --git a/codex-rs/core/src/tools/runtimes/unified_exec.rs b/codex-rs/core/src/tools/runtimes/unified_exec.rs index 5b18476bfc..0f306e6ff2 100644 --- a/codex-rs/core/src/tools/runtimes/unified_exec.rs +++ b/codex-rs/core/src/tools/runtimes/unified_exec.rs @@ -6,12 +6,14 @@ the session manager to spawn PTYs once an ExecEnv is prepared. */ use crate::error::CodexErr; use crate::error::SandboxErr; +use crate::exec::ExecExpiration; use crate::tools::runtimes::build_command_spec; use crate::tools::sandboxing::Approvable; use crate::tools::sandboxing::ApprovalCtx; use crate::tools::sandboxing::ApprovalRequirement; use crate::tools::sandboxing::ProvidesSandboxRetryData; use crate::tools::sandboxing::SandboxAttempt; +use crate::tools::sandboxing::SandboxOverride; use crate::tools::sandboxing::SandboxRetryData; use crate::tools::sandboxing::Sandboxable; use crate::tools::sandboxing::SandboxablePreference; @@ -134,8 +136,19 @@ impl Approvable for UnifiedExecRuntime<'_> { Some(req.approval_requirement.clone()) } - fn wants_escalated_first_attempt(&self, req: &UnifiedExecRequest) -> bool { - req.with_escalated_permissions.unwrap_or(false) + fn sandbox_mode_for_first_attempt(&self, req: &UnifiedExecRequest) -> SandboxOverride { + if req.with_escalated_permissions.unwrap_or(false) + || matches!( + req.approval_requirement, + ApprovalRequirement::Skip { + bypass_sandbox: true + } + ) + { + SandboxOverride::BypassSandboxFirstAttempt + } else { + SandboxOverride::NoOverride + } } } @@ -150,13 +163,13 @@ impl<'a> ToolRuntime for UnifiedExecRunt &req.command, &req.cwd, &req.env, - None, + ExecExpiration::DefaultTimeout, req.with_escalated_permissions, req.justification.clone(), ) .map_err(|_| ToolError::Rejected("missing command line for PTY".to_string()))?; let exec_env = attempt - .env_for(&spec) + .env_for(spec) .map_err(|err| ToolError::Codex(err.into()))?; self.manager .open_session_with_exec_env(&exec_env) diff --git a/codex-rs/core/src/tools/sandboxing.rs b/codex-rs/core/src/tools/sandboxing.rs index e694c7fbef..df10db952e 100644 --- a/codex-rs/core/src/tools/sandboxing.rs +++ b/codex-rs/core/src/tools/sandboxing.rs @@ -89,8 +89,12 @@ pub(crate) struct ApprovalCtx<'a> { // Specifies what tool orchestrator should do with a given tool call. #[derive(Clone, Debug, PartialEq, Eq)] pub(crate) enum ApprovalRequirement { - /// No approval required for this tool call - Skip, + /// No approval required for this tool call. + Skip { + /// The first attempt should skip sandboxing (e.g., when explicitly + /// greenlit by policy). + bypass_sandbox: bool, + }, /// Approval required for this tool call NeedsApproval { reason: Option }, /// Execution forbidden for this tool call @@ -113,10 +117,18 @@ pub(crate) fn default_approval_requirement( if needs_approval { ApprovalRequirement::NeedsApproval { reason: None } } else { - ApprovalRequirement::Skip + ApprovalRequirement::Skip { + bypass_sandbox: false, + } } } +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum SandboxOverride { + NoOverride, + BypassSandboxFirstAttempt, +} + pub(crate) trait Approvable { type ApprovalKey: Hash + Eq + Clone + Debug + Serialize; @@ -124,9 +136,9 @@ pub(crate) trait Approvable { /// Some tools may request to skip the sandbox on the first attempt /// (e.g., when the request explicitly asks for escalated permissions). - /// Defaults to `false`. - fn wants_escalated_first_attempt(&self, _req: &Req) -> bool { - false + /// Defaults to `NoOverride`. + fn sandbox_mode_for_first_attempt(&self, _req: &Req) -> SandboxOverride { + SandboxOverride::NoOverride } fn should_bypass_approval(&self, policy: AskForApproval, already_approved: bool) -> bool { @@ -216,7 +228,7 @@ pub(crate) struct SandboxAttempt<'a> { impl<'a> SandboxAttempt<'a> { pub fn env_for( &self, - spec: &CommandSpec, + spec: CommandSpec, ) -> Result { self.manager.transform( spec, diff --git a/codex-rs/core/src/tools/spec.rs b/codex-rs/core/src/tools/spec.rs index eb4585925d..d07c605e50 100644 --- a/codex-rs/core/src/tools/spec.rs +++ b/codex-rs/core/src/tools/spec.rs @@ -57,8 +57,6 @@ impl ToolsConfig { ConfigShellToolType::Disabled } else if features.enabled(Feature::UnifiedExec) { ConfigShellToolType::UnifiedExec - } else if features.enabled(Feature::ShellCommandTool) { - ConfigShellToolType::ShellCommand } else { model_family.shell_type.clone() }; @@ -1428,6 +1426,24 @@ mod tests { ); } + #[test] + fn test_exp_5_1_defaults() { + assert_model_tools( + "exp-5.1", + &Features::with_defaults(), + &[ + "exec_command", + "write_stdin", + "list_mcp_resources", + "list_mcp_resource_templates", + "read_mcp_resource", + "update_plan", + "apply_patch", + "view_image", + ], + ); + } + #[test] fn test_codex_mini_unified_exec_web_search() { assert_model_tools( @@ -1468,22 +1484,6 @@ mod tests { assert_contains_tool_names(&tools, &subset); } - #[test] - fn test_build_specs_shell_command_present() { - assert_model_tools( - "codex-mini-latest", - Features::with_defaults().enable(Feature::ShellCommandTool), - &[ - "shell_command", - "list_mcp_resources", - "list_mcp_resource_templates", - "read_mcp_resource", - "update_plan", - "view_image", - ], - ); - } - #[test] #[ignore] fn test_parallel_support_flags() { diff --git a/codex-rs/core/src/truncate.rs b/codex-rs/core/src/truncate.rs index bf883c0612..6e38ef6986 100644 --- a/codex-rs/core/src/truncate.rs +++ b/codex-rs/core/src/truncate.rs @@ -296,7 +296,7 @@ fn approx_bytes_for_tokens(tokens: usize) -> usize { tokens.saturating_mul(APPROX_BYTES_PER_TOKEN) } -fn approx_tokens_from_byte_count(bytes: usize) -> u64 { +pub(crate) fn approx_tokens_from_byte_count(bytes: usize) -> u64 { let bytes_u64 = bytes as u64; bytes_u64.saturating_add((APPROX_BYTES_PER_TOKEN as u64).saturating_sub(1)) / (APPROX_BYTES_PER_TOKEN as u64) diff --git a/codex-rs/core/src/unified_exec/errors.rs b/codex-rs/core/src/unified_exec/errors.rs index bfbef93572..02031f22fa 100644 --- a/codex-rs/core/src/unified_exec/errors.rs +++ b/codex-rs/core/src/unified_exec/errors.rs @@ -5,8 +5,9 @@ use thiserror::Error; pub(crate) enum UnifiedExecError { #[error("Failed to create unified exec session: {message}")] CreateSession { message: String }, - #[error("Unknown session id {session_id}")] - UnknownSessionId { session_id: i32 }, + // Called "session" in the model's training. + #[error("Unknown session id {process_id}")] + UnknownSessionId { process_id: String }, #[error("failed to write to stdin")] WriteToStdin, #[error("missing command line for unified exec request")] diff --git a/codex-rs/core/src/unified_exec/mod.rs b/codex-rs/core/src/unified_exec/mod.rs index 390401d789..34b62df342 100644 --- a/codex-rs/core/src/unified_exec/mod.rs +++ b/codex-rs/core/src/unified_exec/mod.rs @@ -22,9 +22,9 @@ //! - `session_manager.rs`: orchestration (approvals, sandboxing, reuse) and request handling. use std::collections::HashMap; +use std::collections::HashSet; use std::path::PathBuf; use std::sync::Arc; -use std::sync::atomic::AtomicI32; use std::time::Duration; use rand::Rng; @@ -46,6 +46,10 @@ pub(crate) const MAX_YIELD_TIME_MS: u64 = 30_000; pub(crate) const DEFAULT_MAX_OUTPUT_TOKENS: usize = 10_000; pub(crate) const UNIFIED_EXEC_OUTPUT_MAX_BYTES: usize = 1024 * 1024; // 1 MiB pub(crate) const UNIFIED_EXEC_OUTPUT_MAX_TOKENS: usize = UNIFIED_EXEC_OUTPUT_MAX_BYTES / 4; +pub(crate) const MAX_UNIFIED_EXEC_SESSIONS: usize = 64; + +// Send a warning message to the models when it reaches this number of sessions. +pub(crate) const WARNING_UNIFIED_EXEC_SESSIONS: usize = 60; pub(crate) struct UnifiedExecContext { pub session: Arc, @@ -66,6 +70,7 @@ impl UnifiedExecContext { #[derive(Debug)] pub(crate) struct ExecCommandRequest { pub command: Vec, + pub process_id: String, pub yield_time_ms: u64, pub max_output_tokens: Option, pub workdir: Option, @@ -76,7 +81,7 @@ pub(crate) struct ExecCommandRequest { #[derive(Debug)] pub(crate) struct WriteStdinRequest<'a> { pub call_id: &'a str, - pub session_id: i32, + pub process_id: &'a str, pub input: &'a str, pub yield_time_ms: u64, pub max_output_tokens: Option, @@ -88,7 +93,7 @@ pub(crate) struct UnifiedExecResponse { pub chunk_id: String, pub wall_time: Duration, pub output: String, - pub session_id: Option, + pub process_id: Option, pub exit_code: Option, pub original_token_count: Option, pub session_command: Option>, @@ -96,18 +101,38 @@ pub(crate) struct UnifiedExecResponse { #[derive(Default)] pub(crate) struct UnifiedExecSessionManager { - next_session_id: AtomicI32, - sessions: Mutex>, + session_store: Mutex, +} + +// Required for mutex sharing. +#[derive(Default)] +pub(crate) struct SessionStore { + sessions: HashMap, + reserved_sessions_id: HashSet, +} + +impl SessionStore { + fn remove(&mut self, session_id: &str) -> Option { + self.reserved_sessions_id.remove(session_id); + self.sessions.remove(session_id) + } + + pub(crate) fn clear(&mut self) { + self.reserved_sessions_id.clear(); + self.sessions.clear(); + } } struct SessionEntry { - session: session::UnifiedExecSession, + session: UnifiedExecSession, session_ref: Arc, turn_ref: Arc, call_id: String, + process_id: String, command: Vec, cwd: PathBuf, started_at: tokio::time::Instant, + last_used: tokio::time::Instant, } pub(crate) fn clamp_yield_time(yield_time_ms: u64) -> u64 { @@ -157,6 +182,11 @@ mod tests { ) -> Result { let context = UnifiedExecContext::new(Arc::clone(session), Arc::clone(turn), "call".to_string()); + let process_id = session + .services + .unified_exec_manager + .allocate_process_id() + .await; session .services @@ -164,6 +194,7 @@ mod tests { .exec_command( ExecCommandRequest { command: vec!["bash".to_string(), "-lc".to_string(), cmd.to_string()], + process_id, yield_time_ms, max_output_tokens: None, workdir: None, @@ -177,7 +208,7 @@ mod tests { async fn write_stdin( session: &Arc, - session_id: i32, + process_id: &str, input: &str, yield_time_ms: u64, ) -> Result { @@ -186,7 +217,7 @@ mod tests { .unified_exec_manager .write_stdin(WriteStdinRequest { call_id: "write-stdin", - session_id, + process_id, input, yield_time_ms, max_output_tokens: None, @@ -219,11 +250,15 @@ mod tests { let (session, turn) = test_session_and_turn(); let open_shell = exec_command(&session, &turn, "bash -i", 2_500).await?; - let session_id = open_shell.session_id.expect("expected session_id"); + let process_id = open_shell + .process_id + .as_ref() + .expect("expected process_id") + .as_str(); write_stdin( &session, - session_id, + process_id, "export CODEX_INTERACTIVE_SHELL_VAR=codex\n", 2_500, ) @@ -231,7 +266,7 @@ mod tests { let out_2 = write_stdin( &session, - session_id, + process_id, "echo $CODEX_INTERACTIVE_SHELL_VAR\n", 2_500, ) @@ -251,11 +286,15 @@ mod tests { let (session, turn) = test_session_and_turn(); let shell_a = exec_command(&session, &turn, "bash -i", 2_500).await?; - let session_a = shell_a.session_id.expect("expected session id"); + let session_a = shell_a + .process_id + .as_ref() + .expect("expected process id") + .clone(); write_stdin( &session, - session_a, + session_a.as_str(), "export CODEX_INTERACTIVE_SHELL_VAR=codex\n", 2_500, ) @@ -263,9 +302,10 @@ mod tests { let out_2 = exec_command(&session, &turn, "echo $CODEX_INTERACTIVE_SHELL_VAR", 2_500).await?; + tokio::time::sleep(Duration::from_secs(2)).await; assert!( - out_2.session_id.is_none(), - "short command should not retain a session" + out_2.process_id.is_none(), + "short command should not report a process id if it exits quickly" ); assert!( !out_2.output.contains("codex"), @@ -274,7 +314,11 @@ mod tests { let out_3 = write_stdin( &session, - session_a, + shell_a + .process_id + .as_ref() + .expect("expected process id") + .as_str(), "echo $CODEX_INTERACTIVE_SHELL_VAR\n", 2_500, ) @@ -294,11 +338,15 @@ mod tests { let (session, turn) = test_session_and_turn(); let open_shell = exec_command(&session, &turn, "bash -i", 2_500).await?; - let session_id = open_shell.session_id.expect("expected session id"); + let process_id = open_shell + .process_id + .as_ref() + .expect("expected process id") + .as_str(); write_stdin( &session, - session_id, + process_id, "export CODEX_INTERACTIVE_SHELL_VAR=codex\n", 2_500, ) @@ -306,7 +354,7 @@ mod tests { let out_2 = write_stdin( &session, - session_id, + process_id, "sleep 5 && echo $CODEX_INTERACTIVE_SHELL_VAR\n", 10, ) @@ -318,7 +366,7 @@ mod tests { tokio::time::sleep(Duration::from_secs(7)).await; - let out_3 = write_stdin(&session, session_id, "", 100).await?; + let out_3 = write_stdin(&session, process_id, "", 100).await?; assert!( out_3.output.contains("codex"), @@ -335,7 +383,7 @@ mod tests { let result = exec_command(&session, &turn, "echo codex", 120_000).await?; - assert!(result.session_id.is_none()); + assert!(result.process_id.is_some()); assert!(result.output.contains("codex")); Ok(()) @@ -348,8 +396,8 @@ mod tests { let result = exec_command(&session, &turn, "echo codex", 2_500).await?; assert!( - result.session_id.is_none(), - "completed command should not retain session" + result.process_id.is_some(), + "completed command should report a process id" ); assert!(result.output.contains("codex")); @@ -357,9 +405,10 @@ mod tests { session .services .unified_exec_manager - .sessions + .session_store .lock() .await + .sessions .is_empty() ); @@ -373,31 +422,36 @@ mod tests { let (session, turn) = test_session_and_turn(); let open_shell = exec_command(&session, &turn, "bash -i", 2_500).await?; - let session_id = open_shell.session_id.expect("expected session id"); + let process_id = open_shell + .process_id + .as_ref() + .expect("expected process id") + .as_str(); - write_stdin(&session, session_id, "exit\n", 2_500).await?; + write_stdin(&session, process_id, "exit\n", 2_500).await?; tokio::time::sleep(Duration::from_millis(200)).await; - let err = write_stdin(&session, session_id, "", 100) + let err = write_stdin(&session, process_id, "", 100) .await .expect_err("expected unknown session error"); match err { - UnifiedExecError::UnknownSessionId { session_id: err_id } => { - assert_eq!(err_id, session_id); + UnifiedExecError::UnknownSessionId { process_id: err_id } => { + assert_eq!(err_id, process_id, "process id should match request"); } other => panic!("expected UnknownSessionId, got {other:?}"), } assert!( - !session + session .services .unified_exec_manager - .sessions + .session_store .lock() .await - .contains_key(&session_id) + .sessions + .is_empty() ); Ok(()) diff --git a/codex-rs/core/src/unified_exec/session_manager.rs b/codex-rs/core/src/unified_exec/session_manager.rs index d9f99b9eaf..72c02cdb99 100644 --- a/codex-rs/core/src/unified_exec/session_manager.rs +++ b/codex-rs/core/src/unified_exec/session_manager.rs @@ -1,6 +1,9 @@ +use rand::Rng; +use std::cmp::Reverse; +use std::collections::HashMap; +use std::collections::HashSet; use std::path::PathBuf; use std::sync::Arc; - use tokio::sync::Notify; use tokio::sync::mpsc; use tokio::time::Duration; @@ -31,11 +34,14 @@ use crate::truncate::approx_token_count; use crate::truncate::formatted_truncate_text; use super::ExecCommandRequest; +use super::MAX_UNIFIED_EXEC_SESSIONS; use super::SessionEntry; +use super::SessionStore; use super::UnifiedExecContext; use super::UnifiedExecError; use super::UnifiedExecResponse; use super::UnifiedExecSessionManager; +use super::WARNING_UNIFIED_EXEC_SESSIONS; use super::WriteStdinRequest; use super::clamp_yield_time; use super::generate_chunk_id; @@ -44,6 +50,24 @@ use super::session::OutputBuffer; use super::session::OutputHandles; use super::session::UnifiedExecSession; +const UNIFIED_EXEC_ENV: [(&str, &str); 8] = [ + ("NO_COLOR", "1"), + ("TERM", "dumb"), + ("LANG", "C.UTF-8"), + ("LC_CTYPE", "C.UTF-8"), + ("LC_ALL", "C.UTF-8"), + ("COLORTERM", ""), + ("PAGER", "cat"), + ("GIT_PAGER", "cat"), +]; + +fn apply_unified_exec_env(mut env: HashMap) -> HashMap { + for (key, value) in UNIFIED_EXEC_ENV { + env.insert(key.to_string(), value.to_string()); + } + env +} + struct PreparedSessionHandles { writer_tx: mpsc::Sender>, output_buffer: OutputBuffer, @@ -53,9 +77,39 @@ struct PreparedSessionHandles { turn_ref: Arc, command: Vec, cwd: PathBuf, + process_id: String, } impl UnifiedExecSessionManager { + pub(crate) async fn allocate_process_id(&self) -> String { + loop { + let mut store = self.session_store.lock().await; + + let process_id = if !cfg!(test) && !cfg!(feature = "deterministic_process_ids") { + // production mode → random + rand::rng().random_range(1_000..100_000).to_string() + } else { + // test or deterministic mode + let next = store + .reserved_sessions_id + .iter() + .filter_map(|s| s.parse::().ok()) + .max() + .map(|m| std::cmp::max(m, 999) + 1) + .unwrap_or(1000); + + next.to_string() + }; + + if store.reserved_sessions_id.contains(&process_id) { + continue; + } + + store.reserved_sessions_id.insert(process_id.clone()); + return process_id; + } + } + pub(crate) async fn exec_command( &self, request: ExecCommandRequest, @@ -97,20 +151,24 @@ impl UnifiedExecSessionManager { let text = String::from_utf8_lossy(&collected).to_string(); let output = formatted_truncate_text(&text, TruncationPolicy::Tokens(max_tokens)); - let chunk_id = generate_chunk_id(); let has_exited = session.has_exited(); - let stored_id = self - .store_session(session, context, &request.command, cwd.clone(), start) + let exit_code = session.exit_code(); + let chunk_id = generate_chunk_id(); + let process_id = if has_exited { + None + } else { + // Only store session if not exited. + self.store_session( + session, + context, + &request.command, + cwd.clone(), + start, + request.process_id.clone(), + ) .await; - let exit_code = self - .sessions - .lock() - .await - .get(&stored_id) - .map(|entry| entry.session.exit_code()); - // Only include a session_id in the response if the process is still alive. - let session_id = if has_exited { None } else { Some(stored_id) }; - + Some(request.process_id.clone()) + }; let original_token_count = approx_token_count(&text); let response = UnifiedExecResponse { @@ -118,18 +176,18 @@ impl UnifiedExecSessionManager { chunk_id, wall_time, output, - session_id, - exit_code: exit_code.flatten(), + process_id: process_id.clone(), + exit_code, original_token_count: Some(original_token_count), session_command: Some(request.command.clone()), }; - if response.session_id.is_some() { + if !has_exited { Self::emit_waiting_status(&context.session, &context.turn, &request.command).await; } // If the command completed during this call, emit an ExecCommandEnd via the emitter. - if response.session_id.is_none() { + if has_exited { let exit = response.exit_code.unwrap_or(-1); Self::emit_exec_end_from_context( context, @@ -138,6 +196,9 @@ impl UnifiedExecSessionManager { response.output.clone(), exit, response.wall_time, + // We always emit the process ID in order to keep consistency between the Begin + // event and the End event. + Some(request.process_id), ) .await; } @@ -149,7 +210,7 @@ impl UnifiedExecSessionManager { &self, request: WriteStdinRequest<'_>, ) -> Result { - let session_id = request.session_id; + let process_id = request.process_id.to_string(); let PreparedSessionHandles { writer_tx, @@ -160,13 +221,15 @@ impl UnifiedExecSessionManager { turn_ref, command: session_command, cwd: session_cwd, - } = self.prepare_session_handles(session_id).await?; + process_id, + } = self.prepare_session_handles(process_id.as_str()).await?; let interaction_emitter = ToolEmitter::unified_exec( &session_command, session_cwd.clone(), ExecCommandSource::UnifiedExecInteraction, (!request.input.is_empty()).then(|| request.input.to_string()), + Some(process_id.clone()), ); let make_event_ctx = || { ToolEventCtx::new( @@ -213,17 +276,21 @@ impl UnifiedExecSessionManager { let original_token_count = approx_token_count(&text); let chunk_id = generate_chunk_id(); - let status = self.refresh_session_state(session_id).await; - let (session_id, exit_code, completion_entry, event_call_id) = match status { - SessionStatus::Alive { exit_code, call_id } => { - (Some(session_id), exit_code, None, call_id) - } + let status = self.refresh_session_state(process_id.as_str()).await; + let (process_id, exit_code, completion_entry, event_call_id) = match status { + SessionStatus::Alive { + exit_code, + call_id, + process_id, + } => (Some(process_id), exit_code, None, call_id), SessionStatus::Exited { exit_code, entry } => { let call_id = entry.call_id.clone(); (None, exit_code, Some(*entry), call_id) } SessionStatus::Unknown => { - return Err(UnifiedExecError::UnknownSessionId { session_id }); + return Err(UnifiedExecError::UnknownSessionId { + process_id: request.process_id.to_string(), + }); } }; @@ -232,7 +299,7 @@ impl UnifiedExecSessionManager { chunk_id, wall_time, output, - session_id, + process_id, exit_code, original_token_count: Some(original_token_count), session_command: Some(session_command.clone()), @@ -253,7 +320,7 @@ impl UnifiedExecSessionManager { ) .await; - if response.session_id.is_some() { + if response.process_id.is_some() { Self::emit_waiting_status(&session_ref, &turn_ref, &session_command).await; } @@ -266,16 +333,17 @@ impl UnifiedExecSessionManager { Ok(response) } - async fn refresh_session_state(&self, session_id: i32) -> SessionStatus { - let mut sessions = self.sessions.lock().await; - let Some(entry) = sessions.get(&session_id) else { + async fn refresh_session_state(&self, process_id: &str) -> SessionStatus { + let mut store = self.session_store.lock().await; + let Some(entry) = store.sessions.get(process_id) else { return SessionStatus::Unknown; }; let exit_code = entry.session.exit_code(); + let process_id = entry.process_id.clone(); if entry.session.has_exited() { - let Some(entry) = sessions.remove(&session_id) else { + let Some(entry) = store.remove(&process_id) else { return SessionStatus::Unknown; }; SessionStatus::Exited { @@ -286,18 +354,24 @@ impl UnifiedExecSessionManager { SessionStatus::Alive { exit_code, call_id: entry.call_id.clone(), + process_id, } } } async fn prepare_session_handles( &self, - session_id: i32, + process_id: &str, ) -> Result { - let sessions = self.sessions.lock().await; - let entry = sessions - .get(&session_id) - .ok_or(UnifiedExecError::UnknownSessionId { session_id })?; + let mut store = self.session_store.lock().await; + let entry = + store + .sessions + .get_mut(process_id) + .ok_or(UnifiedExecError::UnknownSessionId { + process_id: process_id.to_string(), + })?; + entry.last_used = Instant::now(); let OutputHandles { output_buffer, output_notify, @@ -313,6 +387,7 @@ impl UnifiedExecSessionManager { turn_ref: Arc::clone(&entry.turn_ref), command: entry.command.clone(), cwd: entry.cwd.clone(), + process_id: entry.process_id.clone(), }) } @@ -326,6 +401,7 @@ impl UnifiedExecSessionManager { .map_err(|_| UnifiedExecError::WriteToStdin) } + #[allow(clippy::too_many_arguments)] async fn store_session( &self, session: UnifiedExecSession, @@ -333,21 +409,35 @@ impl UnifiedExecSessionManager { command: &[String], cwd: PathBuf, started_at: Instant, - ) -> i32 { - let session_id = self - .next_session_id - .fetch_add(1, std::sync::atomic::Ordering::SeqCst); + process_id: String, + ) { let entry = SessionEntry { session, session_ref: Arc::clone(&context.session), turn_ref: Arc::clone(&context.turn), call_id: context.call_id.clone(), + process_id: process_id.clone(), command: command.to_vec(), cwd, started_at, + last_used: started_at, + }; + let number_sessions = { + let mut store = self.session_store.lock().await; + Self::prune_sessions_if_needed(&mut store); + store.sessions.insert(process_id, entry); + store.sessions.len() + }; + + if number_sessions >= WARNING_UNIFIED_EXEC_SESSIONS { + context + .session + .record_model_warning( + format!("The maximum number of unified exec sessions you can keep open is {WARNING_UNIFIED_EXEC_SESSIONS} and you currently have {number_sessions} sessions open. Reuse older sessions or close them to prevent automatic pruning of old session"), + &context.turn + ) + .await; }; - self.sessions.lock().await.insert(session_id, entry); - session_id } async fn emit_exec_end_from_entry( @@ -375,6 +465,7 @@ impl UnifiedExecSessionManager { entry.cwd, ExecCommandSource::UnifiedExecStartup, None, + Some(entry.process_id.clone()), ); emitter .emit(event_ctx, ToolEventStage::Success(output)) @@ -388,6 +479,7 @@ impl UnifiedExecSessionManager { aggregated_output: String, exit_code: i32, duration: Duration, + process_id: Option, ) { let output = ExecToolCallOutput { exit_code, @@ -403,8 +495,13 @@ impl UnifiedExecSessionManager { &context.call_id, None, ); - let emitter = - ToolEmitter::unified_exec(command, cwd, ExecCommandSource::UnifiedExecStartup, None); + let emitter = ToolEmitter::unified_exec( + command, + cwd, + ExecCommandSource::UnifiedExecStartup, + None, + process_id, + ); emitter .emit(event_ctx, ToolEventStage::Success(output)) .await; @@ -454,21 +551,24 @@ impl UnifiedExecSessionManager { justification: Option, context: &UnifiedExecContext, ) -> Result { + let env = apply_unified_exec_env(create_env(&context.turn.shell_environment_policy)); let mut orchestrator = ToolOrchestrator::new(); let mut runtime = UnifiedExecRuntime::new(self); + let approval_requirement = create_approval_requirement_for_command( + &context.turn.exec_policy, + command, + context.turn.approval_policy, + &context.turn.sandbox_policy, + SandboxPermissions::from(with_escalated_permissions.unwrap_or(false)), + ) + .await; let req = UnifiedExecToolRequest::new( command.to_vec(), cwd, - create_env(&context.turn.shell_environment_policy), + env, with_escalated_permissions, justification, - create_approval_requirement_for_command( - &context.turn.exec_policy, - command, - context.turn.approval_policy, - &context.turn.sandbox_policy, - SandboxPermissions::from(with_escalated_permissions.unwrap_or(false)), - ), + approval_requirement, ); let tool_ctx = ToolCtx { session: context.session.as_ref(), @@ -548,12 +648,66 @@ impl UnifiedExecSessionManager { collected } + + fn prune_sessions_if_needed(store: &mut SessionStore) -> bool { + if store.sessions.len() < MAX_UNIFIED_EXEC_SESSIONS { + return false; + } + + let meta: Vec<(String, Instant, bool)> = store + .sessions + .iter() + .map(|(id, entry)| (id.clone(), entry.last_used, entry.session.has_exited())) + .collect(); + + if let Some(session_id) = Self::session_id_to_prune_from_meta(&meta) { + store.remove(&session_id); + return true; + } + + false + } + + // Centralized pruning policy so we can easily swap strategies later. + fn session_id_to_prune_from_meta(meta: &[(String, Instant, bool)]) -> Option { + if meta.is_empty() { + return None; + } + + let mut by_recency = meta.to_vec(); + by_recency.sort_by_key(|(_, last_used, _)| Reverse(*last_used)); + let protected: HashSet = by_recency + .iter() + .take(8) + .map(|(process_id, _, _)| process_id.clone()) + .collect(); + + let mut lru = meta.to_vec(); + lru.sort_by_key(|(_, last_used, _)| *last_used); + + if let Some((process_id, _, _)) = lru + .iter() + .find(|(process_id, _, exited)| !protected.contains(process_id) && *exited) + { + return Some(process_id.clone()); + } + + lru.into_iter() + .find(|(process_id, _, _)| !protected.contains(process_id)) + .map(|(process_id, _, _)| process_id) + } + + pub(crate) async fn terminate_all_sessions(&self) { + let mut sessions = self.session_store.lock().await; + sessions.clear(); + } } enum SessionStatus { Alive { exit_code: Option, call_id: String, + process_id: String, }, Exited { exit_code: Option, @@ -561,3 +715,107 @@ enum SessionStatus { }, Unknown, } + +#[cfg(test)] +mod tests { + use super::*; + use pretty_assertions::assert_eq; + use tokio::time::Duration; + use tokio::time::Instant; + + #[test] + fn unified_exec_env_injects_defaults() { + let env = apply_unified_exec_env(HashMap::new()); + let expected = HashMap::from([ + ("NO_COLOR".to_string(), "1".to_string()), + ("TERM".to_string(), "dumb".to_string()), + ("LANG".to_string(), "C.UTF-8".to_string()), + ("LC_CTYPE".to_string(), "C.UTF-8".to_string()), + ("LC_ALL".to_string(), "C.UTF-8".to_string()), + ("COLORTERM".to_string(), String::new()), + ("PAGER".to_string(), "cat".to_string()), + ("GIT_PAGER".to_string(), "cat".to_string()), + ]); + + assert_eq!(env, expected); + } + + #[test] + fn unified_exec_env_overrides_existing_values() { + let mut base = HashMap::new(); + base.insert("NO_COLOR".to_string(), "0".to_string()); + base.insert("PATH".to_string(), "/usr/bin".to_string()); + + let env = apply_unified_exec_env(base); + + assert_eq!(env.get("NO_COLOR"), Some(&"1".to_string())); + assert_eq!(env.get("PATH"), Some(&"/usr/bin".to_string())); + } + + #[test] + fn pruning_prefers_exited_sessions_outside_recently_used() { + let now = Instant::now(); + let id = |n: i32| n.to_string(); + let meta = vec![ + (id(1), now - Duration::from_secs(40), false), + (id(2), now - Duration::from_secs(30), true), + (id(3), now - Duration::from_secs(20), false), + (id(4), now - Duration::from_secs(19), false), + (id(5), now - Duration::from_secs(18), false), + (id(6), now - Duration::from_secs(17), false), + (id(7), now - Duration::from_secs(16), false), + (id(8), now - Duration::from_secs(15), false), + (id(9), now - Duration::from_secs(14), false), + (id(10), now - Duration::from_secs(13), false), + ]; + + let candidate = UnifiedExecSessionManager::session_id_to_prune_from_meta(&meta); + + assert_eq!(candidate, Some(id(2))); + } + + #[test] + fn pruning_falls_back_to_lru_when_no_exited() { + let now = Instant::now(); + let id = |n: i32| n.to_string(); + let meta = vec![ + (id(1), now - Duration::from_secs(40), false), + (id(2), now - Duration::from_secs(30), false), + (id(3), now - Duration::from_secs(20), false), + (id(4), now - Duration::from_secs(19), false), + (id(5), now - Duration::from_secs(18), false), + (id(6), now - Duration::from_secs(17), false), + (id(7), now - Duration::from_secs(16), false), + (id(8), now - Duration::from_secs(15), false), + (id(9), now - Duration::from_secs(14), false), + (id(10), now - Duration::from_secs(13), false), + ]; + + let candidate = UnifiedExecSessionManager::session_id_to_prune_from_meta(&meta); + + assert_eq!(candidate, Some(id(1))); + } + + #[test] + fn pruning_protects_recent_sessions_even_if_exited() { + let now = Instant::now(); + let id = |n: i32| n.to_string(); + let meta = vec![ + (id(1), now - Duration::from_secs(40), false), + (id(2), now - Duration::from_secs(30), false), + (id(3), now - Duration::from_secs(20), true), + (id(4), now - Duration::from_secs(19), false), + (id(5), now - Duration::from_secs(18), false), + (id(6), now - Duration::from_secs(17), false), + (id(7), now - Duration::from_secs(16), false), + (id(8), now - Duration::from_secs(15), false), + (id(9), now - Duration::from_secs(14), false), + (id(10), now - Duration::from_secs(13), true), + ]; + + let candidate = UnifiedExecSessionManager::session_id_to_prune_from_meta(&meta); + + // (10) is exited but among the last 8; we should drop the LRU outside that set. + assert_eq!(candidate, Some(id(1))); + } +} diff --git a/codex-rs/core/src/util.rs b/codex-rs/core/src/util.rs index 0bce5b4439..5304a89ac9 100644 --- a/codex-rs/core/src/util.rs +++ b/codex-rs/core/src/util.rs @@ -1,3 +1,5 @@ +use std::path::Path; +use std::path::PathBuf; use std::time::Duration; use rand::Rng; @@ -14,11 +16,11 @@ pub(crate) fn backoff(attempt: u64) -> Duration { Duration::from_millis((base as f64 * jitter) as u64) } -pub(crate) fn error_or_panic(message: String) { +pub(crate) fn error_or_panic(message: impl std::string::ToString) { if cfg!(debug_assertions) || env!("CARGO_PKG_VERSION").contains("alpha") { - panic!("{message}"); + panic!("{}", message.to_string()); } else { - error!("{message}"); + error!("{}", message.to_string()); } } @@ -37,6 +39,14 @@ pub(crate) fn try_parse_error_message(text: &str) -> String { text.to_string() } +pub fn resolve_path(base: &Path, path: &PathBuf) -> PathBuf { + if path.is_absolute() { + path.clone() + } else { + base.join(path) + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/codex-rs/core/tests/common/Cargo.toml b/codex-rs/core/tests/common/Cargo.toml index 65abe23c63..75af1b4dd6 100644 --- a/codex-rs/core/tests/common/Cargo.toml +++ b/codex-rs/core/tests/common/Cargo.toml @@ -1,7 +1,8 @@ [package] -edition = "2024" name = "core_test_support" -version = { workspace = true } +version.workspace = true +edition.workspace = true +license.workspace = true [lib] path = "lib.rs" @@ -9,6 +10,7 @@ path = "lib.rs" [dependencies] anyhow = { workspace = true } assert_cmd = { workspace = true } +base64 = { workspace = true } codex-core = { workspace = true } codex-protocol = { workspace = true } notify = { workspace = true } @@ -18,3 +20,4 @@ tempfile = { workspace = true } tokio = { workspace = true, features = ["time"] } walkdir = { workspace = true } wiremock = { workspace = true } +shlex = { workspace = true } diff --git a/codex-rs/core/tests/common/lib.rs b/codex-rs/core/tests/common/lib.rs index 3f75ed1819..e7b1e71efa 100644 --- a/codex-rs/core/tests/common/lib.rs +++ b/codex-rs/core/tests/common/lib.rs @@ -172,6 +172,15 @@ pub fn sandbox_network_env_var() -> &'static str { codex_core::spawn::CODEX_SANDBOX_NETWORK_DISABLED_ENV_VAR } +pub fn format_with_current_shell(command: &str) -> Vec { + codex_core::shell::default_user_shell().derive_exec_args(command, true) +} + +pub fn format_with_current_shell_display(command: &str) -> String { + let args = format_with_current_shell(command); + shlex::try_join(args.iter().map(String::as_str)).expect("serialize current shell command") +} + pub mod fs_wait { use anyhow::Result; use anyhow::anyhow; diff --git a/codex-rs/core/tests/common/responses.rs b/codex-rs/core/tests/common/responses.rs index 7c5a103f42..a8209b5139 100644 --- a/codex-rs/core/tests/common/responses.rs +++ b/codex-rs/core/tests/common/responses.rs @@ -2,6 +2,7 @@ use std::sync::Arc; use std::sync::Mutex; use anyhow::Result; +use base64::Engine; use serde_json::Value; use wiremock::BodyPrintLimit; use wiremock::Match; @@ -297,12 +298,18 @@ pub fn ev_reasoning_item(id: &str, summary: &[&str], raw_content: &[&str]) -> Va .map(|text| serde_json::json!({"type": "summary_text", "text": text})) .collect(); + let overhead = "b".repeat(550); + let raw_content_joined = raw_content.join(""); + let encrypted_content = + base64::engine::general_purpose::STANDARD.encode(overhead + raw_content_joined.as_str()); + let mut event = serde_json::json!({ "type": "response.output_item.done", "item": { "type": "reasoning", "id": id, "summary": summary_entries, + "encrypted_content": encrypted_content, } }); @@ -424,6 +431,9 @@ pub fn ev_apply_patch_call( ApplyPatchModelOutput::ShellViaHeredoc => { ev_apply_patch_shell_call_via_heredoc(call_id, patch) } + ApplyPatchModelOutput::ShellCommandViaHeredoc => { + ev_apply_patch_shell_command_call_via_heredoc(call_id, patch) + } } } @@ -462,8 +472,11 @@ pub fn ev_apply_patch_function_call(call_id: &str, patch: &str) -> Value { pub fn ev_shell_command_call(call_id: &str, command: &str) -> Value { let args = serde_json::json!({ "command": command }); - let arguments = serde_json::to_string(&args).expect("serialize shell arguments"); + ev_shell_command_call_with_args(call_id, &args) +} +pub fn ev_shell_command_call_with_args(call_id: &str, args: &serde_json::Value) -> Value { + let arguments = serde_json::to_string(args).expect("serialize shell command arguments"); ev_function_call(call_id, "shell_command", &arguments) } @@ -482,6 +495,13 @@ pub fn ev_apply_patch_shell_call_via_heredoc(call_id: &str, patch: &str) -> Valu ev_function_call(call_id, "shell", &arguments) } +pub fn ev_apply_patch_shell_command_call_via_heredoc(call_id: &str, patch: &str) -> Value { + let args = serde_json::json!({ "command": format!("apply_patch <<'EOF'\n{patch}\nEOF\n") }); + let arguments = serde_json::to_string(&args).expect("serialize apply_patch arguments"); + + ev_function_call(call_id, "shell_command", &arguments) +} + pub fn sse_failed(id: &str, code: &str, message: &str) -> String { sse(vec![serde_json::json!({ "type": "response.failed", @@ -498,6 +518,32 @@ pub fn sse_response(body: String) -> ResponseTemplate { .set_body_raw(body, "text/event-stream") } +pub async fn mount_response_once(server: &MockServer, response: ResponseTemplate) -> ResponseMock { + let (mock, response_mock) = base_mock(); + mock.respond_with(response) + .up_to_n_times(1) + .mount(server) + .await; + response_mock +} + +pub async fn mount_response_once_match( + server: &MockServer, + matcher: M, + response: ResponseTemplate, +) -> ResponseMock +where + M: wiremock::Match + Send + Sync + 'static, +{ + let (mock, response_mock) = base_mock(); + mock.and(matcher) + .respond_with(response) + .up_to_n_times(1) + .mount(server) + .await; + response_mock +} + fn base_mock() -> (MockBuilder, ResponseMock) { let response_mock = ResponseMock::new(); let mock = Mock::given(method("POST")) diff --git a/codex-rs/core/tests/common/test_codex.rs b/codex-rs/core/tests/common/test_codex.rs index c51d9f0056..1c5919214f 100644 --- a/codex-rs/core/tests/common/test_codex.rs +++ b/codex-rs/core/tests/common/test_codex.rs @@ -36,6 +36,7 @@ pub enum ApplyPatchModelOutput { Function, Shell, ShellViaHeredoc, + ShellCommandViaHeredoc, } /// A collection of different ways the model can output an apply_patch call @@ -312,7 +313,10 @@ impl TestCodexHarness { ApplyPatchModelOutput::Freeform => self.custom_tool_call_output(call_id).await, ApplyPatchModelOutput::Function | ApplyPatchModelOutput::Shell - | ApplyPatchModelOutput::ShellViaHeredoc => self.function_call_stdout(call_id).await, + | ApplyPatchModelOutput::ShellViaHeredoc + | ApplyPatchModelOutput::ShellCommandViaHeredoc => { + self.function_call_stdout(call_id).await + } } } } diff --git a/codex-rs/core/tests/suite/abort_tasks.rs b/codex-rs/core/tests/suite/abort_tasks.rs index 1774619843..0d4a807a3c 100644 --- a/codex-rs/core/tests/suite/abort_tasks.rs +++ b/codex-rs/core/tests/suite/abort_tasks.rs @@ -17,15 +17,11 @@ use core_test_support::wait_for_event; use regex_lite::Regex; use serde_json::json; -/// Integration test: spawn a long‑running shell tool via a mocked Responses SSE +/// Integration test: spawn a long‑running shell_command tool via a mocked Responses SSE /// function call, then interrupt the session and expect TurnAborted. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn interrupt_long_running_tool_emits_turn_aborted() { - let command = vec![ - "bash".to_string(), - "-lc".to_string(), - "sleep 60".to_string(), - ]; + let command = "sleep 60"; let args = json!({ "command": command, @@ -33,14 +29,19 @@ async fn interrupt_long_running_tool_emits_turn_aborted() { }) .to_string(); let body = sse(vec![ - ev_function_call("call_sleep", "shell", &args), + ev_function_call("call_sleep", "shell_command", &args), ev_completed("done"), ]); let server = start_mock_server().await; mount_sse_once(&server, body).await; - let codex = test_codex().build(&server).await.unwrap().codex; + let codex = test_codex() + .with_model("gpt-5.1") + .build(&server) + .await + .unwrap() + .codex; // Kick off a turn that triggers the function call. codex @@ -67,11 +68,7 @@ async fn interrupt_long_running_tool_emits_turn_aborted() { /// responses server, and ensures the model receives the synthesized abort. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn interrupt_tool_records_history_entries() { - let command = vec![ - "bash".to_string(), - "-lc".to_string(), - "sleep 60".to_string(), - ]; + let command = "sleep 60"; let call_id = "call-history"; let args = json!({ @@ -81,7 +78,7 @@ async fn interrupt_tool_records_history_entries() { .to_string(); let first_body = sse(vec![ ev_response_created("resp-history"), - ev_function_call(call_id, "shell", &args), + ev_function_call(call_id, "shell_command", &args), ev_completed("resp-history"), ]); let follow_up_body = sse(vec![ @@ -92,7 +89,11 @@ async fn interrupt_tool_records_history_entries() { let server = start_mock_server().await; let response_mock = mount_sse_sequence(&server, vec![first_body, follow_up_body]).await; - let fixture = test_codex().build(&server).await.unwrap(); + let fixture = test_codex() + .with_model("gpt-5.1") + .build(&server) + .await + .unwrap(); let codex = Arc::clone(&fixture.codex); codex diff --git a/codex-rs/core/tests/suite/apply_patch_cli.rs b/codex-rs/core/tests/suite/apply_patch_cli.rs index 59ecf421e6..880e74d959 100644 --- a/codex-rs/core/tests/suite/apply_patch_cli.rs +++ b/codex-rs/core/tests/suite/apply_patch_cli.rs @@ -2,13 +2,12 @@ use anyhow::Result; use core_test_support::responses::ev_apply_patch_call; +use core_test_support::responses::ev_shell_command_call; use core_test_support::test_codex::ApplyPatchModelOutput; use pretty_assertions::assert_eq; use std::fs; -use codex_core::config::Config; use codex_core::features::Feature; -use codex_core::model_family::find_family_for_model; use codex_core::protocol::AskForApproval; use codex_core::protocol::EventMsg; use codex_core::protocol::Op; @@ -24,23 +23,24 @@ use core_test_support::responses::ev_response_created; use core_test_support::responses::mount_sse_sequence; use core_test_support::responses::sse; use core_test_support::skip_if_no_network; +use core_test_support::test_codex::TestCodexBuilder; use core_test_support::test_codex::TestCodexHarness; +use core_test_support::test_codex::test_codex; use core_test_support::wait_for_event; use serde_json::json; use test_case::test_case; pub async fn apply_patch_harness() -> Result { - apply_patch_harness_with(|_| {}).await + apply_patch_harness_with(|builder| builder).await } async fn apply_patch_harness_with( - configure: impl FnOnce(&mut Config) + Send + 'static, + configure: impl FnOnce(TestCodexBuilder) -> TestCodexBuilder, ) -> Result { - TestCodexHarness::with_config(|config| { + let builder = configure(test_codex()).with_config(|config| { config.include_apply_patch_tool = true; - configure(config); - }) - .await + }); + TestCodexHarness::with_builder(builder).await } pub async fn mount_apply_patch( @@ -86,11 +86,7 @@ async fn apply_patch_cli_multiple_operations_integration( ) -> Result<()> { skip_if_no_network!(Ok(())); - let harness = apply_patch_harness_with(|config| { - config.model = "gpt-5.1".to_string(); - config.model_family = find_family_for_model("gpt-5.1").expect("gpt-5.1 is valid"); - }) - .await?; + let harness = apply_patch_harness_with(|builder| builder.with_model("gpt-5.1")).await?; // Seed workspace state let modify_path = harness.path("modify.txt"); @@ -132,6 +128,7 @@ D delete.txt #[test_case(ApplyPatchModelOutput::Function)] #[test_case(ApplyPatchModelOutput::Shell)] #[test_case(ApplyPatchModelOutput::ShellViaHeredoc)] +#[test_case(ApplyPatchModelOutput::ShellCommandViaHeredoc)] async fn apply_patch_cli_multiple_chunks(model_output: ApplyPatchModelOutput) -> Result<()> { skip_if_no_network!(Ok(())); @@ -158,6 +155,7 @@ async fn apply_patch_cli_multiple_chunks(model_output: ApplyPatchModelOutput) -> #[test_case(ApplyPatchModelOutput::Function)] #[test_case(ApplyPatchModelOutput::Shell)] #[test_case(ApplyPatchModelOutput::ShellViaHeredoc)] +#[test_case(ApplyPatchModelOutput::ShellCommandViaHeredoc)] async fn apply_patch_cli_moves_file_to_new_directory( model_output: ApplyPatchModelOutput, ) -> Result<()> { @@ -186,6 +184,7 @@ async fn apply_patch_cli_moves_file_to_new_directory( #[test_case(ApplyPatchModelOutput::Function)] #[test_case(ApplyPatchModelOutput::Shell)] #[test_case(ApplyPatchModelOutput::ShellViaHeredoc)] +#[test_case(ApplyPatchModelOutput::ShellCommandViaHeredoc)] async fn apply_patch_cli_updates_file_appends_trailing_newline( model_output: ApplyPatchModelOutput, ) -> Result<()> { @@ -213,6 +212,7 @@ async fn apply_patch_cli_updates_file_appends_trailing_newline( #[test_case(ApplyPatchModelOutput::Function)] #[test_case(ApplyPatchModelOutput::Shell)] #[test_case(ApplyPatchModelOutput::ShellViaHeredoc)] +#[test_case(ApplyPatchModelOutput::ShellCommandViaHeredoc)] async fn apply_patch_cli_insert_only_hunk_modifies_file( model_output: ApplyPatchModelOutput, ) -> Result<()> { @@ -238,6 +238,7 @@ async fn apply_patch_cli_insert_only_hunk_modifies_file( #[test_case(ApplyPatchModelOutput::Function)] #[test_case(ApplyPatchModelOutput::Shell)] #[test_case(ApplyPatchModelOutput::ShellViaHeredoc)] +#[test_case(ApplyPatchModelOutput::ShellCommandViaHeredoc)] async fn apply_patch_cli_move_overwrites_existing_destination( model_output: ApplyPatchModelOutput, ) -> Result<()> { @@ -268,6 +269,7 @@ async fn apply_patch_cli_move_overwrites_existing_destination( #[test_case(ApplyPatchModelOutput::Function)] #[test_case(ApplyPatchModelOutput::Shell)] #[test_case(ApplyPatchModelOutput::ShellViaHeredoc)] +#[test_case(ApplyPatchModelOutput::ShellCommandViaHeredoc)] async fn apply_patch_cli_move_without_content_change_has_no_turn_diff( model_output: ApplyPatchModelOutput, ) -> Result<()> { @@ -325,6 +327,7 @@ async fn apply_patch_cli_move_without_content_change_has_no_turn_diff( #[test_case(ApplyPatchModelOutput::Function)] #[test_case(ApplyPatchModelOutput::Shell)] #[test_case(ApplyPatchModelOutput::ShellViaHeredoc)] +#[test_case(ApplyPatchModelOutput::ShellCommandViaHeredoc)] async fn apply_patch_cli_add_overwrites_existing_file( model_output: ApplyPatchModelOutput, ) -> Result<()> { @@ -350,6 +353,7 @@ async fn apply_patch_cli_add_overwrites_existing_file( #[test_case(ApplyPatchModelOutput::Function)] #[test_case(ApplyPatchModelOutput::Shell)] #[test_case(ApplyPatchModelOutput::ShellViaHeredoc)] +#[test_case(ApplyPatchModelOutput::ShellCommandViaHeredoc)] async fn apply_patch_cli_rejects_invalid_hunk_header( model_output: ApplyPatchModelOutput, ) -> Result<()> { @@ -381,6 +385,7 @@ async fn apply_patch_cli_rejects_invalid_hunk_header( #[test_case(ApplyPatchModelOutput::Function)] #[test_case(ApplyPatchModelOutput::Shell)] #[test_case(ApplyPatchModelOutput::ShellViaHeredoc)] +#[test_case(ApplyPatchModelOutput::ShellCommandViaHeredoc)] async fn apply_patch_cli_reports_missing_context( model_output: ApplyPatchModelOutput, ) -> Result<()> { @@ -414,6 +419,7 @@ async fn apply_patch_cli_reports_missing_context( #[test_case(ApplyPatchModelOutput::Function)] #[test_case(ApplyPatchModelOutput::Shell)] #[test_case(ApplyPatchModelOutput::ShellViaHeredoc)] +#[test_case(ApplyPatchModelOutput::ShellCommandViaHeredoc)] async fn apply_patch_cli_reports_missing_target_file( model_output: ApplyPatchModelOutput, ) -> Result<()> { @@ -449,6 +455,7 @@ async fn apply_patch_cli_reports_missing_target_file( #[test_case(ApplyPatchModelOutput::Function)] #[test_case(ApplyPatchModelOutput::Shell)] #[test_case(ApplyPatchModelOutput::ShellViaHeredoc)] +#[test_case(ApplyPatchModelOutput::ShellCommandViaHeredoc)] async fn apply_patch_cli_delete_missing_file_reports_error( model_output: ApplyPatchModelOutput, ) -> Result<()> { @@ -485,6 +492,7 @@ async fn apply_patch_cli_delete_missing_file_reports_error( #[test_case(ApplyPatchModelOutput::Function)] #[test_case(ApplyPatchModelOutput::Shell)] #[test_case(ApplyPatchModelOutput::ShellViaHeredoc)] +#[test_case(ApplyPatchModelOutput::ShellCommandViaHeredoc)] async fn apply_patch_cli_rejects_empty_patch(model_output: ApplyPatchModelOutput) -> Result<()> { skip_if_no_network!(Ok(())); @@ -509,6 +517,7 @@ async fn apply_patch_cli_rejects_empty_patch(model_output: ApplyPatchModelOutput #[test_case(ApplyPatchModelOutput::Function)] #[test_case(ApplyPatchModelOutput::Shell)] #[test_case(ApplyPatchModelOutput::ShellViaHeredoc)] +#[test_case(ApplyPatchModelOutput::ShellCommandViaHeredoc)] async fn apply_patch_cli_delete_directory_reports_verification_error( model_output: ApplyPatchModelOutput, ) -> Result<()> { @@ -535,6 +544,7 @@ async fn apply_patch_cli_delete_directory_reports_verification_error( #[test_case(ApplyPatchModelOutput::Function)] #[test_case(ApplyPatchModelOutput::Shell)] #[test_case(ApplyPatchModelOutput::ShellViaHeredoc)] +#[test_case(ApplyPatchModelOutput::ShellCommandViaHeredoc)] async fn apply_patch_cli_rejects_path_traversal_outside_workspace( model_output: ApplyPatchModelOutput, ) -> Result<()> { @@ -587,6 +597,7 @@ async fn apply_patch_cli_rejects_path_traversal_outside_workspace( #[test_case(ApplyPatchModelOutput::Function)] #[test_case(ApplyPatchModelOutput::Shell)] #[test_case(ApplyPatchModelOutput::ShellViaHeredoc)] +#[test_case(ApplyPatchModelOutput::ShellCommandViaHeredoc)] async fn apply_patch_cli_rejects_move_path_traversal_outside_workspace( model_output: ApplyPatchModelOutput, ) -> Result<()> { @@ -640,13 +651,16 @@ async fn apply_patch_cli_rejects_move_path_traversal_outside_workspace( #[test_case(ApplyPatchModelOutput::Function)] #[test_case(ApplyPatchModelOutput::Shell)] #[test_case(ApplyPatchModelOutput::ShellViaHeredoc)] +#[test_case(ApplyPatchModelOutput::ShellCommandViaHeredoc)] async fn apply_patch_cli_verification_failure_has_no_side_effects( model_output: ApplyPatchModelOutput, ) -> Result<()> { skip_if_no_network!(Ok(())); - let harness = apply_patch_harness_with(|config| { - config.features.enable(Feature::ApplyPatchFreeform); + let harness = apply_patch_harness_with(|builder| { + builder.with_config(|config| { + config.features.enable(Feature::ApplyPatchFreeform); + }) }) .await?; @@ -667,14 +681,10 @@ async fn apply_patch_cli_verification_failure_has_no_side_effects( } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn apply_patch_shell_heredoc_with_cd_updates_relative_workdir() -> Result<()> { +async fn apply_patch_shell_command_heredoc_with_cd_updates_relative_workdir() -> Result<()> { skip_if_no_network!(Ok(())); - let harness = apply_patch_harness_with(|config| { - config.model = "gpt-5.1".to_string(); - config.model_family = find_family_for_model("gpt-5.1").expect("gpt-5.1 is valid"); - }) - .await?; + let harness = apply_patch_harness_with(|builder| builder.with_model("gpt-5.1")).await?; // Prepare a file inside a subdir; update it via cd && apply_patch heredoc form. let sub = harness.path("sub"); @@ -684,14 +694,10 @@ async fn apply_patch_shell_heredoc_with_cd_updates_relative_workdir() -> Result< let script = "cd sub && apply_patch <<'EOF'\n*** Begin Patch\n*** Update File: in_sub.txt\n@@\n-before\n+after\n*** End Patch\nEOF\n"; let call_id = "shell-heredoc-cd"; - let args = json!({ - "command": ["bash", "-lc", script], - "timeout_ms": 5_000, - }); let bodies = vec![ sse(vec![ ev_response_created("resp-1"), - ev_function_call(call_id, "shell", &serde_json::to_string(&args)?), + ev_shell_command_call(call_id, script), ev_completed("resp-1"), ]), sse(vec![ @@ -706,21 +712,97 @@ async fn apply_patch_shell_heredoc_with_cd_updates_relative_workdir() -> Result< let out = harness.function_call_stdout(call_id).await; assert!( out.contains("Success."), - "expected successful apply_patch invocation via shell: {out}" + "expected successful apply_patch invocation via shell_command: {out}" ); assert_eq!(fs::read_to_string(&target)?, "after\n"); Ok(()) } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn apply_patch_shell_failure_propagates_error_and_skips_diff() -> Result<()> { +async fn apply_patch_shell_command_heredoc_with_cd_emits_turn_diff() -> Result<()> { skip_if_no_network!(Ok(())); - let harness = apply_patch_harness_with(|config| { - config.model = "gpt-5.1".to_string(); - config.model_family = find_family_for_model("gpt-5.1").expect("gpt-5.1 is valid"); + let harness = apply_patch_harness_with(|builder| builder.with_model("gpt-5.1")).await?; + let test = harness.test(); + let codex = test.codex.clone(); + let cwd = test.cwd.clone(); + + // Prepare a file inside a subdir; update it via cd && apply_patch heredoc form. + let sub = test.workspace_path("sub"); + fs::create_dir_all(&sub)?; + let target = sub.join("in_sub.txt"); + fs::write(&target, "before\n")?; + + let script = "cd sub && apply_patch <<'EOF'\n*** Begin Patch\n*** Update File: in_sub.txt\n@@\n-before\n+after\n*** End Patch\nEOF\n"; + let call_id = "shell-heredoc-cd"; + let args = json!({ "command": script, "timeout_ms": 5_000 }); + let bodies = vec![ + sse(vec![ + ev_response_created("resp-1"), + ev_function_call(call_id, "shell_command", &serde_json::to_string(&args)?), + ev_completed("resp-1"), + ]), + sse(vec![ + ev_assistant_message("msg-1", "ok"), + ev_completed("resp-2"), + ]), + ]; + mount_sse_sequence(harness.server(), bodies).await; + + let model = test.session_configured.model.clone(); + codex + .submit(Op::UserTurn { + items: vec![UserInput::Text { + text: "apply via shell heredoc with cd".into(), + }], + final_output_json_schema: None, + cwd: cwd.path().to_path_buf(), + approval_policy: AskForApproval::Never, + sandbox_policy: SandboxPolicy::DangerFullAccess, + model, + effort: None, + summary: ReasoningSummary::Auto, + }) + .await?; + + let mut saw_turn_diff = None; + let mut saw_patch_begin = false; + let mut patch_end_success = None; + wait_for_event(&codex, |event| match event { + EventMsg::PatchApplyBegin(begin) => { + saw_patch_begin = true; + assert_eq!(begin.call_id, call_id); + false + } + EventMsg::PatchApplyEnd(end) => { + assert_eq!(end.call_id, call_id); + patch_end_success = Some(end.success); + false + } + EventMsg::TurnDiff(ev) => { + saw_turn_diff = Some(ev.unified_diff.clone()); + false + } + EventMsg::TaskComplete(_) => true, + _ => false, }) - .await?; + .await; + + assert!(saw_patch_begin, "expected PatchApplyBegin event"); + let patch_end_success = + patch_end_success.expect("expected PatchApplyEnd event to capture success flag"); + assert!(patch_end_success); + + let diff = saw_turn_diff.expect("expected TurnDiff event"); + assert!(diff.contains("diff --git"), "diff header missing: {diff:?}"); + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn apply_patch_shell_command_failure_propagates_error_and_skips_diff() -> Result<()> { + skip_if_no_network!(Ok(())); + + let harness = apply_patch_harness_with(|builder| builder.with_model("gpt-5.1")).await?; let test = harness.test(); let codex = test.codex.clone(); let cwd = test.cwd.clone(); @@ -730,14 +812,11 @@ async fn apply_patch_shell_failure_propagates_error_and_skips_diff() -> Result<( let script = "apply_patch <<'EOF'\n*** Begin Patch\n*** Update File: invalid.txt\n@@\n-nope\n+changed\n*** End Patch\nEOF\n"; let call_id = "shell-apply-failure"; - let args = json!({ - "command": ["bash", "-lc", script], - "timeout_ms": 5_000, - }); + let args = json!({ "command": script, "timeout_ms": 5_000 }); let bodies = vec![ sse(vec![ ev_response_created("resp-1"), - ev_function_call(call_id, "shell", &serde_json::to_string(&args)?), + ev_function_call(call_id, "shell_command", &serde_json::to_string(&args)?), ev_completed("resp-1"), ]), sse(vec![ @@ -780,10 +859,6 @@ async fn apply_patch_shell_failure_propagates_error_and_skips_diff() -> Result<( ); let out = harness.function_call_stdout(call_id).await; - assert!( - out.contains("apply_patch verification failed"), - "expected verification failure message" - ); assert!( out.contains("Failed to find expected lines in"), "expected failure diagnostics: {out}" @@ -797,7 +872,11 @@ async fn apply_patch_shell_failure_propagates_error_and_skips_diff() -> Result<( } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn apply_patch_function_accepts_lenient_heredoc_wrapped_patch() -> Result<()> { +#[test_case(ApplyPatchModelOutput::ShellViaHeredoc)] +#[test_case(ApplyPatchModelOutput::ShellCommandViaHeredoc)] +async fn apply_patch_function_accepts_lenient_heredoc_wrapped_patch( + model_output: ApplyPatchModelOutput, +) -> Result<()> { skip_if_no_network!(Ok(())); let harness = apply_patch_harness().await?; @@ -805,16 +884,8 @@ async fn apply_patch_function_accepts_lenient_heredoc_wrapped_patch() -> Result< let file_name = "lenient.txt"; let patch_inner = format!("*** Begin Patch\n*** Add File: {file_name}\n+lenient\n*** End Patch\n"); - let wrapped = format!("<<'EOF'\n{patch_inner}EOF\n"); let call_id = "apply-lenient"; - mount_apply_patch( - &harness, - call_id, - wrapped.as_str(), - "ok", - ApplyPatchModelOutput::Function, - ) - .await; + mount_apply_patch(&harness, call_id, patch_inner.as_str(), "ok", model_output).await; harness.submit("apply lenient heredoc patch").await?; @@ -828,6 +899,7 @@ async fn apply_patch_function_accepts_lenient_heredoc_wrapped_patch() -> Result< #[test_case(ApplyPatchModelOutput::Function)] #[test_case(ApplyPatchModelOutput::Shell)] #[test_case(ApplyPatchModelOutput::ShellViaHeredoc)] +#[test_case(ApplyPatchModelOutput::ShellCommandViaHeredoc)] async fn apply_patch_cli_end_of_file_anchor(model_output: ApplyPatchModelOutput) -> Result<()> { skip_if_no_network!(Ok(())); @@ -850,6 +922,7 @@ async fn apply_patch_cli_end_of_file_anchor(model_output: ApplyPatchModelOutput) #[test_case(ApplyPatchModelOutput::Function)] #[test_case(ApplyPatchModelOutput::Shell)] #[test_case(ApplyPatchModelOutput::ShellViaHeredoc)] +#[test_case(ApplyPatchModelOutput::ShellCommandViaHeredoc)] async fn apply_patch_cli_missing_second_chunk_context_rejected( model_output: ApplyPatchModelOutput, ) -> Result<()> { @@ -884,6 +957,7 @@ async fn apply_patch_cli_missing_second_chunk_context_rejected( #[test_case(ApplyPatchModelOutput::Function)] #[test_case(ApplyPatchModelOutput::Shell)] #[test_case(ApplyPatchModelOutput::ShellViaHeredoc)] +#[test_case(ApplyPatchModelOutput::ShellCommandViaHeredoc)] async fn apply_patch_emits_turn_diff_event_with_unified_diff( model_output: ApplyPatchModelOutput, ) -> Result<()> { @@ -939,6 +1013,7 @@ async fn apply_patch_emits_turn_diff_event_with_unified_diff( #[test_case(ApplyPatchModelOutput::Function)] #[test_case(ApplyPatchModelOutput::Shell)] #[test_case(ApplyPatchModelOutput::ShellViaHeredoc)] +#[test_case(ApplyPatchModelOutput::ShellCommandViaHeredoc)] async fn apply_patch_turn_diff_for_rename_with_content_change( model_output: ApplyPatchModelOutput, ) -> Result<()> { @@ -1153,6 +1228,7 @@ async fn apply_patch_aggregates_diff_preserves_success_after_failure() -> Result #[test_case(ApplyPatchModelOutput::Function)] #[test_case(ApplyPatchModelOutput::Shell)] #[test_case(ApplyPatchModelOutput::ShellViaHeredoc)] +#[test_case(ApplyPatchModelOutput::ShellCommandViaHeredoc)] async fn apply_patch_change_context_disambiguates_target( model_output: ApplyPatchModelOutput, ) -> Result<()> { diff --git a/codex-rs/core/tests/suite/approvals.rs b/codex-rs/core/tests/suite/approvals.rs index a106d2eae0..caa488a88f 100644 --- a/codex-rs/core/tests/suite/approvals.rs +++ b/codex-rs/core/tests/suite/approvals.rs @@ -2,7 +2,6 @@ use anyhow::Result; use codex_core::features::Feature; -use codex_core::model_family::find_family_for_model; use codex_core::protocol::ApplyPatchApprovalRequestEvent; use codex_core::protocol::AskForApproval; use codex_core::protocol::EventMsg; @@ -71,7 +70,7 @@ enum ActionKind { response_body: &'static str, }, RunCommand { - command: &'static [&'static str], + command: &'static str, }, RunUnifiedExecCommand { command: &'static str, @@ -97,20 +96,12 @@ impl ActionKind { server: &MockServer, call_id: &str, with_escalated_permissions: bool, - ) -> Result<(Value, Option>)> { + ) -> Result<(Value, Option)> { match self { ActionKind::WriteFile { target, content } => { let (path, _) = target.resolve_for_patch(test); let _ = fs::remove_file(&path); - let command = vec![ - "/bin/sh".to_string(), - "-c".to_string(), - format!( - "printf {content:?} > {path:?} && cat {path:?}", - content = content, - path = path - ), - ]; + let command = format!("printf {content:?} > {path:?} && cat {path:?}"); let event = shell_event(call_id, &command, 1_000, with_escalated_permissions)?; Ok((event, Some(command))) } @@ -127,21 +118,18 @@ impl ActionKind { .await; let url = format!("{}{}", server.uri(), endpoint); + let escaped_url = url.replace('\'', "\\'"); let script = format!( - "import sys\nimport urllib.request\nurl = {url:?}\ntry:\n data = urllib.request.urlopen(url, timeout=2).read().decode()\n print('OK:' + data.strip())\nexcept Exception as exc:\n print('ERR:' + exc.__class__.__name__)\n sys.exit(1)", + "import sys\nimport urllib.request\nurl = '{escaped_url}'\ntry:\n data = urllib.request.urlopen(url, timeout=2).read().decode()\n print('OK:' + data.strip())\nexcept Exception as exc:\n print('ERR:' + exc.__class__.__name__)\n sys.exit(1)", ); - let command = vec!["python3".to_string(), "-c".to_string(), script]; + let command = format!("python3 -c \"{script}\""); let event = shell_event(call_id, &command, 1_000, with_escalated_permissions)?; Ok((event, Some(command))) } ActionKind::RunCommand { command } => { - let command: Vec = command - .iter() - .map(std::string::ToString::to_string) - .collect(); - let event = shell_event(call_id, &command, 1_000, with_escalated_permissions)?; - Ok((event, Some(command))) + let event = shell_event(call_id, command, 1_000, with_escalated_permissions)?; + Ok((event, Some(command.to_string()))) } ActionKind::RunUnifiedExecCommand { command, @@ -154,14 +142,7 @@ impl ActionKind { with_escalated_permissions, *justification, )?; - Ok(( - event, - Some(vec![ - "/bin/bash".to_string(), - "-lc".to_string(), - command.to_string(), - ]), - )) + Ok((event, Some(command.to_string()))) } ActionKind::ApplyPatchFunction { target, content } => { let (path, patch_path) = target.resolve_for_patch(test); @@ -185,19 +166,19 @@ fn build_add_file_patch(patch_path: &str, content: &str) -> String { format!("*** Begin Patch\n*** Add File: {patch_path}\n+{content}\n*** End Patch\n") } -fn shell_apply_patch_command(patch: &str) -> Vec { +fn shell_apply_patch_command(patch: &str) -> String { let mut script = String::from("apply_patch <<'PATCH'\n"); script.push_str(patch); if !patch.ends_with('\n') { script.push('\n'); } script.push_str("PATCH\n"); - vec!["bash".to_string(), "-lc".to_string(), script] + script } fn shell_event( call_id: &str, - command: &[String], + command: &str, timeout_ms: u64, with_escalated_permissions: bool, ) -> Result { @@ -209,7 +190,7 @@ fn shell_event( args["with_escalated_permissions"] = json!(true); } let args_str = serde_json::to_string(&args)?; - Ok(ev_function_call(call_id, "shell", &args_str)) + Ok(ev_function_call(call_id, "shell_command", &args_str)) } fn exec_command_event( @@ -296,7 +277,10 @@ impl Expectation { } Expectation::FileCreatedNoExitCode { target, content } => { let (path, _) = target.resolve_for_patch(test); - assert_eq!(result.exit_code, None, "expected no exit code for {path:?}"); + assert!( + result.exit_code.is_none() || result.exit_code == Some(0), + "expected no exit code for {path:?}", + ); assert!( result.stdout.contains(content), "stdout missing {content:?}: {}", @@ -385,8 +369,8 @@ impl Expectation { ); } Expectation::NetworkSuccessNoExitCode { body_contains } => { - assert_eq!( - result.exit_code, None, + assert!( + result.exit_code.is_none() || result.exit_code == Some(0), "expected no exit code for successful network call: {}", result.stdout ); @@ -433,8 +417,8 @@ impl Expectation { ); } Expectation::CommandSuccessNoExitCode { stdout_contains } => { - assert_eq!( - result.exit_code, None, + assert!( + result.exit_code.is_none() || result.exit_code == Some(0), "expected no exit code for trusted command: {}", result.stdout ); @@ -531,10 +515,18 @@ fn parse_result(item: &Value) -> CommandResult { CommandResult { exit_code, stdout } } Err(_) => { + let structured = Regex::new(r"(?s)^Exit code:\s*(-?\d+).*?Output:\n(.*)$").unwrap(); let regex = Regex::new(r"(?s)^.*?Process exited with code (\d+)\n.*?Output:\n(.*)$").unwrap(); // parse freeform output - if let Some(captures) = regex.captures(output_str) { + if let Some(captures) = structured.captures(output_str) { + let exit_code = captures.get(1).unwrap().as_str().parse::().unwrap(); + let output = captures.get(2).unwrap().as_str(); + CommandResult { + exit_code: Some(exit_code), + stdout: output.to_string(), + } + } else if let Some(captures) = regex.captures(output_str) { let exit_code = captures.get(1).unwrap().as_str().parse::().unwrap(); let output = captures.get(2).unwrap().as_str(); CommandResult { @@ -553,7 +545,7 @@ fn parse_result(item: &Value) -> CommandResult { async fn expect_exec_approval( test: &TestCodex, - expected_command: &[String], + expected_command: &str, ) -> ExecApprovalRequestEvent { let event = wait_for_event(&test.codex, |event| { matches!( @@ -565,7 +557,12 @@ async fn expect_exec_approval( match event { EventMsg::ExecApprovalRequest(approval) => { - assert_eq!(approval.command, expected_command); + let last_arg = approval + .command + .last() + .map(std::string::String::as_str) + .unwrap_or_default(); + assert_eq!(last_arg, expected_command); approval } EventMsg::TaskComplete(_) => panic!("expected approval request before completion"), @@ -660,7 +657,7 @@ fn scenarios() -> Vec { features: vec![], model_override: Some("gpt-5.1"), outcome: Outcome::Auto, - expectation: Expectation::FileCreatedNoExitCode { + expectation: Expectation::FileCreated { target: TargetPath::OutsideWorkspace("dfa_on_request_5_1.txt"), content: "danger-on-request", }, @@ -702,7 +699,7 @@ fn scenarios() -> Vec { approval_policy: UnlessTrusted, sandbox_policy: SandboxPolicy::DangerFullAccess, action: ActionKind::RunCommand { - command: &["echo", "trusted-unless"], + command: "echo trusted-unless", }, with_escalated_permissions: false, features: vec![], @@ -717,7 +714,7 @@ fn scenarios() -> Vec { approval_policy: UnlessTrusted, sandbox_policy: SandboxPolicy::DangerFullAccess, action: ActionKind::RunCommand { - command: &["echo", "trusted-unless"], + command: "echo trusted-unless", }, with_escalated_permissions: false, features: vec![], @@ -880,7 +877,7 @@ fn scenarios() -> Vec { approval_policy: OnRequest, sandbox_policy: SandboxPolicy::ReadOnly, action: ActionKind::RunCommand { - command: &["echo", "trusted-read-only"], + command: "echo trusted-read-only", }, with_escalated_permissions: false, features: vec![], @@ -895,7 +892,7 @@ fn scenarios() -> Vec { approval_policy: OnRequest, sandbox_policy: SandboxPolicy::ReadOnly, action: ActionKind::RunCommand { - command: &["echo", "trusted-read-only"], + command: "echo trusted-read-only", }, with_escalated_permissions: false, features: vec![], @@ -1020,7 +1017,7 @@ fn scenarios() -> Vec { }, }, ScenarioSpec { - name: "apply_patch_shell_requires_patch_approval", + name: "apply_patch_shell_command_requires_patch_approval", approval_policy: UnlessTrusted, sandbox_policy: workspace_write(false), action: ActionKind::ApplyPatchShell { @@ -1114,7 +1111,7 @@ fn scenarios() -> Vec { }, }, ScenarioSpec { - name: "apply_patch_shell_outside_requires_patch_approval", + name: "apply_patch_shell_command_outside_requires_patch_approval", approval_policy: OnRequest, sandbox_policy: workspace_write(false), action: ActionKind::ApplyPatchShell { @@ -1229,7 +1226,10 @@ fn scenarios() -> Vec { message_contains: if cfg!(target_os = "linux") { &["Permission denied"] } else { - &["Permission denied|Operation not permitted|Read-only file system"] + &[ + "Permission denied|Operation not permitted|operation not permitted|\ + Read-only file system", + ] }, }, }, @@ -1238,7 +1238,7 @@ fn scenarios() -> Vec { approval_policy: Never, sandbox_policy: SandboxPolicy::ReadOnly, action: ActionKind::RunCommand { - command: &["echo", "trusted-never"], + command: "echo trusted-never", }, with_escalated_permissions: false, features: vec![], @@ -1373,7 +1373,10 @@ fn scenarios() -> Vec { message_contains: if cfg!(target_os = "linux") { &["Permission denied"] } else { - &["Permission denied|Operation not permitted|Read-only file system"] + &[ + "Permission denied|Operation not permitted|operation not permitted|\ + Read-only file system", + ] }, }, }, @@ -1454,14 +1457,11 @@ async fn run_scenario(scenario: &ScenarioSpec) -> Result<()> { let sandbox_policy = scenario.sandbox_policy.clone(); let features = scenario.features.clone(); let model_override = scenario.model_override; + let model = model_override.unwrap_or("gpt-5.1"); - let mut builder = test_codex().with_config(move |config| { + let mut builder = test_codex().with_model(model).with_config(move |config| { config.approval_policy = approval_policy; config.sandbox_policy = sandbox_policy.clone(); - let model = model_override.unwrap_or("gpt-5.1"); - config.model = model.to_string(); - config.model_family = - find_family_for_model(model).expect("model should map to a known family"); for feature in features { config.features.enable(feature); } @@ -1509,7 +1509,7 @@ async fn run_scenario(scenario: &ScenarioSpec) -> Result<()> { expected_reason, } => { let command = expected_command - .as_ref() + .as_deref() .expect("exec approval requires shell command"); let approval = expect_exec_approval(&test, command).await; if let Some(expected_reason) = expected_reason { diff --git a/codex-rs/core/tests/suite/cli_stream.rs b/codex-rs/core/tests/suite/cli_stream.rs index b484eca22a..d7f0fb9830 100644 --- a/codex-rs/core/tests/suite/cli_stream.rs +++ b/codex-rs/core/tests/suite/cli_stream.rs @@ -499,9 +499,20 @@ async fn integration_git_info_unit_test() { "Git info should contain repository_url" ); let repo_url = git_info.repository_url.as_ref().unwrap(); + // Some hosts rewrite remotes (e.g., github.com → git@github.com), so assert against + // the actual remote reported by git instead of a static URL. + let expected_remote_url = std::process::Command::new("git") + .args(["remote", "get-url", "origin"]) + .current_dir(&git_repo) + .output() + .unwrap(); + let expected_remote_url = String::from_utf8(expected_remote_url.stdout) + .unwrap() + .trim() + .to_string(); assert_eq!( - repo_url, "https://github.com/example/integration-test.git", - "Repository URL should match what we configured" + repo_url, &expected_remote_url, + "Repository URL should match git remote get-url output" ); println!("✅ Git info collection test passed!"); diff --git a/codex-rs/core/tests/suite/client.rs b/codex-rs/core/tests/suite/client.rs index 59b7e34171..e074d29755 100644 --- a/codex-rs/core/tests/suite/client.rs +++ b/codex-rs/core/tests/suite/client.rs @@ -15,6 +15,7 @@ use codex_core::WireApi; use codex_core::auth::AuthCredentialsStoreMode; use codex_core::built_in_model_providers; use codex_core::error::CodexErr; +use codex_core::features::Feature; use codex_core::model_family::find_family_for_model; use codex_core::protocol::EventMsg; use codex_core::protocol::Op; @@ -34,6 +35,7 @@ use core_test_support::skip_if_no_network; use core_test_support::test_codex::TestCodex; use core_test_support::test_codex::test_codex; use core_test_support::wait_for_event; +use dunce::canonicalize as normalize_path; use futures::StreamExt; use serde_json::json; use std::io::Write; @@ -620,6 +622,74 @@ async fn includes_user_instructions_message_in_request() { assert_message_ends_with(&request_body["input"][1], ""); } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn skills_append_to_instructions_when_feature_enabled() { + skip_if_no_network!(); + let server = MockServer::start().await; + + let resp_mock = responses::mount_sse_once(&server, sse_completed("resp1")).await; + + let model_provider = ModelProviderInfo { + base_url: Some(format!("{}/v1", server.uri())), + ..built_in_model_providers()["openai"].clone() + }; + + let codex_home = TempDir::new().unwrap(); + let skill_dir = codex_home.path().join("skills/demo"); + std::fs::create_dir_all(&skill_dir).expect("create skill dir"); + std::fs::write( + skill_dir.join("SKILL.md"), + "---\nname: demo\ndescription: build charts\n---\n\n# body\n", + ) + .expect("write skill"); + + let mut config = load_default_config_for_test(&codex_home); + config.model_provider = model_provider; + config.features.enable(Feature::Skills); + config.cwd = codex_home.path().to_path_buf(); + + let conversation_manager = + ConversationManager::with_auth(CodexAuth::from_api_key("Test API Key")); + let codex = conversation_manager + .new_conversation(config) + .await + .expect("create new conversation") + .conversation; + + codex + .submit(Op::UserInput { + items: vec![UserInput::Text { + text: "hello".into(), + }], + }) + .await + .unwrap(); + + wait_for_event(&codex, |ev| matches!(ev, EventMsg::TaskComplete(_))).await; + + let request = resp_mock.single_request(); + let request_body = request.body_json(); + + assert_message_role(&request_body["input"][0], "user"); + let instructions_text = request_body["input"][0]["content"][0]["text"] + .as_str() + .expect("instructions text"); + assert!( + instructions_text.contains("## Skills"), + "expected skills section present" + ); + assert!( + instructions_text.contains("demo: build charts"), + "expected skill summary" + ); + let expected_path = normalize_path(skill_dir.join("SKILL.md")).unwrap(); + let expected_path_str = expected_path.to_string_lossy().replace('\\', "/"); + assert!( + instructions_text.contains(&expected_path_str), + "expected path {expected_path_str} in instructions" + ); +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn includes_configured_effort_in_request() -> anyhow::Result<()> { skip_if_no_network!(Ok(())); @@ -992,7 +1062,7 @@ async fn azure_responses_request_includes_store_and_reasoning_ids() { id: Some("web-search-id".into()), status: Some("completed".into()), action: WebSearchAction::Search { - query: "weather".into(), + query: Some("weather".into()), }, }); prompt.input.push(ResponseItem::FunctionCall { diff --git a/codex-rs/core/tests/suite/codex_delegate.rs b/codex-rs/core/tests/suite/codex_delegate.rs index 6339bfa71a..f5fe1a7df9 100644 --- a/codex-rs/core/tests/suite/codex_delegate.rs +++ b/codex-rs/core/tests/suite/codex_delegate.rs @@ -3,6 +3,7 @@ use codex_core::protocol::EventMsg; use codex_core::protocol::Op; use codex_core::protocol::ReviewDecision; use codex_core::protocol::ReviewRequest; +use codex_core::protocol::ReviewTarget; use codex_core::protocol::SandboxPolicy; use core_test_support::responses::ev_apply_patch_function_call; use core_test_support::responses::ev_assistant_message; @@ -25,17 +26,17 @@ use pretty_assertions::assert_eq; async fn codex_delegate_forwards_exec_approval_and_proceeds_on_approval() { skip_if_no_network!(); - // Sub-agent turn 1: emit a shell function_call requiring approval, then complete. + // Sub-agent turn 1: emit a shell_command function_call requiring approval, then complete. let call_id = "call-exec-1"; let args = serde_json::json!({ - "command": ["bash", "-lc", "rm -rf delegated"], + "command": "rm -rf delegated", "timeout_ms": 1000, "with_escalated_permissions": true, }) .to_string(); let sse1 = sse(vec![ ev_response_created("resp-1"), - ev_function_call(call_id, "shell", &args), + ev_function_call(call_id, "shell_command", &args), ev_completed("resp-1"), ]); @@ -58,7 +59,7 @@ async fn codex_delegate_forwards_exec_approval_and_proceeds_on_approval() { // Build a conversation configured to require approvals so the delegate // routes ExecApprovalRequest via the parent. - let mut builder = test_codex().with_config(|config| { + let mut builder = test_codex().with_model("gpt-5.1").with_config(|config| { config.approval_policy = AskForApproval::OnRequest; config.sandbox_policy = SandboxPolicy::ReadOnly; }); @@ -68,9 +69,10 @@ async fn codex_delegate_forwards_exec_approval_and_proceeds_on_approval() { test.codex .submit(Op::Review { review_request: ReviewRequest { - prompt: "Please review".to_string(), - user_facing_hint: "review".to_string(), - append_to_original_thread: true, + target: ReviewTarget::Custom { + instructions: "Please review".to_string(), + }, + user_facing_hint: None, }, }) .await @@ -133,7 +135,7 @@ async fn codex_delegate_forwards_patch_approval_and_proceeds_on_decision() { let server = start_mock_server().await; mount_sse_sequence(&server, vec![sse1, sse2]).await; - let mut builder = test_codex().with_config(|config| { + let mut builder = test_codex().with_model("gpt-5.1").with_config(|config| { config.approval_policy = AskForApproval::OnRequest; // Use a restricted sandbox so patch approval is required config.sandbox_policy = SandboxPolicy::ReadOnly; @@ -144,9 +146,10 @@ async fn codex_delegate_forwards_patch_approval_and_proceeds_on_decision() { test.codex .submit(Op::Review { review_request: ReviewRequest { - prompt: "Please review".to_string(), - user_facing_hint: "review".to_string(), - append_to_original_thread: true, + target: ReviewTarget::Custom { + instructions: "Please review".to_string(), + }, + user_facing_hint: None, }, }) .await @@ -199,9 +202,10 @@ async fn codex_delegate_ignores_legacy_deltas() { test.codex .submit(Op::Review { review_request: ReviewRequest { - prompt: "Please review".to_string(), - user_facing_hint: "review".to_string(), - append_to_original_thread: true, + target: ReviewTarget::Custom { + instructions: "Please review".to_string(), + }, + user_facing_hint: None, }, }) .await diff --git a/codex-rs/core/tests/suite/compact.rs b/codex-rs/core/tests/suite/compact.rs index 5359f3b1ad..aa74ec8978 100644 --- a/codex-rs/core/tests/suite/compact.rs +++ b/codex-rs/core/tests/suite/compact.rs @@ -7,6 +7,7 @@ use codex_core::built_in_model_providers; use codex_core::compact::SUMMARIZATION_PROMPT; use codex_core::compact::SUMMARY_PREFIX; use codex_core::config::Config; +use codex_core::features::Feature; use codex_core::protocol::EventMsg; use codex_core::protocol::Op; use codex_core::protocol::RolloutItem; @@ -27,6 +28,7 @@ use core_test_support::responses::ev_assistant_message; use core_test_support::responses::ev_completed; use core_test_support::responses::ev_completed_with_tokens; use core_test_support::responses::ev_function_call; +use core_test_support::responses::mount_compact_json_once; use core_test_support::responses::mount_sse_once; use core_test_support::responses::mount_sse_once_match; use core_test_support::responses::mount_sse_sequence; @@ -384,7 +386,7 @@ async fn manual_compact_uses_custom_prompt() { } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn manual_compact_emits_estimated_token_usage_event() { +async fn manual_compact_emits_api_and_local_token_usage_events() { skip_if_no_network!(); let server = start_mock_server().await; @@ -481,9 +483,14 @@ async fn multiple_auto_compact_per_task_runs_after_token_limit_hit() { // mock responses from the model + let reasoning_response_1 = ev_reasoning_item("m1", &["I will create a react app"], &[]); + let encrypted_content_1 = reasoning_response_1["item"]["encrypted_content"] + .as_str() + .unwrap(); + // first chunk of work let model_reasoning_response_1_sse = sse(vec![ - ev_reasoning_item("m1", &["I will create a react app"], &[]), + reasoning_response_1.clone(), ev_local_shell_call("r1-shell", "completed", vec!["echo", "make-react"]), ev_completed_with_tokens("r1", token_count_used), ]); @@ -494,9 +501,14 @@ async fn multiple_auto_compact_per_task_runs_after_token_limit_hit() { ev_completed_with_tokens("r2", token_count_used_after_compaction), ]); + let reasoning_response_2 = ev_reasoning_item("m3", &["I will create a node app"], &[]); + let encrypted_content_2 = reasoning_response_2["item"]["encrypted_content"] + .as_str() + .unwrap(); + // second chunk of work let model_reasoning_response_2_sse = sse(vec![ - ev_reasoning_item("m3", &["I will create a node app"], &[]), + reasoning_response_2.clone(), ev_local_shell_call("r3-shell", "completed", vec!["echo", "make-node"]), ev_completed_with_tokens("r3", token_count_used), ]); @@ -507,6 +519,11 @@ async fn multiple_auto_compact_per_task_runs_after_token_limit_hit() { ev_completed_with_tokens("r4", token_count_used_after_compaction), ]); + let reasoning_response_3 = ev_reasoning_item("m6", &["I will create a python app"], &[]); + let encrypted_content_3 = reasoning_response_3["item"]["encrypted_content"] + .as_str() + .unwrap(); + // third chunk of work let model_reasoning_response_3_sse = sse(vec![ ev_reasoning_item("m6", &["I will create a python app"], &[]), @@ -635,7 +652,7 @@ async fn multiple_auto_compact_per_task_runs_after_token_limit_hit() { }, { "content": null, - "encrypted_content": null, + "encrypted_content": encrypted_content_1, "summary": [ { "text": "I will create a react app", @@ -745,7 +762,7 @@ async fn multiple_auto_compact_per_task_runs_after_token_limit_hit() { }, { "content": null, - "encrypted_content": null, + "encrypted_content": encrypted_content_2, "summary": [ { "text": "I will create a node app", @@ -855,7 +872,7 @@ async fn multiple_auto_compact_per_task_runs_after_token_limit_hit() { }, { "content": null, - "encrypted_content": null, + "encrypted_content": encrypted_content_3, "summary": [ { "text": "I will create a python app", @@ -1879,3 +1896,110 @@ async fn auto_compact_triggers_after_function_call_over_95_percent_usage() { "auto compact request should include the summarization prompt after exceeding 95% (limit {limit})" ); } + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn auto_compact_counts_encrypted_reasoning_before_last_user() { + skip_if_no_network!(); + + let server = start_mock_server().await; + + let first_user = "COUNT_PRE_LAST_REASONING"; + let second_user = "TRIGGER_COMPACT_AT_LIMIT"; + + let pre_last_reasoning_content = "a".repeat(2_400); + let post_last_reasoning_content = "b".repeat(4_000); + + let first_turn = sse(vec![ + ev_reasoning_item("pre-reasoning", &["pre"], &[&pre_last_reasoning_content]), + ev_completed_with_tokens("r1", 10), + ]); + let second_turn = sse(vec![ + ev_reasoning_item("post-reasoning", &["post"], &[&post_last_reasoning_content]), + ev_completed_with_tokens("r2", 80), + ]); + let resume_turn = sse(vec![ + ev_assistant_message("m4", FINAL_REPLY), + ev_completed_with_tokens("r4", 1), + ]); + + let request_log = mount_sse_sequence( + &server, + vec![ + // Turn 1: reasoning before last user (should count). + first_turn, + // Turn 2: reasoning after last user (should be ignored for compaction). + second_turn, + // Turn 3: resume after remote compaction. + resume_turn, + ], + ) + .await; + + let compacted_history = vec![codex_protocol::models::ResponseItem::Message { + id: None, + role: "assistant".to_string(), + content: vec![codex_protocol::models::ContentItem::OutputText { + text: "REMOTE_COMPACT_SUMMARY".to_string(), + }], + }]; + let compact_mock = + mount_compact_json_once(&server, serde_json::json!({ "output": compacted_history })).await; + + let codex = test_codex() + .with_auth(CodexAuth::create_dummy_chatgpt_auth_for_testing()) + .with_config(|config| { + set_test_compact_prompt(config); + config.model_auto_compact_token_limit = Some(300); + config.features.enable(Feature::RemoteCompaction); + }) + .build(&server) + .await + .expect("build codex") + .codex; + + for (idx, user) in [first_user, second_user].into_iter().enumerate() { + codex + .submit(Op::UserInput { + items: vec![UserInput::Text { text: user.into() }], + }) + .await + .unwrap(); + wait_for_event(&codex, |ev| matches!(ev, EventMsg::TaskComplete(_))).await; + + if idx == 0 { + assert!( + compact_mock.requests().is_empty(), + "remote compaction should not run after the first turn" + ); + } + } + + let compact_requests = compact_mock.requests(); + assert_eq!( + compact_requests.len(), + 1, + "remote compaction should run once after the second turn" + ); + assert_eq!( + compact_requests[0].path(), + "/v1/responses/compact", + "remote compaction should hit the compact endpoint" + ); + + let requests = request_log.requests(); + assert_eq!( + requests.len(), + 3, + "conversation should include two user turns and a post-compaction resume" + ); + let second_request_body = requests[1].body_json().to_string(); + assert!( + !second_request_body.contains("REMOTE_COMPACT_SUMMARY"), + "second turn should not include compacted history" + ); + let resume_body = requests[2].body_json().to_string(); + assert!( + resume_body.contains("REMOTE_COMPACT_SUMMARY") || resume_body.contains(FINAL_REPLY), + "resume request should follow remote compact and use compacted history" + ); +} diff --git a/codex-rs/core/tests/suite/compact_remote.rs b/codex-rs/core/tests/suite/compact_remote.rs index dc88bc5747..49fd5791c8 100644 --- a/codex-rs/core/tests/suite/compact_remote.rs +++ b/codex-rs/core/tests/suite/compact_remote.rs @@ -180,13 +180,13 @@ async fn remote_compact_runs_automatically() -> Result<()> { }) .await?; let message = wait_for_event_match(&codex, |ev| match ev { - EventMsg::AgentMessage(ev) => Some(ev.message.clone()), + EventMsg::ContextCompacted(_) => Some(true), _ => None, }) .await; wait_for_event(&codex, |ev| matches!(ev, EventMsg::TaskComplete(_))).await; - assert_eq!(message, "Compact task completed"); + assert!(message); assert_eq!(compact_mock.requests().len(), 1); let follow_up_body = responses_mock.single_request().body_json().to_string(); assert!(follow_up_body.contains("REMOTE_COMPACTED_SUMMARY")); diff --git a/codex-rs/core/tests/suite/exec.rs b/codex-rs/core/tests/suite/exec.rs index ea5ab84879..6c2283107b 100644 --- a/codex-rs/core/tests/suite/exec.rs +++ b/codex-rs/core/tests/suite/exec.rs @@ -32,7 +32,7 @@ async fn run_test_cmd(tmp: TempDir, cmd: Vec<&str>) -> Result) -> Result Result<()> { + // TODO execpolicy doesn't parse powershell commands yet + if cfg!(windows) { + return Ok(()); + } + let mut builder = test_codex().with_config(|config| { let policy_path = config.codex_home.join("policy").join("policy.codexpolicy"); fs::create_dir_all( @@ -40,7 +45,7 @@ async fn execpolicy_blocks_shell_invocation() -> Result<()> { let call_id = "shell-forbidden"; let args = json!({ - "command": ["echo", "blocked"], + "command": "echo blocked", "timeout_ms": 1_000, }); @@ -48,7 +53,7 @@ async fn execpolicy_blocks_shell_invocation() -> Result<()> { &server, sse(vec![ ev_response_created("resp-1"), - ev_function_call(call_id, "shell", &serde_json::to_string(&args)?), + ev_function_call(call_id, "shell_command", &serde_json::to_string(&args)?), ev_completed("resp-1"), ]), ) diff --git a/codex-rs/core/tests/suite/grep_files.rs b/codex-rs/core/tests/suite/grep_files.rs index 6e31ddf486..1b8cc110ef 100644 --- a/codex-rs/core/tests/suite/grep_files.rs +++ b/codex-rs/core/tests/suite/grep_files.rs @@ -1,7 +1,6 @@ #![cfg(not(target_os = "windows"))] use anyhow::Result; -use codex_core::model_family::find_family_for_model; use core_test_support::responses::mount_function_call_agent_response; use core_test_support::responses::start_mock_server; use core_test_support::skip_if_no_network; @@ -127,11 +126,7 @@ async fn grep_files_tool_reports_empty_results() -> Result<()> { #[allow(clippy::expect_used)] async fn build_test_codex(server: &wiremock::MockServer) -> Result { - let mut builder = test_codex().with_config(|config| { - config.model = MODEL_WITH_TOOL.to_string(); - config.model_family = - find_family_for_model(MODEL_WITH_TOOL).expect("model family for test model"); - }); + let mut builder = test_codex().with_model(MODEL_WITH_TOOL); builder.build(server).await } diff --git a/codex-rs/core/tests/suite/mod.rs b/codex-rs/core/tests/suite/mod.rs index 042e96d907..22aeaab1da 100644 --- a/codex-rs/core/tests/suite/mod.rs +++ b/codex-rs/core/tests/suite/mod.rs @@ -50,6 +50,7 @@ mod seatbelt; mod shell_serialization; mod stream_error_allows_next_turn; mod stream_no_completed; +mod text_encoding_fix; mod tool_harness; mod tool_parallelism; mod tools; diff --git a/codex-rs/core/tests/suite/model_tools.rs b/codex-rs/core/tests/suite/model_tools.rs index b44f53a7c3..cb2c5725f2 100644 --- a/codex-rs/core/tests/suite/model_tools.rs +++ b/codex-rs/core/tests/suite/model_tools.rs @@ -1,21 +1,10 @@ #![allow(clippy::unwrap_used)] -use codex_core::CodexAuth; -use codex_core::ConversationManager; -use codex_core::ModelProviderInfo; -use codex_core::built_in_model_providers; -use codex_core::features::Feature; -use codex_core::model_family::find_family_for_model; -use codex_core::protocol::EventMsg; -use codex_core::protocol::Op; -use codex_protocol::user_input::UserInput; -use core_test_support::load_default_config_for_test; use core_test_support::load_sse_fixture_with_id; use core_test_support::responses; +use core_test_support::responses::start_mock_server; use core_test_support::skip_if_no_network; -use core_test_support::wait_for_event; -use tempfile::TempDir; -use wiremock::MockServer; +use core_test_support::test_codex::test_codex; fn sse_completed(id: &str) -> String { load_sse_fixture_with_id("tests/fixtures/completed_template.json", id) @@ -39,46 +28,17 @@ fn tool_identifiers(body: &serde_json::Value) -> Vec { #[allow(clippy::expect_used)] async fn collect_tool_identifiers_for_model(model: &str) -> Vec { - let server = MockServer::start().await; - + let server = start_mock_server().await; let sse = sse_completed(model); let resp_mock = responses::mount_sse_once(&server, sse).await; - let model_provider = ModelProviderInfo { - base_url: Some(format!("{}/v1", server.uri())), - ..built_in_model_providers()["openai"].clone() - }; - - let cwd = TempDir::new().unwrap(); - let codex_home = TempDir::new().unwrap(); - let mut config = load_default_config_for_test(&codex_home); - config.cwd = cwd.path().to_path_buf(); - config.model_provider = model_provider; - config.model = model.to_string(); - config.model_family = - find_family_for_model(model).unwrap_or_else(|| panic!("unknown model family for {model}")); - config.features.disable(Feature::ApplyPatchFreeform); - config.features.disable(Feature::ViewImageTool); - config.features.disable(Feature::WebSearchRequest); - config.features.disable(Feature::UnifiedExec); - - let conversation_manager = - ConversationManager::with_auth(CodexAuth::from_api_key("Test API Key")); - let codex = conversation_manager - .new_conversation(config) + let mut builder = test_codex().with_model(model); + let test = builder + .build(&server) .await - .expect("create new conversation") - .conversation; + .expect("create test Codex conversation"); - codex - .submit(Op::UserInput { - items: vec![UserInput::Text { - text: "hello tools".into(), - }], - }) - .await - .unwrap(); - wait_for_event(&codex, |ev| matches!(ev, EventMsg::TaskComplete(_))).await; + test.submit_turn("hello tools").await.expect("submit turn"); let body = resp_mock.single_request().body_json(); tool_identifiers(&body) @@ -97,7 +57,8 @@ async fn model_selects_expected_tools() { "list_mcp_resources".to_string(), "list_mcp_resource_templates".to_string(), "read_mcp_resource".to_string(), - "update_plan".to_string() + "update_plan".to_string(), + "view_image".to_string() ], "codex-mini-latest should expose the local shell tool", ); @@ -111,7 +72,8 @@ async fn model_selects_expected_tools() { "list_mcp_resource_templates".to_string(), "read_mcp_resource".to_string(), "update_plan".to_string(), - "apply_patch".to_string() + "apply_patch".to_string(), + "view_image".to_string() ], "gpt-5-codex should expose the apply_patch tool", ); @@ -125,7 +87,8 @@ async fn model_selects_expected_tools() { "list_mcp_resource_templates".to_string(), "read_mcp_resource".to_string(), "update_plan".to_string(), - "apply_patch".to_string() + "apply_patch".to_string(), + "view_image".to_string() ], "gpt-5.1-codex should expose the apply_patch tool", ); @@ -139,6 +102,7 @@ async fn model_selects_expected_tools() { "list_mcp_resource_templates".to_string(), "read_mcp_resource".to_string(), "update_plan".to_string(), + "view_image".to_string() ], "gpt-5 should expose the apply_patch tool", ); @@ -152,8 +116,24 @@ async fn model_selects_expected_tools() { "list_mcp_resource_templates".to_string(), "read_mcp_resource".to_string(), "update_plan".to_string(), - "apply_patch".to_string() + "apply_patch".to_string(), + "view_image".to_string() ], "gpt-5.1 should expose the apply_patch tool", ); + let exp_tools = collect_tool_identifiers_for_model("exp-5.1").await; + assert_eq!( + exp_tools, + vec![ + "exec_command".to_string(), + "write_stdin".to_string(), + "list_mcp_resources".to_string(), + "list_mcp_resource_templates".to_string(), + "read_mcp_resource".to_string(), + "update_plan".to_string(), + "apply_patch".to_string(), + "view_image".to_string() + ], + "exp-5.1 should expose the apply_patch tool", + ); } diff --git a/codex-rs/core/tests/suite/prompt_caching.rs b/codex-rs/core/tests/suite/prompt_caching.rs index 55e8f66416..f4455fd022 100644 --- a/codex-rs/core/tests/suite/prompt_caching.rs +++ b/codex-rs/core/tests/suite/prompt_caching.rs @@ -30,18 +30,15 @@ fn text_user_input(text: String) -> serde_json::Value { } fn default_env_context_str(cwd: &str, shell: &Shell) -> String { + let shell_name = shell.name(); format!( r#" - {} + {cwd} on-request read-only restricted -{}"#, - cwd, - match shell.name() { - Some(name) => format!(" {name}\n"), - None => String::new(), - } + {shell_name} +"# ) } @@ -227,7 +224,7 @@ async fn prefixes_context_and_instructions_once_and_consistently_across_requests .await?; wait_for_event(&codex, |ev| matches!(ev, EventMsg::TaskComplete(_))).await; - let shell = default_user_shell().await; + let shell = default_user_shell(); let cwd_str = config.cwd.to_string_lossy(); let expected_env_text = default_env_context_str(&cwd_str, &shell); let expected_ui_text = format!( @@ -345,6 +342,7 @@ async fn overrides_turn_context_but_keeps_cached_prefix_and_key_constant() -> an // After overriding the turn context, the environment context should be emitted again // reflecting the new approval policy and sandbox settings. Omit cwd because it did // not change. + let shell = default_user_shell(); let expected_env_text_2 = format!( r#" never @@ -353,8 +351,10 @@ async fn overrides_turn_context_but_keeps_cached_prefix_and_key_constant() -> an {} + {} "#, - writable.path().to_string_lossy(), + writable.path().display(), + shell.name() ); let expected_env_msg_2 = serde_json::json!({ "type": "message", @@ -522,6 +522,8 @@ async fn per_turn_overrides_keep_cached_prefix_and_key_constant() -> anyhow::Res "role": "user", "content": [ { "type": "input_text", "text": "hello 2" } ] }); + let shell = default_user_shell(); + let expected_env_text_2 = format!( r#" {} @@ -531,9 +533,11 @@ async fn per_turn_overrides_keep_cached_prefix_and_key_constant() -> anyhow::Res {} + {} "#, - new_cwd.path().to_string_lossy(), - writable.path().to_string_lossy(), + new_cwd.path().display(), + writable.path().display(), + shell.name(), ); let expected_env_msg_2 = serde_json::json!({ "type": "message", @@ -610,7 +614,7 @@ async fn send_user_turn_with_no_changes_does_not_send_environment_context() -> a let body1 = req1.single_request().body_json(); let body2 = req2.single_request().body_json(); - let shell = default_user_shell().await; + let shell = default_user_shell(); let default_cwd_lossy = default_cwd.to_string_lossy(); let expected_ui_text = format!( "# AGENTS.md instructions for {default_cwd_lossy}\n\n\nbe consistent and helpful\n" @@ -697,7 +701,7 @@ async fn send_user_turn_with_changes_sends_environment_context() -> anyhow::Resu let body1 = req1.single_request().body_json(); let body2 = req2.single_request().body_json(); - let shell = default_user_shell().await; + let shell = default_user_shell(); let expected_ui_text = format!( "# AGENTS.md instructions for {}\n\n\nbe consistent and helpful\n", default_cwd.to_string_lossy() @@ -717,14 +721,15 @@ async fn send_user_turn_with_changes_sends_environment_context() -> anyhow::Resu ]); assert_eq!(body1["input"], expected_input_1); - let expected_env_msg_2 = text_user_input( + let shell_name = shell.name(); + let expected_env_msg_2 = text_user_input(format!( r#" never danger-full-access enabled + {shell_name} "# - .to_string(), - ); + )); let expected_user_message_2 = text_user_input("hello 2".to_string()); let expected_input_2 = serde_json::Value::Array(vec![ expected_ui_msg, diff --git a/codex-rs/core/tests/suite/review.rs b/codex-rs/core/tests/suite/review.rs index 3904f18f8e..7216e19251 100644 --- a/codex-rs/core/tests/suite/review.rs +++ b/codex-rs/core/tests/suite/review.rs @@ -16,8 +16,10 @@ use codex_core::protocol::ReviewFinding; use codex_core::protocol::ReviewLineRange; use codex_core::protocol::ReviewOutputEvent; use codex_core::protocol::ReviewRequest; +use codex_core::protocol::ReviewTarget; use codex_core::protocol::RolloutItem; use codex_core::protocol::RolloutLine; +use codex_core::review_format::render_review_output_text; use codex_protocol::user_input::UserInput; use core_test_support::load_default_config_for_test; use core_test_support::load_sse_fixture_with_id_from_str; @@ -80,9 +82,10 @@ async fn review_op_emits_lifecycle_and_review_output() { codex .submit(Op::Review { review_request: ReviewRequest { - prompt: "Please review my changes".to_string(), - user_facing_hint: "my changes".to_string(), - append_to_original_thread: true, + target: ReviewTarget::Custom { + instructions: "Please review my changes".to_string(), + }, + user_facing_hint: None, }, }) .await @@ -124,22 +127,36 @@ async fn review_op_emits_lifecycle_and_review_output() { let mut saw_header = false; let mut saw_finding_line = false; + let expected_assistant_text = render_review_output_text(&expected); + let mut saw_assistant_plain = false; + let mut saw_assistant_xml = false; for line in text.lines() { if line.trim().is_empty() { continue; } let v: serde_json::Value = serde_json::from_str(line).expect("jsonl line"); let rl: RolloutLine = serde_json::from_value(v).expect("rollout line"); - if let RolloutItem::ResponseItem(ResponseItem::Message { role, content, .. }) = rl.item - && role == "user" - { - for c in content { - if let ContentItem::InputText { text } = c { - if text.contains("full review output from reviewer model") { - saw_header = true; + if let RolloutItem::ResponseItem(ResponseItem::Message { role, content, .. }) = rl.item { + if role == "user" { + for c in content { + if let ContentItem::InputText { text } = c { + if text.contains("full review output from reviewer model") { + saw_header = true; + } + if text.contains("- Prefer Stylize helpers — /tmp/file.rs:10-20") { + saw_finding_line = true; + } } - if text.contains("- Prefer Stylize helpers — /tmp/file.rs:10-20") { - saw_finding_line = true; + } + } else if role == "assistant" { + for c in content { + if let ContentItem::OutputText { text } = c { + if text.contains("") { + saw_assistant_xml = true; + } + if text == expected_assistant_text { + saw_assistant_plain = true; + } } } } @@ -150,6 +167,14 @@ async fn review_op_emits_lifecycle_and_review_output() { saw_finding_line, "formatted finding line missing from rollout" ); + assert!( + saw_assistant_plain, + "assistant review output missing from rollout" + ); + assert!( + !saw_assistant_xml, + "assistant review output contains user_action markup" + ); server.verify().await; } @@ -177,9 +202,10 @@ async fn review_op_with_plain_text_emits_review_fallback() { codex .submit(Op::Review { review_request: ReviewRequest { - prompt: "Plain text review".to_string(), - user_facing_hint: "plain text review".to_string(), - append_to_original_thread: true, + target: ReviewTarget::Custom { + instructions: "Plain text review".to_string(), + }, + user_facing_hint: None, }, }) .await @@ -236,9 +262,10 @@ async fn review_filters_agent_message_related_events() { codex .submit(Op::Review { review_request: ReviewRequest { - prompt: "Filter streaming events".to_string(), - user_facing_hint: "Filter streaming events".to_string(), - append_to_original_thread: true, + target: ReviewTarget::Custom { + instructions: "Filter streaming events".to_string(), + }, + user_facing_hint: None, }, }) .await @@ -247,7 +274,7 @@ async fn review_filters_agent_message_related_events() { let mut saw_entered = false; let mut saw_exited = false; - // Drain until TaskComplete; assert filtered events never surface. + // Drain until TaskComplete; assert streaming-related events never surface. wait_for_event(&codex, |event| match event { EventMsg::TaskComplete(_) => true, EventMsg::EnteredReviewMode(_) => { @@ -265,12 +292,6 @@ async fn review_filters_agent_message_related_events() { EventMsg::AgentMessageDelta(_) => { panic!("unexpected AgentMessageDelta surfaced during review") } - EventMsg::ItemCompleted(ev) => match &ev.item { - codex_protocol::items::TurnItem::AgentMessage(_) => { - panic!("unexpected ItemCompleted for TurnItem::AgentMessage surfaced during review") - } - _ => false, - }, _ => false, }) .await; @@ -279,8 +300,9 @@ async fn review_filters_agent_message_related_events() { server.verify().await; } -/// When the model returns structured JSON in a review, ensure no AgentMessage -/// is emitted; the UI consumes the structured result via ExitedReviewMode. +/// When the model returns structured JSON in a review, ensure only a single +/// non-streaming AgentMessage is emitted; the UI consumes the structured +/// result via ExitedReviewMode plus a final assistant message. // Windows CI only: bump to 4 workers to prevent SSE/event starvation and test timeouts. #[cfg_attr(windows, tokio::test(flavor = "multi_thread", worker_threads = 4))] #[cfg_attr(not(windows), tokio::test(flavor = "multi_thread", worker_threads = 2))] @@ -321,21 +343,25 @@ async fn review_does_not_emit_agent_message_on_structured_output() { codex .submit(Op::Review { review_request: ReviewRequest { - prompt: "check structured".to_string(), - user_facing_hint: "check structured".to_string(), - append_to_original_thread: true, + target: ReviewTarget::Custom { + instructions: "check structured".to_string(), + }, + user_facing_hint: None, }, }) .await .unwrap(); - // Drain events until TaskComplete; ensure none are AgentMessage. + // Drain events until TaskComplete; ensure we only see a final + // AgentMessage (no streaming assistant messages). let mut saw_entered = false; let mut saw_exited = false; + let mut agent_messages = 0; wait_for_event(&codex, |event| match event { EventMsg::TaskComplete(_) => true, EventMsg::AgentMessage(_) => { - panic!("unexpected AgentMessage during review with structured output") + agent_messages += 1; + false } EventMsg::EnteredReviewMode(_) => { saw_entered = true; @@ -348,6 +374,7 @@ async fn review_does_not_emit_agent_message_on_structured_output() { _ => false, }) .await; + assert_eq!(1, agent_messages, "expected exactly one AgentMessage event"); assert!(saw_entered && saw_exited, "missing review lifecycle events"); server.verify().await; @@ -375,9 +402,10 @@ async fn review_uses_custom_review_model_from_config() { codex .submit(Op::Review { review_request: ReviewRequest { - prompt: "use custom model".to_string(), - user_facing_hint: "use custom model".to_string(), - append_to_original_thread: true, + target: ReviewTarget::Custom { + instructions: "use custom model".to_string(), + }, + user_facing_hint: None, }, }) .await @@ -493,9 +521,10 @@ async fn review_input_isolated_from_parent_history() { codex .submit(Op::Review { review_request: ReviewRequest { - prompt: review_prompt.clone(), - user_facing_hint: review_prompt.clone(), - append_to_original_thread: true, + target: ReviewTarget::Custom { + instructions: review_prompt.clone(), + }, + user_facing_hint: None, }, }) .await @@ -583,11 +612,10 @@ async fn review_input_isolated_from_parent_history() { server.verify().await; } -/// After a review thread finishes, its conversation should not leak into the -/// parent session. A subsequent parent turn must not include any review -/// messages in its request `input`. +/// After a review thread finishes, its conversation should be visible in the +/// parent session so later turns can reference the results. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn review_history_does_not_leak_into_parent_session() { +async fn review_history_surfaces_in_parent_session() { skip_if_no_network!(); // Respond to both the review request and the subsequent parent request. @@ -606,9 +634,10 @@ async fn review_history_does_not_leak_into_parent_session() { codex .submit(Op::Review { review_request: ReviewRequest { - prompt: "Start a review".to_string(), - user_facing_hint: "Start a review".to_string(), - append_to_original_thread: true, + target: ReviewTarget::Custom { + instructions: "Start a review".to_string(), + }, + user_facing_hint: None, }, }) .await @@ -651,20 +680,26 @@ async fn review_history_does_not_leak_into_parent_session() { let last_text = last["content"][0]["text"].as_str().unwrap(); assert_eq!(last_text, followup); - // Ensure no review-thread content leaked into the parent request - let contains_review_prompt = input - .iter() - .any(|msg| msg["content"][0]["text"].as_str().unwrap_or_default() == "Start a review"); + // Ensure review-thread content is present for downstream turns. + let contains_review_rollout_user = input.iter().any(|msg| { + msg["content"][0]["text"] + .as_str() + .unwrap_or_default() + .contains("User initiated a review task.") + }); let contains_review_assistant = input.iter().any(|msg| { - msg["content"][0]["text"].as_str().unwrap_or_default() == "review assistant output" + msg["content"][0]["text"] + .as_str() + .unwrap_or_default() + .contains("review assistant output") }); assert!( - !contains_review_prompt, - "review prompt leaked into parent turn input" + contains_review_rollout_user, + "review rollout user message missing from parent turn input" ); assert!( - !contains_review_assistant, - "review assistant output leaked into parent turn input" + contains_review_assistant, + "review assistant output missing from parent turn input" ); server.verify().await; diff --git a/codex-rs/core/tests/suite/shell_serialization.rs b/codex-rs/core/tests/suite/shell_serialization.rs index 24c90bde2b..77c835b320 100644 --- a/codex-rs/core/tests/suite/shell_serialization.rs +++ b/codex-rs/core/tests/suite/shell_serialization.rs @@ -2,6 +2,7 @@ #![allow(clippy::expect_used)] use anyhow::Result; +use codex_core::config::Config; use codex_core::features::Feature; use codex_core::model_family::find_family_for_model; use codex_core::protocol::SandboxPolicy; @@ -40,6 +41,20 @@ const FIXTURE_JSON: &str = r#"{ } "#; +fn configure_shell_command_model(output_type: ShellModelOutput, config: &mut Config) { + if !matches!(output_type, ShellModelOutput::ShellCommand) { + return; + } + + if let Some(shell_command_family) = find_family_for_model("test-gpt-5-codex") { + if config.model_family.shell_type == shell_command_family.shell_type { + return; + } + config.model = shell_command_family.slug.clone(); + config.model_family = shell_command_family; + } +} + fn shell_responses( call_id: &str, command: Vec<&str>, @@ -108,14 +123,9 @@ async fn shell_output_stays_json_without_freeform_apply_patch( skip_if_no_network!(Ok(())); let server = start_mock_server().await; - let mut builder = test_codex().with_config(move |config| { + let mut builder = test_codex().with_model("gpt-5").with_config(move |config| { config.features.disable(Feature::ApplyPatchFreeform); - config.model = "gpt-5".to_string(); - config.model_family = find_family_for_model("gpt-5").expect("gpt-5 is a model family"); - if matches!(output_type, ShellModelOutput::ShellCommand) { - config.features.enable(Feature::ShellCommandTool); - } - let _ = output_type; + configure_shell_command_model(output_type, config); }); let test = builder.build(&server).await?; @@ -170,10 +180,7 @@ async fn shell_output_is_structured_with_freeform_apply_patch( let server = start_mock_server().await; let mut builder = test_codex().with_config(move |config| { config.features.enable(Feature::ApplyPatchFreeform); - if matches!(output_type, ShellModelOutput::ShellCommand) { - config.features.enable(Feature::ShellCommandTool); - } - let _ = output_type; + configure_shell_command_model(output_type, config); }); let test = builder.build(&server).await?; @@ -219,14 +226,9 @@ async fn shell_output_preserves_fixture_json_without_serialization( skip_if_no_network!(Ok(())); let server = start_mock_server().await; - let mut builder = test_codex().with_config(move |config| { + let mut builder = test_codex().with_model("gpt-5").with_config(move |config| { config.features.disable(Feature::ApplyPatchFreeform); - config.model = "gpt-5".to_string(); - config.model_family = find_family_for_model("gpt-5").expect("gpt-5 is a model family"); - if matches!(output_type, ShellModelOutput::ShellCommand) { - config.features.enable(Feature::ShellCommandTool); - } - let _ = output_type; + configure_shell_command_model(output_type, config); }); let test = builder.build(&server).await?; @@ -293,10 +295,7 @@ async fn shell_output_structures_fixture_with_serialization( let server = start_mock_server().await; let mut builder = test_codex().with_config(move |config| { config.features.enable(Feature::ApplyPatchFreeform); - if matches!(output_type, ShellModelOutput::ShellCommand) { - config.features.enable(Feature::ShellCommandTool); - } - let _ = output_type; + configure_shell_command_model(output_type, config); }); let test = builder.build(&server).await?; @@ -358,15 +357,12 @@ async fn shell_output_for_freeform_tool_records_duration( let server = start_mock_server().await; let mut builder = test_codex().with_config(move |config| { config.include_apply_patch_tool = true; - if matches!(output_type, ShellModelOutput::ShellCommand) { - config.features.enable(Feature::ShellCommandTool); - } - let _ = output_type; + configure_shell_command_model(output_type, config); }); let test = builder.build(&server).await?; let call_id = "shell-structured"; - let responses = shell_responses(call_id, vec!["/bin/bash", "-c", "sleep 1"], output_type)?; + let responses = shell_responses(call_id, vec!["/bin/sh", "-c", "sleep 1"], output_type)?; let mock = mount_sse_sequence(&server, responses).await; test.submit_turn_with_policy( @@ -412,16 +408,12 @@ async fn shell_output_reserializes_truncated_content(output_type: ShellModelOutp skip_if_no_network!(Ok(())); let server = start_mock_server().await; - let mut builder = test_codex().with_config(move |config| { - config.model = "gpt-5.1-codex".to_string(); - config.model_family = - find_family_for_model("gpt-5.1-codex").expect("gpt-5.1-codex is a model family"); - config.tool_output_token_limit = Some(200); - if matches!(output_type, ShellModelOutput::ShellCommand) { - config.features.enable(Feature::ShellCommandTool); - } - let _ = output_type; - }); + let mut builder = test_codex() + .with_model("gpt-5.1-codex") + .with_config(move |config| { + config.tool_output_token_limit = Some(200); + configure_shell_command_model(output_type, config); + }); let test = builder.build(&server).await?; let call_id = "shell-truncated"; @@ -717,15 +709,12 @@ async fn shell_output_is_structured_for_nonzero_exit(output_type: ShellModelOutp skip_if_no_network!(Ok(())); let server = start_mock_server().await; - let mut builder = test_codex().with_config(move |config| { - config.model = "gpt-5.1-codex".to_string(); - config.model_family = - find_family_for_model("gpt-5.1-codex").expect("gpt-5.1-codex is a model family"); - config.include_apply_patch_tool = true; - if matches!(output_type, ShellModelOutput::ShellCommand) { - config.features.enable(Feature::ShellCommandTool); - } - }); + let mut builder = test_codex() + .with_model("gpt-5.1-codex") + .with_config(move |config| { + config.include_apply_patch_tool = true; + configure_shell_command_model(output_type, config); + }); let test = builder.build(&server).await?; let call_id = "shell-nonzero-exit"; @@ -760,7 +749,7 @@ async fn shell_command_output_is_freeform() -> Result<()> { let server = start_mock_server().await; let mut builder = test_codex().with_config(move |config| { - config.features.enable(Feature::ShellCommandTool); + configure_shell_command_model(ShellModelOutput::ShellCommand, config); }); let test = builder.build(&server).await?; @@ -812,11 +801,7 @@ async fn shell_command_output_is_not_truncated_under_10k_bytes() -> Result<()> { skip_if_no_network!(Ok(())); let server = start_mock_server().await; - let mut builder = test_codex() - .with_model("gpt-5.1") - .with_config(move |config| { - config.features.enable(Feature::ShellCommandTool); - }); + let mut builder = test_codex().with_model("gpt-5.1"); let test = builder.build(&server).await?; let call_id = "shell-command"; @@ -866,11 +851,7 @@ async fn shell_command_output_is_not_truncated_over_10k_bytes() -> Result<()> { skip_if_no_network!(Ok(())); let server = start_mock_server().await; - let mut builder = test_codex() - .with_model("gpt-5.1") - .with_config(move |config| { - config.features.enable(Feature::ShellCommandTool); - }); + let mut builder = test_codex().with_model("gpt-5.1"); let test = builder.build(&server).await?; let call_id = "shell-command"; @@ -920,12 +901,11 @@ async fn local_shell_call_output_is_structured() -> Result<()> { skip_if_no_network!(Ok(())); let server = start_mock_server().await; - let mut builder = test_codex().with_config(|config| { - config.model = "gpt-5.1-codex".to_string(); - config.model_family = - find_family_for_model("gpt-5.1-codex").expect("gpt-5.1-codex is a model family"); - config.include_apply_patch_tool = true; - }); + let mut builder = test_codex() + .with_model("gpt-5.1-codex") + .with_config(|config| { + config.include_apply_patch_tool = true; + }); let test = builder.build(&server).await?; let call_id = "local-shell-call"; diff --git a/codex-rs/core/tests/suite/text_encoding_fix.rs b/codex-rs/core/tests/suite/text_encoding_fix.rs new file mode 100644 index 0000000000..ecebb1e429 --- /dev/null +++ b/codex-rs/core/tests/suite/text_encoding_fix.rs @@ -0,0 +1,77 @@ +//! Integration test for the text encoding fix for issue #6178. +//! +//! These tests simulate VSCode's shell preview on Windows/WSL where the output +//! may be encoded with a legacy code page before it reaches Codex. + +use codex_core::exec::StreamOutput; +use pretty_assertions::assert_eq; + +#[test] +fn test_utf8_shell_output() { + // Baseline: UTF-8 output should bypass the detector and remain unchanged. + assert_eq!(decode_shell_output("пример".as_bytes()), "пример"); +} + +#[test] +fn test_cp1251_shell_output() { + // VS Code shells on Windows frequently surface CP1251 bytes for Cyrillic text. + assert_eq!(decode_shell_output(b"\xEF\xF0\xE8\xEC\xE5\xF0"), "пример"); +} + +#[test] +fn test_cp866_shell_output() { + // Native cmd.exe still defaults to CP866; make sure we recognize that too. + assert_eq!(decode_shell_output(b"\xAF\xE0\xA8\xAC\xA5\xE0"), "пример"); +} + +#[test] +fn test_windows_1252_smart_decoding() { + // Smart detection should turn fancy quotes/dashes into the proper Unicode glyphs. + assert_eq!( + decode_shell_output(b"\x93\x94 test \x96 dash"), + "\u{201C}\u{201D} test \u{2013} dash" + ); +} + +#[test] +fn test_smart_decoding_improves_over_lossy_utf8() { + // Regression guard: String::from_utf8_lossy() alone used to emit replacement chars here. + let bytes = b"\x93\x94 test \x96 dash"; + assert!( + String::from_utf8_lossy(bytes).contains('\u{FFFD}'), + "lossy UTF-8 should inject replacement chars" + ); + assert_eq!( + decode_shell_output(bytes), + "\u{201C}\u{201D} test \u{2013} dash", + "smart decoding should keep curly quotes intact" + ); +} + +#[test] +fn test_mixed_ascii_and_legacy_encoding() { + // Commands tend to mix ASCII status text with Latin-1 bytes (e.g. café). + assert_eq!(decode_shell_output(b"Output: caf\xE9"), "Output: café"); // codespell:ignore caf +} + +#[test] +fn test_pure_latin1_shell_output() { + // Latin-1 by itself should still decode correctly (regression coverage for the older tests). + assert_eq!(decode_shell_output(b"caf\xE9"), "café"); // codespell:ignore caf +} + +#[test] +fn test_invalid_bytes_still_fall_back_to_lossy() { + // If detection fails, we still want the user to see replacement characters. + let bytes = b"\xFF\xFE\xFD"; + assert_eq!(decode_shell_output(bytes), String::from_utf8_lossy(bytes)); +} + +fn decode_shell_output(bytes: &[u8]) -> String { + StreamOutput { + text: bytes.to_vec(), + truncated_after_lines: None, + } + .from_utf8_lossy() + .text +} diff --git a/codex-rs/core/tests/suite/tool_harness.rs b/codex-rs/core/tests/suite/tool_harness.rs index 02c7494cc0..a36ab03a41 100644 --- a/codex-rs/core/tests/suite/tool_harness.rs +++ b/codex-rs/core/tests/suite/tool_harness.rs @@ -4,7 +4,6 @@ use std::fs; use assert_matches::assert_matches; use codex_core::features::Feature; -use codex_core::model_family::find_family_for_model; use codex_core::protocol::AskForApproval; use codex_core::protocol::EventMsg; use codex_core::protocol::Op; @@ -54,10 +53,7 @@ async fn shell_tool_executes_command_and_streams_output() -> anyhow::Result<()> let server = start_mock_server().await; - let mut builder = test_codex().with_config(|config| { - config.model = "gpt-5".to_string(); - config.model_family = find_family_for_model("gpt-5").expect("gpt-5 is a valid model"); - }); + let mut builder = test_codex().with_model("gpt-5"); let TestCodex { codex, cwd, diff --git a/codex-rs/core/tests/suite/tool_parallelism.rs b/codex-rs/core/tests/suite/tool_parallelism.rs index 807bb9d82c..abb55f3a14 100644 --- a/codex-rs/core/tests/suite/tool_parallelism.rs +++ b/codex-rs/core/tests/suite/tool_parallelism.rs @@ -4,7 +4,6 @@ use std::time::Duration; use std::time::Instant; -use codex_core::model_family::find_family_for_model; use codex_core::protocol::AskForApproval; use codex_core::protocol::EventMsg; use codex_core::protocol::Op; @@ -56,11 +55,7 @@ async fn run_turn_and_measure(test: &TestCodex, prompt: &str) -> anyhow::Result< #[allow(clippy::expect_used)] async fn build_codex_with_test_tool(server: &wiremock::MockServer) -> anyhow::Result { - let mut builder = test_codex().with_config(|config| { - config.model = "test-gpt-5.1-codex".to_string(); - config.model_family = - find_family_for_model("test-gpt-5.1-codex").expect("test-gpt-5.1-codex model family"); - }); + let mut builder = test_codex().with_model("test-gpt-5.1-codex"); builder.build(server).await } @@ -146,10 +141,11 @@ async fn non_parallel_tools_run_serially() -> anyhow::Result<()> { skip_if_no_network!(Ok(())); let server = start_mock_server().await; - let test = test_codex().build(&server).await?; + let mut builder = test_codex().with_model("gpt-5.1"); + let test = builder.build(&server).await?; let shell_args = json!({ - "command": ["/bin/sh", "-c", "sleep 0.3"], + "command": "sleep 0.3", "timeout_ms": 1_000, }); let args_one = serde_json::to_string(&shell_args)?; @@ -157,8 +153,8 @@ async fn non_parallel_tools_run_serially() -> anyhow::Result<()> { let first_response = sse(vec![ json!({"type": "response.created", "response": {"id": "resp-1"}}), - ev_function_call("call-1", "shell", &args_one), - ev_function_call("call-2", "shell", &args_two), + ev_function_call("call-1", "shell_command", &args_one), + ev_function_call("call-2", "shell_command", &args_two), ev_completed("resp-1"), ]); let second_response = sse(vec![ @@ -167,7 +163,7 @@ async fn non_parallel_tools_run_serially() -> anyhow::Result<()> { ]); mount_sse_sequence(&server, vec![first_response, second_response]).await; - let duration = run_turn_and_measure(&test, "run shell twice").await?; + let duration = run_turn_and_measure(&test, "run shell_command twice").await?; assert_serial_duration(duration); Ok(()) @@ -185,14 +181,14 @@ async fn mixed_tools_fall_back_to_serial() -> anyhow::Result<()> { }) .to_string(); let shell_args = serde_json::to_string(&json!({ - "command": ["/bin/sh", "-c", "sleep 0.3"], + "command": "sleep 0.3", "timeout_ms": 1_000, }))?; let first_response = sse(vec![ json!({"type": "response.created", "response": {"id": "resp-1"}}), ev_function_call("call-1", "test_sync_tool", &sync_args), - ev_function_call("call-2", "shell", &shell_args), + ev_function_call("call-2", "shell_command", &shell_args), ev_completed("resp-1"), ]); let second_response = sse(vec![ @@ -215,7 +211,7 @@ async fn tool_results_grouped() -> anyhow::Result<()> { let test = build_codex_with_test_tool(&server).await?; let shell_args = serde_json::to_string(&json!({ - "command": ["/bin/sh", "-c", "echo 'shell output'"], + "command": "echo 'shell output'", "timeout_ms": 1_000, }))?; @@ -223,9 +219,9 @@ async fn tool_results_grouped() -> anyhow::Result<()> { &server, sse(vec![ json!({"type": "response.created", "response": {"id": "resp-1"}}), - ev_function_call("call-1", "shell", &shell_args), - ev_function_call("call-2", "shell", &shell_args), - ev_function_call("call-3", "shell", &shell_args), + ev_function_call("call-1", "shell_command", &shell_args), + ev_function_call("call-2", "shell_command", &shell_args), + ev_function_call("call-3", "shell_command", &shell_args), ev_completed("resp-1"), ]), ) diff --git a/codex-rs/core/tests/suite/tools.rs b/codex-rs/core/tests/suite/tools.rs index 549d6e7d57..1f743c422f 100644 --- a/codex-rs/core/tests/suite/tools.rs +++ b/codex-rs/core/tests/suite/tools.rs @@ -8,7 +8,6 @@ use std::time::Instant; use anyhow::Context; use anyhow::Result; use codex_core::features::Feature; -use codex_core::model_family::find_family_for_model; use codex_core::protocol::AskForApproval; use codex_core::protocol::SandboxPolicy; use core_test_support::assert_regex_match; @@ -96,10 +95,7 @@ async fn shell_escalated_permissions_rejected_then_ok() -> Result<()> { skip_if_no_network!(Ok(())); let server = start_mock_server().await; - let mut builder = test_codex().with_config(|config| { - config.model = "gpt-5".to_string(); - config.model_family = find_family_for_model("gpt-5").expect("gpt-5 is a valid model"); - }); + let mut builder = test_codex().with_model("gpt-5"); let test = builder.build(&server).await?; let command = ["/bin/echo", "shell ok"]; @@ -196,11 +192,7 @@ async fn sandbox_denied_shell_returns_original_output() -> Result<()> { skip_if_no_network!(Ok(())); let server = start_mock_server().await; - let mut builder = test_codex().with_config(|config| { - config.model = "gpt-5.1-codex".to_string(); - config.model_family = - find_family_for_model("gpt-5.1-codex").expect("gpt-5.1-codex model family"); - }); + let mut builder = test_codex().with_model("gpt-5.1-codex"); let fixture = builder.build(&server).await?; let call_id = "sandbox-denied-shell"; @@ -350,10 +342,7 @@ async fn shell_timeout_includes_timeout_prefix_and_metadata() -> Result<()> { skip_if_no_network!(Ok(())); let server = start_mock_server().await; - let mut builder = test_codex().with_config(|config| { - config.model = "gpt-5".to_string(); - config.model_family = find_family_for_model("gpt-5").expect("gpt-5 is a valid model"); - }); + let mut builder = test_codex().with_model("gpt-5"); let test = builder.build(&server).await?; let call_id = "shell-timeout"; @@ -424,9 +413,7 @@ async fn shell_timeout_handles_background_grandchild_stdout() -> Result<()> { skip_if_no_network!(Ok(())); let server = start_mock_server().await; - let mut builder = test_codex().with_config(|config| { - config.model = "gpt-5.1".to_string(); - config.model_family = find_family_for_model("gpt-5.1").expect("gpt-5.1 is a valid model"); + let mut builder = test_codex().with_model("gpt-5.1").with_config(|config| { config.sandbox_policy = SandboxPolicy::DangerFullAccess; }); let test = builder.build(&server).await?; diff --git a/codex-rs/core/tests/suite/truncation.rs b/codex-rs/core/tests/suite/truncation.rs index ed9e7cde3b..e5ab28a5eb 100644 --- a/codex-rs/core/tests/suite/truncation.rs +++ b/codex-rs/core/tests/suite/truncation.rs @@ -6,7 +6,6 @@ use anyhow::Result; use codex_core::config::types::McpServerConfig; use codex_core::config::types::McpServerTransportConfig; use codex_core::features::Feature; -use codex_core::model_family::find_family_for_model; use codex_core::protocol::AskForApproval; use codex_core::protocol::EventMsg; use codex_core::protocol::Op; @@ -38,12 +37,7 @@ async fn truncate_function_error_trims_respond_to_model() -> Result<()> { skip_if_no_network!(Ok(())); let server = start_mock_server().await; - let mut builder = test_codex().with_config(|config| { - // Use the test model that wires function tools like grep_files - config.model = "test-gpt-5.1-codex".to_string(); - config.model_family = - find_family_for_model("test-gpt-5.1-codex").expect("model family for test model"); - }); + let mut builder = test_codex().with_model("test-gpt-5.1-codex"); let test = builder.build(&server).await?; // Construct a very long, non-existent path to force a RespondToModel error with a large message @@ -98,7 +92,7 @@ async fn truncate_function_error_trims_respond_to_model() -> Result<()> { Ok(()) } -// Verifies that a standard tool call (shell) exceeding the model formatting +// Verifies that a standard tool call (shell_command) exceeding the model formatting // limits is truncated before being sent back to the model. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn tool_call_output_configured_limit_chars_type() -> Result<()> { @@ -106,7 +100,7 @@ async fn tool_call_output_configured_limit_chars_type() -> Result<()> { let server = start_mock_server().await; - // Use a model that exposes the generic shell tool. + // Use a model that exposes the shell_command tool. let mut builder = test_codex().with_model("gpt-5.1").with_config(|config| { config.tool_output_token_limit = Some(100_000); }); @@ -114,28 +108,22 @@ async fn tool_call_output_configured_limit_chars_type() -> Result<()> { let fixture = builder.build(&server).await?; let call_id = "shell-too-large"; - let args = if cfg!(windows) { - serde_json::json!({ - "command": [ - "powershell", - "-Command", - "for ($i=1; $i -le 100000; $i++) { Write-Output $i }" - ], - "timeout_ms": 5_000, - }) + let command = if cfg!(windows) { + "for ($i=1; $i -le 100000; $i++) { Write-Output $i }" } else { - serde_json::json!({ - "command": ["/bin/sh", "-c", "seq 1 100000"], - "timeout_ms": 5_000, - }) + "seq 1 100000" }; + let args = serde_json::json!({ + "command": command, + "timeout_ms": 5_000, + }); // First response: model tells us to run the tool; second: complete the turn. mount_sse_once( &server, sse(vec![ responses::ev_response_created("resp-1"), - responses::ev_function_call(call_id, "shell", &serde_json::to_string(&args)?), + responses::ev_function_call(call_id, "shell_command", &serde_json::to_string(&args)?), responses::ev_completed("resp-1"), ]), ) @@ -167,7 +155,10 @@ async fn tool_call_output_configured_limit_chars_type() -> Result<()> { "expected truncated shell output to be plain text" ); - assert_eq!(output.len(), 400097, "we should be almost 100k tokens"); + assert!( + (400000..=401000).contains(&output.len()), + "we should be almost 100k tokens" + ); assert!( !output.contains("tokens truncated"), @@ -177,7 +168,7 @@ async fn tool_call_output_configured_limit_chars_type() -> Result<()> { Ok(()) } -// Verifies that a standard tool call (shell) exceeding the model formatting +// Verifies that a standard tool call (shell_command) exceeding the model formatting // limits is truncated before being sent back to the model. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn tool_call_output_exceeds_limit_truncated_chars_limit() -> Result<()> { @@ -185,34 +176,28 @@ async fn tool_call_output_exceeds_limit_truncated_chars_limit() -> Result<()> { let server = start_mock_server().await; - // Use a model that exposes the generic shell tool. + // Use a model that exposes the shell_command tool. let mut builder = test_codex().with_model("gpt-5.1"); let fixture = builder.build(&server).await?; let call_id = "shell-too-large"; - let args = if cfg!(windows) { - serde_json::json!({ - "command": [ - "powershell", - "-Command", - "for ($i=1; $i -le 100000; $i++) { Write-Output $i }" - ], - "timeout_ms": 5_000, - }) + let command = if cfg!(windows) { + "for ($i=1; $i -le 100000; $i++) { Write-Output $i }" } else { - serde_json::json!({ - "command": ["/bin/sh", "-c", "seq 1 100000"], - "timeout_ms": 5_000, - }) + "seq 1 100000" }; + let args = serde_json::json!({ + "command": command, + "timeout_ms": 5_000, + }); // First response: model tells us to run the tool; second: complete the turn. mount_sse_once( &server, sse(vec![ responses::ev_response_created("resp-1"), - responses::ev_function_call(call_id, "shell", &serde_json::to_string(&args)?), + responses::ev_function_call(call_id, "shell_command", &serde_json::to_string(&args)?), responses::ev_completed("resp-1"), ]), ) @@ -244,15 +229,20 @@ async fn tool_call_output_exceeds_limit_truncated_chars_limit() -> Result<()> { "expected truncated shell output to be plain text" ); - assert_eq!(output.len(), 9976); // ~10k characters - let truncated_pattern = r#"(?s)^Exit code: 0\nWall time: 0 seconds\nTotal output lines: 100000\nOutput:\n.*?…\d+ chars truncated….*$"#; + let truncated_pattern = r#"(?s)^Exit code: 0\nWall time: [0-9]+(?:\.[0-9]+)? seconds\nTotal output lines: 100000\nOutput:\n.*?…\d+ chars truncated….*$"#; assert_regex_match(truncated_pattern, &output); + let len = output.len(); + assert!( + (9_900..=10_100).contains(&len), + "expected ~10k chars after truncation, got {len}" + ); + Ok(()) } -// Verifies that a standard tool call (shell) exceeding the model formatting +// Verifies that a standard tool call (shell_command) exceeding the model formatting // limits is truncated before being sent back to the model. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn tool_call_output_exceeds_limit_truncated_for_model() -> Result<()> { @@ -260,37 +250,27 @@ async fn tool_call_output_exceeds_limit_truncated_for_model() -> Result<()> { let server = start_mock_server().await; - // Use a model that exposes the generic shell tool. - let mut builder = test_codex().with_config(|config| { - config.model = "gpt-5.1-codex".to_string(); - config.model_family = - find_family_for_model("gpt-5.1-codex").expect("gpt-5.1-codex is a model family"); - }); + // Use a model that exposes the shell_command tool. + let mut builder = test_codex().with_model("gpt-5.1-codex"); let fixture = builder.build(&server).await?; let call_id = "shell-too-large"; - let args = if cfg!(windows) { - serde_json::json!({ - "command": [ - "powershell", - "-Command", - "for ($i=1; $i -le 100000; $i++) { Write-Output $i }" - ], - "timeout_ms": 5_000, - }) + let command = if cfg!(windows) { + "for ($i=1; $i -le 100000; $i++) { Write-Output $i }" } else { - serde_json::json!({ - "command": ["/bin/sh", "-c", "seq 1 100000"], - "timeout_ms": 5_000, - }) + "seq 1 100000" }; + let args = serde_json::json!({ + "command": command, + "timeout_ms": 5_000, + }); // First response: model tells us to run the tool; second: complete the turn. mount_sse_once( &server, sse(vec![ responses::ev_response_created("resp-1"), - responses::ev_function_call(call_id, "shell", &serde_json::to_string(&args)?), + responses::ev_function_call(call_id, "shell_command", &serde_json::to_string(&args)?), responses::ev_completed("resp-1"), ]), ) @@ -340,41 +320,31 @@ $"#; Ok(()) } -// Ensures shell tool outputs that exceed the line limit are truncated only once. +// Ensures shell_command outputs that exceed the line limit are truncated only once. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn tool_call_output_truncated_only_once() -> Result<()> { skip_if_no_network!(Ok(())); let server = start_mock_server().await; - let mut builder = test_codex().with_config(|config| { - config.model = "gpt-5.1-codex".to_string(); - config.model_family = - find_family_for_model("gpt-5.1-codex").expect("gpt-5.1-codex is a model family"); - }); + let mut builder = test_codex().with_model("gpt-5.1-codex"); let fixture = builder.build(&server).await?; let call_id = "shell-single-truncation"; - let args = if cfg!(windows) { - serde_json::json!({ - "command": [ - "powershell", - "-Command", - "for ($i=1; $i -le 10000; $i++) { Write-Output $i }" - ], - "timeout_ms": 5_000, - }) + let command = if cfg!(windows) { + "for ($i=1; $i -le 10000; $i++) { Write-Output $i }" } else { - serde_json::json!({ - "command": ["/bin/sh", "-c", "seq 1 10000"], - "timeout_ms": 5_000, - }) + "seq 1 10000" }; + let args = serde_json::json!({ + "command": command, + "timeout_ms": 5_000, + }); mount_sse_once( &server, sse(vec![ responses::ev_response_created("resp-1"), - responses::ev_function_call(call_id, "shell", &serde_json::to_string(&args)?), + responses::ev_function_call(call_id, "shell_command", &serde_json::to_string(&args)?), responses::ev_completed("resp-1"), ]), ) @@ -604,17 +574,16 @@ async fn token_policy_marker_reports_tokens() -> Result<()> { skip_if_no_network!(Ok(())); let server = start_mock_server().await; - let mut builder = test_codex().with_config(|config| { - config.model = "gpt-5.1-codex".to_string(); // token policy - config.model_family = - find_family_for_model("gpt-5.1-codex").expect("model family for gpt-5.1-codex"); - config.tool_output_token_limit = Some(50); // small budget to force truncation - }); + let mut builder = test_codex() + .with_model("gpt-5.1-codex") + .with_config(|config| { + config.tool_output_token_limit = Some(50); // small budget to force truncation + }); let fixture = builder.build(&server).await?; let call_id = "shell-token-marker"; let args = json!({ - "command": ["/bin/sh", "-c", "seq 1 150"], + "command": "seq 1 150", "timeout_ms": 5_000, }); @@ -622,7 +591,7 @@ async fn token_policy_marker_reports_tokens() -> Result<()> { &server, sse(vec![ ev_response_created("resp-1"), - ev_function_call(call_id, "shell", &serde_json::to_string(&args)?), + ev_function_call(call_id, "shell_command", &serde_json::to_string(&args)?), ev_completed("resp-1"), ]), ) @@ -645,7 +614,7 @@ async fn token_policy_marker_reports_tokens() -> Result<()> { .function_call_output_text(call_id) .context("shell output present")?; - let pattern = r#"(?s)^\{"output":"Total output lines: 150\\n\\n1\\n2\\n3\\n4\\n5\\n.*?…\d+ tokens truncated…7\\n138\\n139\\n140\\n141\\n142\\n143\\n144\\n145\\n146\\n147\\n148\\n149\\n150\\n","metadata":\{"exit_code":0,"duration_seconds":0\.0\}\}$"#; + let pattern = r"(?s)^Exit code: 0\nWall time: [0-9]+(?:\.[0-9]+)? seconds\nTotal output lines: 150\nOutput:\n1\n2\n3\n4\n5\n6\n7\n8\n9\n10\n11\n12\n13\n14\n15\n16\n17\n18\n19.*tokens truncated.*129\n130\n131\n132\n133\n134\n135\n136\n137\n138\n139\n140\n141\n142\n143\n144\n145\n146\n147\n148\n149\n150\n$"; assert_regex_match(pattern, &output); @@ -658,16 +627,14 @@ async fn byte_policy_marker_reports_bytes() -> Result<()> { skip_if_no_network!(Ok(())); let server = start_mock_server().await; - let mut builder = test_codex().with_config(|config| { - config.model = "gpt-5.1".to_string(); // byte policy - config.model_family = find_family_for_model("gpt-5.1").expect("model family for gpt-5.1"); + let mut builder = test_codex().with_model("gpt-5.1").with_config(|config| { config.tool_output_token_limit = Some(50); // ~200 byte cap }); let fixture = builder.build(&server).await?; let call_id = "shell-byte-marker"; let args = json!({ - "command": ["/bin/sh", "-c", "seq 1 150"], + "command": "seq 1 150", "timeout_ms": 5_000, }); @@ -675,7 +642,7 @@ async fn byte_policy_marker_reports_bytes() -> Result<()> { &server, sse(vec![ ev_response_created("resp-1"), - ev_function_call(call_id, "shell", &serde_json::to_string(&args)?), + ev_function_call(call_id, "shell_command", &serde_json::to_string(&args)?), ev_completed("resp-1"), ]), ) @@ -698,30 +665,29 @@ async fn byte_policy_marker_reports_bytes() -> Result<()> { .function_call_output_text(call_id) .context("shell output present")?; - let pattern = r#"(?s)^\{"output":"Total output lines: 150\\n\\n1\\n2\\n3\\n4\\n5.*?…\d+ chars truncated…7\\n138\\n139\\n140\\n141\\n142\\n143\\n144\\n145\\n146\\n147\\n148\\n149\\n150\\n","metadata":\{"exit_code":0,"duration_seconds":0\.0\}\}$"#; + let pattern = r"(?s)^Exit code: 0\nWall time: [0-9]+(?:\.[0-9]+)? seconds\nTotal output lines: 150\nOutput:\n1\n2\n3\n4\n5\n6\n7\n8\n9\n10\n11\n12\n13\n14\n15\n16\n17\n18\n19.*chars truncated.*129\n130\n131\n132\n133\n134\n135\n136\n137\n138\n139\n140\n141\n142\n143\n144\n145\n146\n147\n148\n149\n150\n$"; assert_regex_match(pattern, &output); Ok(()) } -// Shell tool output should remain intact when the config opts into a large token budget. +// shell_command output should remain intact when the config opts into a large token budget. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn shell_tool_output_not_truncated_with_custom_limit() -> Result<()> { +async fn shell_command_output_not_truncated_with_custom_limit() -> Result<()> { skip_if_no_network!(Ok(())); let server = start_mock_server().await; - let mut builder = test_codex().with_config(|config| { - config.model = "gpt-5.1-codex".to_string(); - config.model_family = - find_family_for_model("gpt-5.1-codex").expect("model family for gpt-5.1-codex"); - config.tool_output_token_limit = Some(50_000); // ample budget - }); + let mut builder = test_codex() + .with_model("gpt-5.1-codex") + .with_config(|config| { + config.tool_output_token_limit = Some(50_000); // ample budget + }); let fixture = builder.build(&server).await?; let call_id = "shell-no-trunc"; let args = json!({ - "command": ["/bin/sh", "-c", "seq 1 1000"], + "command": "seq 1 1000", "timeout_ms": 5_000, }); let expected_body: String = (1..=1000).map(|i| format!("{i}\n")).collect(); @@ -730,7 +696,7 @@ async fn shell_tool_output_not_truncated_with_custom_limit() -> Result<()> { &server, sse(vec![ ev_response_created("resp-1"), - ev_function_call(call_id, "shell", &serde_json::to_string(&args)?), + ev_function_call(call_id, "shell_command", &serde_json::to_string(&args)?), ev_completed("resp-1"), ]), ) diff --git a/codex-rs/core/tests/suite/undo.rs b/codex-rs/core/tests/suite/undo.rs index 6dcb8e81e0..4fcd138cb4 100644 --- a/codex-rs/core/tests/suite/undo.rs +++ b/codex-rs/core/tests/suite/undo.rs @@ -9,9 +9,7 @@ use anyhow::Context; use anyhow::Result; use anyhow::bail; use codex_core::CodexConversation; -use codex_core::config::Config; use codex_core::features::Feature; -use codex_core::model_family::find_family_for_model; use codex_core::protocol::EventMsg; use codex_core::protocol::Op; use codex_core::protocol::UndoCompletedEvent; @@ -23,18 +21,17 @@ use core_test_support::responses::mount_sse_sequence; use core_test_support::responses::sse; use core_test_support::skip_if_no_network; use core_test_support::test_codex::TestCodexHarness; +use core_test_support::test_codex::test_codex; use core_test_support::wait_for_event_match; use pretty_assertions::assert_eq; #[allow(clippy::expect_used)] async fn undo_harness() -> Result { - TestCodexHarness::with_config(|config: &mut Config| { + let builder = test_codex().with_model("gpt-5.1").with_config(|config| { config.include_apply_patch_tool = true; - config.model = "gpt-5.1".to_string(); - config.model_family = find_family_for_model("gpt-5.1").expect("gpt-5.1 is valid"); config.features.enable(Feature::GhostCommit); - }) - .await + }); + TestCodexHarness::with_builder(builder).await } fn git(path: &Path, args: &[&str]) -> Result<()> { diff --git a/codex-rs/core/tests/suite/unified_exec.rs b/codex-rs/core/tests/suite/unified_exec.rs index d3059aba42..dc7bdb6b10 100644 --- a/codex-rs/core/tests/suite/unified_exec.rs +++ b/codex-rs/core/tests/suite/unified_exec.rs @@ -1,5 +1,7 @@ #![cfg(not(target_os = "windows"))] use std::collections::HashMap; +use std::ffi::OsStr; +use std::fs; use std::sync::OnceLock; use anyhow::Context; @@ -23,6 +25,7 @@ use core_test_support::responses::start_mock_server; use core_test_support::skip_if_no_network; use core_test_support::skip_if_sandbox; use core_test_support::test_codex::TestCodex; +use core_test_support::test_codex::TestCodexHarness; use core_test_support::test_codex::test_codex; use core_test_support::wait_for_event; use core_test_support::wait_for_event_match; @@ -44,7 +47,7 @@ fn extract_output_text(item: &Value) -> Option<&str> { struct ParsedUnifiedExecOutput { chunk_id: Option, wall_time_seconds: f64, - session_id: Option, + process_id: Option, exit_code: Option, original_token_count: Option, output: String, @@ -59,7 +62,7 @@ fn parse_unified_exec_output(raw: &str) -> Result { r#"(?:Chunk ID: (?P[^\n]+)\n)?"#, r#"Wall time: (?P-?\d+(?:\.\d+)?) seconds\n"#, r#"(?:Process exited with code (?P-?\d+)\n)?"#, - r#"(?:Process running with session ID (?P-?\d+)\n)?"#, + r#"(?:Process running with session ID (?P-?\d+)\n)?"#, r#"(?:Original token count: (?P\d+)\n)?"#, r#"Output:\n?(?P.*)$"#, )) @@ -92,15 +95,9 @@ fn parse_unified_exec_output(raw: &str) -> Result { }) .transpose()?; - let session_id = captures - .name("session_id") - .map(|value| { - value - .as_str() - .parse::() - .context("failed to parse session id from unified exec output") - }) - .transpose()?; + let process_id = captures + .name("process_id") + .map(|value| value.as_str().to_string()); let original_token_count = captures .name("original_token_count") @@ -121,7 +118,7 @@ fn parse_unified_exec_output(raw: &str) -> Result { Ok(ParsedUnifiedExecOutput { chunk_id, wall_time_seconds, - session_id, + process_id, exit_code, original_token_count, output, @@ -154,6 +151,130 @@ fn collect_tool_outputs(bodies: &[Value]) -> Result Result<()> { + skip_if_no_network!(Ok(())); + skip_if_sandbox!(Ok(())); + + let builder = test_codex().with_config(|config| { + config.include_apply_patch_tool = true; + config.use_experimental_unified_exec_tool = true; + config.features.enable(Feature::UnifiedExec); + }); + let harness = TestCodexHarness::with_builder(builder).await?; + + let patch = + "*** Begin Patch\n*** Add File: uexec_apply.txt\n+hello from unified exec\n*** End Patch"; + let command = format!("apply_patch <<'EOF'\n{patch}\nEOF\n"); + let call_id = "uexec-apply-patch"; + let args = json!({ + "cmd": command, + "yield_time_ms": 250, + }); + + let responses = vec![ + sse(vec![ + ev_response_created("resp-1"), + ev_function_call(call_id, "exec_command", &serde_json::to_string(&args)?), + ev_completed("resp-1"), + ]), + sse(vec![ + ev_response_created("resp-2"), + ev_assistant_message("msg-1", "done"), + ev_completed("resp-2"), + ]), + ]; + mount_sse_sequence(harness.server(), responses).await; + + let test = harness.test(); + let codex = test.codex.clone(); + let cwd = test.cwd_path().to_path_buf(); + let session_model = test.session_configured.model.clone(); + + codex + .submit(Op::UserTurn { + items: vec![UserInput::Text { + text: "apply patch via unified exec".into(), + }], + final_output_json_schema: None, + cwd, + approval_policy: AskForApproval::Never, + sandbox_policy: SandboxPolicy::DangerFullAccess, + model: session_model, + effort: None, + summary: ReasoningSummary::Auto, + }) + .await?; + + let mut saw_patch_begin = false; + let mut patch_end = None; + let mut saw_exec_begin = false; + let mut saw_exec_end = false; + wait_for_event(&codex, |event| match event { + EventMsg::PatchApplyBegin(begin) if begin.call_id == call_id => { + saw_patch_begin = true; + assert!( + begin + .changes + .keys() + .any(|path| path.file_name() == Some(OsStr::new("uexec_apply.txt"))), + "expected apply_patch changes to target uexec_apply.txt", + ); + false + } + EventMsg::PatchApplyEnd(end) if end.call_id == call_id => { + patch_end = Some(end.clone()); + false + } + EventMsg::ExecCommandBegin(event) if event.call_id == call_id => { + saw_exec_begin = true; + false + } + EventMsg::ExecCommandEnd(event) if event.call_id == call_id => { + saw_exec_end = true; + false + } + EventMsg::TaskComplete(_) => true, + _ => false, + }) + .await; + + assert!( + saw_patch_begin, + "expected apply_patch to emit PatchApplyBegin" + ); + let patch_end = patch_end.expect("expected apply_patch to emit PatchApplyEnd"); + assert!( + patch_end.success, + "expected apply_patch to finish successfully: stdout={:?} stderr={:?}", + patch_end.stdout, patch_end.stderr, + ); + assert!( + !saw_exec_begin, + "apply_patch should be intercepted before exec_command begin" + ); + assert!( + !saw_exec_end, + "apply_patch should not emit exec_command end events" + ); + + let output = harness.function_call_stdout(call_id).await; + assert!( + output.contains("Success. Updated the following files:"), + "expected apply_patch output, got: {output:?}" + ); + assert!( + output.contains("A uexec_apply.txt"), + "expected apply_patch file summary, got: {output:?}" + ); + assert_eq!( + fs::read_to_string(harness.path("uexec_apply.txt"))?, + "hello from unified exec\n" + ); + + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn unified_exec_emits_exec_command_begin_event() -> Result<()> { skip_if_no_network!(Ok(())); @@ -174,6 +295,7 @@ async fn unified_exec_emits_exec_command_begin_event() -> Result<()> { let call_id = "uexec-begin-event"; let args = json!({ + "shell": "bash".to_string(), "cmd": "/bin/echo hello unified exec".to_string(), "yield_time_ms": 250, }); @@ -215,14 +337,8 @@ async fn unified_exec_emits_exec_command_begin_event() -> Result<()> { }) .await; - assert_eq!( - begin_event.command, - vec![ - "/bin/bash".to_string(), - "-lc".to_string(), - "/bin/echo hello unified exec".to_string() - ] - ); + assert_command(&begin_event.command, "-lc", "/bin/echo hello unified exec"); + assert_eq!(begin_event.cwd, cwd.path()); wait_for_event(&codex, |event| matches!(event, EventMsg::TaskComplete(_))).await; @@ -230,6 +346,82 @@ async fn unified_exec_emits_exec_command_begin_event() -> Result<()> { Ok(()) } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn unified_exec_resolves_relative_workdir() -> Result<()> { + skip_if_no_network!(Ok(())); + skip_if_sandbox!(Ok(())); + + let server = start_mock_server().await; + + let mut builder = test_codex().with_model("gpt-5").with_config(|config| { + config.use_experimental_unified_exec_tool = true; + config.features.enable(Feature::UnifiedExec); + }); + let TestCodex { + codex, + cwd, + session_configured, + .. + } = builder.build(&server).await?; + + let workdir_rel = std::path::PathBuf::from("uexec_relative_workdir"); + std::fs::create_dir_all(cwd.path().join(&workdir_rel))?; + + let call_id = "uexec-workdir-relative"; + let args = json!({ + "cmd": "pwd", + "yield_time_ms": 250, + "workdir": workdir_rel.to_string_lossy().to_string(), + }); + + let responses = vec![ + sse(vec![ + ev_response_created("resp-1"), + ev_function_call(call_id, "exec_command", &serde_json::to_string(&args)?), + ev_completed("resp-1"), + ]), + sse(vec![ + ev_response_created("resp-2"), + ev_assistant_message("msg-1", "finished"), + ev_completed("resp-2"), + ]), + ]; + mount_sse_sequence(&server, responses).await; + + let session_model = session_configured.model.clone(); + + codex + .submit(Op::UserTurn { + items: vec![UserInput::Text { + text: "run relative workdir test".into(), + }], + final_output_json_schema: None, + cwd: cwd.path().to_path_buf(), + approval_policy: AskForApproval::Never, + sandbox_policy: SandboxPolicy::DangerFullAccess, + model: session_model, + effort: None, + summary: ReasoningSummary::Auto, + }) + .await?; + + let begin_event = wait_for_event_match(&codex, |msg| match msg { + EventMsg::ExecCommandBegin(event) if event.call_id == call_id => Some(event.clone()), + _ => None, + }) + .await; + + assert_eq!( + begin_event.cwd, + cwd.path().join(workdir_rel), + "exec_command cwd should resolve relative workdir against turn cwd", + ); + + wait_for_event(&codex, |event| matches!(event, EventMsg::TaskComplete(_))).await; + + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[ignore = "flaky"] async fn unified_exec_respects_workdir_override() -> Result<()> { @@ -335,7 +527,7 @@ async fn unified_exec_emits_exec_command_end_event() -> Result<()> { let poll_call_id = "uexec-end-event-poll"; let poll_args = json!({ "chars": "", - "session_id": 0, + "session_id": 1000, "yield_time_ms": 250, }); @@ -493,7 +685,7 @@ async fn unified_exec_emits_output_delta_for_write_stdin() -> Result<()> { let stdin_call_id = "uexec-stdin-delta"; let stdin_args = json!({ "chars": "echo WSTDIN-MARK\\n", - "session_id": 0, + "session_id": 1000, "yield_time_ms": 800, }); @@ -585,14 +777,15 @@ async fn unified_exec_emits_begin_for_write_stdin() -> Result<()> { let open_call_id = "uexec-open-for-begin"; let open_args = json!({ - "cmd": "/bin/sh -c echo ready".to_string(), + "shell": "bash".to_string(), + "cmd": "bash -i".to_string(), "yield_time_ms": 200, }); let stdin_call_id = "uexec-stdin-begin"; let stdin_args = json!({ "chars": "echo hello", - "session_id": 0, + "session_id": 1000, "yield_time_ms": 400, }); @@ -646,14 +839,7 @@ async fn unified_exec_emits_begin_for_write_stdin() -> Result<()> { }) .await; - assert_eq!( - begin_event.command, - vec![ - "/bin/bash".to_string(), - "-lc".to_string(), - "/bin/sh -c echo ready".to_string() - ] - ); + assert_command(&begin_event.command, "-lc", "bash -i"); assert_eq!( begin_event.interaction_input, Some("echo hello".to_string()) @@ -687,14 +873,15 @@ async fn unified_exec_emits_begin_event_for_write_stdin_requests() -> Result<()> let open_call_id = "uexec-open-session"; let open_args = json!({ - "cmd": "/bin/sh -c echo ready".to_string(), + "shell": "bash".to_string(), + "cmd": "bash -i".to_string(), "yield_time_ms": 250, }); let poll_call_id = "uexec-poll-empty"; let poll_args = json!({ "chars": "", - "session_id": 0, + "session_id": 1000, "yield_time_ms": 150, }); @@ -762,14 +949,9 @@ async fn unified_exec_emits_begin_event_for_write_stdin_requests() -> Result<()> .iter() .find(|ev| ev.call_id == open_call_id) .expect("missing exec_command begin"); - assert_eq!( - open_event.command, - vec![ - "/bin/bash".to_string(), - "-lc".to_string(), - "/bin/sh -c echo ready".to_string() - ] - ); + + assert_command(&open_event.command, "-lc", "bash -i"); + assert!( open_event.interaction_input.is_none(), "startup begin events should not include interaction input" @@ -780,14 +962,9 @@ async fn unified_exec_emits_begin_event_for_write_stdin_requests() -> Result<()> .iter() .find(|ev| ev.call_id == poll_call_id) .expect("missing write_stdin begin"); - assert_eq!( - poll_event.command, - vec![ - "/bin/bash".to_string(), - "-lc".to_string(), - "/bin/sh -c echo ready".to_string() - ] - ); + + assert_command(&poll_event.command, "-lc", "bash -i"); + assert!( poll_event.interaction_input.is_none(), "poll begin events should omit interaction input" @@ -880,8 +1057,8 @@ async fn exec_command_reports_chunk_and_exit_metadata() -> Result<()> { ); assert!( - metadata.session_id.is_none(), - "exec_command for a completed process should not include session_id" + metadata.process_id.is_none(), + "exec_command for a completed process should not include process_id" ); let exit_code = metadata.exit_code.expect("expected exit_code"); @@ -973,7 +1150,7 @@ async fn unified_exec_respects_early_exit_notifications() -> Result<()> { .expect("missing early exit unified_exec output"); assert!( - output.session_id.is_none(), + output.process_id.is_none(), "short-lived process should not keep a session alive" ); assert_eq!( @@ -1023,12 +1200,12 @@ async fn write_stdin_returns_exit_metadata_and_clears_session() -> Result<()> { }); let send_args = serde_json::json!({ "chars": "hello unified exec\n", - "session_id": 0, + "session_id": 1000, "yield_time_ms": 500, }); let exit_args = serde_json::json!({ "chars": "\u{0004}", - "session_id": 0, + "session_id": 1000, "yield_time_ms": 500, }); @@ -1099,12 +1276,13 @@ async fn write_stdin_returns_exit_metadata_and_clears_session() -> Result<()> { let start_output = outputs .get(start_call_id) .expect("missing start output for exec_command"); - let session_id = start_output - .session_id - .expect("expected session id from exec_command"); + let process_id = start_output + .process_id + .clone() + .expect("expected process id from exec_command"); assert!( - session_id >= 0, - "session_id should be non-negative, got {session_id}" + process_id.len() > 3, + "process_id should be at least 4 digits, got {process_id}" ); assert!( start_output.exit_code.is_none(), @@ -1120,11 +1298,12 @@ async fn write_stdin_returns_exit_metadata_and_clears_session() -> Result<()> { "expected echoed output from cat, got {echoed:?}" ); let echoed_session = send_output - .session_id - .expect("write_stdin should return session id while process is running"); + .process_id + .clone() + .expect("write_stdin should return process id while process is running"); assert_eq!( - echoed_session, session_id, - "write_stdin should reuse existing session id" + echoed_session, process_id, + "write_stdin should reuse existing process id" ); assert!( send_output.exit_code.is_none(), @@ -1135,8 +1314,8 @@ async fn write_stdin_returns_exit_metadata_and_clears_session() -> Result<()> { .get(exit_call_id) .expect("missing exit metadata output"); assert!( - exit_output.session_id.is_none(), - "session_id should be omitted once the process exits" + exit_output.process_id.is_none(), + "process_id should be omitted once the process exits" ); let exit_code = exit_output .exit_code @@ -1182,14 +1361,14 @@ async fn unified_exec_emits_end_event_when_session_dies_via_stdin() -> Result<() let echo_call_id = "uexec-end-on-exit-echo"; let echo_args = serde_json::json!({ "chars": "bye-END\n", - "session_id": 0, + "session_id": 1000, "yield_time_ms": 300, }); let exit_call_id = "uexec-end-on-exit"; let exit_args = serde_json::json!({ "chars": "\u{0004}", - "session_id": 0, + "session_id": 1000, "yield_time_ms": 500, }); @@ -1285,7 +1464,7 @@ async fn unified_exec_reuses_session_via_stdin() -> Result<()> { let second_call_id = "uexec-stdin"; let second_args = serde_json::json!({ "chars": "hello unified exec\n", - "session_id": 0, + "session_id": 1000, "yield_time_ms": 500, }); @@ -1347,17 +1526,20 @@ async fn unified_exec_reuses_session_via_stdin() -> Result<()> { let start_output = outputs .get(first_call_id) .expect("missing first unified_exec output"); - let session_id = start_output.session_id.unwrap_or_default(); + let process_id = start_output.process_id.clone().unwrap_or_default(); assert!( - session_id >= 0, - "expected session id in first unified_exec response" + !process_id.is_empty(), + "expected process id in first unified_exec response" ); assert!(start_output.output.is_empty()); let reuse_output = outputs .get(second_call_id) .expect("missing reused unified_exec output"); - assert_eq!(reuse_output.session_id.unwrap_or_default(), session_id); + assert_eq!( + reuse_output.process_id.clone().unwrap_or_default(), + process_id + ); let echoed = reuse_output.output.as_str(); assert!( echoed.contains("hello unified exec"), @@ -1413,7 +1595,7 @@ PY let second_call_id = "uexec-lag-poll"; let second_args = serde_json::json!({ "chars": "", - "session_id": 0, + "session_id": 1000, "yield_time_ms": 2_000, }); @@ -1480,9 +1662,9 @@ PY let start_output = outputs .get(first_call_id) .expect("missing initial unified_exec output"); - let session_id = start_output.session_id.unwrap_or_default(); + let process_id = start_output.process_id.clone().unwrap_or_default(); assert!( - session_id >= 0, + !process_id.is_empty(), "expected session id from initial unified_exec response" ); @@ -1524,7 +1706,7 @@ async fn unified_exec_timeout_and_followup_poll() -> Result<()> { let second_call_id = "uexec-poll"; let second_args = serde_json::json!({ "chars": "", - "session_id": 0, + "session_id": 1000, "yield_time_ms": 800, }); @@ -1589,7 +1771,7 @@ async fn unified_exec_timeout_and_followup_poll() -> Result<()> { let outputs = collect_tool_outputs(&bodies)?; let first_output = outputs.get(first_call_id).expect("missing timeout output"); - assert_eq!(first_output.session_id, Some(0)); + assert!(first_output.process_id.is_some()); assert!(first_output.output.is_empty()); let poll_output = outputs.get(second_call_id).expect("missing poll output"); @@ -1622,8 +1804,8 @@ async fn unified_exec_formats_large_output_summary() -> Result<()> { } = builder.build(&server).await?; let script = r#"python3 - <<'PY' -for i in range(10000): - print("token token ") +import sys +sys.stdout.write("token token \n" * 5000) PY "#; @@ -1760,3 +1942,176 @@ async fn unified_exec_runs_under_sandbox() -> Result<()> { Ok(()) } + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore] +async fn unified_exec_prunes_exited_sessions_first() -> Result<()> { + skip_if_no_network!(Ok(())); + skip_if_sandbox!(Ok(())); + + let server = start_mock_server().await; + + let mut builder = test_codex().with_config(|config| { + config.use_experimental_unified_exec_tool = true; + config.features.enable(Feature::UnifiedExec); + }); + let TestCodex { + codex, + cwd, + session_configured, + .. + } = builder.build(&server).await?; + + const MAX_SESSIONS_FOR_TEST: i32 = 64; + const FILLER_SESSIONS: i32 = MAX_SESSIONS_FOR_TEST - 1; + + let keep_call_id = "uexec-prune-keep"; + let keep_args = serde_json::json!({ + "cmd": "/bin/cat", + "yield_time_ms": 250, + }); + + let prune_call_id = "uexec-prune-target"; + // Give the sleeper time to exit before the filler sessions trigger pruning. + let prune_args = serde_json::json!({ + "cmd": "sleep 1", + "yield_time_ms": 1_250, + }); + + let mut events = vec![ev_response_created("resp-prune-1")]; + events.push(ev_function_call( + keep_call_id, + "exec_command", + &serde_json::to_string(&keep_args)?, + )); + events.push(ev_function_call( + prune_call_id, + "exec_command", + &serde_json::to_string(&prune_args)?, + )); + + for idx in 0..FILLER_SESSIONS { + let filler_args = serde_json::json!({ + "cmd": format!("echo filler {idx}"), + "yield_time_ms": 250, + }); + let call_id = format!("uexec-prune-fill-{idx}"); + events.push(ev_function_call( + &call_id, + "exec_command", + &serde_json::to_string(&filler_args)?, + )); + } + + let keep_write_call_id = "uexec-prune-keep-write"; + let keep_write_args = serde_json::json!({ + "chars": "still alive\n", + "session_id": 1000, + "yield_time_ms": 500, + }); + events.push(ev_function_call( + keep_write_call_id, + "write_stdin", + &serde_json::to_string(&keep_write_args)?, + )); + + let probe_call_id = "uexec-prune-probe"; + let probe_args = serde_json::json!({ + "chars": "should fail\n", + "session_id": 1001, + "yield_time_ms": 500, + }); + events.push(ev_function_call( + probe_call_id, + "write_stdin", + &serde_json::to_string(&probe_args)?, + )); + + events.push(ev_completed("resp-prune-1")); + let first_response = sse(events); + let completion_response = sse(vec![ + ev_response_created("resp-prune-2"), + ev_assistant_message("msg-prune", "done"), + ev_completed("resp-prune-2"), + ]); + let response_mock = + mount_sse_sequence(&server, vec![first_response, completion_response]).await; + + let session_model = session_configured.model.clone(); + + codex + .submit(Op::UserTurn { + items: vec![UserInput::Text { + text: "fill session cache".into(), + }], + final_output_json_schema: None, + cwd: cwd.path().to_path_buf(), + approval_policy: AskForApproval::Never, + sandbox_policy: SandboxPolicy::DangerFullAccess, + model: session_model, + effort: None, + summary: ReasoningSummary::Auto, + }) + .await?; + + wait_for_event(&codex, |event| matches!(event, EventMsg::TaskComplete(_))).await; + + let requests = response_mock.requests(); + assert!( + !requests.is_empty(), + "expected at least one response request" + ); + + let keep_start = requests + .iter() + .find_map(|req| req.function_call_output_text(keep_call_id)) + .expect("missing initial keep session output"); + let keep_start_output = parse_unified_exec_output(&keep_start)?; + assert!(keep_start_output.process_id.is_some()); + assert!(keep_start_output.exit_code.is_none()); + + let prune_start = requests + .iter() + .find_map(|req| req.function_call_output_text(prune_call_id)) + .expect("missing initial prune session output"); + let prune_start_output = parse_unified_exec_output(&prune_start)?; + assert!(prune_start_output.process_id.is_some()); + assert!(prune_start_output.exit_code.is_none()); + + let keep_write = requests + .iter() + .find_map(|req| req.function_call_output_text(keep_write_call_id)) + .expect("missing keep write output"); + let keep_write_output = parse_unified_exec_output(&keep_write)?; + assert!(keep_write_output.process_id.is_some()); + assert!( + keep_write_output.output.contains("still alive"), + "expected cat session to echo input, got {:?}", + keep_write_output.output + ); + + let pruned_probe = requests + .iter() + .find_map(|req| req.function_call_output_text(probe_call_id)) + .expect("missing probe output"); + assert!( + pruned_probe.contains("UnknownSessionId") || pruned_probe.contains("Unknown process id"), + "expected probe to fail after pruning, got {pruned_probe:?}" + ); + + Ok(()) +} + +fn assert_command(command: &[String], expected_args: &str, expected_cmd: &str) { + assert_eq!(command.len(), 3); + let shell_path = &command[0]; + assert!( + shell_path == "/bin/bash" + || shell_path == "/usr/bin/bash" + || shell_path == "/usr/local/bin/bash" + || shell_path.ends_with("/bash"), + "unexpected bash path: {shell_path}" + ); + assert_eq!(command[1], expected_args); + assert_eq!(command[2], expected_cmd); +} diff --git a/codex-rs/core/tests/suite/user_shell_cmd.rs b/codex-rs/core/tests/suite/user_shell_cmd.rs index e6fbdee9e5..964cc58d50 100644 --- a/codex-rs/core/tests/suite/user_shell_cmd.rs +++ b/codex-rs/core/tests/suite/user_shell_cmd.rs @@ -1,7 +1,6 @@ use anyhow::Context; use codex_core::ConversationManager; use codex_core::NewConversation; -use codex_core::model_family::find_family_for_model; use codex_core::protocol::EventMsg; use codex_core::protocol::ExecCommandEndEvent; use codex_core::protocol::ExecCommandSource; @@ -275,27 +274,22 @@ async fn user_shell_command_is_truncated_only_once() -> anyhow::Result<()> { let server = start_mock_server().await; - let mut builder = test_codex().with_config(|config| { - config.tool_output_token_limit = Some(100); - config.model = "gpt-5.1-codex".to_string(); - config.model_family = - find_family_for_model("gpt-5-codex").expect("gpt-5-codex is a model family"); - }); + let mut builder = test_codex() + .with_model("gpt-5.1-codex") + .with_config(|config| { + config.tool_output_token_limit = Some(100); + }); let fixture = builder.build(&server).await?; let call_id = "user-shell-double-truncation"; let args = if cfg!(windows) { serde_json::json!({ - "command": [ - "powershell", - "-Command", - "for ($i=1; $i -le 2000; $i++) { Write-Output $i }" - ], + "command": "for ($i=1; $i -le 2000; $i++) { Write-Output $i }", "timeout_ms": 5_000, }) } else { serde_json::json!({ - "command": ["/bin/sh", "-c", "seq 1 2000"], + "command": "seq 1 2000", "timeout_ms": 5_000, }) }; @@ -304,7 +298,7 @@ async fn user_shell_command_is_truncated_only_once() -> anyhow::Result<()> { &server, sse(vec![ ev_response_created("resp-1"), - ev_function_call(call_id, "shell", &serde_json::to_string(&args)?), + ev_function_call(call_id, "shell_command", &serde_json::to_string(&args)?), ev_completed("resp-1"), ]), ) @@ -319,19 +313,22 @@ async fn user_shell_command_is_truncated_only_once() -> anyhow::Result<()> { .await; fixture - .submit_turn_with_policy("trigger big shell output", SandboxPolicy::DangerFullAccess) + .submit_turn_with_policy( + "trigger big shell_command output", + SandboxPolicy::DangerFullAccess, + ) .await?; let output = mock2 .single_request() .function_call_output_text(call_id) - .context("function_call_output present for shell call")?; + .context("function_call_output present for shell_command call")?; let truncation_headers = output.matches("Total output lines:").count(); assert_eq!( truncation_headers, 1, - "shell output should carry only one truncation header: {output}" + "shell_command output should carry only one truncation header: {output}" ); Ok(()) diff --git a/codex-rs/core/tests/suite/view_image.rs b/codex-rs/core/tests/suite/view_image.rs index 394c5c2f14..6c0f6dcc80 100644 --- a/codex-rs/core/tests/suite/view_image.rs +++ b/codex-rs/core/tests/suite/view_image.rs @@ -474,3 +474,82 @@ async fn view_image_tool_errors_when_file_missing() -> anyhow::Result<()> { Ok(()) } + +#[cfg(not(debug_assertions))] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn replaces_invalid_local_image_after_bad_request() -> anyhow::Result<()> { + skip_if_no_network!(Ok(())); + + let server = start_mock_server().await; + + const INVALID_IMAGE_ERROR: &str = + "The image data you provided does not represent a valid image"; + + let invalid_image_mock = responses::mount_response_once_match( + &server, + body_string_contains("\"input_image\""), + ResponseTemplate::new(400) + .insert_header("content-type", "text/plain") + .set_body_string(INVALID_IMAGE_ERROR), + ) + .await; + + let success_response = sse(vec![ + ev_response_created("resp-2"), + ev_assistant_message("msg-1", "done"), + ev_completed("resp-2"), + ]); + + let completion_mock = responses::mount_sse_once(&server, success_response).await; + + let TestCodex { + codex, + cwd, + session_configured, + .. + } = test_codex().build(&server).await?; + + let rel_path = "assets/poisoned.png"; + let abs_path = cwd.path().join(rel_path); + if let Some(parent) = abs_path.parent() { + std::fs::create_dir_all(parent)?; + } + let image = ImageBuffer::from_pixel(1024, 512, Rgba([10u8, 20, 30, 255])); + image.save(&abs_path)?; + + let session_model = session_configured.model.clone(); + + codex + .submit(Op::UserTurn { + items: vec![UserInput::LocalImage { + path: abs_path.clone(), + }], + final_output_json_schema: None, + cwd: cwd.path().to_path_buf(), + approval_policy: AskForApproval::Never, + sandbox_policy: SandboxPolicy::DangerFullAccess, + model: session_model, + effort: None, + summary: ReasoningSummary::Auto, + }) + .await?; + + wait_for_event(&codex, |event| matches!(event, EventMsg::TaskComplete(_))).await; + + let first_body = invalid_image_mock.single_request().body_json(); + assert!( + find_image_message(&first_body).is_some(), + "initial request should include the uploaded image" + ); + + let second_request = completion_mock.single_request(); + let second_body = second_request.body_json(); + assert!( + find_image_message(&second_body).is_none(), + "second request should replace the invalid image" + ); + let user_texts = second_request.message_input_texts("user"); + assert!(user_texts.iter().any(|text| text == "Invalid image")); + + Ok(()) +} diff --git a/codex-rs/default.nix b/codex-rs/default.nix index 26971f1846..867e57ee2d 100644 --- a/codex-rs/default.nix +++ b/codex-rs/default.nix @@ -22,6 +22,7 @@ rustPlatform.buildRustPackage (_: { cargoLock.outputHashes = { "ratatui-0.29.0" = "sha256-HBvT5c8GsiCxMffNjJGLmHnvG77A6cqEL+1ARurBXho="; "crossterm-0.28.1" = "sha256-6qCtfSMuXACKFb9ATID39XyFDIEMFDmbx6SSmNe+728="; + "rmcp-0.9.0" = "sha256-0iPrpf0Ha/facO3p5e0hUKHBqGp/iS+C+OdS+pRKMOU="; }; meta = with lib; { diff --git a/codex-rs/deny.toml b/codex-rs/deny.toml new file mode 100644 index 0000000000..f9d411f841 --- /dev/null +++ b/codex-rs/deny.toml @@ -0,0 +1,272 @@ +# This template contains all of the possible sections and their default values + +# Note that all fields that take a lint level have these possible values: +# * deny - An error will be produced and the check will fail +# * warn - A warning will be produced, but the check will not fail +# * allow - No warning or error will be produced, though in some cases a note +# will be + +# The values provided in this template are the default values that will be used +# when any section or field is not specified in your own configuration + +# Root options + +# The graph table configures how the dependency graph is constructed and thus +# which crates the checks are performed against +[graph] +# If 1 or more target triples (and optionally, target_features) are specified, +# only the specified targets will be checked when running `cargo deny check`. +# This means, if a particular package is only ever used as a target specific +# dependency, such as, for example, the `nix` crate only being used via the +# `target_family = "unix"` configuration, that only having windows targets in +# this list would mean the nix crate, as well as any of its exclusive +# dependencies not shared by any other crates, would be ignored, as the target +# list here is effectively saying which targets you are building for. +targets = [ + # The triple can be any string, but only the target triples built in to + # rustc (as of 1.40) can be checked against actual config expressions + #"x86_64-unknown-linux-musl", + # You can also specify which target_features you promise are enabled for a + # particular target. target_features are currently not validated against + # the actual valid features supported by the target architecture. + #{ triple = "wasm32-unknown-unknown", features = ["atomics"] }, +] +# When creating the dependency graph used as the source of truth when checks are +# executed, this field can be used to prune crates from the graph, removing them +# from the view of cargo-deny. This is an extremely heavy hammer, as if a crate +# is pruned from the graph, all of its dependencies will also be pruned unless +# they are connected to another crate in the graph that hasn't been pruned, +# so it should be used with care. The identifiers are [Package ID Specifications] +# (https://doc.rust-lang.org/cargo/reference/pkgid-spec.html) +#exclude = [] +# If true, metadata will be collected with `--all-features`. Note that this can't +# be toggled off if true, if you want to conditionally enable `--all-features` it +# is recommended to pass `--all-features` on the cmd line instead +all-features = false +# If true, metadata will be collected with `--no-default-features`. The same +# caveat with `all-features` applies +no-default-features = false +# If set, these feature will be enabled when collecting metadata. If `--features` +# is specified on the cmd line they will take precedence over this option. +#features = [] + +# The output table provides options for how/if diagnostics are outputted +[output] +# When outputting inclusion graphs in diagnostics that include features, this +# option can be used to specify the depth at which feature edges will be added. +# This option is included since the graphs can be quite large and the addition +# of features from the crate(s) to all of the graph roots can be far too verbose. +# This option can be overridden via `--feature-depth` on the cmd line +feature-depth = 1 + +# This section is considered when running `cargo deny check advisories` +# More documentation for the advisories section can be found here: +# https://embarkstudios.github.io/cargo-deny/checks/advisories/cfg.html +[advisories] +# The path where the advisory databases are cloned/fetched into +#db-path = "$CARGO_HOME/advisory-dbs" +# The url(s) of the advisory databases to use +#db-urls = ["https://github.com/rustsec/advisory-db"] +# A list of advisory IDs to ignore. Note that ignored advisories will still +# output a note when they are encountered. +ignore = [ + { id = "RUSTSEC-2024-0388", reason = "derivative is unmaintained; pulled in via starlark v0.13.0 used by execpolicy/cli/core; no fixed release yet" }, + { id = "RUSTSEC-2025-0057", reason = "fxhash is unmaintained; pulled in via starlark_map/starlark v0.13.0 used by execpolicy/cli/core; no fixed release yet" }, + { id = "RUSTSEC-2024-0436", reason = "paste is unmaintained; pulled in via ratatui/rmcp/starlark used by tui/execpolicy; no fixed release yet" }, +] +# If this is true, then cargo deny will use the git executable to fetch advisory database. +# If this is false, then it uses a built-in git library. +# Setting this to true can be helpful if you have special authentication requirements that cargo-deny does not support. +# See Git Authentication for more information about setting up git authentication. +#git-fetch-with-cli = true + +# This section is considered when running `cargo deny check licenses` +# More documentation for the licenses section can be found here: +# https://embarkstudios.github.io/cargo-deny/checks/licenses/cfg.html +[licenses] +# List of explicitly allowed licenses +# See https://spdx.org/licenses/ for list of possible licenses +# [possible values: any SPDX 3.11 short identifier (+ optional exception)]. +allow = [ + # Apache-2.0 - https://www.apache.org/licenses/LICENSE-2.0 + # Used by: allocative, anyhow, arboard, askama, assert_cmd, assert_matches, async-channel, async-trait, base64, chardetng, chrono, clap, clap_complete, color-eyre, ctor, diffy, dirs, dunce, encoding_rs, env_logger, escargot, eventsource-stream, futures, http, image, indexmap, insta, itertools, keyring, landlock, lazy_static, libc, log, maplit, multimap, once_cell, opentelemetry, opentelemetry-appender-tracing, opentelemetry-otlp, opentelemetry-semantic-conventions, opentelemetry_sdk, pathdiff, predicates, pretty_assertions, rand, regex-lite, reqwest, seccompiler, serde, serde_json, serde_with, sha1, sha2, shlex, similar, socket2, starlark, supports-color, sys-locale, tempfile, test-log, thiserror, time, tiny_http, toml, toml_edit, unicode-segmentation, unicode-width, url, uuid, webbrowser, wiremock, zeroize + "Apache-2.0", + # BSD-2-Clause - https://opensource.org/license/bsd-2-clause + # Used by: transitive only + "BSD-2-Clause", + # BSD-3-Clause - https://opensource.org/license/bsd-3-clause + # Used by: encoding_rs, seccompiler + "BSD-3-Clause", + # BSL-1.0 - https://www.boost.org/users/license.html + # Used by: transitive only + "BSL-1.0", + # CC0-1.0 - https://creativecommons.org/publicdomain/zero/1.0/ + # Used by: dunce, notify + "CC0-1.0", + # CDLA-Permissive-2.0 - https://cdla.dev/permissive-2-0/ + # Used by: transitive only + "CDLA-Permissive-2.0", + # ISC - https://opensource.org/license/isc-license-txt + # Used by: transitive only + "ISC", + # MIT - https://opensource.org/license/mit + # Used by: allocative, ansi-to-tui, anyhow, arboard, askama, assert_cmd, assert_matches, async-channel, async-stream, async-trait, axum, base64, bytes, chardetng, chrono, clap, clap_complete, color-eyre, crossterm, ctor, derive_more, diffy, dirs, dotenvy, encoding_rs, env-flags, env_logger, escargot, eventsource-stream, futures, http, ignore, image, indexmap, itertools, keyring, landlock, lazy_static, libc, log, lru, maplit, mime_guess, multimap, once_cell, openssl-sys, os_info, owo-colors, path-absolutize, pathdiff, portable-pty, predicates, pretty_assertions, pulldown-cmark, rand, ratatui, ratatui-macros, regex-lite, reqwest, rmcp, schemars, serde, serde_json, serde_with, serial_test, sha1, sha2, shlex, socket2, strum, strum_macros, sys-locale, tempfile, test-log, textwrap, thiserror, time, tiny_http, tokio, tokio-stream, tokio-test, tokio-util, toml, toml_edit, tonic, tracing, tracing-appender, tracing-subscriber, tracing-test, tree-sitter, tree-sitter-bash, tree-sitter-highlight, ts-rs, uds_windows, unicode-segmentation, unicode-width, url, urlencoding, uuid, vt100, walkdir, webbrowser, which, wildmatch, wiremock, zeroize + "MIT", + # MIT-0 - https://opensource.org/license/mit-0 + # Used by: dunce + "MIT-0", + # MPL-2.0 - https://www.mozilla.org/MPL/2.0/ + # Used by: nucleo-matcher + "MPL-2.0", + # Unicode-3.0 - https://opensource.org/license/unicode + # Used by: icu_decimal, icu_locale_core, icu_provider + "Unicode-3.0", + # Unlicense - https://opensource.org/license/unlicense/ + # Used by: ignore, walkdir + "Unlicense", + # Zlib - https://opensource.org/license/zlib + # Used by: transitive only + "Zlib", +] +# The confidence threshold for detecting a license from license text. +# The higher the value, the more closely the license text must be to the +# canonical license text of a valid SPDX license file. +# [possible values: any between 0.0 and 1.0]. +confidence-threshold = 0.8 +# Allow 1 or more licenses on a per-crate basis, so that particular licenses +# aren't accepted for every possible crate as with the normal allow list +exceptions = [ + # Each entry is the crate and version constraint, and its specific allow + # list + #{ allow = ["Zlib"], crate = "adler32" }, +] + +# Some crates don't have (easily) machine readable licensing information, +# adding a clarification entry for it allows you to manually specify the +# licensing information +#[[licenses.clarify]] +# The package spec the clarification applies to +#crate = "ring" +# The SPDX expression for the license requirements of the crate +#expression = "MIT AND ISC AND OpenSSL" +# One or more files in the crate's source used as the "source of truth" for +# the license expression. If the contents match, the clarification will be used +# when running the license check, otherwise the clarification will be ignored +# and the crate will be checked normally, which may produce warnings or errors +# depending on the rest of your configuration +#license-files = [ +# Each entry is a crate relative path, and the (opaque) hash of its contents +#{ path = "LICENSE", hash = 0xbd0eed23 } +#] + +[licenses.private] +# If true, ignores workspace crates that aren't published, or are only +# published to private registries. +# To see how to mark a crate as unpublished (to the official registry), +# visit https://doc.rust-lang.org/cargo/reference/manifest.html#the-publish-field. +ignore = false +# One or more private registries that you might publish crates to, if a crate +# is only published to private registries, and ignore is true, the crate will +# not have its license(s) checked +registries = [ + #"https://sekretz.com/registry +] + +# This section is considered when running `cargo deny check bans`. +# More documentation about the 'bans' section can be found here: +# https://embarkstudios.github.io/cargo-deny/checks/bans/cfg.html +[bans] +# Lint level for when multiple versions of the same crate are detected +multiple-versions = "warn" +# Lint level for when a crate version requirement is `*` +wildcards = "allow" +# The graph highlighting used when creating dotgraphs for crates +# with multiple versions +# * lowest-version - The path to the lowest versioned duplicate is highlighted +# * simplest-path - The path to the version with the fewest edges is highlighted +# * all - Both lowest-version and simplest-path are used +highlight = "all" +# The default lint level for `default` features for crates that are members of +# the workspace that is being checked. This can be overridden by allowing/denying +# `default` on a crate-by-crate basis if desired. +workspace-default-features = "allow" +# The default lint level for `default` features for external crates that are not +# members of the workspace. This can be overridden by allowing/denying `default` +# on a crate-by-crate basis if desired. +external-default-features = "allow" +# List of crates that are allowed. Use with care! +allow = [ + #"ansi_term@0.11.0", + #{ crate = "ansi_term@0.11.0", reason = "you can specify a reason it is allowed" }, +] +# List of crates to deny +deny = [ + #"ansi_term@0.11.0", + #{ crate = "ansi_term@0.11.0", reason = "you can specify a reason it is banned" }, + # Wrapper crates can optionally be specified to allow the crate when it + # is a direct dependency of the otherwise banned crate + #{ crate = "ansi_term@0.11.0", wrappers = ["this-crate-directly-depends-on-ansi_term"] }, +] + +# List of features to allow/deny +# Each entry the name of a crate and a version range. If version is +# not specified, all versions will be matched. +#[[bans.features]] +#crate = "reqwest" +# Features to not allow +#deny = ["json"] +# Features to allow +#allow = [ +# "rustls", +# "__rustls", +# "__tls", +# "hyper-rustls", +# "rustls", +# "rustls-pemfile", +# "rustls-tls-webpki-roots", +# "tokio-rustls", +# "webpki-roots", +#] +# If true, the allowed features must exactly match the enabled feature set. If +# this is set there is no point setting `deny` +#exact = true + +# Certain crates/versions that will be skipped when doing duplicate detection. +skip = [ + #"ansi_term@0.11.0", + #{ crate = "ansi_term@0.11.0", reason = "you can specify a reason why it can't be updated/removed" }, +] +# Similarly to `skip` allows you to skip certain crates during duplicate +# detection. Unlike skip, it also includes the entire tree of transitive +# dependencies starting at the specified crate, up to a certain depth, which is +# by default infinite. +skip-tree = [ + #"ansi_term@0.11.0", # will be skipped along with _all_ of its direct and transitive dependencies + #{ crate = "ansi_term@0.11.0", depth = 20 }, +] + +# This section is considered when running `cargo deny check sources`. +# More documentation about the 'sources' section can be found here: +# https://embarkstudios.github.io/cargo-deny/checks/sources/cfg.html +[sources] +# Lint level for what to happen when a crate from a crate registry that is not +# in the allow list is encountered +unknown-registry = "warn" +# Lint level for what to happen when a crate from a git repository that is not +# in the allow list is encountered +unknown-git = "warn" +# List of URLs for allowed crate registries. Defaults to the crates.io index +# if not specified. If it is specified but empty, no registries are allowed. +allow-registry = ["https://github.com/rust-lang/crates.io-index"] +# List of URLs for allowed Git repositories +allow-git = [] + +[sources.allow-org] +# github.com organizations to allow git sources for +github = [ + "nornagon", # ratatui and crossterm forks +] +# gitlab.com organizations to allow git sources for +gitlab = [] +# bitbucket.org organizations to allow git sources for +bitbucket = [] diff --git a/codex-rs/exec-server/Cargo.toml b/codex-rs/exec-server/Cargo.toml index 54cead4118..5f8032595e 100644 --- a/codex-rs/exec-server/Cargo.toml +++ b/codex-rs/exec-server/Cargo.toml @@ -1,7 +1,8 @@ [package] -edition = "2024" name = "codex-exec-server" -version = { workspace = true } +version.workspace = true +edition.workspace = true +license.workspace = true [[bin]] name = "codex-execve-wrapper" @@ -49,6 +50,7 @@ tokio = { workspace = true, features = [ "rt-multi-thread", "signal", ] } +tokio-util = { workspace = true } tracing = { workspace = true } tracing-subscriber = { workspace = true, features = ["env-filter", "fmt"] } diff --git a/codex-rs/exec-server/src/posix.rs b/codex-rs/exec-server/src/posix.rs index ea0fab1a7e..3a4a4b9525 100644 --- a/codex-rs/exec-server/src/posix.rs +++ b/codex-rs/exec-server/src/posix.rs @@ -71,12 +71,14 @@ mod escalation_policy; mod mcp; mod mcp_escalation_policy; mod socket; +mod stopwatch; /// Default value of --execve option relative to the current executable. /// Note this must match the name of the binary as specified in Cargo.toml. const CODEX_EXECVE_WRAPPER_EXE_NAME: &str = "codex-execve-wrapper"; #[derive(Parser)] +#[clap(version)] struct McpServerCli { /// Executable to delegate execve(2) calls to in Bash. #[arg(long = "execve")] diff --git a/codex-rs/exec-server/src/posix/escalate_server.rs b/codex-rs/exec-server/src/posix/escalate_server.rs index 1f76687ce0..b71142d5b1 100644 --- a/codex-rs/exec-server/src/posix/escalate_server.rs +++ b/codex-rs/exec-server/src/posix/escalate_server.rs @@ -8,11 +8,10 @@ use std::time::Duration; use anyhow::Context as _; use path_absolutize::Absolutize as _; -use codex_core::exec::SandboxType; +use codex_core::SandboxState; use codex_core::exec::process_exec_tool_call; -use codex_core::get_platform_sandbox; -use codex_core::protocol::SandboxPolicy; use tokio::process::Command; +use tokio_util::sync::CancellationToken; use crate::posix::escalate_protocol::BASH_EXEC_WRAPPER_ENV_VAR; use crate::posix::escalate_protocol::ESCALATE_SOCKET_ENV_VAR; @@ -22,8 +21,10 @@ use crate::posix::escalate_protocol::EscalateResponse; use crate::posix::escalate_protocol::SuperExecMessage; use crate::posix::escalate_protocol::SuperExecResult; use crate::posix::escalation_policy::EscalationPolicy; +use crate::posix::mcp::ExecParams; use crate::posix::socket::AsyncDatagramSocket; use crate::posix::socket::AsyncSocket; +use codex_core::exec::ExecExpiration; pub(crate) struct EscalateServer { bash_path: PathBuf, @@ -45,17 +46,16 @@ impl EscalateServer { pub async fn exec( &self, - command: String, - env: HashMap, - workdir: PathBuf, - timeout_ms: Option, + params: ExecParams, + cancel_rx: CancellationToken, + sandbox_state: &SandboxState, ) -> anyhow::Result { let (escalate_server, escalate_client) = AsyncDatagramSocket::pair()?; let client_socket = escalate_client.into_inner(); client_socket.set_cloexec(false)?; let escalate_task = tokio::spawn(escalate_task(escalate_server, self.policy.clone())); - let mut env = env.clone(); + let mut env = std::env::vars().collect::>(); env.insert( ESCALATE_SOCKET_ENV_VAR.to_string(), client_socket.as_raw_fd().to_string(), @@ -65,30 +65,33 @@ impl EscalateServer { self.execve_wrapper.to_string_lossy().to_string(), ); - // TODO: use the sandbox policy and cwd from the calling client. - // Note that sandbox_cwd is ignored for ReadOnly, but needs to be legit - // for `SandboxPolicy::WorkspaceWrite`. - let sandbox_policy = SandboxPolicy::ReadOnly; - let sandbox_cwd = PathBuf::from("/__NONEXISTENT__"); - + let ExecParams { + command, + workdir, + timeout_ms: _, + login, + } = params; let result = process_exec_tool_call( codex_core::exec::ExecParams { command: vec![ self.bash_path.to_string_lossy().to_string(), - "-c".to_string(), + if login == Some(false) { + "-c".to_string() + } else { + "-lc".to_string() + }, command, ], cwd: PathBuf::from(&workdir), - timeout_ms, + expiration: ExecExpiration::Cancellation(cancel_rx), env, with_escalated_permissions: None, justification: None, arg0: None, }, - get_platform_sandbox().unwrap_or(SandboxType::None), - &sandbox_policy, - &sandbox_cwd, - &None, + &sandbox_state.sandbox_policy, + &sandbox_state.sandbox_cwd, + &sandbox_state.codex_linux_sandbox_exe, None, ) .await?; diff --git a/codex-rs/exec-server/src/posix/mcp.rs b/codex-rs/exec-server/src/posix/mcp.rs index f5785dc5d0..134fdc01c0 100644 --- a/codex-rs/exec-server/src/posix/mcp.rs +++ b/codex-rs/exec-server/src/posix/mcp.rs @@ -1,8 +1,13 @@ use std::path::PathBuf; +use std::sync::Arc; use std::time::Duration; use anyhow::Context as _; use anyhow::Result; +use codex_core::MCP_SANDBOX_STATE_CAPABILITY; +use codex_core::MCP_SANDBOX_STATE_NOTIFICATION; +use codex_core::SandboxState; +use codex_core::protocol::SandboxPolicy; use rmcp::ErrorData as McpError; use rmcp::RoleServer; use rmcp::ServerHandler; @@ -17,15 +22,20 @@ use rmcp::tool; use rmcp::tool_handler; use rmcp::tool_router; use rmcp::transport::stdio; +use tokio::sync::RwLock; +use tracing::debug; use crate::posix::escalate_server::EscalateServer; use crate::posix::escalate_server::{self}; use crate::posix::mcp_escalation_policy::ExecPolicy; use crate::posix::mcp_escalation_policy::McpEscalationPolicy; +use crate::posix::stopwatch::Stopwatch; /// Path to our patched bash. const CODEX_BASH_PATH_ENV_VAR: &str = "CODEX_BASH_PATH"; +const SANDBOX_STATE_CAPABILITY_VERSION: &str = "1.0.0"; + pub(crate) fn get_bash_path() -> Result { std::env::var(CODEX_BASH_PATH_ENV_VAR) .map(PathBuf::from) @@ -40,6 +50,8 @@ pub struct ExecParams { pub workdir: String, /// The timeout for the command in milliseconds. pub timeout_ms: Option, + /// Launch Bash with -lc instead of -c: defaults to true. + pub login: Option, } #[derive(Debug, serde::Serialize, schemars::JsonSchema)] @@ -67,6 +79,7 @@ pub struct ExecTool { bash_path: PathBuf, execve_wrapper: PathBuf, policy: ExecPolicy, + sandbox_state: Arc>>, } #[tool_router] @@ -77,6 +90,7 @@ impl ExecTool { bash_path, execve_wrapper, policy, + sandbox_state: Arc::new(RwLock::new(None)), } } @@ -87,19 +101,31 @@ impl ExecTool { context: RequestContext, Parameters(params): Parameters, ) -> Result { + let effective_timeout = Duration::from_millis( + params + .timeout_ms + .unwrap_or(codex_core::exec::DEFAULT_EXEC_COMMAND_TIMEOUT_MS), + ); + let stopwatch = Stopwatch::new(effective_timeout); + let cancel_token = stopwatch.cancellation_token(); + let sandbox_state = + self.sandbox_state + .read() + .await + .clone() + .unwrap_or_else(|| SandboxState { + sandbox_policy: SandboxPolicy::ReadOnly, + codex_linux_sandbox_exe: None, + sandbox_cwd: PathBuf::from(¶ms.workdir), + }); let escalate_server = EscalateServer::new( self.bash_path.clone(), self.execve_wrapper.clone(), - McpEscalationPolicy::new(self.policy, context), + McpEscalationPolicy::new(self.policy, context, stopwatch.clone()), ); + let result = escalate_server - .exec( - params.command, - // TODO: use ShellEnvironmentPolicy - std::env::vars().collect(), - PathBuf::from(¶ms.workdir), - params.timeout_ms, - ) + .exec(params, cancel_token, &sandbox_state) .await .map_err(|e| McpError::internal_error(e.to_string(), None))?; Ok(CallToolResult::success(vec![Content::json( @@ -111,9 +137,22 @@ impl ExecTool { #[tool_handler] impl ServerHandler for ExecTool { fn get_info(&self) -> ServerInfo { + let mut experimental_capabilities = ExperimentalCapabilities::new(); + let mut sandbox_state_capability = JsonObject::new(); + sandbox_state_capability.insert( + "version".to_string(), + serde_json::Value::String(SANDBOX_STATE_CAPABILITY_VERSION.to_string()), + ); + experimental_capabilities.insert( + MCP_SANDBOX_STATE_CAPABILITY.to_string(), + sandbox_state_capability, + ); ServerInfo { protocol_version: ProtocolVersion::V_2025_06_18, - capabilities: ServerCapabilities::builder().enable_tools().build(), + capabilities: ServerCapabilities::builder() + .enable_tools() + .enable_experimental_with(experimental_capabilities) + .build(), server_info: Implementation::from_build_env(), instructions: Some( "This server provides a tool to execute shell commands and return their output." @@ -129,6 +168,31 @@ impl ServerHandler for ExecTool { ) -> Result { Ok(self.get_info()) } + + async fn on_custom_notification( + &self, + notification: rmcp::model::CustomClientNotification, + _context: rmcp::service::NotificationContext, + ) { + let rmcp::model::CustomClientNotification { method, params, .. } = notification; + if method == MCP_SANDBOX_STATE_NOTIFICATION + && let Some(params) = params + { + match serde_json::from_value::(params) { + Ok(sandbox_state) => { + debug!( + ?sandbox_state.sandbox_policy, + "received sandbox state notification" + ); + let mut state = self.sandbox_state.write().await; + *state = Some(sandbox_state); + } + Err(err) => { + tracing::warn!(?err, "failed to deserialize sandbox state notification"); + } + } + } + } } pub(crate) async fn serve( @@ -139,3 +203,50 @@ pub(crate) async fn serve( let tool = ExecTool::new(bash_path, execve_wrapper, policy); tool.serve(stdio()).await } + +#[cfg(test)] +mod tests { + use super::*; + use pretty_assertions::assert_eq; + use serde_json::json; + + /// Verify that the way we use serde does not compromise the desired JSON + /// schema via schemars. In particular, ensure that the `login` and + /// `timeout_ms` fields are optional. + #[test] + fn exec_params_json_schema_matches_expected() { + let schema = schemars::schema_for!(ExecParams); + let actual = serde_json::to_value(schema).expect("schema should serialize"); + + assert_eq!( + actual, + json!({ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "ExecParams", + "type": "object", + "properties": { + "command": { + "description": "The bash string to execute.", + "type": "string" + }, + "login": { + "description": "Launch Bash with -lc instead of -c: defaults to true.", + "type": ["boolean", "null"] + }, + "timeout_ms": { + "description": "The timeout for the command in milliseconds.", + "format": "uint64", + "minimum": 0, + "type": ["integer", "null"] + }, + "workdir": { + "description": + "The working directory to execute the command in. Must be an absolute path.", + "type": "string" + } + }, + "required": ["command", "workdir"] + }) + ); + } +} diff --git a/codex-rs/exec-server/src/posix/mcp_escalation_policy.rs b/codex-rs/exec-server/src/posix/mcp_escalation_policy.rs index 069948ea06..9e059fdba5 100644 --- a/codex-rs/exec-server/src/posix/mcp_escalation_policy.rs +++ b/codex-rs/exec-server/src/posix/mcp_escalation_policy.rs @@ -10,6 +10,7 @@ use rmcp::service::RequestContext; use crate::posix::escalate_protocol::EscalateAction; use crate::posix::escalation_policy::EscalationPolicy; +use crate::posix::stopwatch::Stopwatch; /// This is the policy which decides how to handle an exec() call. /// @@ -34,11 +35,20 @@ pub(crate) enum ExecPolicyOutcome { pub(crate) struct McpEscalationPolicy { policy: ExecPolicy, context: RequestContext, + stopwatch: Stopwatch, } impl McpEscalationPolicy { - pub(crate) fn new(policy: ExecPolicy, context: RequestContext) -> Self { - Self { policy, context } + pub(crate) fn new( + policy: ExecPolicy, + context: RequestContext, + stopwatch: Stopwatch, + ) -> Self { + Self { + policy, + context, + stopwatch, + } } async fn prompt( @@ -54,25 +64,34 @@ impl McpEscalationPolicy { } else { format!("{} {}", file.display(), args) }; - context - .peer - .create_elicitation(CreateElicitationRequestParam { - message: format!("Allow agent to run `{command}` in `{}`?", workdir.display()), - requested_schema: ElicitationSchema::builder() - .title("Execution Permission Request") - .optional_string_with("reason", |schema| { - schema.description("Optional reason for allowing or denying execution") + self.stopwatch + .pause_for(async { + context + .peer + .create_elicitation(CreateElicitationRequestParam { + message: format!( + "Allow agent to run `{command}` in `{}`?", + workdir.display() + ), + requested_schema: ElicitationSchema::builder() + .title("Execution Permission Request") + .optional_string_with("reason", |schema| { + schema.description( + "Optional reason for allowing or denying execution", + ) + }) + .build() + .map_err(|e| { + McpError::internal_error( + format!("failed to build elicitation schema: {e}"), + None, + ) + })?, }) - .build() - .map_err(|e| { - McpError::internal_error( - format!("failed to build elicitation schema: {e}"), - None, - ) - })?, + .await + .map_err(|e| McpError::internal_error(e.to_string(), None)) }) .await - .map_err(|e| McpError::internal_error(e.to_string(), None)) } } diff --git a/codex-rs/exec-server/src/posix/stopwatch.rs b/codex-rs/exec-server/src/posix/stopwatch.rs new file mode 100644 index 0000000000..de29a45685 --- /dev/null +++ b/codex-rs/exec-server/src/posix/stopwatch.rs @@ -0,0 +1,211 @@ +use std::future::Future; +use std::sync::Arc; +use std::time::Duration; +use std::time::Instant; + +use tokio::sync::Mutex; +use tokio::sync::Notify; +use tokio_util::sync::CancellationToken; + +#[derive(Clone, Debug)] +pub(crate) struct Stopwatch { + limit: Duration, + inner: Arc>, + notify: Arc, +} + +#[derive(Debug)] +struct StopwatchState { + elapsed: Duration, + running_since: Option, + active_pauses: u32, +} + +impl Stopwatch { + pub(crate) fn new(limit: Duration) -> Self { + Self { + inner: Arc::new(Mutex::new(StopwatchState { + elapsed: Duration::ZERO, + running_since: Some(Instant::now()), + active_pauses: 0, + })), + notify: Arc::new(Notify::new()), + limit, + } + } + + pub(crate) fn cancellation_token(&self) -> CancellationToken { + let limit = self.limit; + let token = CancellationToken::new(); + let cancel = token.clone(); + let inner = Arc::clone(&self.inner); + let notify = Arc::clone(&self.notify); + tokio::spawn(async move { + loop { + let (remaining, running) = { + let guard = inner.lock().await; + let elapsed = guard.elapsed + + guard + .running_since + .map(|since| since.elapsed()) + .unwrap_or_default(); + if elapsed >= limit { + break; + } + (limit - elapsed, guard.running_since.is_some()) + }; + + if !running { + notify.notified().await; + continue; + } + + let sleep = tokio::time::sleep(remaining); + tokio::pin!(sleep); + tokio::select! { + _ = &mut sleep => { + break; + } + _ = notify.notified() => { + continue; + } + } + } + cancel.cancel(); + }); + token + } + + /// Runs `fut`, pausing the stopwatch while the future is pending. The clock + /// resumes automatically when the future completes. Nested/overlapping + /// calls are reference-counted so the stopwatch only resumes when every + /// pause is lifted. + pub(crate) async fn pause_for(&self, fut: F) -> T + where + F: Future, + { + self.pause().await; + let result = fut.await; + self.resume().await; + result + } + + async fn pause(&self) { + let mut guard = self.inner.lock().await; + guard.active_pauses += 1; + if guard.active_pauses == 1 + && let Some(since) = guard.running_since.take() + { + guard.elapsed += since.elapsed(); + self.notify.notify_waiters(); + } + } + + async fn resume(&self) { + let mut guard = self.inner.lock().await; + if guard.active_pauses == 0 { + return; + } + guard.active_pauses -= 1; + if guard.active_pauses == 0 && guard.running_since.is_none() { + guard.running_since = Some(Instant::now()); + self.notify.notify_waiters(); + } + } +} + +#[cfg(test)] +mod tests { + use super::Stopwatch; + use tokio::time::Duration; + use tokio::time::Instant; + use tokio::time::sleep; + use tokio::time::timeout; + + #[tokio::test] + async fn cancellation_receiver_fires_after_limit() { + let stopwatch = Stopwatch::new(Duration::from_millis(50)); + let token = stopwatch.cancellation_token(); + let start = Instant::now(); + token.cancelled().await; + assert!(start.elapsed() >= Duration::from_millis(50)); + } + + #[tokio::test] + async fn pause_prevents_timeout_until_resumed() { + let stopwatch = Stopwatch::new(Duration::from_millis(50)); + let token = stopwatch.cancellation_token(); + + let pause_handle = tokio::spawn({ + let stopwatch = stopwatch.clone(); + async move { + stopwatch + .pause_for(async { + sleep(Duration::from_millis(100)).await; + }) + .await; + } + }); + + assert!( + timeout(Duration::from_millis(30), token.cancelled()) + .await + .is_err() + ); + + pause_handle.await.expect("pause task should finish"); + + token.cancelled().await; + } + + #[tokio::test] + async fn overlapping_pauses_only_resume_once() { + let stopwatch = Stopwatch::new(Duration::from_millis(50)); + let token = stopwatch.cancellation_token(); + + // First pause. + let pause1 = { + let stopwatch = stopwatch.clone(); + tokio::spawn(async move { + stopwatch + .pause_for(async { + sleep(Duration::from_millis(80)).await; + }) + .await; + }) + }; + + // Overlapping pause that ends sooner. + let pause2 = { + let stopwatch = stopwatch.clone(); + tokio::spawn(async move { + stopwatch + .pause_for(async { + sleep(Duration::from_millis(30)).await; + }) + .await; + }) + }; + + // While both pauses are active, the cancellation should not fire. + assert!( + timeout(Duration::from_millis(40), token.cancelled()) + .await + .is_err() + ); + + pause2.await.expect("short pause should complete"); + + // Still paused because the long pause is active. + assert!( + timeout(Duration::from_millis(30), token.cancelled()) + .await + .is_err() + ); + + pause1.await.expect("long pause should complete"); + + // Now the stopwatch should resume and hit the limit shortly after. + token.cancelled().await; + } +} diff --git a/codex-rs/exec/Cargo.toml b/codex-rs/exec/Cargo.toml index 4a0ddf7dde..f26bec0e67 100644 --- a/codex-rs/exec/Cargo.toml +++ b/codex-rs/exec/Cargo.toml @@ -1,7 +1,8 @@ [package] -edition = "2024" name = "codex-exec" -version = { workspace = true } +version.workspace = true +edition.workspace = true +license.workspace = true [[bin]] name = "codex-exec" diff --git a/codex-rs/exec/src/cli.rs b/codex-rs/exec/src/cli.rs index ef20bf6fc7..392ebb0cd6 100644 --- a/codex-rs/exec/src/cli.rs +++ b/codex-rs/exec/src/cli.rs @@ -91,6 +91,9 @@ pub struct Cli { pub enum Command { /// Resume a previous session by id or pick the most recent with --last. Resume(ResumeArgs), + + /// Run a code review against the current repository. + Review(ReviewArgs), } #[derive(Parser, Debug)] @@ -101,7 +104,7 @@ pub struct ResumeArgs { pub session_id: Option, /// Resume the most recent recorded session (newest) without specifying an id. - #[arg(long = "last", default_value_t = false, conflicts_with = "session_id")] + #[arg(long = "last", default_value_t = false)] pub last: bool, /// Prompt to send after resuming the session. If `-` is used, read from stdin. @@ -109,6 +112,41 @@ pub struct ResumeArgs { pub prompt: Option, } +#[derive(Parser, Debug)] +pub struct ReviewArgs { + /// Review staged, unstaged, and untracked changes. + #[arg( + long = "uncommitted", + default_value_t = false, + conflicts_with_all = ["base", "commit", "prompt"] + )] + pub uncommitted: bool, + + /// Review changes against the given base branch. + #[arg( + long = "base", + value_name = "BRANCH", + conflicts_with_all = ["uncommitted", "commit", "prompt"] + )] + pub base: Option, + + /// Review the changes introduced by a commit. + #[arg( + long = "commit", + value_name = "SHA", + conflicts_with_all = ["uncommitted", "base", "prompt"] + )] + pub commit: Option, + + /// Optional commit title to display in the review summary. + #[arg(long = "title", value_name = "TITLE", requires = "commit")] + pub commit_title: Option, + + /// Custom review instructions. If `-` is used, read from stdin. + #[arg(value_name = "PROMPT", value_hint = clap::ValueHint::Other)] + pub prompt: Option, +} + #[derive(Debug, Clone, Copy, Default, PartialEq, Eq, ValueEnum)] #[value(rename_all = "kebab-case")] pub enum Color { diff --git a/codex-rs/exec/src/event_processor_with_human_output.rs b/codex-rs/exec/src/event_processor_with_human_output.rs index 58d7a9c5fd..e5308b379f 100644 --- a/codex-rs/exec/src/event_processor_with_human_output.rs +++ b/codex-rs/exec/src/event_processor_with_human_output.rs @@ -161,7 +161,7 @@ impl EventProcessor for EventProcessorWithHumanOutput { fn process_event(&mut self, event: Event) -> CodexStatus { let Event { id: _, msg } = event; match msg { - EventMsg::Error(ErrorEvent { message }) => { + EventMsg::Error(ErrorEvent { message, .. }) => { let prefix = "ERROR:".style(self.red); ts_msg!(self, "{prefix} {message}"); } @@ -221,12 +221,25 @@ impl EventProcessor for EventProcessorWithHumanOutput { EventMsg::BackgroundEvent(BackgroundEventEvent { message }) => { ts_msg!(self, "{}", message.style(self.dimmed)); } - EventMsg::StreamError(StreamErrorEvent { message }) => { + EventMsg::StreamError(StreamErrorEvent { message, .. }) => { ts_msg!(self, "{}", message.style(self.dimmed)); } EventMsg::TaskStarted(_) => { // Ignore. } + EventMsg::ElicitationRequest(ev) => { + ts_msg!( + self, + "{} {}", + "elicitation request".style(self.magenta), + ev.server_name.style(self.dimmed) + ); + ts_msg!( + self, + "{}", + "auto-cancelling (not supported in exec mode)".style(self.dimmed) + ); + } EventMsg::TaskComplete(TaskCompleteEvent { last_agent_message }) => { let last_message = last_agent_message.as_deref(); if let Some(output_file) = self.last_message_path.as_deref() { @@ -546,6 +559,9 @@ impl EventProcessor for EventProcessorWithHumanOutput { ts_msg!(self, "task aborted: review ended"); } }, + EventMsg::ContextCompacted(_) => { + ts_msg!(self, "context compacted"); + } EventMsg::ShutdownComplete => return CodexStatus::Shutdown, EventMsg::WebSearchBegin(_) | EventMsg::ExecApprovalRequest(_) diff --git a/codex-rs/exec/src/exec_events.rs b/codex-rs/exec/src/exec_events.rs index 23f471e224..f3726dad76 100644 --- a/codex-rs/exec/src/exec_events.rs +++ b/codex-rs/exec/src/exec_events.rs @@ -144,6 +144,7 @@ pub enum CommandExecutionStatus { InProgress, Completed, Failed, + Declined, } /// A command executed by the agent. diff --git a/codex-rs/exec/src/lib.rs b/codex-rs/exec/src/lib.rs index a003b4ff21..0cf5aaf788 100644 --- a/codex-rs/exec/src/lib.rs +++ b/codex-rs/exec/src/lib.rs @@ -11,6 +11,8 @@ pub mod event_processor_with_jsonl_output; pub mod exec_events; pub use cli::Cli; +pub use cli::Command; +pub use cli::ReviewArgs; use codex_common::oss::ensure_oss_provider_ready; use codex_common::oss::get_default_model_for_oss_provider; use codex_core::AuthManager; @@ -29,7 +31,10 @@ use codex_core::protocol::AskForApproval; use codex_core::protocol::Event; use codex_core::protocol::EventMsg; use codex_core::protocol::Op; +use codex_core::protocol::ReviewRequest; +use codex_core::protocol::ReviewTarget; use codex_core::protocol::SessionSource; +use codex_protocol::approvals::ElicitationAction; use codex_protocol::config_types::SandboxMode; use codex_protocol::user_input::UserInput; use event_processor_with_human_output::EventProcessorWithHumanOutput; @@ -52,6 +57,16 @@ use crate::event_processor::EventProcessor; use codex_core::default_client::set_default_originator; use codex_core::find_conversation_path_by_id_str; +enum InitialOperation { + UserTurn { + items: Vec, + output_schema: Option, + }, + Review { + review_request: ReviewRequest, + }, +} + pub async fn run_main(cli: Cli, codex_linux_sandbox_exe: Option) -> anyhow::Result<()> { if let Err(err) = set_default_originator("codex_exec".to_string()) { tracing::warn!(?err, "Failed to set codex exec originator override {err:?}"); @@ -78,50 +93,6 @@ pub async fn run_main(cli: Cli, codex_linux_sandbox_exe: Option) -> any config_overrides, } = cli; - // Determine the prompt source (parent or subcommand) and read from stdin if needed. - let prompt_arg = match &command { - // Allow prompt before the subcommand by falling back to the parent-level prompt - // when the Resume subcommand did not provide its own prompt. - Some(ExecCommand::Resume(args)) => args.prompt.clone().or(prompt), - None => prompt, - }; - - let prompt = match prompt_arg { - Some(p) if p != "-" => p, - // Either `-` was passed or no positional arg. - maybe_dash => { - // When no arg (None) **and** stdin is a TTY, bail out early – unless the - // user explicitly forced reading via `-`. - let force_stdin = matches!(maybe_dash.as_deref(), Some("-")); - - if std::io::stdin().is_terminal() && !force_stdin { - eprintln!( - "No prompt provided. Either specify one as an argument or pipe the prompt into stdin." - ); - std::process::exit(1); - } - - // Ensure the user knows we are waiting on stdin, as they may - // have gotten into this state by mistake. If so, and they are not - // writing to stdin, Codex will hang indefinitely, so this should - // help them debug in that case. - if !force_stdin { - eprintln!("Reading prompt from stdin..."); - } - let mut buffer = String::new(); - if let Err(e) = std::io::stdin().read_to_string(&mut buffer) { - eprintln!("Failed to read prompt from stdin: {e}"); - std::process::exit(1); - } else if buffer.trim().is_empty() { - eprintln!("No prompt provided via stdin."); - std::process::exit(1); - } - buffer - } - }; - - let output_schema = load_output_schema(output_schema_path); - let (stdout_with_ansi, stderr_with_ansi) = match color { cli::Color::Always => (true, true), cli::Color::Never => (false, false), @@ -314,8 +285,8 @@ pub async fn run_main(cli: Cli, codex_linux_sandbox_exe: Option) -> any conversation_id: _, conversation, session_configured, - } = if let Some(ExecCommand::Resume(args)) = command { - let resume_path = resolve_resume_path(&config, &args).await?; + } = if let Some(ExecCommand::Resume(args)) = command.as_ref() { + let resume_path = resolve_resume_path(&config, args).await?; if let Some(path) = resume_path { conversation_manager @@ -331,9 +302,64 @@ pub async fn run_main(cli: Cli, codex_linux_sandbox_exe: Option) -> any .new_conversation(config.clone()) .await? }; - // Print the effective configuration and prompt so users can see what Codex + let (initial_operation, prompt_summary) = match (command, prompt, images) { + (Some(ExecCommand::Review(review_cli)), _, _) => { + let review_request = build_review_request(review_cli)?; + let summary = codex_core::review_prompts::user_facing_hint(&review_request.target); + (InitialOperation::Review { review_request }, summary) + } + (Some(ExecCommand::Resume(args)), root_prompt, imgs) => { + let prompt_arg = args + .prompt + .clone() + .or_else(|| { + if args.last { + args.session_id.clone() + } else { + None + } + }) + .or(root_prompt); + let prompt_text = resolve_prompt(prompt_arg); + let mut items: Vec = imgs + .into_iter() + .map(|path| UserInput::LocalImage { path }) + .collect(); + items.push(UserInput::Text { + text: prompt_text.clone(), + }); + let output_schema = load_output_schema(output_schema_path.clone()); + ( + InitialOperation::UserTurn { + items, + output_schema, + }, + prompt_text, + ) + } + (None, root_prompt, imgs) => { + let prompt_text = resolve_prompt(root_prompt); + let mut items: Vec = imgs + .into_iter() + .map(|path| UserInput::LocalImage { path }) + .collect(); + items.push(UserInput::Text { + text: prompt_text.clone(), + }); + let output_schema = load_output_schema(output_schema_path); + ( + InitialOperation::UserTurn { + items, + output_schema, + }, + prompt_text, + ) + } + }; + + // Print the effective configuration and initial request so users can see what Codex // is using. - event_processor.print_config_summary(&config, &prompt, &session_configured); + event_processor.print_config_summary(&config, &prompt_summary, &session_configured); info!("Codex initialized with event: {session_configured:?}"); @@ -376,31 +402,48 @@ pub async fn run_main(cli: Cli, codex_linux_sandbox_exe: Option) -> any }); } - // Package images and prompt into a single user input turn. - let mut items: Vec = images - .into_iter() - .map(|path| UserInput::LocalImage { path }) - .collect(); - items.push(UserInput::Text { text: prompt }); - let initial_prompt_task_id = conversation - .submit(Op::UserTurn { + match initial_operation { + InitialOperation::UserTurn { items, - cwd: default_cwd, - approval_policy: default_approval_policy, - sandbox_policy: default_sandbox_policy, - model: default_model, - effort: default_effort, - summary: default_summary, - final_output_json_schema: output_schema, - }) - .await?; - info!("Sent prompt with event ID: {initial_prompt_task_id}"); + output_schema, + } => { + let task_id = conversation + .submit(Op::UserTurn { + items, + cwd: default_cwd, + approval_policy: default_approval_policy, + sandbox_policy: default_sandbox_policy, + model: default_model, + effort: default_effort, + summary: default_summary, + final_output_json_schema: output_schema, + }) + .await?; + info!("Sent prompt with event ID: {task_id}"); + task_id + } + InitialOperation::Review { review_request } => { + let task_id = conversation.submit(Op::Review { review_request }).await?; + info!("Sent review request with event ID: {task_id}"); + task_id + } + }; // Run the loop until the task is complete. // Track whether a fatal error was reported by the server so we can // exit with a non-zero status for automation-friendly signaling. let mut error_seen = false; while let Some(event) = rx.recv().await { + if let EventMsg::ElicitationRequest(ev) = &event.msg { + // Automatically cancel elicitation requests in exec mode. + conversation + .submit(Op::ResolveElicitation { + server_name: ev.server_name.clone(), + request_id: ev.id.clone(), + decision: ElicitationAction::Cancel, + }) + .await?; + } if matches!(event.msg, EventMsg::Error(_)) { error_seen = true; } @@ -478,3 +521,130 @@ fn load_output_schema(path: Option) -> Option { } } } + +fn resolve_prompt(prompt_arg: Option) -> String { + match prompt_arg { + Some(p) if p != "-" => p, + maybe_dash => { + let force_stdin = matches!(maybe_dash.as_deref(), Some("-")); + + if std::io::stdin().is_terminal() && !force_stdin { + eprintln!( + "No prompt provided. Either specify one as an argument or pipe the prompt into stdin." + ); + std::process::exit(1); + } + + if !force_stdin { + eprintln!("Reading prompt from stdin..."); + } + let mut buffer = String::new(); + if let Err(e) = std::io::stdin().read_to_string(&mut buffer) { + eprintln!("Failed to read prompt from stdin: {e}"); + std::process::exit(1); + } else if buffer.trim().is_empty() { + eprintln!("No prompt provided via stdin."); + std::process::exit(1); + } + buffer + } + } +} + +fn build_review_request(args: ReviewArgs) -> anyhow::Result { + let target = if args.uncommitted { + ReviewTarget::UncommittedChanges + } else if let Some(branch) = args.base { + ReviewTarget::BaseBranch { branch } + } else if let Some(sha) = args.commit { + ReviewTarget::Commit { + sha, + title: args.commit_title, + } + } else if let Some(prompt_arg) = args.prompt { + let prompt = resolve_prompt(Some(prompt_arg)).trim().to_string(); + if prompt.is_empty() { + anyhow::bail!("Review prompt cannot be empty"); + } + ReviewTarget::Custom { + instructions: prompt, + } + } else { + anyhow::bail!( + "Specify --uncommitted, --base, --commit, or provide custom review instructions" + ); + }; + + Ok(ReviewRequest { + target, + user_facing_hint: None, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use pretty_assertions::assert_eq; + + #[test] + fn builds_uncommitted_review_request() { + let request = build_review_request(ReviewArgs { + uncommitted: true, + base: None, + commit: None, + commit_title: None, + prompt: None, + }) + .expect("builds uncommitted review request"); + + let expected = ReviewRequest { + target: ReviewTarget::UncommittedChanges, + user_facing_hint: None, + }; + + assert_eq!(request, expected); + } + + #[test] + fn builds_commit_review_request_with_title() { + let request = build_review_request(ReviewArgs { + uncommitted: false, + base: None, + commit: Some("123456789".to_string()), + commit_title: Some("Add review command".to_string()), + prompt: None, + }) + .expect("builds commit review request"); + + let expected = ReviewRequest { + target: ReviewTarget::Commit { + sha: "123456789".to_string(), + title: Some("Add review command".to_string()), + }, + user_facing_hint: None, + }; + + assert_eq!(request, expected); + } + + #[test] + fn builds_custom_review_request_trims_prompt() { + let request = build_review_request(ReviewArgs { + uncommitted: false, + base: None, + commit: None, + commit_title: None, + prompt: Some(" custom review instructions ".to_string()), + }) + .expect("builds custom review request"); + + let expected = ReviewRequest { + target: ReviewTarget::Custom { + instructions: "custom review instructions".to_string(), + }, + user_facing_hint: None, + }; + + assert_eq!(request, expected); + } +} diff --git a/codex-rs/exec/tests/event_processor_with_json_output.rs b/codex-rs/exec/tests/event_processor_with_json_output.rs index cbcd5458bc..7b2d3c18ba 100644 --- a/codex-rs/exec/tests/event_processor_with_json_output.rs +++ b/codex-rs/exec/tests/event_processor_with_json_output.rs @@ -47,6 +47,7 @@ use codex_exec::exec_events::WebSearchItem; use codex_protocol::plan_tool::PlanItemArg; use codex_protocol::plan_tool::StepStatus; use codex_protocol::plan_tool::UpdatePlanArgs; +use codex_protocol::protocol::CodexErrorInfo; use mcp_types::CallToolResult; use mcp_types::ContentBlock; use mcp_types::TextContent; @@ -539,6 +540,7 @@ fn error_event_produces_error() { "e1", EventMsg::Error(codex_core::protocol::ErrorEvent { message: "boom".to_string(), + codex_error_info: Some(CodexErrorInfo::Other), }), )); assert_eq!( @@ -578,6 +580,7 @@ fn stream_error_event_produces_error() { "e1", EventMsg::StreamError(codex_core::protocol::StreamErrorEvent { message: "retrying".to_string(), + codex_error_info: Some(CodexErrorInfo::Other), }), )); assert_eq!( @@ -596,6 +599,7 @@ fn error_followed_by_task_complete_produces_turn_failed() { "e1", EventMsg::Error(ErrorEvent { message: "boom".to_string(), + codex_error_info: Some(CodexErrorInfo::Other), }), ); assert_eq!( @@ -633,6 +637,7 @@ fn exec_command_end_success_produces_completed_command_item() { "c1", EventMsg::ExecCommandBegin(ExecCommandBeginEvent { call_id: "1".to_string(), + process_id: None, turn_id: "turn-1".to_string(), command: command.clone(), cwd: cwd.clone(), @@ -662,6 +667,7 @@ fn exec_command_end_success_produces_completed_command_item() { "c2", EventMsg::ExecCommandEnd(ExecCommandEndEvent { call_id: "1".to_string(), + process_id: None, turn_id: "turn-1".to_string(), command, cwd, @@ -705,6 +711,7 @@ fn exec_command_end_failure_produces_failed_command_item() { "c1", EventMsg::ExecCommandBegin(ExecCommandBeginEvent { call_id: "2".to_string(), + process_id: None, turn_id: "turn-1".to_string(), command: command.clone(), cwd: cwd.clone(), @@ -733,6 +740,7 @@ fn exec_command_end_failure_produces_failed_command_item() { "c2", EventMsg::ExecCommandEnd(ExecCommandEndEvent { call_id: "2".to_string(), + process_id: None, turn_id: "turn-1".to_string(), command, cwd, @@ -773,6 +781,7 @@ fn exec_command_end_without_begin_is_ignored() { "c1", EventMsg::ExecCommandEnd(ExecCommandEndEvent { call_id: "no-begin".to_string(), + process_id: None, turn_id: "turn-1".to_string(), command: Vec::new(), cwd: PathBuf::from("."), diff --git a/codex-rs/exec/tests/suite/resume.rs b/codex-rs/exec/tests/suite/resume.rs index 1a56edded3..e37b38606a 100644 --- a/codex-rs/exec/tests/suite/resume.rs +++ b/codex-rs/exec/tests/suite/resume.rs @@ -123,6 +123,60 @@ fn exec_resume_last_appends_to_existing_file() -> anyhow::Result<()> { Ok(()) } +#[test] +fn exec_resume_last_accepts_prompt_after_flag_in_json_mode() -> anyhow::Result<()> { + let test = test_codex_exec(); + let fixture = + Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/cli_responses_fixture.sse"); + + // 1) First run: create a session with a unique marker in the content. + let marker = format!("resume-last-json-{}", Uuid::new_v4()); + let prompt = format!("echo {marker}"); + + test.cmd() + .env("CODEX_RS_SSE_FIXTURE", &fixture) + .env("OPENAI_BASE_URL", "http://unused.local") + .arg("--skip-git-repo-check") + .arg("-C") + .arg(env!("CARGO_MANIFEST_DIR")) + .arg(&prompt) + .assert() + .success(); + + // Find the created session file containing the marker. + let sessions_dir = test.home_path().join("sessions"); + let path = find_session_file_containing_marker(&sessions_dir, &marker) + .expect("no session file found after first run"); + + // 2) Second run: resume the most recent file and pass the prompt after --last. + let marker2 = format!("resume-last-json-2-{}", Uuid::new_v4()); + let prompt2 = format!("echo {marker2}"); + + test.cmd() + .env("CODEX_RS_SSE_FIXTURE", &fixture) + .env("OPENAI_BASE_URL", "http://unused.local") + .arg("--skip-git-repo-check") + .arg("-C") + .arg(env!("CARGO_MANIFEST_DIR")) + .arg("--json") + .arg("resume") + .arg("--last") + .arg(&prompt2) + .assert() + .success(); + + let resumed_path = find_session_file_containing_marker(&sessions_dir, &marker2) + .expect("no resumed session file containing marker2"); + assert_eq!( + resumed_path, path, + "resume --last should append to existing file" + ); + let content = std::fs::read_to_string(&resumed_path)?; + assert!(content.contains(&marker)); + assert!(content.contains(&marker2)); + Ok(()) +} + #[test] fn exec_resume_by_id_appends_to_existing_file() -> anyhow::Result<()> { let test = test_codex_exec(); diff --git a/codex-rs/execpolicy-legacy/Cargo.toml b/codex-rs/execpolicy-legacy/Cargo.toml index 89e6e43b41..17628a0478 100644 --- a/codex-rs/execpolicy-legacy/Cargo.toml +++ b/codex-rs/execpolicy-legacy/Cargo.toml @@ -1,8 +1,9 @@ [package] -edition = "2024" name = "codex-execpolicy-legacy" +version.workspace = true +edition.workspace = true +license.workspace = true description = "Legacy exec policy engine for validating proposed exec calls." -version = { workspace = true } [[bin]] name = "codex-execpolicy-legacy" diff --git a/codex-rs/execpolicy/Cargo.toml b/codex-rs/execpolicy/Cargo.toml index bececed4b2..890c23faa7 100644 --- a/codex-rs/execpolicy/Cargo.toml +++ b/codex-rs/execpolicy/Cargo.toml @@ -1,7 +1,8 @@ [package] name = "codex-execpolicy" -version = { workspace = true } -edition = "2024" +version.workspace = true +edition.workspace = true +license.workspace = true description = "Codex exec policy: prefix-based Starlark rules for command decisions." [lib] @@ -27,3 +28,4 @@ thiserror = { workspace = true } [dev-dependencies] pretty_assertions = { workspace = true } +tempfile = { workspace = true } diff --git a/codex-rs/execpolicy/README.md b/codex-rs/execpolicy/README.md index f508c8ca64..9fd9c63306 100644 --- a/codex-rs/execpolicy/README.md +++ b/codex-rs/execpolicy/README.md @@ -20,18 +20,18 @@ prefix_rule( ``` ## CLI -- Provide one or more policy files (for example `src/default.codexpolicy`) to check a command: +- From the Codex CLI, run `codex execpolicy check` subcommand with one or more policy files (for example `src/default.codexpolicy`) to check a command: +```bash +codex execpolicy check --policy path/to/policy.codexpolicy git status +``` +- Pass multiple `--policy` flags to merge rules, evaluated in the order provided, and use `--pretty` for formatted JSON. +- You can also run the standalone dev binary directly during development: ```bash cargo run -p codex-execpolicy -- check --policy path/to/policy.codexpolicy git status ``` -- Pass multiple `--policy` flags to merge rules, evaluated in the order provided: -```bash -cargo run -p codex-execpolicy -- check --policy base.codexpolicy --policy overrides.codexpolicy git status -``` -- Output is JSON by default; pass `--pretty` for pretty-printed JSON - Example outcomes: - Match: `{"match": { ... "decision": "allow" ... }}` - - No match: `"noMatch"` + - No match: `{"noMatch": {}}` ## Response shapes - Match: @@ -53,8 +53,10 @@ cargo run -p codex-execpolicy -- check --policy base.codexpolicy --policy overri - No match: ```json -"noMatch" +{"noMatch": {}} ``` - `matchedRules` lists every rule whose prefix matched the command; `matchedPrefix` is the exact prefix that matched. - The effective `decision` is the strictest severity across all matches (`forbidden` > `prompt` > `allow`). + +Note: `execpolicy` commands are still in preview. The API may have breaking changes in the future. diff --git a/codex-rs/execpolicy/src/amend.rs b/codex-rs/execpolicy/src/amend.rs new file mode 100644 index 0000000000..36a02cebf9 --- /dev/null +++ b/codex-rs/execpolicy/src/amend.rs @@ -0,0 +1,226 @@ +use std::fs::OpenOptions; +use std::io::Read; +use std::io::Seek; +use std::io::SeekFrom; +use std::io::Write; +use std::path::Path; +use std::path::PathBuf; + +use serde_json; +use thiserror::Error; + +#[derive(Debug, Error)] +pub enum AmendError { + #[error("prefix rule requires at least one token")] + EmptyPrefix, + #[error("policy path has no parent: {path}")] + MissingParent { path: PathBuf }, + #[error("failed to create policy directory {dir}: {source}")] + CreatePolicyDir { + dir: PathBuf, + source: std::io::Error, + }, + #[error("failed to format prefix tokens: {source}")] + SerializePrefix { source: serde_json::Error }, + #[error("failed to open policy file {path}: {source}")] + OpenPolicyFile { + path: PathBuf, + source: std::io::Error, + }, + #[error("failed to write to policy file {path}: {source}")] + WritePolicyFile { + path: PathBuf, + source: std::io::Error, + }, + #[error("failed to lock policy file {path}: {source}")] + LockPolicyFile { + path: PathBuf, + source: std::io::Error, + }, + #[error("failed to seek policy file {path}: {source}")] + SeekPolicyFile { + path: PathBuf, + source: std::io::Error, + }, + #[error("failed to read policy file {path}: {source}")] + ReadPolicyFile { + path: PathBuf, + source: std::io::Error, + }, + #[error("failed to read metadata for policy file {path}: {source}")] + PolicyMetadata { + path: PathBuf, + source: std::io::Error, + }, +} + +/// Note this thread uses advisory file locking and performs blocking I/O, so it should be used with +/// [`tokio::task::spawn_blocking`] when called from an async context. +pub fn blocking_append_allow_prefix_rule( + policy_path: &Path, + prefix: &[String], +) -> Result<(), AmendError> { + if prefix.is_empty() { + return Err(AmendError::EmptyPrefix); + } + + let tokens = prefix + .iter() + .map(serde_json::to_string) + .collect::, _>>() + .map_err(|source| AmendError::SerializePrefix { source })?; + let pattern = format!("[{}]", tokens.join(", ")); + let rule = format!(r#"prefix_rule(pattern={pattern}, decision="allow")"#); + + let dir = policy_path + .parent() + .ok_or_else(|| AmendError::MissingParent { + path: policy_path.to_path_buf(), + })?; + match std::fs::create_dir(dir) { + Ok(()) => {} + Err(ref source) if source.kind() == std::io::ErrorKind::AlreadyExists => {} + Err(source) => { + return Err(AmendError::CreatePolicyDir { + dir: dir.to_path_buf(), + source, + }); + } + } + append_locked_line(policy_path, &rule) +} + +fn append_locked_line(policy_path: &Path, line: &str) -> Result<(), AmendError> { + let mut file = OpenOptions::new() + .create(true) + .read(true) + .append(true) + .open(policy_path) + .map_err(|source| AmendError::OpenPolicyFile { + path: policy_path.to_path_buf(), + source, + })?; + file.lock().map_err(|source| AmendError::LockPolicyFile { + path: policy_path.to_path_buf(), + source, + })?; + + let len = file + .metadata() + .map_err(|source| AmendError::PolicyMetadata { + path: policy_path.to_path_buf(), + source, + })? + .len(); + + // Ensure file ends in a newline before appending. + if len > 0 { + file.seek(SeekFrom::End(-1)) + .map_err(|source| AmendError::SeekPolicyFile { + path: policy_path.to_path_buf(), + source, + })?; + let mut last = [0; 1]; + file.read_exact(&mut last) + .map_err(|source| AmendError::ReadPolicyFile { + path: policy_path.to_path_buf(), + source, + })?; + + if last[0] != b'\n' { + file.write_all(b"\n") + .map_err(|source| AmendError::WritePolicyFile { + path: policy_path.to_path_buf(), + source, + })?; + } + } + + file.write_all(format!("{line}\n").as_bytes()) + .map_err(|source| AmendError::WritePolicyFile { + path: policy_path.to_path_buf(), + source, + })?; + + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use pretty_assertions::assert_eq; + use tempfile::tempdir; + + #[test] + fn appends_rule_and_creates_directories() { + let tmp = tempdir().expect("create temp dir"); + let policy_path = tmp.path().join("policy").join("default.codexpolicy"); + + blocking_append_allow_prefix_rule( + &policy_path, + &[String::from("echo"), String::from("Hello, world!")], + ) + .expect("append rule"); + + let contents = + std::fs::read_to_string(&policy_path).expect("default.codexpolicy should exist"); + assert_eq!( + contents, + r#"prefix_rule(pattern=["echo", "Hello, world!"], decision="allow") +"# + ); + } + + #[test] + fn appends_rule_without_duplicate_newline() { + let tmp = tempdir().expect("create temp dir"); + let policy_path = tmp.path().join("policy").join("default.codexpolicy"); + std::fs::create_dir_all(policy_path.parent().unwrap()).expect("create policy dir"); + std::fs::write( + &policy_path, + r#"prefix_rule(pattern=["ls"], decision="allow") +"#, + ) + .expect("write seed rule"); + + blocking_append_allow_prefix_rule( + &policy_path, + &[String::from("echo"), String::from("Hello, world!")], + ) + .expect("append rule"); + + let contents = std::fs::read_to_string(&policy_path).expect("read policy"); + assert_eq!( + contents, + r#"prefix_rule(pattern=["ls"], decision="allow") +prefix_rule(pattern=["echo", "Hello, world!"], decision="allow") +"# + ); + } + + #[test] + fn inserts_newline_when_missing_before_append() { + let tmp = tempdir().expect("create temp dir"); + let policy_path = tmp.path().join("policy").join("default.codexpolicy"); + std::fs::create_dir_all(policy_path.parent().unwrap()).expect("create policy dir"); + std::fs::write( + &policy_path, + r#"prefix_rule(pattern=["ls"], decision="allow")"#, + ) + .expect("write seed rule without newline"); + + blocking_append_allow_prefix_rule( + &policy_path, + &[String::from("echo"), String::from("Hello, world!")], + ) + .expect("append rule"); + + let contents = std::fs::read_to_string(&policy_path).expect("read policy"); + assert_eq!( + contents, + r#"prefix_rule(pattern=["ls"], decision="allow") +prefix_rule(pattern=["echo", "Hello, world!"], decision="allow") +"# + ); + } +} diff --git a/codex-rs/execpolicy/src/execpolicycheck.rs b/codex-rs/execpolicy/src/execpolicycheck.rs new file mode 100644 index 0000000000..0b5e0dcafc --- /dev/null +++ b/codex-rs/execpolicy/src/execpolicycheck.rs @@ -0,0 +1,67 @@ +use std::fs; +use std::path::PathBuf; + +use anyhow::Context; +use anyhow::Result; +use clap::Parser; + +use crate::Evaluation; +use crate::Policy; +use crate::PolicyParser; + +/// Arguments for evaluating a command against one or more execpolicy files. +#[derive(Debug, Parser, Clone)] +pub struct ExecPolicyCheckCommand { + /// Paths to execpolicy files to evaluate (repeatable). + #[arg(short = 'p', long = "policy", value_name = "PATH", required = true)] + pub policies: Vec, + + /// Pretty-print the JSON output. + #[arg(long)] + pub pretty: bool, + + /// Command tokens to check against the policy. + #[arg( + value_name = "COMMAND", + required = true, + trailing_var_arg = true, + allow_hyphen_values = true + )] + pub command: Vec, +} + +impl ExecPolicyCheckCommand { + /// Load the policies for this command, evaluate the command, and render JSON output. + pub fn run(&self) -> Result<()> { + let policy = load_policies(&self.policies)?; + let evaluation = policy.check(&self.command); + + let json = format_evaluation_json(&evaluation, self.pretty)?; + println!("{json}"); + + Ok(()) + } +} + +pub fn format_evaluation_json(evaluation: &Evaluation, pretty: bool) -> Result { + if pretty { + serde_json::to_string_pretty(evaluation).map_err(Into::into) + } else { + serde_json::to_string(evaluation).map_err(Into::into) + } +} + +pub fn load_policies(policy_paths: &[PathBuf]) -> Result { + let mut parser = PolicyParser::new(); + + for policy_path in policy_paths { + let policy_file_contents = fs::read_to_string(policy_path) + .with_context(|| format!("failed to read policy at {}", policy_path.display()))?; + let policy_identifier = policy_path.to_string_lossy().to_string(); + parser + .parse(&policy_identifier, &policy_file_contents) + .with_context(|| format!("failed to parse policy at {}", policy_path.display()))?; + } + + Ok(parser.build()) +} diff --git a/codex-rs/execpolicy/src/lib.rs b/codex-rs/execpolicy/src/lib.rs index 1b789fd862..31062f1cb6 100644 --- a/codex-rs/execpolicy/src/lib.rs +++ b/codex-rs/execpolicy/src/lib.rs @@ -1,12 +1,17 @@ +pub mod amend; pub mod decision; pub mod error; +pub mod execpolicycheck; pub mod parser; pub mod policy; pub mod rule; +pub use amend::AmendError; +pub use amend::blocking_append_allow_prefix_rule; pub use decision::Decision; pub use error::Error; pub use error::Result; +pub use execpolicycheck::ExecPolicyCheckCommand; pub use parser::PolicyParser; pub use policy::Evaluation; pub use policy::Policy; diff --git a/codex-rs/execpolicy/src/main.rs b/codex-rs/execpolicy/src/main.rs index 318abb5cae..e1373b6d16 100644 --- a/codex-rs/execpolicy/src/main.rs +++ b/codex-rs/execpolicy/src/main.rs @@ -1,66 +1,22 @@ -use std::fs; -use std::path::PathBuf; - -use anyhow::Context; use anyhow::Result; use clap::Parser; -use codex_execpolicy::PolicyParser; +use codex_execpolicy::ExecPolicyCheckCommand; /// CLI for evaluating exec policies #[derive(Parser)] #[command(name = "codex-execpolicy")] enum Cli { /// Evaluate a command against a policy. - Check { - #[arg(short, long = "policy", value_name = "PATH", required = true)] - policies: Vec, - - /// Pretty-print the JSON output. - #[arg(long)] - pretty: bool, - - /// Command tokens to check. - #[arg( - value_name = "COMMAND", - required = true, - trailing_var_arg = true, - allow_hyphen_values = true - )] - command: Vec, - }, + Check(ExecPolicyCheckCommand), } fn main() -> Result<()> { let cli = Cli::parse(); match cli { - Cli::Check { - policies, - command, - pretty, - } => cmd_check(policies, command, pretty), + Cli::Check(cmd) => cmd_check(cmd), } } -fn cmd_check(policy_paths: Vec, args: Vec, pretty: bool) -> Result<()> { - let policy = load_policies(&policy_paths)?; - - let eval = policy.check(&args); - let json = if pretty { - serde_json::to_string_pretty(&eval)? - } else { - serde_json::to_string(&eval)? - }; - println!("{json}"); - Ok(()) -} - -fn load_policies(policy_paths: &[PathBuf]) -> Result { - let mut parser = PolicyParser::new(); - for policy_path in policy_paths { - let policy_file_contents = fs::read_to_string(policy_path) - .with_context(|| format!("failed to read policy at {}", policy_path.display()))?; - let policy_identifier = policy_path.to_string_lossy().to_string(); - parser.parse(&policy_identifier, &policy_file_contents)?; - } - Ok(parser.build()) +fn cmd_check(cmd: ExecPolicyCheckCommand) -> Result<()> { + cmd.run() } diff --git a/codex-rs/execpolicy/src/policy.rs b/codex-rs/execpolicy/src/policy.rs index a543e5c4b0..10858c9fad 100644 --- a/codex-rs/execpolicy/src/policy.rs +++ b/codex-rs/execpolicy/src/policy.rs @@ -1,9 +1,15 @@ use crate::decision::Decision; +use crate::error::Error; +use crate::error::Result; +use crate::rule::PatternToken; +use crate::rule::PrefixPattern; +use crate::rule::PrefixRule; use crate::rule::RuleMatch; use crate::rule::RuleRef; use multimap::MultiMap; use serde::Deserialize; use serde::Serialize; +use std::sync::Arc; #[derive(Clone, Debug)] pub struct Policy { @@ -23,13 +29,34 @@ impl Policy { &self.rules_by_program } + pub fn add_prefix_rule(&mut self, prefix: &[String], decision: Decision) -> Result<()> { + let (first_token, rest) = prefix + .split_first() + .ok_or_else(|| Error::InvalidPattern("prefix cannot be empty".to_string()))?; + + let rule: RuleRef = Arc::new(PrefixRule { + pattern: PrefixPattern { + first: Arc::from(first_token.as_str()), + rest: rest + .iter() + .map(|token| PatternToken::Single(token.clone())) + .collect::>() + .into(), + }, + decision, + }); + + self.rules_by_program.insert(first_token.clone(), rule); + Ok(()) + } + pub fn check(&self, cmd: &[String]) -> Evaluation { let rules = match cmd.first() { Some(first) => match self.rules_by_program.get_vec(first) { Some(rules) => rules, - None => return Evaluation::NoMatch, + None => return Evaluation::NoMatch {}, }, - None => return Evaluation::NoMatch, + None => return Evaluation::NoMatch {}, }; let matched_rules: Vec = @@ -39,7 +66,7 @@ impl Policy { decision, matched_rules, }, - None => Evaluation::NoMatch, + None => Evaluation::NoMatch {}, } } @@ -52,7 +79,7 @@ impl Policy { .into_iter() .flat_map(|command| match self.check(command.as_ref()) { Evaluation::Match { matched_rules, .. } => matched_rules, - Evaluation::NoMatch => Vec::new(), + Evaluation::NoMatch { .. } => Vec::new(), }) .collect(); @@ -61,7 +88,7 @@ impl Policy { decision, matched_rules, }, - None => Evaluation::NoMatch, + None => Evaluation::NoMatch {}, } } } @@ -69,7 +96,7 @@ impl Policy { #[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub enum Evaluation { - NoMatch, + NoMatch {}, Match { decision: Decision, #[serde(rename = "matchedRules")] diff --git a/codex-rs/execpolicy/tests/basic.rs b/codex-rs/execpolicy/tests/basic.rs index 301f962616..9a7ec58b1e 100644 --- a/codex-rs/execpolicy/tests/basic.rs +++ b/codex-rs/execpolicy/tests/basic.rs @@ -1,8 +1,12 @@ use std::any::Any; use std::sync::Arc; +use anyhow::Context; +use anyhow::Result; use codex_execpolicy::Decision; +use codex_execpolicy::Error; use codex_execpolicy::Evaluation; +use codex_execpolicy::Policy; use codex_execpolicy::PolicyParser; use codex_execpolicy::RuleMatch; use codex_execpolicy::RuleRef; @@ -35,16 +39,14 @@ fn rule_snapshots(rules: &[RuleRef]) -> Vec { } #[test] -fn basic_match() { +fn basic_match() -> Result<()> { let policy_src = r#" prefix_rule( pattern = ["git", "status"], ) "#; let mut parser = PolicyParser::new(); - parser - .parse("test.codexpolicy", policy_src) - .expect("parse policy"); + parser.parse("test.codexpolicy", policy_src)?; let policy = parser.build(); let cmd = tokens(&["git", "status"]); let evaluation = policy.check(&cmd); @@ -58,10 +60,54 @@ prefix_rule( }, evaluation ); + Ok(()) } #[test] -fn parses_multiple_policy_files() { +fn add_prefix_rule_extends_policy() -> Result<()> { + let mut policy = Policy::empty(); + policy.add_prefix_rule(&tokens(&["ls", "-l"]), Decision::Prompt)?; + + let rules = rule_snapshots(policy.rules().get_vec("ls").context("missing ls rules")?); + assert_eq!( + vec![RuleSnapshot::Prefix(PrefixRule { + pattern: PrefixPattern { + first: Arc::from("ls"), + rest: vec![PatternToken::Single(String::from("-l"))].into(), + }, + decision: Decision::Prompt, + })], + rules + ); + + let evaluation = policy.check(&tokens(&["ls", "-l", "/tmp"])); + assert_eq!( + Evaluation::Match { + decision: Decision::Prompt, + matched_rules: vec![RuleMatch::PrefixRuleMatch { + matched_prefix: tokens(&["ls", "-l"]), + decision: Decision::Prompt, + }], + }, + evaluation + ); + Ok(()) +} + +#[test] +fn add_prefix_rule_rejects_empty_prefix() -> Result<()> { + let mut policy = Policy::empty(); + let result = policy.add_prefix_rule(&[], Decision::Allow); + + match result.unwrap_err() { + Error::InvalidPattern(message) => assert_eq!(message, "prefix cannot be empty"), + other => panic!("expected InvalidPattern(..), got {other:?}"), + } + Ok(()) +} + +#[test] +fn parses_multiple_policy_files() -> Result<()> { let first_policy = r#" prefix_rule( pattern = ["git"], @@ -75,15 +121,11 @@ prefix_rule( ) "#; let mut parser = PolicyParser::new(); - parser - .parse("first.codexpolicy", first_policy) - .expect("parse policy"); - parser - .parse("second.codexpolicy", second_policy) - .expect("parse policy"); + parser.parse("first.codexpolicy", first_policy)?; + parser.parse("second.codexpolicy", second_policy)?; let policy = parser.build(); - let git_rules = rule_snapshots(policy.rules().get_vec("git").expect("git rules")); + let git_rules = rule_snapshots(policy.rules().get_vec("git").context("missing git rules")?); assert_eq!( vec![ RuleSnapshot::Prefix(PrefixRule { @@ -133,23 +175,27 @@ prefix_rule( }, commit_eval ); + Ok(()) } #[test] -fn only_first_token_alias_expands_to_multiple_rules() { +fn only_first_token_alias_expands_to_multiple_rules() -> Result<()> { let policy_src = r#" prefix_rule( pattern = [["bash", "sh"], ["-c", "-l"]], ) "#; let mut parser = PolicyParser::new(); - parser - .parse("test.codexpolicy", policy_src) - .expect("parse policy"); + parser.parse("test.codexpolicy", policy_src)?; let policy = parser.build(); - let bash_rules = rule_snapshots(policy.rules().get_vec("bash").expect("bash rules")); - let sh_rules = rule_snapshots(policy.rules().get_vec("sh").expect("sh rules")); + let bash_rules = rule_snapshots( + policy + .rules() + .get_vec("bash") + .context("missing bash rules")?, + ); + let sh_rules = rule_snapshots(policy.rules().get_vec("sh").context("missing sh rules")?); assert_eq!( vec![RuleSnapshot::Prefix(PrefixRule { pattern: PrefixPattern { @@ -194,22 +240,21 @@ prefix_rule( }, sh_eval ); + Ok(()) } #[test] -fn tail_aliases_are_not_cartesian_expanded() { +fn tail_aliases_are_not_cartesian_expanded() -> Result<()> { let policy_src = r#" prefix_rule( pattern = ["npm", ["i", "install"], ["--legacy-peer-deps", "--no-save"]], ) "#; let mut parser = PolicyParser::new(); - parser - .parse("test.codexpolicy", policy_src) - .expect("parse policy"); + parser.parse("test.codexpolicy", policy_src)?; let policy = parser.build(); - let rules = rule_snapshots(policy.rules().get_vec("npm").expect("npm rules")); + let rules = rule_snapshots(policy.rules().get_vec("npm").context("missing npm rules")?); assert_eq!( vec![RuleSnapshot::Prefix(PrefixRule { pattern: PrefixPattern { @@ -251,10 +296,11 @@ prefix_rule( }, npm_install ); + Ok(()) } #[test] -fn match_and_not_match_examples_are_enforced() { +fn match_and_not_match_examples_are_enforced() -> Result<()> { let policy_src = r#" prefix_rule( pattern = ["git", "status"], @@ -266,9 +312,7 @@ prefix_rule( ) "#; let mut parser = PolicyParser::new(); - parser - .parse("test.codexpolicy", policy_src) - .expect("parse policy"); + parser.parse("test.codexpolicy", policy_src)?; let policy = parser.build(); let match_eval = policy.check(&tokens(&["git", "status"])); assert_eq!( @@ -288,11 +332,12 @@ prefix_rule( "color.status=always", "status", ])); - assert_eq!(Evaluation::NoMatch, no_match_eval); + assert_eq!(Evaluation::NoMatch {}, no_match_eval); + Ok(()) } #[test] -fn strictest_decision_wins_across_matches() { +fn strictest_decision_wins_across_matches() -> Result<()> { let policy_src = r#" prefix_rule( pattern = ["git"], @@ -304,9 +349,7 @@ prefix_rule( ) "#; let mut parser = PolicyParser::new(); - parser - .parse("test.codexpolicy", policy_src) - .expect("parse policy"); + parser.parse("test.codexpolicy", policy_src)?; let policy = parser.build(); let commit = policy.check(&tokens(&["git", "commit", "-m", "hi"])); @@ -326,10 +369,11 @@ prefix_rule( }, commit ); + Ok(()) } #[test] -fn strictest_decision_across_multiple_commands() { +fn strictest_decision_across_multiple_commands() -> Result<()> { let policy_src = r#" prefix_rule( pattern = ["git"], @@ -341,9 +385,7 @@ prefix_rule( ) "#; let mut parser = PolicyParser::new(); - parser - .parse("test.codexpolicy", policy_src) - .expect("parse policy"); + parser.parse("test.codexpolicy", policy_src)?; let policy = parser.build(); let commands = vec![ @@ -372,4 +414,5 @@ prefix_rule( }, evaluation ); + Ok(()) } diff --git a/codex-rs/feedback/Cargo.toml b/codex-rs/feedback/Cargo.toml index b104f51225..0697704072 100644 --- a/codex-rs/feedback/Cargo.toml +++ b/codex-rs/feedback/Cargo.toml @@ -1,7 +1,8 @@ [package] -edition.workspace = true name = "codex-feedback" version.workspace = true +edition.workspace = true +license.workspace = true [dependencies] anyhow = { workspace = true } diff --git a/codex-rs/feedback/src/lib.rs b/codex-rs/feedback/src/lib.rs index e1ccc3aac9..eaa949717a 100644 --- a/codex-rs/feedback/src/lib.rs +++ b/codex-rs/feedback/src/lib.rs @@ -10,6 +10,7 @@ use std::time::Duration; use anyhow::Result; use anyhow::anyhow; use codex_protocol::ConversationId; +use codex_protocol::protocol::SessionSource; use tracing_subscriber::fmt::writer::MakeWriter; const DEFAULT_MAX_BYTES: usize = 4 * 1024 * 1024; // 4 MiB @@ -174,6 +175,7 @@ impl CodexLogSnapshot { reason: Option<&str>, include_logs: bool, rollout_path: Option<&std::path::Path>, + session_source: Option, ) -> Result<()> { use std::collections::BTreeMap; use std::fs; @@ -203,6 +205,9 @@ impl CodexLogSnapshot { (String::from("classification"), classification.to_string()), (String::from("cli_version"), cli_version.to_string()), ]); + if let Some(source) = session_source.as_ref() { + tags.insert(String::from("session_source"), source.to_string()); + } if let Some(r) = reason { tags.insert(String::from("reason"), r.to_string()); } diff --git a/codex-rs/file-search/Cargo.toml b/codex-rs/file-search/Cargo.toml index 40671389e1..e0dea1c139 100644 --- a/codex-rs/file-search/Cargo.toml +++ b/codex-rs/file-search/Cargo.toml @@ -1,7 +1,8 @@ [package] -edition = "2024" name = "codex-file-search" -version = { workspace = true } +version.workspace = true +edition.workspace = true +license.workspace = true [[bin]] name = "codex-file-search" diff --git a/codex-rs/keyring-store/Cargo.toml b/codex-rs/keyring-store/Cargo.toml index 932693de50..dd3686ee59 100644 --- a/codex-rs/keyring-store/Cargo.toml +++ b/codex-rs/keyring-store/Cargo.toml @@ -1,7 +1,8 @@ [package] -edition = "2024" name = "codex-keyring-store" -version = { workspace = true } +version.workspace = true +edition.workspace = true +license.workspace = true [lints] workspace = true diff --git a/codex-rs/linux-sandbox/Cargo.toml b/codex-rs/linux-sandbox/Cargo.toml index 264f15e751..cac8604f5d 100644 --- a/codex-rs/linux-sandbox/Cargo.toml +++ b/codex-rs/linux-sandbox/Cargo.toml @@ -1,7 +1,8 @@ [package] -edition = "2024" name = "codex-linux-sandbox" -version = { workspace = true } +version.workspace = true +edition.workspace = true +license.workspace = true [[bin]] name = "codex-linux-sandbox" diff --git a/codex-rs/linux-sandbox/tests/suite/landlock.rs b/codex-rs/linux-sandbox/tests/suite/landlock.rs index 508dba08cf..e145aa2f73 100644 --- a/codex-rs/linux-sandbox/tests/suite/landlock.rs +++ b/codex-rs/linux-sandbox/tests/suite/landlock.rs @@ -3,7 +3,6 @@ use codex_core::config::types::ShellEnvironmentPolicy; use codex_core::error::CodexErr; use codex_core::error::SandboxErr; use codex_core::exec::ExecParams; -use codex_core::exec::SandboxType; use codex_core::exec::process_exec_tool_call; use codex_core::exec_env::create_env; use codex_core::protocol::SandboxPolicy; @@ -40,7 +39,7 @@ async fn run_cmd(cmd: &[&str], writable_roots: &[PathBuf], timeout_ms: u64) { let params = ExecParams { command: cmd.iter().copied().map(str::to_owned).collect(), cwd, - timeout_ms: Some(timeout_ms), + expiration: timeout_ms.into(), env: create_env_from_core_vars(), with_escalated_permissions: None, justification: None, @@ -60,7 +59,6 @@ async fn run_cmd(cmd: &[&str], writable_roots: &[PathBuf], timeout_ms: u64) { let codex_linux_sandbox_exe = Some(PathBuf::from(sandbox_program)); let res = process_exec_tool_call( params, - SandboxType::LinuxSeccomp, &sandbox_policy, sandbox_cwd.as_path(), &codex_linux_sandbox_exe, @@ -143,7 +141,7 @@ async fn assert_network_blocked(cmd: &[&str]) { cwd, // Give the tool a generous 2-second timeout so even slow DNS timeouts // do not stall the suite. - timeout_ms: Some(NETWORK_TIMEOUT_MS), + expiration: NETWORK_TIMEOUT_MS.into(), env: create_env_from_core_vars(), with_escalated_permissions: None, justification: None, @@ -155,7 +153,6 @@ async fn assert_network_blocked(cmd: &[&str]) { let codex_linux_sandbox_exe: Option = Some(PathBuf::from(sandbox_program)); let result = process_exec_tool_call( params, - SandboxType::LinuxSeccomp, &sandbox_policy, sandbox_cwd.as_path(), &codex_linux_sandbox_exe, diff --git a/codex-rs/lmstudio/Cargo.toml b/codex-rs/lmstudio/Cargo.toml index 4035d5529b..3ac7614c22 100644 --- a/codex-rs/lmstudio/Cargo.toml +++ b/codex-rs/lmstudio/Cargo.toml @@ -2,6 +2,7 @@ name = "codex-lmstudio" version.workspace = true edition.workspace = true +license.workspace = true [lib] name = "codex_lmstudio" @@ -13,7 +14,7 @@ codex-core = { path = "../core" } reqwest = { version = "0.12", features = ["json", "stream"] } serde_json = "1" tokio = { version = "1", features = ["rt"] } -tracing = { version = "0.1.41", features = ["log"] } +tracing = { version = "0.1.43", features = ["log"] } which = "6.0" [dev-dependencies] diff --git a/codex-rs/login/Cargo.toml b/codex-rs/login/Cargo.toml index 563515c637..94af9e19af 100644 --- a/codex-rs/login/Cargo.toml +++ b/codex-rs/login/Cargo.toml @@ -1,7 +1,8 @@ [package] -edition = "2024" name = "codex-login" -version = { workspace = true } +version.workspace = true +edition.workspace = true +license.workspace = true [lints] workspace = true diff --git a/codex-rs/login/src/device_code_auth.rs b/codex-rs/login/src/device_code_auth.rs index acaf30ba0c..a121de7ebd 100644 --- a/codex-rs/login/src/device_code_auth.rs +++ b/codex-rs/login/src/device_code_auth.rs @@ -8,11 +8,10 @@ use std::time::Instant; use crate::pkce::PkceCodes; use crate::server::ServerOptions; -use std::io::Write; -use std::io::{self}; +use std::io; -const ANSI_YELLOW: &str = "\x1b[93m"; -const ANSI_BOLD: &str = "\x1b[1m"; +const ANSI_BLUE: &str = "\x1b[94m"; +const ANSI_GRAY: &str = "\x1b[90m"; const ANSI_RESET: &str = "\x1b[0m"; #[derive(Deserialize)] @@ -138,14 +137,16 @@ async fn poll_for_token( } } -fn print_colored_warning_device_code() { - let mut stdout = io::stdout().lock(); - let _ = write!( - stdout, - "{ANSI_YELLOW}{ANSI_BOLD}Only use device code authentication when browser login is not available.{ANSI_RESET}{ANSI_YELLOW}\n\ -{ANSI_BOLD}Keep the code secret; do not share it.{ANSI_RESET}{ANSI_RESET}\n\n" +fn print_device_code_prompt(code: &str) { + println!( + "\nWelcome to Codex [v{ANSI_GRAY}{version}{ANSI_RESET}]\n{ANSI_GRAY}OpenAI's command-line coding agent{ANSI_RESET}\n\ +\nFollow these steps to sign in with ChatGPT using device code authorization:\n\ +\n1. Open this link in your browser\n {ANSI_BLUE}https://auth.openai.com/codex/device{ANSI_RESET}\n\ +\n2. Enter this one-time code {ANSI_GRAY}(expires in 15 minutes){ANSI_RESET}\n {ANSI_BLUE}{code}{ANSI_RESET}\n\ +\n{ANSI_GRAY}Device codes are a common phishing target. Never share this code.{ANSI_RESET}\n", + version = env!("CARGO_PKG_VERSION"), + code = code ); - let _ = stdout.flush(); } /// Full device code login flow. @@ -153,13 +154,9 @@ pub async fn run_device_code_login(opts: ServerOptions) -> std::io::Result<()> { let client = reqwest::Client::new(); let base_url = opts.issuer.trim_end_matches('/'); let api_base_url = format!("{}/api/accounts", opts.issuer.trim_end_matches('/')); - print_colored_warning_device_code(); let uc = request_user_code(&client, &api_base_url, &opts.client_id).await?; - println!( - "To authenticate:\n 1. Open in your browser: {ANSI_BOLD}https://auth.openai.com/codex/device{ANSI_RESET}\n 2. Enter the one-time code below within 15 minutes:\n\n {ANSI_BOLD}{}{ANSI_RESET}\n", - uc.user_code - ); + print_device_code_prompt(&uc.user_code); let code_resp = poll_for_token( &client, diff --git a/codex-rs/mcp-server/Cargo.toml b/codex-rs/mcp-server/Cargo.toml index 484af6d8e8..00a2204578 100644 --- a/codex-rs/mcp-server/Cargo.toml +++ b/codex-rs/mcp-server/Cargo.toml @@ -1,7 +1,8 @@ [package] -edition = "2024" name = "codex-mcp-server" -version = { workspace = true } +version.workspace = true +edition.workspace = true +license.workspace = true [[bin]] name = "codex-mcp-server" diff --git a/codex-rs/mcp-server/src/codex_tool_runner.rs b/codex-rs/mcp-server/src/codex_tool_runner.rs index 2e0075643d..b5e30d24ce 100644 --- a/codex-rs/mcp-server/src/codex_tool_runner.rs +++ b/codex-rs/mcp-server/src/codex_tool_runner.rs @@ -208,6 +208,10 @@ async fn run_codex_tool_session_inner( EventMsg::Warning(_) => { continue; } + EventMsg::ElicitationRequest(_) => { + // TODO: forward elicitation requests to the client? + continue; + } EventMsg::ApplyPatchApprovalRequest(ApplyPatchApprovalRequestEvent { call_id, turn_id: _, @@ -302,6 +306,7 @@ async fn run_codex_tool_session_inner( | EventMsg::UndoStarted(_) | EventMsg::UndoCompleted(_) | EventMsg::ExitedReviewMode(_) + | EventMsg::ContextCompacted(_) | EventMsg::SaveSessionResponse(_) | EventMsg::DeprecationNotice(_) => { // For now, we do not do anything extra for these diff --git a/codex-rs/mcp-server/tests/common/Cargo.toml b/codex-rs/mcp-server/tests/common/Cargo.toml index 7c2bc2266a..c549a03fc9 100644 --- a/codex-rs/mcp-server/tests/common/Cargo.toml +++ b/codex-rs/mcp-server/tests/common/Cargo.toml @@ -1,7 +1,8 @@ [package] -edition = "2024" name = "mcp_test_support" -version = { workspace = true } +version.workspace = true +edition.workspace = true +license.workspace = true [lib] path = "lib.rs" @@ -23,3 +24,5 @@ tokio = { workspace = true, features = [ "rt-multi-thread", ] } wiremock = { workspace = true } +core_test_support = { path = "../../../core/tests/common" } +shlex = { workspace = true } diff --git a/codex-rs/mcp-server/tests/common/lib.rs b/codex-rs/mcp-server/tests/common/lib.rs index d088b184ea..d8d6b67384 100644 --- a/codex-rs/mcp-server/tests/common/lib.rs +++ b/codex-rs/mcp-server/tests/common/lib.rs @@ -2,12 +2,13 @@ mod mcp_process; mod mock_model_server; mod responses; +pub use core_test_support::format_with_current_shell; pub use mcp_process::McpProcess; use mcp_types::JSONRPCResponse; pub use mock_model_server::create_mock_chat_completions_server; pub use responses::create_apply_patch_sse_response; pub use responses::create_final_assistant_message_sse_response; -pub use responses::create_shell_sse_response; +pub use responses::create_shell_command_sse_response; use serde::de::DeserializeOwned; pub fn to_response(response: JSONRPCResponse) -> anyhow::Result { diff --git a/codex-rs/mcp-server/tests/common/responses.rs b/codex-rs/mcp-server/tests/common/responses.rs index 9a827fb986..0a9183c043 100644 --- a/codex-rs/mcp-server/tests/common/responses.rs +++ b/codex-rs/mcp-server/tests/common/responses.rs @@ -1,17 +1,18 @@ use serde_json::json; use std::path::Path; -pub fn create_shell_sse_response( +pub fn create_shell_command_sse_response( command: Vec, workdir: Option<&Path>, timeout_ms: Option, call_id: &str, ) -> anyhow::Result { - // The `arguments`` for the `shell` tool is a serialized JSON object. + // The `arguments` for the `shell_command` tool is a serialized JSON object. + let command_str = shlex::try_join(command.iter().map(String::as_str))?; let tool_call_arguments = serde_json::to_string(&json!({ - "command": command, + "command": command_str, "workdir": workdir.map(|w| w.to_string_lossy()), - "timeout": timeout_ms + "timeout_ms": timeout_ms }))?; let tool_call = json!({ "choices": [ @@ -21,7 +22,7 @@ pub fn create_shell_sse_response( { "id": call_id, "function": { - "name": "shell", + "name": "shell_command", "arguments": tool_call_arguments } } @@ -62,10 +63,10 @@ pub fn create_apply_patch_sse_response( patch_content: &str, call_id: &str, ) -> anyhow::Result { - // Use shell command to call apply_patch with heredoc format - let shell_command = format!("apply_patch <<'EOF'\n{patch_content}\nEOF"); + // Use shell_command to call apply_patch with heredoc format + let command = format!("apply_patch <<'EOF'\n{patch_content}\nEOF"); let tool_call_arguments = serde_json::to_string(&json!({ - "command": ["bash", "-lc", shell_command] + "command": command }))?; let tool_call = json!({ @@ -76,7 +77,7 @@ pub fn create_apply_patch_sse_response( { "id": call_id, "function": { - "name": "shell", + "name": "shell_command", "arguments": tool_call_arguments } } diff --git a/codex-rs/mcp-server/tests/suite/codex_tool.rs b/codex-rs/mcp-server/tests/suite/codex_tool.rs index ae0c23f1d8..f65495c473 100644 --- a/codex-rs/mcp-server/tests/suite/codex_tool.rs +++ b/codex-rs/mcp-server/tests/suite/codex_tool.rs @@ -30,7 +30,8 @@ use mcp_test_support::McpProcess; use mcp_test_support::create_apply_patch_sse_response; use mcp_test_support::create_final_assistant_message_sse_response; use mcp_test_support::create_mock_chat_completions_server; -use mcp_test_support::create_shell_sse_response; +use mcp_test_support::create_shell_command_sse_response; +use mcp_test_support::format_with_current_shell; // Allow ample time on slower CI or under load to avoid flakes. const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(20); @@ -71,13 +72,16 @@ async fn shell_command_approval_triggers_elicitation() -> anyhow::Result<()> { "-c".to_string(), format!("import pathlib; pathlib.Path('{created_filename}').touch()"), ]; + let expected_shell_command = format_with_current_shell(&format!( + "python3 -c \"import pathlib; pathlib.Path('{created_filename}').touch()\"" + )); let McpHandle { process: mut mcp_process, server: _server, dir: _dir, } = create_mcp_process(vec![ - create_shell_sse_response( + create_shell_command_sse_response( shell_command.clone(), Some(workdir_for_shell_function_call.path()), Some(5_000), @@ -111,7 +115,7 @@ async fn shell_command_approval_triggers_elicitation() -> anyhow::Result<()> { )?; let expected_elicitation_request = create_expected_elicitation_request( elicitation_request_id.clone(), - shell_command.clone(), + expected_shell_command, workdir_for_shell_function_call.path(), codex_request_id.to_string(), params.codex_event_id.clone(), @@ -218,6 +222,12 @@ async fn test_patch_approval_triggers_elicitation() { } async fn patch_approval_triggers_elicitation() -> anyhow::Result<()> { + if cfg!(windows) { + // powershell apply_patch shell calls are not parsed into apply patch approvals + + return Ok(()); + } + let cwd = TempDir::new()?; let test_file = cwd.path().join("destination_file.txt"); std::fs::write(&test_file, "original content\n")?; diff --git a/codex-rs/mcp-types/Cargo.toml b/codex-rs/mcp-types/Cargo.toml index 8ea4fdfee5..92cf539611 100644 --- a/codex-rs/mcp-types/Cargo.toml +++ b/codex-rs/mcp-types/Cargo.toml @@ -1,7 +1,8 @@ [package] -edition = "2024" name = "mcp-types" -version = { workspace = true } +version.workspace = true +edition.workspace = true +license.workspace = true [lints] workspace = true diff --git a/codex-rs/mcp-types/generate_mcp_types.py b/codex-rs/mcp-types/generate_mcp_types.py index 7535b4c760..5aaac81cd7 100755 --- a/codex-rs/mcp-types/generate_mcp_types.py +++ b/codex-rs/mcp-types/generate_mcp_types.py @@ -38,6 +38,13 @@ SERVER_NOTIFICATION_TYPE_NAMES: list[str] = [] # order to compile without warnings. LARGE_ENUMS = {"ServerResult"} +# some types need setting a default value for `r#type` +# ref: [#7417](https://github.com/openai/codex/pull/7417) +default_type_values: dict[str, str] = { + "ToolInputSchema": "object", + "ToolOutputSchema": "object", +} + def main() -> int: parser = argparse.ArgumentParser( @@ -351,6 +358,14 @@ class StructField: out.append(f" pub {self.name}: {self.type_name},\n") +def append_serde_attr(existing: str | None, fragment: str) -> str: + if existing is None: + return f"#[serde({fragment})]" + assert existing.startswith("#[serde(") and existing.endswith(")]"), existing + body = existing[len("#[serde(") : -2] + return f"#[serde({body}, {fragment})]" + + def define_struct( name: str, properties: dict[str, Any], @@ -359,6 +374,14 @@ def define_struct( ) -> list[str]: out: list[str] = [] + type_default_fn: str | None = None + if name in default_type_values: + snake_name = to_snake_case(name) or name + type_default_fn = f"{snake_name}_type_default_str" + out.append(f"fn {type_default_fn}() -> String {{\n") + out.append(f' "{default_type_values[name]}".to_string()\n') + out.append("}\n\n") + fields: list[StructField] = [] for prop_name, prop in properties.items(): if prop_name == "_meta": @@ -380,6 +403,10 @@ def define_struct( if is_optional: prop_type = f"Option<{prop_type}>" rs_prop = rust_prop_name(prop_name, is_optional) + + if prop_name == "type" and type_default_fn: + rs_prop.serde = append_serde_attr(rs_prop.serde, f'default = "{type_default_fn}"') + if prop_type.startswith("&'static str"): fields.append(StructField("const", rs_prop.name, prop_type, rs_prop.serde, rs_prop.ts)) else: diff --git a/codex-rs/mcp-types/src/lib.rs b/codex-rs/mcp-types/src/lib.rs index e390322fc8..7418bea854 100644 --- a/codex-rs/mcp-types/src/lib.rs +++ b/codex-rs/mcp-types/src/lib.rs @@ -1474,6 +1474,10 @@ pub struct Tool { pub title: Option, } +fn tool_output_schema_type_default_str() -> String { + "object".to_string() +} + /// An optional JSON Schema object defining the structure of the tool's output returned in /// the structuredContent field of a CallToolResult. #[derive(Debug, Clone, PartialEq, Deserialize, Serialize, JsonSchema, TS)] @@ -1484,9 +1488,14 @@ pub struct ToolOutputSchema { #[serde(default, skip_serializing_if = "Option::is_none")] #[ts(optional)] pub required: Option>, + #[serde(default = "tool_output_schema_type_default_str")] pub r#type: String, // &'static str = "object" } +fn tool_input_schema_type_default_str() -> String { + "object".to_string() +} + /// A JSON Schema object defining the expected parameters for the tool. #[derive(Debug, Clone, PartialEq, Deserialize, Serialize, JsonSchema, TS)] pub struct ToolInputSchema { @@ -1496,6 +1505,7 @@ pub struct ToolInputSchema { #[serde(default, skip_serializing_if = "Option::is_none")] #[ts(optional)] pub required: Option>, + #[serde(default = "tool_input_schema_type_default_str")] pub r#type: String, // &'static str = "object" } diff --git a/codex-rs/ollama/Cargo.toml b/codex-rs/ollama/Cargo.toml index 14dd6d2fcd..ee16bd5e05 100644 --- a/codex-rs/ollama/Cargo.toml +++ b/codex-rs/ollama/Cargo.toml @@ -1,7 +1,8 @@ [package] -edition = "2024" name = "codex-ollama" -version = { workspace = true } +version.workspace = true +edition.workspace = true +license.workspace = true [lib] name = "codex_ollama" diff --git a/codex-rs/otel/Cargo.toml b/codex-rs/otel/Cargo.toml index 2ab170c917..059e8e38e6 100644 --- a/codex-rs/otel/Cargo.toml +++ b/codex-rs/otel/Cargo.toml @@ -1,7 +1,8 @@ [package] -edition = "2024" name = "codex-otel" -version = { workspace = true } +version.workspace = true +edition.workspace = true +license.workspace = true [lib] doctest = false diff --git a/codex-rs/otel/src/otel_event_manager.rs b/codex-rs/otel/src/otel_event_manager.rs index 5d9cbd4997..b6bc07e79f 100644 --- a/codex-rs/otel/src/otel_event_manager.rs +++ b/codex-rs/otel/src/otel_event_manager.rs @@ -88,7 +88,6 @@ impl OtelEventManager { reasoning_effort: Option, reasoning_summary: ReasoningSummary, context_window: Option, - max_output_tokens: Option, auto_compact_token_limit: Option, approval_policy: AskForApproval, sandbox_policy: SandboxPolicy, @@ -111,7 +110,6 @@ impl OtelEventManager { reasoning_effort = reasoning_effort.map(|e| e.to_string()), reasoning_summary = %reasoning_summary, context_window = context_window, - max_output_tokens = max_output_tokens, auto_compact_token_limit = auto_compact_token_limit, approval_policy = %approval_policy, sandbox_policy = %sandbox_policy, @@ -133,7 +131,18 @@ impl OtelEventManager { Ok(response) => (Some(response.status().as_u16()), None), Err(error) => (error.status().map(|s| s.as_u16()), Some(error.to_string())), }; + self.record_api_request(attempt, status, error.as_deref(), duration); + response + } + + pub fn record_api_request( + &self, + attempt: u64, + status: Option, + error: Option<&str>, + duration: Duration, + ) { tracing::event!( tracing::Level::INFO, event.name = "codex.api_request", @@ -151,8 +160,6 @@ impl OtelEventManager { error.message = error, attempt = attempt, ); - - response } pub fn log_sse_event( diff --git a/codex-rs/process-hardening/Cargo.toml b/codex-rs/process-hardening/Cargo.toml index 2ba4b0d5ca..2a867572df 100644 --- a/codex-rs/process-hardening/Cargo.toml +++ b/codex-rs/process-hardening/Cargo.toml @@ -1,7 +1,8 @@ [package] -edition = "2024" name = "codex-process-hardening" -version = { workspace = true } +version.workspace = true +edition.workspace = true +license.workspace = true [lib] name = "codex_process_hardening" diff --git a/codex-rs/process-hardening/src/lib.rs b/codex-rs/process-hardening/src/lib.rs index 0a624fb387..772647671f 100644 --- a/codex-rs/process-hardening/src/lib.rs +++ b/codex-rs/process-hardening/src/lib.rs @@ -29,6 +29,7 @@ const PTRACE_DENY_ATTACH_FAILED_EXIT_CODE: i32 = 6; target_os = "android", target_os = "macos", target_os = "freebsd", + target_os = "netbsd", target_os = "openbsd" ))] const SET_RLIMIT_CORE_FAILED_EXIT_CODE: i32 = 7; diff --git a/codex-rs/protocol/Cargo.toml b/codex-rs/protocol/Cargo.toml index 00ed100e08..08f8375357 100644 --- a/codex-rs/protocol/Cargo.toml +++ b/codex-rs/protocol/Cargo.toml @@ -1,7 +1,8 @@ [package] -edition = "2024" name = "codex-protocol" -version = { workspace = true } +version.workspace = true +edition.workspace = true +license.workspace = true [lib] name = "codex_protocol" diff --git a/codex-rs/protocol/src/approvals.rs b/codex-rs/protocol/src/approvals.rs index 25f5e90e9e..9ef3a9e368 100644 --- a/codex-rs/protocol/src/approvals.rs +++ b/codex-rs/protocol/src/approvals.rs @@ -3,6 +3,7 @@ use std::path::PathBuf; use crate::parse_command::ParsedCommand; use crate::protocol::FileChange; +use mcp_types::RequestId; use schemars::JsonSchema; use serde::Deserialize; use serde::Serialize; @@ -53,6 +54,24 @@ pub struct ExecApprovalRequestEvent { pub parsed_cmd: Vec, } +#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, TS)] +pub struct ElicitationRequestEvent { + pub server_name: String, + pub id: RequestId, + pub message: String, + // TODO: MCP servers can request we fill out a schema for the elicitation. We don't support + // this yet. + // pub requested_schema: ElicitRequestParamsRequestedSchema, +} + +#[derive(Debug, Clone, Copy, Deserialize, Serialize, PartialEq, Eq, JsonSchema, TS)] +#[serde(rename_all = "lowercase")] +pub enum ElicitationAction { + Accept, + Decline, + Cancel, +} + #[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, TS)] pub struct ApplyPatchApprovalRequestEvent { /// Responses API call id for the associated patch apply call, if available. diff --git a/codex-rs/protocol/src/models.rs b/codex-rs/protocol/src/models.rs index 2e623eefef..daf98152d5 100644 --- a/codex-rs/protocol/src/models.rs +++ b/codex-rs/protocol/src/models.rs @@ -132,6 +132,7 @@ pub enum ResponseItem { GhostSnapshot { ghost_commit: GhostCommit, }, + #[serde(alias = "compaction")] CompactionSummary { encrypted_content: String, }, @@ -230,8 +231,24 @@ pub struct LocalShellExecAction { #[serde(tag = "type", rename_all = "snake_case")] pub enum WebSearchAction { Search { - query: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional)] + query: Option, }, + OpenPage { + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional)] + url: Option, + }, + FindInPage { + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional)] + url: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional)] + pattern: Option, + }, + #[serde(other)] Other, } @@ -521,6 +538,7 @@ mod tests { use anyhow::Result; use mcp_types::ImageContent; use mcp_types::TextContent; + use pretty_assertions::assert_eq; use tempfile::tempdir; #[test] @@ -634,6 +652,87 @@ mod tests { Ok(()) } + #[test] + fn deserializes_compaction_alias() -> Result<()> { + let json = r#"{"type":"compaction","encrypted_content":"abc"}"#; + + let item: ResponseItem = serde_json::from_str(json)?; + + assert_eq!( + item, + ResponseItem::CompactionSummary { + encrypted_content: "abc".into(), + } + ); + Ok(()) + } + + #[test] + fn roundtrips_web_search_call_actions() -> Result<()> { + let cases = vec![ + ( + r#"{ + "type": "web_search_call", + "status": "completed", + "action": { + "type": "search", + "query": "weather seattle" + } + }"#, + WebSearchAction::Search { + query: Some("weather seattle".into()), + }, + Some("completed".into()), + ), + ( + r#"{ + "type": "web_search_call", + "status": "open", + "action": { + "type": "open_page", + "url": "https://example.com" + } + }"#, + WebSearchAction::OpenPage { + url: Some("https://example.com".into()), + }, + Some("open".into()), + ), + ( + r#"{ + "type": "web_search_call", + "status": "in_progress", + "action": { + "type": "find_in_page", + "url": "https://example.com/docs", + "pattern": "installation" + } + }"#, + WebSearchAction::FindInPage { + url: Some("https://example.com/docs".into()), + pattern: Some("installation".into()), + }, + Some("in_progress".into()), + ), + ]; + + for (json_literal, expected_action, expected_status) in cases { + let parsed: ResponseItem = serde_json::from_str(json_literal)?; + let expected = ResponseItem::WebSearchCall { + id: None, + status: expected_status.clone(), + action: expected_action.clone(), + }; + assert_eq!(parsed, expected); + + let serialized = serde_json::to_value(&parsed)?; + let original_value: serde_json::Value = serde_json::from_str(json_literal)?; + assert_eq!(serialized, original_value); + } + + Ok(()) + } + #[test] fn deserialize_shell_tool_call_params() -> Result<()> { let json = r#"{ diff --git a/codex-rs/protocol/src/protocol.rs b/codex-rs/protocol/src/protocol.rs index a4eff723b6..53b2984723 100644 --- a/codex-rs/protocol/src/protocol.rs +++ b/codex-rs/protocol/src/protocol.rs @@ -11,6 +11,7 @@ use std::str::FromStr; use std::time::Duration; use crate::ConversationId; +use crate::approvals::ElicitationRequestEvent; use crate::config_types::ReasoningEffort as ReasoningEffortConfig; use crate::config_types::ReasoningSummary as ReasoningSummaryConfig; use crate::custom_prompts::CustomPrompt; @@ -23,6 +24,7 @@ use crate::parse_command::ParsedCommand; use crate::plan_tool::UpdatePlanArgs; use crate::user_input::UserInput; use mcp_types::CallToolResult; +use mcp_types::RequestId; use mcp_types::Resource as McpResource; use mcp_types::ResourceTemplate as McpResourceTemplate; use mcp_types::Tool as McpTool; @@ -35,6 +37,7 @@ use strum_macros::Display; use ts_rs::TS; pub use crate::approvals::ApplyPatchApprovalRequestEvent; +pub use crate::approvals::ElicitationAction; pub use crate::approvals::ExecApprovalRequestEvent; pub use crate::approvals::SandboxCommandAssessment; pub use crate::approvals::SandboxRiskLevel; @@ -156,6 +159,16 @@ pub enum Op { decision: ReviewDecision, }, + /// Resolve an MCP elicitation request. + ResolveElicitation { + /// Name of the MCP server that issued the request. + server_name: String, + /// Request identifier from the MCP server. + request_id: RequestId, + /// User's decision for the request. + decision: ElicitationAction, + }, + /// Append an entry to the persistent cross-session message history. /// /// Note the entry is not guaranteed to be logged if the user has @@ -445,6 +458,9 @@ pub enum EventMsg { /// indicates the task continued but the user should still be notified. Warning(WarningEvent), + /// Conversation history was compacted (either automatically or manually). + ContextCompacted(ContextCompactedEvent), + /// Agent has started a task TaskStarted(TaskStartedEvent), @@ -508,6 +524,8 @@ pub enum EventMsg { ExecApprovalRequest(ExecApprovalRequestEvent), + ElicitationRequest(ElicitationRequestEvent), + ApplyPatchApprovalRequest(ApplyPatchApprovalRequestEvent), /// Notification advising the user that something they are using has been @@ -568,6 +586,35 @@ pub enum EventMsg { ReasoningRawContentDelta(ReasoningRawContentDeltaEvent), } +/// Codex errors that we expose to clients. +#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq, JsonSchema, TS)] +#[serde(rename_all = "snake_case")] +#[ts(rename_all = "snake_case")] +pub enum CodexErrorInfo { + ContextWindowExceeded, + UsageLimitExceeded, + HttpConnectionFailed { + http_status_code: Option, + }, + /// Failed to connect to the response SSE stream. + ResponseStreamConnectionFailed { + http_status_code: Option, + }, + InternalServerError, + Unauthorized, + BadRequest, + SandboxError, + /// The response SSE stream disconnected in the middle of a turnbefore completion. + ResponseStreamDisconnected { + http_status_code: Option, + }, + /// Reached the retry limit for responses. + ResponseTooManyFailedAttempts { + http_status_code: Option, + }, + Other, +} + #[derive(Debug, Clone, Deserialize, Serialize, TS, JsonSchema)] pub struct RawResponseItemEvent { pub item: ResponseItem, @@ -692,6 +739,8 @@ pub struct ExitedReviewModeEvent { #[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, TS)] pub struct ErrorEvent { pub message: String, + #[serde(default)] + pub codex_error_info: Option, } #[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, TS)] @@ -699,6 +748,9 @@ pub struct WarningEvent { pub message: String, } +#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, TS)] +pub struct ContextCompactedEvent; + #[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, TS)] pub struct TaskCompleteEvent { pub last_agent_message: Option, @@ -1098,6 +1150,29 @@ pub enum SubAgentSource { Other(String), } +impl fmt::Display for SessionSource { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + SessionSource::Cli => f.write_str("cli"), + SessionSource::VSCode => f.write_str("vscode"), + SessionSource::Exec => f.write_str("exec"), + SessionSource::Mcp => f.write_str("mcp"), + SessionSource::SubAgent(sub_source) => write!(f, "subagent_{sub_source}"), + SessionSource::Unknown => f.write_str("unknown"), + } + } +} + +impl fmt::Display for SubAgentSource { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + SubAgentSource::Review => f.write_str("review"), + SubAgentSource::Compact => f.write_str("compact"), + SubAgentSource::Other(other) => f.write_str(other), + } + } +} + #[derive(Serialize, Deserialize, Clone, Debug, JsonSchema, TS)] pub struct SessionMeta { pub id: ConversationId, @@ -1197,13 +1272,47 @@ pub struct GitInfo { pub repository_url: Option, } +#[derive(Debug, Clone, Copy, Deserialize, Serialize, PartialEq, Eq, JsonSchema, TS)] +#[serde(rename_all = "snake_case")] +pub enum ReviewDelivery { + Inline, + Detached, +} + +#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, JsonSchema, TS)] +#[serde(tag = "type", rename_all = "camelCase")] +#[ts(tag = "type")] +pub enum ReviewTarget { + /// Review the working tree: staged, unstaged, and untracked files. + UncommittedChanges, + + /// Review changes between the current branch and the given base branch. + #[serde(rename_all = "camelCase")] + #[ts(rename_all = "camelCase")] + BaseBranch { branch: String }, + + /// Review the changes introduced by a specific commit. + #[serde(rename_all = "camelCase")] + #[ts(rename_all = "camelCase")] + Commit { + sha: String, + /// Optional human-readable label (e.g., commit subject) for UIs. + title: Option, + }, + + /// Arbitrary instructions provided by the user. + #[serde(rename_all = "camelCase")] + #[ts(rename_all = "camelCase")] + Custom { instructions: String }, +} + #[derive(Debug, Clone, Deserialize, Serialize, PartialEq, JsonSchema, TS)] /// Review request sent to the review session. pub struct ReviewRequest { - pub prompt: String, - pub user_facing_hint: String, - #[serde(default)] - pub append_to_original_thread: bool, + pub target: ReviewTarget, + #[serde(skip_serializing_if = "Option::is_none")] + #[ts(optional)] + pub user_facing_hint: Option, } /// Structured review result produced by a child review session. @@ -1269,6 +1378,10 @@ impl Default for ExecCommandSource { pub struct ExecCommandBeginEvent { /// Identifier so this can be paired with the ExecCommandEnd event. pub call_id: String, + /// Identifier for the underlying PTY process (when available). + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional)] + pub process_id: Option, /// Turn ID that this command belongs to. pub turn_id: String, /// The command to be executed. @@ -1289,6 +1402,10 @@ pub struct ExecCommandBeginEvent { pub struct ExecCommandEndEvent { /// Identifier for the ExecCommandBegin that finished. pub call_id: String, + /// Identifier for the underlying PTY process (when available). + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional)] + pub process_id: Option, /// Turn ID that this command belongs to. pub turn_id: String, /// The command that was executed. @@ -1379,6 +1496,8 @@ pub struct UndoCompletedEvent { #[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, TS)] pub struct StreamErrorEvent { pub message: String, + #[serde(default)] + pub codex_error_info: Option, } #[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, TS)] diff --git a/codex-rs/responses-api-proxy/Cargo.toml b/codex-rs/responses-api-proxy/Cargo.toml index 651019a816..e0ea600033 100644 --- a/codex-rs/responses-api-proxy/Cargo.toml +++ b/codex-rs/responses-api-proxy/Cargo.toml @@ -1,7 +1,8 @@ [package] -edition = "2024" name = "codex-responses-api-proxy" -version = { workspace = true } +version.workspace = true +edition.workspace = true +license.workspace = true [lib] name = "codex_responses_api_proxy" diff --git a/codex-rs/rmcp-client/Cargo.toml b/codex-rs/rmcp-client/Cargo.toml index 5c3f1dc0c8..f2525c18d3 100644 --- a/codex-rs/rmcp-client/Cargo.toml +++ b/codex-rs/rmcp-client/Cargo.toml @@ -1,7 +1,8 @@ [package] -edition = "2024" name = "codex-rmcp-client" -version = { workspace = true } +version.workspace = true +edition.workspace = true +license.workspace = true [lints] workspace = true diff --git a/codex-rs/rmcp-client/src/lib.rs b/codex-rs/rmcp-client/src/lib.rs index 87ce86b464..ac617f3d29 100644 --- a/codex-rs/rmcp-client/src/lib.rs +++ b/codex-rs/rmcp-client/src/lib.rs @@ -17,4 +17,8 @@ pub use oauth::delete_oauth_tokens; pub(crate) use oauth::load_oauth_tokens; pub use oauth::save_oauth_tokens; pub use perform_oauth_login::perform_oauth_login; +pub use rmcp::model::ElicitationAction; +pub use rmcp_client::Elicitation; +pub use rmcp_client::ElicitationResponse; pub use rmcp_client::RmcpClient; +pub use rmcp_client::SendElicitation; diff --git a/codex-rs/rmcp-client/src/logging_client_handler.rs b/codex-rs/rmcp-client/src/logging_client_handler.rs index 85d237b0e9..0d2c3aaa97 100644 --- a/codex-rs/rmcp-client/src/logging_client_handler.rs +++ b/codex-rs/rmcp-client/src/logging_client_handler.rs @@ -1,13 +1,15 @@ +use std::sync::Arc; + use rmcp::ClientHandler; use rmcp::RoleClient; use rmcp::model::CancelledNotificationParam; use rmcp::model::ClientInfo; use rmcp::model::CreateElicitationRequestParam; use rmcp::model::CreateElicitationResult; -use rmcp::model::ElicitationAction; use rmcp::model::LoggingLevel; use rmcp::model::LoggingMessageNotificationParam; use rmcp::model::ProgressNotificationParam; +use rmcp::model::RequestId; use rmcp::model::ResourceUpdatedNotificationParam; use rmcp::service::NotificationContext; use rmcp::service::RequestContext; @@ -16,32 +18,36 @@ use tracing::error; use tracing::info; use tracing::warn; -#[derive(Debug, Clone)] +use crate::rmcp_client::SendElicitation; + +#[derive(Clone)] pub(crate) struct LoggingClientHandler { client_info: ClientInfo, + send_elicitation: Arc, } impl LoggingClientHandler { - pub(crate) fn new(client_info: ClientInfo) -> Self { - Self { client_info } + pub(crate) fn new(client_info: ClientInfo, send_elicitation: SendElicitation) -> Self { + Self { + client_info, + send_elicitation: Arc::new(send_elicitation), + } } } impl ClientHandler for LoggingClientHandler { - // TODO (CODEX-3571): support elicitations. async fn create_elicitation( &self, request: CreateElicitationRequestParam, - _context: RequestContext, + context: RequestContext, ) -> Result { - info!( - "MCP server requested elicitation ({}). Elicitations are not supported yet. Declining.", - request.message - ); - Ok(CreateElicitationResult { - action: ElicitationAction::Decline, - content: None, - }) + let id = match context.id { + RequestId::String(id) => mcp_types::RequestId::String(id.to_string()), + RequestId::Number(id) => mcp_types::RequestId::Integer(id), + }; + (self.send_elicitation)(id, request) + .await + .map_err(|err| rmcp::ErrorData::internal_error(err.to_string(), None)) } async fn on_cancelled( diff --git a/codex-rs/rmcp-client/src/rmcp_client.rs b/codex-rs/rmcp-client/src/rmcp_client.rs index d7d3477b00..bcf7b49e93 100644 --- a/codex-rs/rmcp-client/src/rmcp_client.rs +++ b/codex-rs/rmcp-client/src/rmcp_client.rs @@ -9,6 +9,7 @@ use std::time::Duration; use anyhow::Result; use anyhow::anyhow; use futures::FutureExt; +use futures::future::BoxFuture; use mcp_types::CallToolRequestParams; use mcp_types::CallToolResult; use mcp_types::InitializeRequestParams; @@ -21,8 +22,14 @@ use mcp_types::ListToolsRequestParams; use mcp_types::ListToolsResult; use mcp_types::ReadResourceRequestParams; use mcp_types::ReadResourceResult; +use mcp_types::RequestId; use reqwest::header::HeaderMap; use rmcp::model::CallToolRequestParam; +use rmcp::model::ClientNotification; +use rmcp::model::CreateElicitationRequestParam; +use rmcp::model::CreateElicitationResult; +use rmcp::model::CustomClientNotification; +use rmcp::model::Extensions; use rmcp::model::InitializeRequestParam; use rmcp::model::PaginatedRequestParam; use rmcp::model::ReadResourceRequestParam; @@ -77,6 +84,14 @@ enum ClientState { }, } +pub type Elicitation = CreateElicitationRequestParam; +pub type ElicitationResponse = CreateElicitationResult; + +/// Interface for sending elicitation requests to the UI and awaiting a response. +pub type SendElicitation = Box< + dyn Fn(RequestId, Elicitation) -> BoxFuture<'static, Result> + Send + Sync, +>; + /// MCP client implemented on top of the official `rmcp` SDK. /// https://github.com/modelcontextprotocol/rust-sdk pub struct RmcpClient { @@ -200,9 +215,10 @@ impl RmcpClient { &self, params: InitializeRequestParams, timeout: Option, + send_elicitation: SendElicitation, ) -> Result { let rmcp_params: InitializeRequestParam = convert_to_rmcp(params.clone())?; - let client_handler = LoggingClientHandler::new(rmcp_params); + let client_handler = LoggingClientHandler::new(rmcp_params, send_elicitation); let (transport, oauth_persistor) = { let mut guard = self.state.lock().await; @@ -348,6 +364,25 @@ impl RmcpClient { Ok(converted) } + pub async fn send_custom_notification( + &self, + method: &str, + params: Option, + ) -> Result<()> { + let service: Arc> = self.service().await?; + service.service(); + service + .send_notification(ClientNotification::CustomClientNotification( + CustomClientNotification { + method: method.to_string(), + params, + extensions: Extensions::new(), + }, + )) + .await?; + Ok(()) + } + async fn service(&self) -> Result>> { let guard = self.state.lock().await; match &*guard { diff --git a/codex-rs/rmcp-client/tests/resources.rs b/codex-rs/rmcp-client/tests/resources.rs index 2117f9b14c..fda21d14e2 100644 --- a/codex-rs/rmcp-client/tests/resources.rs +++ b/codex-rs/rmcp-client/tests/resources.rs @@ -2,8 +2,11 @@ use std::ffi::OsString; use std::path::PathBuf; use std::time::Duration; +use codex_rmcp_client::ElicitationAction; +use codex_rmcp_client::ElicitationResponse; use codex_rmcp_client::RmcpClient; use escargot::CargoBuild; +use futures::FutureExt as _; use mcp_types::ClientCapabilities; use mcp_types::Implementation; use mcp_types::InitializeRequestParams; @@ -55,7 +58,19 @@ async fn rmcp_client_can_list_and_read_resources() -> anyhow::Result<()> { .await?; client - .initialize(init_params(), Some(Duration::from_secs(5))) + .initialize( + init_params(), + Some(Duration::from_secs(5)), + Box::new(|_, _| { + async { + Ok(ElicitationResponse { + action: ElicitationAction::Accept, + content: Some(json!({})), + }) + } + .boxed() + }), + ) .await?; let list = client diff --git a/codex-rs/scripts/create_github_release b/codex-rs/scripts/create_github_release index 47d6d3ce8c..e7b8972db5 100755 --- a/codex-rs/scripts/create_github_release +++ b/codex-rs/scripts/create_github_release @@ -298,8 +298,12 @@ def create_tag_ref(version: str, tag_sha: str) -> None: def determine_version(args: argparse.Namespace) -> str: latest_version = get_latest_release_version() - major, minor, patch = parse_semver(latest_version) - next_minor_version = format_version(major, minor + 1, patch) + # When determining the next version after the current release, + # we should always increment the minor version, but reset the + # patch to zero. In practice, `patch` should only be non-zero if + # --emergency-version-override was used. + major, minor, _patch = parse_semver(latest_version) + next_minor_version = format_version(major, minor + 1, 0) if args.publish_release: return next_minor_version diff --git a/codex-rs/stdio-to-uds/Cargo.toml b/codex-rs/stdio-to-uds/Cargo.toml index 4f7132883d..71ed3722c3 100644 --- a/codex-rs/stdio-to-uds/Cargo.toml +++ b/codex-rs/stdio-to-uds/Cargo.toml @@ -1,7 +1,8 @@ [package] -edition = "2024" name = "codex-stdio-to-uds" -version = { workspace = true } +version.workspace = true +edition.workspace = true +license.workspace = true [[bin]] name = "codex-stdio-to-uds" diff --git a/codex-rs/tui/Cargo.toml b/codex-rs/tui/Cargo.toml index 01fef9cc74..be4f5aead7 100644 --- a/codex-rs/tui/Cargo.toml +++ b/codex-rs/tui/Cargo.toml @@ -1,7 +1,8 @@ [package] -edition = "2024" name = "codex-tui" -version = { workspace = true } +version.workspace = true +edition.workspace = true +license.workspace = true [[bin]] name = "codex-tui" @@ -90,6 +91,7 @@ unicode-width = { workspace = true } url = { workspace = true } codex-windows-sandbox = { workspace = true } +tokio-util = { workspace = true, features = ["time"] } [target.'cfg(unix)'.dependencies] libc = { workspace = true } diff --git a/codex-rs/tui/src/app.rs b/codex-rs/tui/src/app.rs index 749a145b75..02c6c919d2 100644 --- a/codex-rs/tui/src/app.rs +++ b/codex-rs/tui/src/app.rs @@ -16,6 +16,8 @@ use crate::pager_overlay::Overlay; use crate::render::highlight::highlight_bash_to_lines; use crate::render::renderable::Renderable; use crate::resume_picker::ResumeSelection; +use crate::skill_error_prompt::SkillErrorPromptOutcome; +use crate::skill_error_prompt::run_skill_error_prompt; use crate::tui; use crate::tui::TuiEvent; use crate::update_action::UpdateAction; @@ -32,12 +34,13 @@ use codex_core::config::edit::ConfigEditsBuilder; #[cfg(target_os = "windows")] use codex_core::features::Feature; use codex_core::model_family::find_family_for_model; +use codex_core::protocol::EventMsg; use codex_core::protocol::FinalOutput; -#[cfg(target_os = "windows")] use codex_core::protocol::Op; use codex_core::protocol::SessionSource; use codex_core::protocol::TokenUsage; use codex_core::protocol_config_types::ReasoningEffort as ReasoningEffortConfig; +use codex_core::skills::load_skills; use codex_protocol::ConversationId; use color_eyre::eyre::Result; use color_eyre::eyre::WrapErr; @@ -46,6 +49,8 @@ use crossterm::event::KeyEvent; use crossterm::event::KeyEventKind; use ratatui::style::Stylize; use ratatui::text::Line; +use ratatui::widgets::Paragraph; +use ratatui::widgets::Wrap; use std::path::PathBuf; use std::sync::Arc; use std::sync::atomic::AtomicBool; @@ -219,11 +224,23 @@ pub(crate) struct App { /// Set when the user confirms an update; propagated on exit. pub(crate) pending_update_action: Option, + /// Ignore the next ShutdownComplete event when we're intentionally + /// stopping a conversation (e.g., before starting a new one). + suppress_shutdown_complete: bool, + // One-shot suppression of the next world-writable scan after user confirmation. skip_world_writable_scan_once: bool, } impl App { + async fn shutdown_current_conversation(&mut self) { + if let Some(conversation_id) = self.chat_widget.conversation_id() { + self.suppress_shutdown_complete = true; + self.chat_widget.submit_op(Op::Shutdown); + self.server.remove_conversation(&conversation_id).await; + } + } + #[allow(clippy::too_many_arguments)] pub async fn run( tui: &mut tui::Tui, @@ -234,6 +251,7 @@ impl App { initial_images: Vec, resume_selection: ResumeSelection, feedback: codex_feedback::CodexFeedback, + is_first_run: bool, ) -> Result { use tokio_stream::StreamExt; let (app_event_tx, mut app_event_rx) = unbounded_channel(); @@ -252,6 +270,20 @@ impl App { SessionSource::Cli, )); + let skills_outcome = load_skills(&config); + if !skills_outcome.errors.is_empty() { + match run_skill_error_prompt(tui, &skills_outcome.errors).await { + SkillErrorPromptOutcome::Exit => { + return Ok(AppExitInfo { + token_usage: TokenUsage::default(), + conversation_id: None, + update_action: None, + }); + } + SkillErrorPromptOutcome::Continue => {} + } + } + let enhanced_keys_supported = tui.enhanced_keys_supported(); let mut chat_widget = match resume_selection { @@ -265,6 +297,7 @@ impl App { enhanced_keys_supported, auth_manager: auth_manager.clone(), feedback: feedback.clone(), + is_first_run, }; ChatWidget::new(init, conversation_manager.clone()) } @@ -288,6 +321,7 @@ impl App { enhanced_keys_supported, auth_manager: auth_manager.clone(), feedback: feedback.clone(), + is_first_run, }; ChatWidget::new_from_existing( init, @@ -341,6 +375,7 @@ impl App { backtrack: BacktrackState::default(), feedback: feedback.clone(), pending_update_action: None, + suppress_shutdown_complete: false, skip_world_writable_scan_once: false, }; @@ -363,7 +398,8 @@ impl App { let env_map: std::collections::HashMap = std::env::vars().collect(); let tx = app.app_event_tx.clone(); let logs_base_dir = app.config.codex_home.clone(); - Self::spawn_world_writable_scan(cwd, env_map, logs_base_dir, tx); + let sandbox_policy = app.config.sandbox_policy.clone(); + Self::spawn_world_writable_scan(cwd, env_map, logs_base_dir, sandbox_policy, tx); } } @@ -450,6 +486,7 @@ impl App { self.chat_widget.token_usage(), self.chat_widget.conversation_id(), ); + self.shutdown_current_conversation().await; let init = crate::chatwidget::ChatWidgetInit { config: self.config.clone(), frame_requester: tui.frame_requester(), @@ -459,6 +496,7 @@ impl App { enhanced_keys_supported: self.enhanced_keys_supported, auth_manager: self.auth_manager.clone(), feedback: self.feedback.clone(), + is_first_run: false, }; self.chat_widget = ChatWidget::new(init, self.server.clone()); if let Some(summary) = summary { @@ -520,6 +558,12 @@ impl App { self.chat_widget.on_commit_tick(); } AppEvent::CodexEvent(event) => { + if self.suppress_shutdown_complete + && matches!(event.msg, EventMsg::ShutdownComplete) + { + self.suppress_shutdown_complete = false; + return Ok(true); + } self.chat_widget.handle_codex_event(event); } AppEvent::ConversationHistory(ev) => { @@ -737,7 +781,14 @@ impl App { std::env::vars().collect(); let tx = self.app_event_tx.clone(); let logs_base_dir = self.config.codex_home.clone(); - Self::spawn_world_writable_scan(cwd, env_map, logs_base_dir, tx); + let sandbox_policy = self.config.sandbox_policy.clone(); + Self::spawn_world_writable_scan( + cwd, + env_map, + logs_base_dir, + sandbox_policy, + tx, + ); } } } @@ -841,6 +892,23 @@ impl App { "E X E C".to_string(), )); } + ApprovalRequest::McpElicitation { + server_name, + message, + .. + } => { + let _ = tui.enter_alt_screen(); + let paragraph = Paragraph::new(vec![ + Line::from(vec!["Server: ".into(), server_name.bold()]), + Line::from(""), + Line::from(message), + ]) + .wrap(Wrap { trim: false }); + self.overlay = Some(Overlay::new_static_with_renderables( + vec![Box::new(paragraph)], + "E L I C I T A T I O N".to_string(), + )); + } }, } Ok(true) @@ -931,47 +999,23 @@ impl App { cwd: PathBuf, env_map: std::collections::HashMap, logs_base_dir: PathBuf, + sandbox_policy: codex_core::protocol::SandboxPolicy, tx: AppEventSender, ) { - #[inline] - fn normalize_windows_path_for_display(p: &std::path::Path) -> String { - let canon = dunce::canonicalize(p).unwrap_or_else(|_| p.to_path_buf()); - canon.display().to_string().replace('/', "\\") - } tokio::task::spawn_blocking(move || { - let result = codex_windows_sandbox::preflight_audit_everyone_writable( + let result = codex_windows_sandbox::apply_world_writable_scan_and_denies( + &logs_base_dir, &cwd, &env_map, + &sandbox_policy, Some(logs_base_dir.as_path()), ); - if let Ok(ref paths) = result - && !paths.is_empty() - { - let as_strings: Vec = paths - .iter() - .map(|p| normalize_windows_path_for_display(p)) - .collect(); - let sample_paths: Vec = as_strings.iter().take(3).cloned().collect(); - let extra_count = if as_strings.len() > sample_paths.len() { - as_strings.len() - sample_paths.len() - } else { - 0 - }; - + if result.is_err() { + // Scan failed: warn without examples. tx.send(AppEvent::OpenWorldWritableWarningConfirmation { preset: None, - sample_paths, - extra_count, - failed_scan: false, - }); - } else if result.is_err() { - // Scan failed: still warn, but with no examples and mark as failed. - let sample_paths: Vec = Vec::new(); - let extra_count = 0usize; - tx.send(AppEvent::OpenWorldWritableWarningConfirmation { - preset: None, - sample_paths, - extra_count, + sample_paths: Vec::new(), + extra_count: 0usize, failed_scan: true, }); } @@ -1016,6 +1060,8 @@ mod tests { use codex_core::CodexAuth; use codex_core::ConversationManager; use codex_core::protocol::AskForApproval; + use codex_core::protocol::Event; + use codex_core::protocol::EventMsg; use codex_core::protocol::SandboxPolicy; use codex_core::protocol::SessionConfiguredEvent; use codex_protocol::ConversationId; @@ -1051,10 +1097,51 @@ mod tests { backtrack: BacktrackState::default(), feedback: codex_feedback::CodexFeedback::new(), pending_update_action: None, + suppress_shutdown_complete: false, skip_world_writable_scan_once: false, } } + fn make_test_app_with_channels() -> ( + App, + tokio::sync::mpsc::UnboundedReceiver, + tokio::sync::mpsc::UnboundedReceiver, + ) { + let (chat_widget, app_event_tx, rx, op_rx) = make_chatwidget_manual_with_sender(); + let config = chat_widget.config_ref().clone(); + let server = Arc::new(ConversationManager::with_auth(CodexAuth::from_api_key( + "Test API Key", + ))); + let auth_manager = + AuthManager::from_auth_for_testing(CodexAuth::from_api_key("Test API Key")); + let file_search = FileSearchManager::new(config.cwd.clone(), app_event_tx.clone()); + + ( + App { + server, + app_event_tx, + chat_widget, + auth_manager, + config, + active_profile: None, + file_search, + transcript_cells: Vec::new(), + overlay: None, + deferred_history_lines: Vec::new(), + has_emitted_history_lines: false, + enhanced_keys_supported: false, + commit_anim_running: Arc::new(AtomicBool::new(false)), + backtrack: BacktrackState::default(), + feedback: codex_feedback::CodexFeedback::new(), + pending_update_action: None, + suppress_shutdown_complete: false, + skip_world_writable_scan_once: false, + }, + rx, + op_rx, + ) + } + #[test] fn model_migration_prompt_only_shows_for_deprecated_models() { assert!(should_show_model_migration_prompt("gpt-5", "gpt-5.1", None)); @@ -1177,6 +1264,42 @@ mod tests { assert_eq!(prefill, "follow-up (edited)"); } + #[tokio::test] + async fn new_session_requests_shutdown_for_previous_conversation() { + let (mut app, mut app_event_rx, mut op_rx) = make_test_app_with_channels(); + + let conversation_id = ConversationId::new(); + let event = SessionConfiguredEvent { + session_id: conversation_id, + model: "gpt-test".to_string(), + model_provider_id: "test-provider".to_string(), + approval_policy: AskForApproval::Never, + sandbox_policy: SandboxPolicy::ReadOnly, + cwd: PathBuf::from("/home/user/project"), + reasoning_effort: None, + history_log_id: 0, + history_entry_count: 0, + initial_messages: None, + rollout_path: PathBuf::new(), + }; + + app.chat_widget.handle_codex_event(Event { + id: String::new(), + msg: EventMsg::SessionConfigured(event), + }); + + while app_event_rx.try_recv().is_ok() {} + while op_rx.try_recv().is_ok() {} + + app.shutdown_current_conversation().await; + + match op_rx.try_recv() { + Ok(Op::Shutdown) => {} + Ok(other) => panic!("expected Op::Shutdown, got {other:?}"), + Err(_) => panic!("expected shutdown op to be sent"), + } + } + #[test] fn session_summary_skip_zero_usage() { assert!(session_summary(TokenUsage::default(), None).is_none()); diff --git a/codex-rs/tui/src/app_backtrack.rs b/codex-rs/tui/src/app_backtrack.rs index b161867e44..677f29abdd 100644 --- a/codex-rs/tui/src/app_backtrack.rs +++ b/codex-rs/tui/src/app_backtrack.rs @@ -347,6 +347,7 @@ impl App { enhanced_keys_supported: self.enhanced_keys_supported, auth_manager: self.auth_manager.clone(), feedback: self.feedback.clone(), + is_first_run: false, }; self.chat_widget = crate::chatwidget::ChatWidget::new_from_existing(init, conv, session_configured); diff --git a/codex-rs/tui/src/bottom_pane/approval_overlay.rs b/codex-rs/tui/src/bottom_pane/approval_overlay.rs index ef709f0051..c6006a9ae7 100644 --- a/codex-rs/tui/src/bottom_pane/approval_overlay.rs +++ b/codex-rs/tui/src/bottom_pane/approval_overlay.rs @@ -16,6 +16,7 @@ use crate::key_hint::KeyBinding; use crate::render::highlight::highlight_bash_to_lines; use crate::render::renderable::ColumnRenderable; use crate::render::renderable::Renderable; +use codex_core::protocol::ElicitationAction; use codex_core::protocol::FileChange; use codex_core::protocol::Op; use codex_core::protocol::ReviewDecision; @@ -25,6 +26,7 @@ use crossterm::event::KeyCode; use crossterm::event::KeyEvent; use crossterm::event::KeyEventKind; use crossterm::event::KeyModifiers; +use mcp_types::RequestId; use ratatui::buffer::Buffer; use ratatui::layout::Rect; use ratatui::style::Stylize; @@ -48,6 +50,11 @@ pub(crate) enum ApprovalRequest { cwd: PathBuf, changes: HashMap, }, + McpElicitation { + server_name: String, + request_id: RequestId, + message: String, + }, } /// Modal overlay asking the user to approve or deny one or more requests. @@ -105,6 +112,10 @@ impl ApprovalOverlay { patch_options(), "Would you like to make the following edits?".to_string(), ), + ApprovalVariant::McpElicitation { server_name, .. } => ( + elicitation_options(), + format!("{server_name} needs your approval."), + ), }; let header = Box::new(ColumnRenderable::with([ @@ -149,13 +160,23 @@ impl ApprovalOverlay { return; }; if let Some(variant) = self.current_variant.as_ref() { - match (&variant, option.decision) { - (ApprovalVariant::Exec { id, command }, decision) => { - self.handle_exec_decision(id, command, decision); + match (&variant, &option.decision) { + (ApprovalVariant::Exec { id, command }, ApprovalDecision::Review(decision)) => { + self.handle_exec_decision(id, command, *decision); } - (ApprovalVariant::ApplyPatch { id, .. }, decision) => { - self.handle_patch_decision(id, decision); + (ApprovalVariant::ApplyPatch { id, .. }, ApprovalDecision::Review(decision)) => { + self.handle_patch_decision(id, *decision); } + ( + ApprovalVariant::McpElicitation { + server_name, + request_id, + }, + ApprovalDecision::McpElicitation(decision), + ) => { + self.handle_elicitation_decision(server_name, request_id, *decision); + } + _ => {} } } @@ -179,6 +200,20 @@ impl ApprovalOverlay { })); } + fn handle_elicitation_decision( + &self, + server_name: &str, + request_id: &RequestId, + decision: ElicitationAction, + ) { + self.app_event_tx + .send(AppEvent::CodexOp(Op::ResolveElicitation { + server_name: server_name.to_string(), + request_id: request_id.clone(), + decision, + })); + } + fn advance_queue(&mut self) { if let Some(next) = self.queue.pop() { self.set_current(next); @@ -244,6 +279,16 @@ impl BottomPaneView for ApprovalOverlay { ApprovalVariant::ApplyPatch { id, .. } => { self.handle_patch_decision(id, ReviewDecision::Abort); } + ApprovalVariant::McpElicitation { + server_name, + request_id, + } => { + self.handle_elicitation_decision( + server_name, + request_id, + ElicitationAction::Cancel, + ); + } } } self.queue.clear(); @@ -336,6 +381,25 @@ impl From for ApprovalRequestState { header: Box::new(ColumnRenderable::with(header)), } } + ApprovalRequest::McpElicitation { + server_name, + request_id, + message, + } => { + let header = Paragraph::new(vec![ + Line::from(vec!["Server: ".into(), server_name.clone().bold()]), + Line::from(""), + Line::from(message), + ]) + .wrap(Wrap { trim: false }); + Self { + variant: ApprovalVariant::McpElicitation { + server_name, + request_id, + }, + header: Box::new(header), + } + } } } } @@ -364,14 +428,29 @@ fn render_risk_lines(risk: &SandboxCommandAssessment) -> Vec> { #[derive(Clone)] enum ApprovalVariant { - Exec { id: String, command: Vec }, - ApplyPatch { id: String }, + Exec { + id: String, + command: Vec, + }, + ApplyPatch { + id: String, + }, + McpElicitation { + server_name: String, + request_id: RequestId, + }, +} + +#[derive(Clone)] +enum ApprovalDecision { + Review(ReviewDecision), + McpElicitation(ElicitationAction), } #[derive(Clone)] struct ApprovalOption { label: String, - decision: ReviewDecision, + decision: ApprovalDecision, display_shortcut: Option, additional_shortcuts: Vec, } @@ -388,19 +467,19 @@ fn exec_options() -> Vec { vec![ ApprovalOption { label: "Yes, proceed".to_string(), - decision: ReviewDecision::Approved, + decision: ApprovalDecision::Review(ReviewDecision::Approved), display_shortcut: None, additional_shortcuts: vec![key_hint::plain(KeyCode::Char('y'))], }, ApprovalOption { label: "Yes, and don't ask again for this command".to_string(), - decision: ReviewDecision::ApprovedForSession, + decision: ApprovalDecision::Review(ReviewDecision::ApprovedForSession), display_shortcut: None, additional_shortcuts: vec![key_hint::plain(KeyCode::Char('a'))], }, ApprovalOption { label: "No, and tell Codex what to do differently".to_string(), - decision: ReviewDecision::Abort, + decision: ApprovalDecision::Review(ReviewDecision::Abort), display_shortcut: Some(key_hint::plain(KeyCode::Esc)), additional_shortcuts: vec![key_hint::plain(KeyCode::Char('n'))], }, @@ -411,19 +490,42 @@ fn patch_options() -> Vec { vec![ ApprovalOption { label: "Yes, proceed".to_string(), - decision: ReviewDecision::Approved, + decision: ApprovalDecision::Review(ReviewDecision::Approved), display_shortcut: None, additional_shortcuts: vec![key_hint::plain(KeyCode::Char('y'))], }, ApprovalOption { label: "No, and tell Codex what to do differently".to_string(), - decision: ReviewDecision::Abort, + decision: ApprovalDecision::Review(ReviewDecision::Abort), display_shortcut: Some(key_hint::plain(KeyCode::Esc)), additional_shortcuts: vec![key_hint::plain(KeyCode::Char('n'))], }, ] } +fn elicitation_options() -> Vec { + vec![ + ApprovalOption { + label: "Yes, provide the requested info".to_string(), + decision: ApprovalDecision::McpElicitation(ElicitationAction::Accept), + display_shortcut: None, + additional_shortcuts: vec![key_hint::plain(KeyCode::Char('y'))], + }, + ApprovalOption { + label: "No, but continue without it".to_string(), + decision: ApprovalDecision::McpElicitation(ElicitationAction::Decline), + display_shortcut: None, + additional_shortcuts: vec![key_hint::plain(KeyCode::Char('n'))], + }, + ApprovalOption { + label: "Cancel this request".to_string(), + decision: ApprovalDecision::McpElicitation(ElicitationAction::Cancel), + display_shortcut: Some(key_hint::plain(KeyCode::Esc)), + additional_shortcuts: vec![key_hint::plain(KeyCode::Char('c'))], + }, + ] +} + #[cfg(test)] mod tests { use super::*; diff --git a/codex-rs/tui/src/bottom_pane/chat_composer.rs b/codex-rs/tui/src/bottom_pane/chat_composer.rs index d956c70c4f..8a372fd05f 100644 --- a/codex-rs/tui/src/bottom_pane/chat_composer.rs +++ b/codex-rs/tui/src/bottom_pane/chat_composer.rs @@ -104,6 +104,7 @@ pub(crate) struct ChatComposer { dismissed_file_popup_token: Option, current_file_query: Option, pending_pastes: Vec<(String, String)>, + large_paste_counters: HashMap, has_focus: bool, attached_images: Vec, placeholder_text: String, @@ -116,6 +117,7 @@ pub(crate) struct ChatComposer { footer_mode: FooterMode, footer_hint_override: Option>, context_window_percent: Option, + context_window_used_tokens: Option, } /// Popup state – at most one can be visible at any time. @@ -149,6 +151,7 @@ impl ChatComposer { dismissed_file_popup_token: None, current_file_query: None, pending_pastes: Vec::new(), + large_paste_counters: HashMap::new(), has_focus: has_input_focus, attached_images: Vec::new(), placeholder_text, @@ -159,6 +162,7 @@ impl ChatComposer { footer_mode: FooterMode::ShortcutSummary, footer_hint_override: None, context_window_percent: None, + context_window_used_tokens: None, }; // Apply configuration via the setter to keep side-effects centralized. this.set_disable_paste_burst(disable_paste_burst); @@ -223,7 +227,7 @@ impl ChatComposer { pub fn handle_paste(&mut self, pasted: String) -> bool { let char_count = pasted.chars().count(); if char_count > LARGE_PASTE_CHAR_THRESHOLD { - let placeholder = format!("[Pasted Content {char_count} chars]"); + let placeholder = self.next_large_paste_placeholder(char_count); self.textarea.insert_element(&placeholder); self.pending_pastes.push((placeholder, pasted)); } else if char_count > 1 && self.handle_paste_image_path(pasted.clone()) { @@ -376,6 +380,17 @@ impl ChatComposer { self.set_has_focus(has_focus); } + fn next_large_paste_placeholder(&mut self, char_count: usize) -> String { + let base = format!("[Pasted Content {char_count} chars]"); + let next_suffix = self.large_paste_counters.entry(char_count).or_insert(0); + *next_suffix += 1; + if *next_suffix == 1 { + base + } else { + format!("{base} #{next_suffix}") + } + } + pub(crate) fn insert_str(&mut self, text: &str) { self.textarea.insert_str(text); self.sync_command_popup(); @@ -970,21 +985,17 @@ impl ChatComposer { return (InputResult::None, true); } } - // If we have pending placeholder pastes, submit immediately to expand them. + // If we have pending placeholder pastes, replace them in the textarea text + // and continue to the normal submission flow to handle slash commands. if !self.pending_pastes.is_empty() { let mut text = self.textarea.text().to_string(); - self.textarea.set_text(""); for (placeholder, actual) in &self.pending_pastes { if text.contains(placeholder) { text = text.replace(placeholder, actual); } } + self.textarea.set_text(&text); self.pending_pastes.clear(); - if text.is_empty() { - return (InputResult::None, true); - } - self.history.record_local_submission(&text); - return (InputResult::Submitted(text), true); } // During a paste-like burst, treat Enter as a newline instead of submit. @@ -1407,6 +1418,7 @@ impl ChatComposer { use_shift_enter_hint: self.use_shift_enter_hint, is_task_running: self.is_task_running, context_window_percent: self.context_window_percent, + context_window_used_tokens: self.context_window_used_tokens, } } @@ -1537,10 +1549,13 @@ impl ChatComposer { self.is_task_running = running; } - pub(crate) fn set_context_window_percent(&mut self, percent: Option) { - if self.context_window_percent != percent { - self.context_window_percent = percent; + pub(crate) fn set_context_window(&mut self, percent: Option, used_tokens: Option) { + if self.context_window_percent == percent && self.context_window_used_tokens == used_tokens + { + return; } + self.context_window_percent = percent; + self.context_window_used_tokens = used_tokens; } pub(crate) fn set_esc_backtrack_hint(&mut self, show: bool) { @@ -2726,6 +2741,83 @@ mod tests { ); } + #[test] + fn deleting_duplicate_length_pastes_removes_only_target() { + use crossterm::event::KeyCode; + use crossterm::event::KeyEvent; + use crossterm::event::KeyModifiers; + + let (tx, _rx) = unbounded_channel::(); + let sender = AppEventSender::new(tx); + let mut composer = ChatComposer::new( + true, + sender, + false, + "Ask Codex to do anything".to_string(), + false, + ); + + let paste = "x".repeat(LARGE_PASTE_CHAR_THRESHOLD + 4); + let placeholder_base = format!("[Pasted Content {} chars]", paste.chars().count()); + let placeholder_second = format!("{placeholder_base} #2"); + + composer.handle_paste(paste.clone()); + composer.handle_paste(paste.clone()); + assert_eq!( + composer.textarea.text(), + format!("{placeholder_base}{placeholder_second}") + ); + assert_eq!(composer.pending_pastes.len(), 2); + + composer.textarea.set_cursor(composer.textarea.text().len()); + composer.handle_key_event(KeyEvent::new(KeyCode::Backspace, KeyModifiers::NONE)); + + assert_eq!(composer.textarea.text(), placeholder_base); + assert_eq!(composer.pending_pastes.len(), 1); + assert_eq!(composer.pending_pastes[0].0, placeholder_base); + assert_eq!(composer.pending_pastes[0].1, paste); + } + + #[test] + fn large_paste_numbering_does_not_reuse_after_deletion() { + use crossterm::event::KeyCode; + use crossterm::event::KeyEvent; + use crossterm::event::KeyModifiers; + + let (tx, _rx) = unbounded_channel::(); + let sender = AppEventSender::new(tx); + let mut composer = ChatComposer::new( + true, + sender, + false, + "Ask Codex to do anything".to_string(), + false, + ); + + let paste = "x".repeat(LARGE_PASTE_CHAR_THRESHOLD + 4); + let base = format!("[Pasted Content {} chars]", paste.chars().count()); + let second = format!("{base} #2"); + let third = format!("{base} #3"); + + composer.handle_paste(paste.clone()); + composer.handle_paste(paste.clone()); + assert_eq!(composer.textarea.text(), format!("{base}{second}")); + + composer.textarea.set_cursor(base.len()); + composer.handle_key_event(KeyEvent::new(KeyCode::Backspace, KeyModifiers::NONE)); + assert_eq!(composer.textarea.text(), second); + assert_eq!(composer.pending_pastes.len(), 1); + assert_eq!(composer.pending_pastes[0].0, second); + + composer.textarea.set_cursor(composer.textarea.text().len()); + composer.handle_paste(paste); + + assert_eq!(composer.textarea.text(), format!("{second}{third}")); + assert_eq!(composer.pending_pastes.len(), 2); + assert_eq!(composer.pending_pastes[0].0, second); + assert_eq!(composer.pending_pastes[1].0, third); + } + #[test] fn test_partial_placeholder_deletion() { use crossterm::event::KeyCode; @@ -3087,6 +3179,70 @@ mod tests { assert!(composer.textarea.is_empty()); } + #[test] + fn custom_prompt_with_large_paste_expands_correctly() { + use crossterm::event::KeyCode; + use crossterm::event::KeyEvent; + use crossterm::event::KeyModifiers; + + let (tx, _rx) = unbounded_channel::(); + let sender = AppEventSender::new(tx); + let mut composer = ChatComposer::new( + true, + sender, + false, + "Ask Codex to do anything".to_string(), + false, + ); + + // Create a custom prompt with positional args (no named args like $USER) + composer.set_custom_prompts(vec![CustomPrompt { + name: "code-review".to_string(), + path: "/tmp/code-review.md".to_string().into(), + content: "Please review the following code:\n\n$1".to_string(), + description: None, + argument_hint: None, + }]); + + // Type the slash command + let command_text = "/prompts:code-review "; + composer.textarea.set_text(command_text); + composer.textarea.set_cursor(command_text.len()); + + // Paste large content (>3000 chars) to trigger placeholder + let large_content = "x".repeat(LARGE_PASTE_CHAR_THRESHOLD + 3000); + composer.handle_paste(large_content.clone()); + + // Verify placeholder was created + let placeholder = format!("[Pasted Content {} chars]", large_content.chars().count()); + assert_eq!( + composer.textarea.text(), + format!("/prompts:code-review {}", placeholder) + ); + assert_eq!(composer.pending_pastes.len(), 1); + assert_eq!(composer.pending_pastes[0].0, placeholder); + assert_eq!(composer.pending_pastes[0].1, large_content); + + // Submit by pressing Enter + let (result, _needs_redraw) = + composer.handle_key_event(KeyEvent::new(KeyCode::Enter, KeyModifiers::NONE)); + + // Verify the custom prompt was expanded with the large content as positional arg + match result { + InputResult::Submitted(text) => { + // The prompt should be expanded, with the large content replacing $1 + assert_eq!( + text, + format!("Please review the following code:\n\n{}", large_content), + "Expected prompt expansion with large content as $1" + ); + } + _ => panic!("expected Submitted, got: {result:?}"), + } + assert!(composer.textarea.is_empty()); + assert!(composer.pending_pastes.is_empty()); + } + #[test] fn slash_path_input_submits_without_command_error() { use crossterm::event::KeyCode; diff --git a/codex-rs/tui/src/bottom_pane/feedback_view.rs b/codex-rs/tui/src/bottom_pane/feedback_view.rs index a8476df0ab..c563ab8e90 100644 --- a/codex-rs/tui/src/bottom_pane/feedback_view.rs +++ b/codex-rs/tui/src/bottom_pane/feedback_view.rs @@ -19,6 +19,7 @@ use crate::app_event::FeedbackCategory; use crate::app_event_sender::AppEventSender; use crate::history_cell; use crate::render::renderable::Renderable; +use codex_core::protocol::SessionSource; use super::CancellationEvent; use super::bottom_pane_view::BottomPaneView; @@ -26,7 +27,8 @@ use super::popup_consts::standard_popup_hint_line; use super::textarea::TextArea; use super::textarea::TextAreaState; -const BASE_ISSUE_URL: &str = "https://github.com/openai/codex/issues/new?template=2-bug-report.yml"; +const BASE_BUG_ISSUE_URL: &str = + "https://github.com/openai/codex/issues/new?template=2-bug-report.yml"; /// Minimal input overlay to collect an optional feedback note, then upload /// both logs and rollout with classification + metadata. @@ -84,30 +86,43 @@ impl FeedbackNoteView { } else { None }, + Some(SessionSource::Cli), ); match result { Ok(()) => { - let issue_url = format!("{BASE_ISSUE_URL}&steps=Uploaded%20thread:%20{thread_id}"); let prefix = if self.include_logs { "• Feedback uploaded." } else { "• Feedback recorded (no logs)." }; - self.app_event_tx.send(AppEvent::InsertHistoryCell(Box::new( - history_cell::PlainHistoryCell::new(vec![ - Line::from(format!( - "{prefix} Please open an issue using the following URL:" - )), + let issue_url = issue_url_for_category(self.category, &thread_id); + let mut lines = vec![Line::from(match issue_url.as_ref() { + Some(_) => format!("{prefix} Please open an issue using the following URL:"), + None => format!("{prefix} Thanks for the feedback!"), + })]; + if let Some(url) = issue_url { + lines.extend([ "".into(), - Line::from(vec![" ".into(), issue_url.cyan().underlined()]), + Line::from(vec![" ".into(), url.cyan().underlined()]), "".into(), Line::from(vec![ " Or mention your thread ID ".into(), std::mem::take(&mut thread_id).bold(), " in an existing issue.".into(), ]), - ]), + ]); + } else { + lines.extend([ + "".into(), + Line::from(vec![ + " Thread ID: ".into(), + std::mem::take(&mut thread_id).bold(), + ]), + ]); + } + self.app_event_tx.send(AppEvent::InsertHistoryCell(Box::new( + history_cell::PlainHistoryCell::new(lines), ))); } Err(e) => { @@ -320,6 +335,15 @@ fn feedback_classification(category: FeedbackCategory) -> &'static str { } } +fn issue_url_for_category(category: FeedbackCategory, thread_id: &str) -> Option { + match category { + FeedbackCategory::Bug | FeedbackCategory::BadResult | FeedbackCategory::Other => Some( + format!("{BASE_BUG_ISSUE_URL}&steps=Uploaded%20thread:%20{thread_id}"), + ), + FeedbackCategory::GoodResult => None, + } +} + // Build the selection popup params for feedback categories. pub(crate) fn feedback_selection_params( app_event_tx: AppEventSender, @@ -514,4 +538,22 @@ mod tests { let rendered = render(&view, 60); insta::assert_snapshot!("feedback_view_other", rendered); } + + #[test] + fn issue_url_available_for_bug_bad_result_and_other() { + let bug_url = issue_url_for_category(FeedbackCategory::Bug, "thread-1"); + assert!( + bug_url + .as_deref() + .is_some_and(|url| url.contains("template=2-bug-report")) + ); + + let bad_result_url = issue_url_for_category(FeedbackCategory::BadResult, "thread-2"); + assert!(bad_result_url.is_some()); + + let other_url = issue_url_for_category(FeedbackCategory::Other, "thread-3"); + assert!(other_url.is_some()); + + assert!(issue_url_for_category(FeedbackCategory::GoodResult, "t").is_none()); + } } diff --git a/codex-rs/tui/src/bottom_pane/footer.rs b/codex-rs/tui/src/bottom_pane/footer.rs index 79d7c60fa7..11a97c783b 100644 --- a/codex-rs/tui/src/bottom_pane/footer.rs +++ b/codex-rs/tui/src/bottom_pane/footer.rs @@ -1,6 +1,7 @@ use crate::key_hint; use crate::key_hint::KeyBinding; use crate::render::line_utils::prefix_lines; +use crate::status::format_tokens_compact; use crate::ui_consts::FOOTER_INDENT_COLS; use crossterm::event::KeyCode; use ratatui::buffer::Buffer; @@ -18,6 +19,7 @@ pub(crate) struct FooterProps { pub(crate) use_shift_enter_hint: bool, pub(crate) is_task_running: bool, pub(crate) context_window_percent: Option, + pub(crate) context_window_used_tokens: Option, } #[derive(Clone, Copy, Debug, Eq, PartialEq)] @@ -81,7 +83,10 @@ fn footer_lines(props: FooterProps) -> Vec> { is_task_running: props.is_task_running, })], FooterMode::ShortcutSummary => { - let mut line = context_window_line(props.context_window_percent); + let mut line = context_window_line( + props.context_window_percent, + props.context_window_used_tokens, + ); line.push_span(" · ".dim()); line.extend(vec![ key_hint::plain(KeyCode::Char('?')).into(), @@ -94,7 +99,10 @@ fn footer_lines(props: FooterProps) -> Vec> { esc_backtrack_hint: props.esc_backtrack_hint, }), FooterMode::EscHint => vec![esc_hint_line(props.esc_backtrack_hint)], - FooterMode::ContextOnly => vec![context_window_line(props.context_window_percent)], + FooterMode::ContextOnly => vec![context_window_line( + props.context_window_percent, + props.context_window_used_tokens, + )], } } @@ -221,9 +229,18 @@ fn build_columns(entries: Vec>) -> Vec> { .collect() } -fn context_window_line(percent: Option) -> Line<'static> { - let percent = percent.unwrap_or(100).clamp(0, 100); - Line::from(vec![Span::from(format!("{percent}% context left")).dim()]) +fn context_window_line(percent: Option, used_tokens: Option) -> Line<'static> { + if let Some(percent) = percent { + let percent = percent.clamp(0, 100); + return Line::from(vec![Span::from(format!("{percent}% context left")).dim()]); + } + + if let Some(tokens) = used_tokens { + let used_fmt = format_tokens_compact(tokens); + return Line::from(vec![Span::from(format!("{used_fmt} used")).dim()]); + } + + Line::from(vec![Span::from("100% context left").dim()]) } #[derive(Clone, Copy, Debug, Eq, PartialEq)] @@ -400,6 +417,7 @@ mod tests { use_shift_enter_hint: false, is_task_running: false, context_window_percent: None, + context_window_used_tokens: None, }, ); @@ -411,6 +429,7 @@ mod tests { use_shift_enter_hint: true, is_task_running: false, context_window_percent: None, + context_window_used_tokens: None, }, ); @@ -422,6 +441,7 @@ mod tests { use_shift_enter_hint: false, is_task_running: false, context_window_percent: None, + context_window_used_tokens: None, }, ); @@ -433,6 +453,7 @@ mod tests { use_shift_enter_hint: false, is_task_running: true, context_window_percent: None, + context_window_used_tokens: None, }, ); @@ -444,6 +465,7 @@ mod tests { use_shift_enter_hint: false, is_task_running: false, context_window_percent: None, + context_window_used_tokens: None, }, ); @@ -455,6 +477,7 @@ mod tests { use_shift_enter_hint: false, is_task_running: false, context_window_percent: None, + context_window_used_tokens: None, }, ); @@ -466,6 +489,19 @@ mod tests { use_shift_enter_hint: false, is_task_running: true, context_window_percent: Some(72), + context_window_used_tokens: None, + }, + ); + + snapshot_footer( + "footer_context_tokens_used", + FooterProps { + mode: FooterMode::ShortcutSummary, + esc_backtrack_hint: false, + use_shift_enter_hint: false, + is_task_running: false, + context_window_percent: None, + context_window_used_tokens: Some(123_456), }, ); } diff --git a/codex-rs/tui/src/bottom_pane/list_selection_view.rs b/codex-rs/tui/src/bottom_pane/list_selection_view.rs index 88fbaf5b53..d294a47265 100644 --- a/codex-rs/tui/src/bottom_pane/list_selection_view.rs +++ b/codex-rs/tui/src/bottom_pane/list_selection_view.rs @@ -52,6 +52,7 @@ pub(crate) struct SelectionViewParams { pub is_searchable: bool, pub search_placeholder: Option, pub header: Box, + pub initial_selected_idx: Option, } impl Default for SelectionViewParams { @@ -64,6 +65,7 @@ impl Default for SelectionViewParams { is_searchable: false, search_placeholder: None, header: Box::new(()), + initial_selected_idx: None, } } } @@ -80,6 +82,7 @@ pub(crate) struct ListSelectionView { filtered_indices: Vec, last_selected_actual_idx: Option, header: Box, + initial_selected_idx: Option, } impl ListSelectionView { @@ -110,6 +113,7 @@ impl ListSelectionView { filtered_indices: Vec::new(), last_selected_actual_idx: None, header, + initial_selected_idx: params.initial_selected_idx, }; s.apply_filter(); s @@ -132,7 +136,8 @@ impl ListSelectionView { (!self.is_searchable) .then(|| self.items.iter().position(|item| item.is_current)) .flatten() - }); + }) + .or_else(|| self.initial_selected_idx.take()); if self.is_searchable && !self.search_query.is_empty() { let query_lower = self.search_query.to_lowercase(); diff --git a/codex-rs/tui/src/bottom_pane/mod.rs b/codex-rs/tui/src/bottom_pane/mod.rs index 6738d7672d..b4255fd979 100644 --- a/codex-rs/tui/src/bottom_pane/mod.rs +++ b/codex-rs/tui/src/bottom_pane/mod.rs @@ -69,12 +69,14 @@ pub(crate) struct BottomPane { is_task_running: bool, ctrl_c_quit_hint: bool, esc_backtrack_hint: bool, + animations_enabled: bool, /// Inline status indicator shown above the composer while a task is running. status: Option, /// Queued user messages to show above the composer while a turn is running. queued_user_messages: QueuedUserMessages, context_window_percent: Option, + context_window_used_tokens: Option, } pub(crate) struct BottomPaneParams { @@ -84,29 +86,40 @@ pub(crate) struct BottomPaneParams { pub(crate) enhanced_keys_supported: bool, pub(crate) placeholder_text: String, pub(crate) disable_paste_burst: bool, + pub(crate) animations_enabled: bool, } impl BottomPane { pub fn new(params: BottomPaneParams) -> Self { - let enhanced_keys_supported = params.enhanced_keys_supported; + let BottomPaneParams { + app_event_tx, + frame_requester, + has_input_focus, + enhanced_keys_supported, + placeholder_text, + disable_paste_burst, + animations_enabled, + } = params; Self { composer: ChatComposer::new( - params.has_input_focus, - params.app_event_tx.clone(), + has_input_focus, + app_event_tx.clone(), enhanced_keys_supported, - params.placeholder_text, - params.disable_paste_burst, + placeholder_text, + disable_paste_burst, ), view_stack: Vec::new(), - app_event_tx: params.app_event_tx, - frame_requester: params.frame_requester, - has_input_focus: params.has_input_focus, + app_event_tx, + frame_requester, + has_input_focus, is_task_running: false, ctrl_c_quit_hint: false, status: None, queued_user_messages: QueuedUserMessages::new(), esc_backtrack_hint: false, + animations_enabled, context_window_percent: None, + context_window_used_tokens: None, } } @@ -119,6 +132,11 @@ impl BottomPane { self.context_window_percent } + #[cfg(test)] + pub(crate) fn context_window_used_tokens(&self) -> Option { + self.context_window_used_tokens + } + fn active_view(&self) -> Option<&dyn BottomPaneView> { self.view_stack.last().map(std::convert::AsRef::as_ref) } @@ -294,6 +312,7 @@ impl BottomPane { self.status = Some(StatusIndicatorWidget::new( self.app_event_tx.clone(), self.frame_requester.clone(), + self.animations_enabled, )); } if let Some(status) = self.status.as_mut() { @@ -319,6 +338,7 @@ impl BottomPane { self.status = Some(StatusIndicatorWidget::new( self.app_event_tx.clone(), self.frame_requester.clone(), + self.animations_enabled, )); self.request_redraw(); } @@ -331,13 +351,16 @@ impl BottomPane { } } - pub(crate) fn set_context_window_percent(&mut self, percent: Option) { - if self.context_window_percent == percent { + pub(crate) fn set_context_window(&mut self, percent: Option, used_tokens: Option) { + if self.context_window_percent == percent && self.context_window_used_tokens == used_tokens + { return; } self.context_window_percent = percent; - self.composer.set_context_window_percent(percent); + self.context_window_used_tokens = used_tokens; + self.composer + .set_context_window(percent, self.context_window_used_tokens); self.request_redraw(); } @@ -554,6 +577,7 @@ mod tests { enhanced_keys_supported: false, placeholder_text: "Ask Codex to do anything".to_string(), disable_paste_burst: false, + animations_enabled: true, }); pane.push_approval_request(exec_request()); assert_eq!(CancellationEvent::Handled, pane.on_ctrl_c()); @@ -574,6 +598,7 @@ mod tests { enhanced_keys_supported: false, placeholder_text: "Ask Codex to do anything".to_string(), disable_paste_burst: false, + animations_enabled: true, }); // Create an approval modal (active view). @@ -605,6 +630,7 @@ mod tests { enhanced_keys_supported: false, placeholder_text: "Ask Codex to do anything".to_string(), disable_paste_burst: false, + animations_enabled: true, }); // Start a running task so the status indicator is active above the composer. @@ -670,6 +696,7 @@ mod tests { enhanced_keys_supported: false, placeholder_text: "Ask Codex to do anything".to_string(), disable_paste_burst: false, + animations_enabled: true, }); // Begin a task: show initial status. @@ -695,6 +722,7 @@ mod tests { enhanced_keys_supported: false, placeholder_text: "Ask Codex to do anything".to_string(), disable_paste_burst: false, + animations_enabled: true, }); // Activate spinner (status view replaces composer) with no live ring. @@ -724,6 +752,7 @@ mod tests { enhanced_keys_supported: false, placeholder_text: "Ask Codex to do anything".to_string(), disable_paste_burst: false, + animations_enabled: true, }); pane.set_task_running(true); @@ -750,6 +779,7 @@ mod tests { enhanced_keys_supported: false, placeholder_text: "Ask Codex to do anything".to_string(), disable_paste_burst: false, + animations_enabled: true, }); pane.set_task_running(true); diff --git a/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__footer__tests__footer_context_tokens_used.snap b/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__footer__tests__footer_context_tokens_used.snap new file mode 100644 index 0000000000..a77ca5565b --- /dev/null +++ b/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__footer__tests__footer_context_tokens_used.snap @@ -0,0 +1,5 @@ +--- +source: tui/src/bottom_pane/footer.rs +expression: terminal.backend() +--- +" 123K used · ? for shortcuts " diff --git a/codex-rs/tui/src/chatwidget.rs b/codex-rs/tui/src/chatwidget.rs index 6ecb3891c7..51d70edbfe 100644 --- a/codex-rs/tui/src/chatwidget.rs +++ b/codex-rs/tui/src/chatwidget.rs @@ -1,4 +1,5 @@ use std::collections::HashMap; +use std::collections::HashSet; use std::collections::VecDeque; use std::path::PathBuf; use std::sync::Arc; @@ -19,6 +20,7 @@ use codex_core::protocol::AgentReasoningRawContentDeltaEvent; use codex_core::protocol::AgentReasoningRawContentEvent; use codex_core::protocol::ApplyPatchApprovalRequestEvent; use codex_core::protocol::BackgroundEventEvent; +use codex_core::protocol::CreditsSnapshot; use codex_core::protocol::DeprecationNoticeEvent; use codex_core::protocol::ErrorEvent; use codex_core::protocol::Event; @@ -39,6 +41,7 @@ use codex_core::protocol::Op; use codex_core::protocol::PatchApplyBeginEvent; use codex_core::protocol::RateLimitSnapshot; use codex_core::protocol::ReviewRequest; +use codex_core::protocol::ReviewTarget; use codex_core::protocol::StreamErrorEvent; use codex_core::protocol::TaskCompleteEvent; use codex_core::protocol::TokenUsage; @@ -53,6 +56,7 @@ use codex_core::protocol::WarningEvent; use codex_core::protocol::WebSearchBeginEvent; use codex_core::protocol::WebSearchEndEvent; use codex_protocol::ConversationId; +use codex_protocol::approvals::ElicitationRequestEvent; use codex_protocol::parse_command::ParsedCommand; use codex_protocol::user_input::UserInput; use crossterm::event::KeyCode; @@ -139,6 +143,20 @@ struct RunningCommand { source: ExecCommandSource, } +struct UnifiedExecWaitState { + command_display: String, +} + +impl UnifiedExecWaitState { + fn new(command_display: String) -> Self { + Self { command_display } + } + + fn is_duplicate(&self, command_display: &str) -> bool { + self.command_display == command_display + } +} + const RATE_LIMIT_WARNING_THRESHOLDS: [f64; 3] = [75.0, 90.0, 95.0]; const NUDGE_MODEL_SLUG: &str = "gpt-5.1-codex-mini"; const RATE_LIMIT_SWITCH_PROMPT_THRESHOLD: f64 = 90.0; @@ -238,6 +256,7 @@ pub(crate) struct ChatWidgetInit { pub(crate) enhanced_keys_supported: bool, pub(crate) auth_manager: Arc, pub(crate) feedback: codex_feedback::CodexFeedback, + pub(crate) is_first_run: bool, } #[derive(Default)] @@ -265,6 +284,8 @@ pub(crate) struct ChatWidget { // Stream lifecycle controller stream_controller: Option, running_commands: HashMap, + suppressed_exec_calls: HashSet, + last_unified_wait: Option, task_complete_pending: bool, mcp_startup_status: Option>, // Queue of interruptive UI events deferred during an active write cycle @@ -478,6 +499,8 @@ impl ChatWidget { // Mark task stopped and request redraw now that all content is in history. self.bottom_pane.set_task_running(false); self.running_commands.clear(); + self.suppressed_exec_calls.clear(); + self.last_unified_wait = None; self.request_redraw(); // If there is a queued user message, send exactly one now to begin the next turn. @@ -494,7 +517,7 @@ impl ChatWidget { match info { Some(info) => self.apply_token_info(info), None => { - self.bottom_pane.set_context_window_percent(None); + self.bottom_pane.set_context_window(None, None); self.token_info = None; } } @@ -502,7 +525,8 @@ impl ChatWidget { fn apply_token_info(&mut self, info: TokenUsageInfo) { let percent = self.context_remaining_percent(&info); - self.bottom_pane.set_context_window_percent(percent); + let used_tokens = self.context_used_tokens(&info, percent.is_some()); + self.bottom_pane.set_context_window(percent, used_tokens); self.token_info = Some(info); } @@ -515,12 +539,20 @@ impl ChatWidget { }) } + fn context_used_tokens(&self, info: &TokenUsageInfo, percent_known: bool) -> Option { + if percent_known { + return None; + } + + Some(info.total_token_usage.tokens_in_context_window()) + } + fn restore_pre_review_token_info(&mut self) { if let Some(saved) = self.pre_review_token_info.take() { match saved { Some(info) => self.apply_token_info(info), None => { - self.bottom_pane.set_context_window_percent(None); + self.bottom_pane.set_context_window(None, None); self.token_info = None; } } @@ -528,7 +560,19 @@ impl ChatWidget { } pub(crate) fn on_rate_limit_snapshot(&mut self, snapshot: Option) { - if let Some(snapshot) = snapshot { + if let Some(mut snapshot) = snapshot { + if snapshot.credits.is_none() { + snapshot.credits = self + .rate_limit_snapshot + .as_ref() + .and_then(|display| display.credits.as_ref()) + .map(|credits| CreditsSnapshot { + has_credits: credits.has_credits, + unlimited: credits.unlimited, + balance: credits.balance.clone(), + }); + } + let warnings = self.rate_limit_warnings.take_warnings( snapshot .secondary @@ -587,6 +631,8 @@ impl ChatWidget { // Reset running state and clear streaming buffers. self.bottom_pane.set_task_running(false); self.running_commands.clear(); + self.suppressed_exec_calls.clear(); + self.last_unified_wait = None; self.stream_controller = None; self.maybe_show_pending_rate_limit_prompt(); } @@ -733,6 +779,14 @@ impl ChatWidget { ); } + fn on_elicitation_request(&mut self, ev: ElicitationRequestEvent) { + let ev2 = ev.clone(); + self.defer_or_handle( + |q| q.push_elicitation(ev), + |s| s.handle_elicitation_request_now(ev2), + ); + } + fn on_exec_command_begin(&mut self, ev: ExecCommandBeginEvent) { self.flush_answer_stream_with_separator(); let ev2 = ev.clone(); @@ -938,6 +992,9 @@ impl ChatWidget { pub(crate) fn handle_exec_end_now(&mut self, ev: ExecCommandEndEvent) { let running = self.running_commands.remove(&ev.call_id); + if self.suppressed_exec_calls.remove(&ev.call_id) { + return; + } let (command, parsed, source) = match running { Some(rc) => (rc.command, rc.parsed_cmd, rc.source), None => ( @@ -962,6 +1019,7 @@ impl ChatWidget { parsed, source, None, + self.config.animations, ))); } @@ -1038,6 +1096,22 @@ impl ChatWidget { }); } + pub(crate) fn handle_elicitation_request_now(&mut self, ev: ElicitationRequestEvent) { + self.flush_answer_stream_with_separator(); + + self.notify(Notification::ElicitationRequested { + server_name: ev.server_name.clone(), + }); + + let request = ApprovalRequest::McpElicitation { + server_name: ev.server_name, + request_id: ev.id, + message: ev.message, + }; + self.bottom_pane.push_approval_request(request); + self.request_redraw(); + } + pub(crate) fn handle_exec_begin_now(&mut self, ev: ExecCommandBeginEvent) { // Ensure the status indicator is visible while the command runs. self.running_commands.insert( @@ -1048,6 +1122,27 @@ impl ChatWidget { source: ev.source, }, ); + let is_wait_interaction = matches!(ev.source, ExecCommandSource::UnifiedExecInteraction) + && ev + .interaction_input + .as_deref() + .map(str::is_empty) + .unwrap_or(true); + let command_display = ev.command.join(" "); + let should_suppress_unified_wait = is_wait_interaction + && self + .last_unified_wait + .as_ref() + .is_some_and(|wait| wait.is_duplicate(&command_display)); + if is_wait_interaction { + self.last_unified_wait = Some(UnifiedExecWaitState::new(command_display)); + } else { + self.last_unified_wait = None; + } + if should_suppress_unified_wait { + self.suppressed_exec_calls.insert(ev.call_id); + return; + } let interaction_input = ev.interaction_input.clone(); if let Some(cell) = self .active_cell @@ -1071,6 +1166,7 @@ impl ChatWidget { ev.parsed_cmd, ev.source, interaction_input, + self.config.animations, ))); } @@ -1083,6 +1179,7 @@ impl ChatWidget { self.active_cell = Some(Box::new(history_cell::new_active_mcp_tool_call( ev.call_id, ev.invocation, + self.config.animations, ))); self.request_redraw(); } @@ -1104,7 +1201,11 @@ impl ChatWidget { Some(cell) if cell.call_id() == call_id => cell.complete(duration, result), _ => { self.flush_active_cell(); - let mut cell = history_cell::new_active_mcp_tool_call(call_id, invocation); + let mut cell = history_cell::new_active_mcp_tool_call( + call_id, + invocation, + self.config.animations, + ); let extra_cell = cell.complete(duration, result); self.active_cell = Some(Box::new(cell)); extra_cell @@ -1130,6 +1231,7 @@ impl ChatWidget { enhanced_keys_supported, auth_manager, feedback, + is_first_run, } = common; let mut rng = rand::rng(); let placeholder = EXAMPLE_PROMPTS[rng.random_range(0..EXAMPLE_PROMPTS.len())].to_string(); @@ -1146,6 +1248,7 @@ impl ChatWidget { enhanced_keys_supported, placeholder_text: placeholder, disable_paste_burst: config.disable_paste_burst, + animations_enabled: config.animations, }), active_cell: None, config: config.clone(), @@ -1162,6 +1265,8 @@ impl ChatWidget { rate_limit_poller: None, stream_controller: None, running_commands: HashMap::new(), + suppressed_exec_calls: HashSet::new(), + last_unified_wait: None, task_complete_pending: false, mcp_startup_status: None, interrupts: InterruptManager::new(), @@ -1171,7 +1276,7 @@ impl ChatWidget { retry_status_header: None, conversation_id: None, queued_user_messages: VecDeque::new(), - show_welcome_banner: true, + show_welcome_banner: is_first_run, suppress_session_configured_redraw: false, pending_notification: None, is_review_mode: false, @@ -1202,6 +1307,7 @@ impl ChatWidget { enhanced_keys_supported, auth_manager, feedback, + .. } = common; let mut rng = rand::rng(); let placeholder = EXAMPLE_PROMPTS[rng.random_range(0..EXAMPLE_PROMPTS.len())].to_string(); @@ -1220,6 +1326,7 @@ impl ChatWidget { enhanced_keys_supported, placeholder_text: placeholder, disable_paste_burst: config.disable_paste_burst, + animations_enabled: config.animations, }), active_cell: None, config: config.clone(), @@ -1236,6 +1343,8 @@ impl ChatWidget { rate_limit_poller: None, stream_controller: None, running_commands: HashMap::new(), + suppressed_exec_calls: HashSet::new(), + last_unified_wait: None, task_complete_pending: false, mcp_startup_status: None, interrupts: InterruptManager::new(), @@ -1245,7 +1354,7 @@ impl ChatWidget { retry_status_header: None, conversation_id: None, queued_user_messages: VecDeque::new(), - show_welcome_banner: true, + show_welcome_banner: false, suppress_session_configured_redraw: true, pending_notification: None, is_review_mode: false, @@ -1277,7 +1386,9 @@ impl ChatWidget { modifiers, kind: KeyEventKind::Press, .. - } if modifiers.contains(KeyModifiers::CONTROL) && c.eq_ignore_ascii_case(&'v') => { + } if modifiers.intersects(KeyModifiers::CONTROL | KeyModifiers::ALT) + && c.eq_ignore_ascii_case(&'v') => + { match paste_image_to_temp_png() { Ok((path, info)) => { self.attach_image( @@ -1683,7 +1794,7 @@ impl ChatWidget { self.on_rate_limit_snapshot(ev.rate_limits); } EventMsg::Warning(WarningEvent { message }) => self.on_warning(message), - EventMsg::Error(ErrorEvent { message }) => self.on_error(message), + EventMsg::Error(ErrorEvent { message, .. }) => self.on_error(message), EventMsg::McpStartupUpdate(ev) => self.on_mcp_startup_update(ev), EventMsg::McpStartupComplete(ev) => self.on_mcp_startup_complete(ev), EventMsg::SaveSessionResponse(ev) => { @@ -1711,6 +1822,9 @@ impl ChatWidget { EventMsg::ApplyPatchApprovalRequest(ev) => { self.on_apply_patch_approval_request(id.unwrap_or_default(), ev) } + EventMsg::ElicitationRequest(ev) => { + self.on_elicitation_request(ev); + } EventMsg::ExecCommandBegin(ev) => self.on_exec_command_begin(ev), EventMsg::ExecCommandOutputDelta(delta) => self.on_exec_command_output_delta(delta), EventMsg::PatchApplyBegin(ev) => self.on_patch_apply_begin(ev), @@ -1732,7 +1846,9 @@ impl ChatWidget { } EventMsg::UndoStarted(ev) => self.on_undo_started(ev), EventMsg::UndoCompleted(ev) => self.on_undo_completed(ev), - EventMsg::StreamError(StreamErrorEvent { message }) => self.on_stream_error(message), + EventMsg::StreamError(StreamErrorEvent { message, .. }) => { + self.on_stream_error(message) + } EventMsg::UserMessage(ev) => { if from_replay { self.on_user_message_event(ev); @@ -1742,6 +1858,7 @@ impl ChatWidget { self.on_entered_review_mode(review_request) } EventMsg::ExitedReviewMode(review) => self.on_exited_review_mode(review), + EventMsg::ContextCompacted(_) => self.on_agent_message("Context compacted".to_owned()), EventMsg::RawResponseItem(_) | EventMsg::ItemStarted(_) | EventMsg::ItemCompleted(_) @@ -1757,7 +1874,10 @@ impl ChatWidget { self.pre_review_token_info = Some(self.token_info.clone()); } self.is_review_mode = true; - let banner = format!(">> Code review started: {} <<", review.user_facing_hint); + let hint = review + .user_facing_hint + .unwrap_or_else(|| codex_core::review_prompts::user_facing_hint(&review.target)); + let banner = format!(">> Code review started: {hint} <<"); self.add_to_history(history_cell::new_review_status_line(banner)); self.request_redraw(); } @@ -2143,6 +2263,14 @@ impl ChatWidget { } else { default_choice }; + let selection_choice = highlight_choice.or(default_choice); + let initial_selected_idx = choices + .iter() + .position(|choice| choice.stored == selection_choice) + .or_else(|| { + selection_choice + .and_then(|effort| choices.iter().position(|choice| choice.display == effort)) + }); let mut items: Vec = Vec::new(); for choice in choices.iter() { let effort = choice.display; @@ -2219,6 +2347,7 @@ impl ChatWidget { header: Box::new(header), footer_hint: Some(standard_popup_hint_line()), items, + initial_selected_idx, ..Default::default() }); } @@ -2268,7 +2397,7 @@ impl ChatWidget { let presets: Vec = builtin_approval_presets(); for preset in presets.into_iter() { let is_current = - current_approval == preset.approval && current_sandbox == preset.sandbox; + Self::preset_matches_current(current_approval, ¤t_sandbox, &preset); let name = preset.label.to_string(); let description_text = preset.description; let description = Some(description_text.to_string()); @@ -2356,6 +2485,28 @@ impl ChatWidget { })] } + fn preset_matches_current( + current_approval: AskForApproval, + current_sandbox: &SandboxPolicy, + preset: &ApprovalPreset, + ) -> bool { + if current_approval != preset.approval { + return false; + } + matches!( + (&preset.sandbox, current_sandbox), + (SandboxPolicy::ReadOnly, SandboxPolicy::ReadOnly) + | ( + SandboxPolicy::DangerFullAccess, + SandboxPolicy::DangerFullAccess + ) + | ( + SandboxPolicy::WorkspaceWrite { .. }, + SandboxPolicy::WorkspaceWrite { .. } + ) + ) + } + #[cfg(target_os = "windows")] pub(crate) fn world_writable_warning_details(&self) -> Option<(Vec, usize, bool)> { if self @@ -2366,11 +2517,18 @@ impl ChatWidget { { return None; } - let cwd = match std::env::current_dir() { - Ok(cwd) => cwd, - Err(_) => return Some((Vec::new(), 0, true)), - }; - codex_windows_sandbox::world_writable_warning_details(self.config.codex_home.as_path(), cwd) + let cwd = self.config.cwd.clone(); + let env_map: std::collections::HashMap = std::env::vars().collect(); + match codex_windows_sandbox::apply_world_writable_scan_and_denies( + self.config.codex_home.as_path(), + cwd.as_path(), + &env_map, + &self.config.sandbox_policy, + Some(self.config.codex_home.as_path()), + ) { + Ok(_) => None, + Err(_) => Some((Vec::new(), 0, true)), + } } #[cfg(not(target_os = "windows"))] @@ -2796,17 +2954,14 @@ impl ChatWidget { items.push(SelectionItem { name: "Review uncommitted changes".to_string(), - actions: vec![Box::new( - move |tx: &AppEventSender| { - tx.send(AppEvent::CodexOp(Op::Review { - review_request: ReviewRequest { - prompt: "Review the current code changes (staged, unstaged, and untracked files) and provide prioritized findings.".to_string(), - user_facing_hint: "current changes".to_string(), - append_to_original_thread: true, - }, - })); - }, - )], + actions: vec![Box::new(move |tx: &AppEventSender| { + tx.send(AppEvent::CodexOp(Op::Review { + review_request: ReviewRequest { + target: ReviewTarget::UncommittedChanges, + user_facing_hint: None, + }, + })); + })], dismiss_on_select: true, ..Default::default() }); @@ -2855,11 +3010,10 @@ impl ChatWidget { actions: vec![Box::new(move |tx3: &AppEventSender| { tx3.send(AppEvent::CodexOp(Op::Review { review_request: ReviewRequest { - prompt: format!( - "Review the code changes against the base branch '{branch}'. Start by finding the merge diff between the current branch and {branch}'s upstream e.g. (`git merge-base HEAD \"$(git rev-parse --abbrev-ref \"{branch}@{{upstream}}\")\"`), then run `git diff` against that SHA to see what changes we would merge into the {branch} branch. Provide prioritized, actionable findings." - ), - user_facing_hint: format!("changes against '{branch}'"), - append_to_original_thread: true, + target: ReviewTarget::BaseBranch { + branch: branch.clone(), + }, + user_facing_hint: None, }, })); })], @@ -2886,21 +3040,18 @@ impl ChatWidget { for entry in commits { let subject = entry.subject.clone(); let sha = entry.sha.clone(); - let short = sha.chars().take(7).collect::(); let search_val = format!("{subject} {sha}"); items.push(SelectionItem { name: subject.clone(), actions: vec![Box::new(move |tx3: &AppEventSender| { - let hint = format!("commit {short}"); - let prompt = format!( - "Review the code changes introduced by commit {sha} (\"{subject}\"). Provide prioritized, actionable findings." - ); tx3.send(AppEvent::CodexOp(Op::Review { review_request: ReviewRequest { - prompt, - user_facing_hint: hint, - append_to_original_thread: true, + target: ReviewTarget::Commit { + sha: sha.clone(), + title: Some(subject.clone()), + }, + user_facing_hint: None, }, })); })], @@ -2933,9 +3084,10 @@ impl ChatWidget { } tx.send(AppEvent::CodexOp(Op::Review { review_request: ReviewRequest { - prompt: trimmed.clone(), - user_facing_hint: trimmed, - append_to_original_thread: true, + target: ReviewTarget::Custom { + instructions: trimmed, + }, + user_facing_hint: None, }, })); }), @@ -3008,6 +3160,7 @@ enum Notification { AgentTurnComplete { response: String }, ExecApprovalRequested { command: String }, EditApprovalRequested { cwd: PathBuf, changes: Vec }, + ElicitationRequested { server_name: String }, } impl Notification { @@ -3031,6 +3184,9 @@ impl Notification { } ) } + Notification::ElicitationRequested { server_name } => { + format!("Approval requested by {server_name}") + } } } @@ -3038,7 +3194,8 @@ impl Notification { match self { Notification::AgentTurnComplete { .. } => "agent-turn-complete", Notification::ExecApprovalRequested { .. } - | Notification::EditApprovalRequested { .. } => "approval-requested", + | Notification::EditApprovalRequested { .. } + | Notification::ElicitationRequested { .. } => "approval-requested", } } @@ -3132,21 +3289,18 @@ pub(crate) fn show_review_commit_picker_with_entries( for entry in entries { let subject = entry.subject.clone(); let sha = entry.sha.clone(); - let short = sha.chars().take(7).collect::(); let search_val = format!("{subject} {sha}"); items.push(SelectionItem { name: subject.clone(), actions: vec![Box::new(move |tx3: &AppEventSender| { - let hint = format!("commit {short}"); - let prompt = format!( - "Review the code changes introduced by commit {sha} (\"{subject}\"). Provide prioritized, actionable findings." - ); tx3.send(AppEvent::CodexOp(Op::Review { review_request: ReviewRequest { - prompt, - user_facing_hint: hint, - append_to_original_thread: true, + target: ReviewTarget::Commit { + sha: sha.clone(), + title: Some(subject.clone()), + }, + user_facing_hint: None, }, })); })], diff --git a/codex-rs/tui/src/chatwidget/agent.rs b/codex-rs/tui/src/chatwidget/agent.rs index 3c326fed86..240972347f 100644 --- a/codex-rs/tui/src/chatwidget/agent.rs +++ b/codex-rs/tui/src/chatwidget/agent.rs @@ -4,7 +4,6 @@ use codex_core::CodexConversation; use codex_core::ConversationManager; use codex_core::NewConversation; use codex_core::config::Config; -use codex_core::protocol::ErrorEvent; use codex_core::protocol::Event; use codex_core::protocol::EventMsg; use codex_core::protocol::Op; @@ -37,7 +36,7 @@ pub(crate) fn spawn_agent( eprintln!("{message}"); app_event_tx_clone.send(AppEvent::CodexEvent(Event { id: "".to_string(), - msg: EventMsg::Error(ErrorEvent { message }), + msg: EventMsg::Error(err.to_error_event(None)), })); app_event_tx_clone.send(AppEvent::ExitRequest); tracing::error!("failed to initialize codex: {err}"); diff --git a/codex-rs/tui/src/chatwidget/interrupts.rs b/codex-rs/tui/src/chatwidget/interrupts.rs index 531de3e646..dc1e683ea5 100644 --- a/codex-rs/tui/src/chatwidget/interrupts.rs +++ b/codex-rs/tui/src/chatwidget/interrupts.rs @@ -7,6 +7,7 @@ use codex_core::protocol::ExecCommandEndEvent; use codex_core::protocol::McpToolCallBeginEvent; use codex_core::protocol::McpToolCallEndEvent; use codex_core::protocol::PatchApplyEndEvent; +use codex_protocol::approvals::ElicitationRequestEvent; use super::ChatWidget; @@ -14,6 +15,7 @@ use super::ChatWidget; pub(crate) enum QueuedInterrupt { ExecApproval(String, ExecApprovalRequestEvent), ApplyPatchApproval(String, ApplyPatchApprovalRequestEvent), + Elicitation(ElicitationRequestEvent), ExecBegin(ExecCommandBeginEvent), ExecEnd(ExecCommandEndEvent), McpBegin(McpToolCallBeginEvent), @@ -51,6 +53,10 @@ impl InterruptManager { .push_back(QueuedInterrupt::ApplyPatchApproval(id, ev)); } + pub(crate) fn push_elicitation(&mut self, ev: ElicitationRequestEvent) { + self.queue.push_back(QueuedInterrupt::Elicitation(ev)); + } + pub(crate) fn push_exec_begin(&mut self, ev: ExecCommandBeginEvent) { self.queue.push_back(QueuedInterrupt::ExecBegin(ev)); } @@ -78,6 +84,7 @@ impl InterruptManager { QueuedInterrupt::ApplyPatchApproval(id, ev) => { chat.handle_apply_patch_approval_now(id, ev) } + QueuedInterrupt::Elicitation(ev) => chat.handle_elicitation_request_now(ev), QueuedInterrupt::ExecBegin(ev) => chat.handle_exec_begin_now(ev), QueuedInterrupt::ExecEnd(ev) => chat.handle_exec_end_now(ev), QueuedInterrupt::McpBegin(ev) => chat.handle_mcp_begin_now(ev), diff --git a/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__user_shell_ls_output.snap b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__user_shell_ls_output.snap new file mode 100644 index 0000000000..c67cd637d7 --- /dev/null +++ b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__user_shell_ls_output.snap @@ -0,0 +1,7 @@ +--- +source: tui/src/chatwidget/tests.rs +expression: blob +--- +• You ran ls + └ file1 + file2 diff --git a/codex-rs/tui/src/chatwidget/tests.rs b/codex-rs/tui/src/chatwidget/tests.rs index 840d3f5d4a..8657df94f1 100644 --- a/codex-rs/tui/src/chatwidget/tests.rs +++ b/codex-rs/tui/src/chatwidget/tests.rs @@ -18,6 +18,7 @@ use codex_core::protocol::AgentReasoningDeltaEvent; use codex_core::protocol::AgentReasoningEvent; use codex_core::protocol::ApplyPatchApprovalRequestEvent; use codex_core::protocol::BackgroundEventEvent; +use codex_core::protocol::CreditsSnapshot; use codex_core::protocol::Event; use codex_core::protocol::EventMsg; use codex_core::protocol::ExecApprovalRequestEvent; @@ -35,6 +36,7 @@ use codex_core::protocol::ReviewFinding; use codex_core::protocol::ReviewLineRange; use codex_core::protocol::ReviewOutputEvent; use codex_core::protocol::ReviewRequest; +use codex_core::protocol::ReviewTarget; use codex_core::protocol::StreamErrorEvent; use codex_core::protocol::TaskCompleteEvent; use codex_core::protocol::TaskStartedEvent; @@ -50,11 +52,13 @@ use codex_protocol::parse_command::ParsedCommand; use codex_protocol::plan_tool::PlanItemArg; use codex_protocol::plan_tool::StepStatus; use codex_protocol::plan_tool::UpdatePlanArgs; +use codex_protocol::protocol::CodexErrorInfo; use crossterm::event::KeyCode; use crossterm::event::KeyEvent; use crossterm::event::KeyModifiers; use insta::assert_snapshot; use pretty_assertions::assert_eq; +use std::collections::HashSet; use std::path::PathBuf; use tempfile::NamedTempFile; use tempfile::tempdir; @@ -151,9 +155,10 @@ fn entered_review_mode_uses_request_hint() { chat.handle_codex_event(Event { id: "review-start".into(), msg: EventMsg::EnteredReviewMode(ReviewRequest { - prompt: "Review the latest changes".to_string(), - user_facing_hint: "feature branch".to_string(), - append_to_original_thread: true, + target: ReviewTarget::BaseBranch { + branch: "feature".to_string(), + }, + user_facing_hint: Some("feature branch".to_string()), }), }); @@ -171,9 +176,8 @@ fn entered_review_mode_defaults_to_current_changes_banner() { chat.handle_codex_event(Event { id: "review-start".into(), msg: EventMsg::EnteredReviewMode(ReviewRequest { - prompt: "Review the current changes".to_string(), - user_facing_hint: "current changes".to_string(), - append_to_original_thread: true, + target: ReviewTarget::UncommittedChanges, + user_facing_hint: None, }), }); @@ -239,9 +243,10 @@ fn review_restores_context_window_indicator() { chat.handle_codex_event(Event { id: "review-start".into(), msg: EventMsg::EnteredReviewMode(ReviewRequest { - prompt: "Review the latest changes".to_string(), - user_facing_hint: "feature branch".to_string(), - append_to_original_thread: true, + target: ReviewTarget::BaseBranch { + branch: "feature".to_string(), + }, + user_facing_hint: Some("feature branch".to_string()), }), }); @@ -293,6 +298,41 @@ fn token_count_none_resets_context_indicator() { assert_eq!(chat.bottom_pane.context_window_percent(), None); } +#[test] +fn context_indicator_shows_used_tokens_when_window_unknown() { + let (mut chat, _rx, _ops) = make_chatwidget_manual(); + + chat.config.model_context_window = None; + let auto_compact_limit = 200_000; + chat.config.model_auto_compact_token_limit = Some(auto_compact_limit); + + // No model window, so the indicator should fall back to showing tokens used. + let total_tokens = 106_000; + let token_usage = TokenUsage { + total_tokens, + ..TokenUsage::default() + }; + let token_info = TokenUsageInfo { + total_token_usage: token_usage.clone(), + last_token_usage: token_usage, + model_context_window: None, + }; + + chat.handle_codex_event(Event { + id: "token-usage".into(), + msg: EventMsg::TokenCount(TokenCountEvent { + info: Some(token_info), + rate_limits: None, + }), + }); + + assert_eq!(chat.bottom_pane.context_window_percent(), None); + assert_eq!( + chat.bottom_pane.context_window_used_tokens(), + Some(total_tokens) + ); +} + #[cfg_attr( target_os = "macos", ignore = "system configuration APIs are blocked under macOS seatbelt" @@ -315,6 +355,7 @@ async fn helpers_are_available_and_do_not_panic() { enhanced_keys_supported: false, auth_manager, feedback: codex_feedback::CodexFeedback::new(), + is_first_run: true, }; let mut w = ChatWidget::new(init, conversation_manager); // Basic construction sanity. @@ -338,6 +379,7 @@ fn make_chatwidget_manual() -> ( enhanced_keys_supported: false, placeholder_text: "Ask Codex to do anything".to_string(), disable_paste_burst: false, + animations_enabled: cfg.animations, }); let auth_manager = AuthManager::from_auth_for_testing(CodexAuth::from_api_key("test")); let widget = ChatWidget { @@ -356,6 +398,8 @@ fn make_chatwidget_manual() -> ( rate_limit_poller: None, stream_controller: None, running_commands: HashMap::new(), + suppressed_exec_calls: HashSet::new(), + last_unified_wait: None, task_complete_pending: false, mcp_startup_status: None, interrupts: InterruptManager::new(), @@ -479,6 +523,53 @@ fn test_rate_limit_warnings_monthly() { ); } +#[test] +fn rate_limit_snapshot_keeps_prior_credits_when_missing_from_headers() { + let (mut chat, _rx, _op_rx) = make_chatwidget_manual(); + + chat.on_rate_limit_snapshot(Some(RateLimitSnapshot { + primary: None, + secondary: None, + credits: Some(CreditsSnapshot { + has_credits: true, + unlimited: false, + balance: Some("17.5".to_string()), + }), + })); + let initial_balance = chat + .rate_limit_snapshot + .as_ref() + .and_then(|snapshot| snapshot.credits.as_ref()) + .and_then(|credits| credits.balance.as_deref()); + assert_eq!(initial_balance, Some("17.5")); + + chat.on_rate_limit_snapshot(Some(RateLimitSnapshot { + primary: Some(RateLimitWindow { + used_percent: 80.0, + window_minutes: Some(60), + resets_at: Some(123), + }), + secondary: None, + credits: None, + })); + + let display = chat + .rate_limit_snapshot + .as_ref() + .expect("rate limits should be cached"); + let credits = display + .credits + .as_ref() + .expect("credits should persist when headers omit them"); + + assert_eq!(credits.balance.as_deref(), Some("17.5")); + assert!(!credits.unlimited); + assert_eq!( + display.primary.as_ref().map(|window| window.used_percent), + Some(80.0) + ); +} + #[test] fn rate_limit_switch_prompt_skips_when_on_lower_cost_model() { let (mut chat, _, _) = make_chatwidget_manual(); @@ -717,6 +808,7 @@ fn begin_exec_with_source( let interaction_input = None; let event = ExecCommandBeginEvent { call_id: call_id.to_string(), + process_id: None, turn_id: "turn-1".to_string(), command, cwd, @@ -755,11 +847,13 @@ fn end_exec( parsed_cmd, source, interaction_input, + process_id, } = begin_event; chat.handle_codex_event(Event { id: call_id.clone(), msg: EventMsg::ExecCommandEnd(ExecCommandEndEvent { call_id, + process_id, turn_id, command, cwd, @@ -1307,12 +1401,13 @@ fn custom_prompt_submit_sends_review_op() { match evt { AppEvent::CodexOp(Op::Review { review_request }) => { assert_eq!( - review_request.prompt, - "please audit dependencies".to_string() - ); - assert_eq!( - review_request.user_facing_hint, - "please audit dependencies".to_string() + review_request, + ReviewRequest { + target: ReviewTarget::Custom { + instructions: "please audit dependencies".to_string(), + }, + user_facing_hint: None, + } ); } other => panic!("unexpected app event: {other:?}"), @@ -1562,6 +1657,29 @@ fn approvals_selection_popup_snapshot() { assert_snapshot!("approvals_selection_popup", popup); } +#[test] +fn preset_matching_ignores_extra_writable_roots() { + let preset = builtin_approval_presets() + .into_iter() + .find(|p| p.id == "auto") + .expect("auto preset exists"); + let current_sandbox = SandboxPolicy::WorkspaceWrite { + writable_roots: vec![PathBuf::from("C:\\extra")], + network_access: false, + exclude_tmpdir_env_var: false, + exclude_slash_tmp: false, + }; + + assert!( + ChatWidget::preset_matches_current(AskForApproval::OnRequest, ¤t_sandbox, &preset), + "WorkspaceWrite with extra roots should still match the Agent preset" + ); + assert!( + !ChatWidget::preset_matches_current(AskForApproval::Never, ¤t_sandbox, &preset), + "approval mismatch should prevent matching the preset" + ); +} + #[test] fn full_access_confirmation_popup_snapshot() { let (mut chat, _rx, _op_rx) = make_chatwidget_manual(); @@ -1790,6 +1908,28 @@ fn exec_history_extends_previous_when_consecutive() { assert_snapshot!("exploring_step6_finish_cat_bar", active_blob(&chat)); } +#[test] +fn user_shell_command_renders_output_not_exploring() { + let (mut chat, mut rx, _op_rx) = make_chatwidget_manual(); + + let begin_ls = begin_exec_with_source( + &mut chat, + "user-shell-ls", + "ls", + ExecCommandSource::UserShell, + ); + end_exec(&mut chat, begin_ls, "file1\nfile2\n", "", 0); + + let cells = drain_insert_history(&mut rx); + assert_eq!( + cells.len(), + 1, + "expected a single history cell for the user command" + ); + let blob = lines_to_single_string(cells.first().unwrap()); + assert_snapshot!("user_shell_ls_output", blob); +} + #[test] fn disabled_slash_command_while_task_running_snapshot() { // Build a chat widget and simulate an active task @@ -2624,6 +2764,7 @@ fn stream_error_updates_status_indicator() { id: "sub-1".into(), msg: EventMsg::StreamError(StreamErrorEvent { message: msg.to_string(), + codex_error_info: Some(CodexErrorInfo::Other), }), }); @@ -2829,6 +2970,7 @@ fn chatwidget_exec_and_status_layout_vt100_snapshot() { id: "c1".into(), msg: EventMsg::ExecCommandBegin(ExecCommandBeginEvent { call_id: "c1".into(), + process_id: None, turn_id: "turn-1".into(), command: command.clone(), cwd: cwd.clone(), @@ -2841,6 +2983,7 @@ fn chatwidget_exec_and_status_layout_vt100_snapshot() { id: "c1".into(), msg: EventMsg::ExecCommandEnd(ExecCommandEndEvent { call_id: "c1".into(), + process_id: None, turn_id: "turn-1".into(), command, cwd, diff --git a/codex-rs/tui/src/clipboard_paste.rs b/codex-rs/tui/src/clipboard_paste.rs index 4945eea350..2a669f5c4f 100644 --- a/codex-rs/tui/src/clipboard_paste.rs +++ b/codex-rs/tui/src/clipboard_paste.rs @@ -181,6 +181,14 @@ pub fn normalize_pasted_path(pasted: &str) -> Option { drive || unc }; if looks_like_windows_path { + #[cfg(target_os = "linux")] + { + if is_probably_wsl() + && let Some(converted) = convert_windows_path_to_wsl(pasted) + { + return Some(converted); + } + } return Some(PathBuf::from(pasted)); } @@ -193,6 +201,41 @@ pub fn normalize_pasted_path(pasted: &str) -> Option { None } +#[cfg(target_os = "linux")] +fn is_probably_wsl() -> bool { + std::env::var_os("WSL_DISTRO_NAME").is_some() + || std::env::var_os("WSL_INTEROP").is_some() + || std::env::var_os("WSLENV").is_some() +} + +#[cfg(target_os = "linux")] +fn convert_windows_path_to_wsl(input: &str) -> Option { + if input.starts_with("\\\\") { + return None; + } + + let drive_letter = input.chars().next()?.to_ascii_lowercase(); + if !drive_letter.is_ascii_lowercase() { + return None; + } + + if input.get(1..2) != Some(":") { + return None; + } + + let mut result = PathBuf::from(format!("/mnt/{drive_letter}")); + for component in input + .get(2..)? + .trim_start_matches(['\\', '/']) + .split(['\\', '/']) + .filter(|component| !component.is_empty()) + { + result.push(component); + } + + Some(result) +} + /// Infer an image format for the provided path based on its extension. pub fn pasted_image_format(path: &Path) -> EncodedImageFormat { match path @@ -210,6 +253,40 @@ pub fn pasted_image_format(path: &Path) -> EncodedImageFormat { #[cfg(test)] mod pasted_paths_tests { use super::*; + #[cfg(target_os = "linux")] + use std::ffi::OsString; + + #[cfg(target_os = "linux")] + struct EnvVarGuard { + key: &'static str, + original: Option, + } + + #[cfg(target_os = "linux")] + impl EnvVarGuard { + fn set(key: &'static str, value: &str) -> Self { + let original = std::env::var_os(key); + unsafe { + std::env::set_var(key, value); + } + Self { key, original } + } + } + + #[cfg(target_os = "linux")] + impl Drop for EnvVarGuard { + fn drop(&mut self) { + if let Some(original) = &self.original { + unsafe { + std::env::set_var(self.key, original); + } + } else { + unsafe { + std::env::remove_var(self.key); + } + } + } + } #[cfg(not(windows))] #[test] @@ -223,7 +300,17 @@ mod pasted_paths_tests { fn normalize_file_url_windows() { let input = r"C:\Temp\example.png"; let result = normalize_pasted_path(input).expect("should parse file URL"); - assert_eq!(result, PathBuf::from(r"C:\Temp\example.png")); + #[cfg(target_os = "linux")] + let expected = if is_probably_wsl() + && let Some(converted) = convert_windows_path_to_wsl(input) + { + converted + } else { + PathBuf::from(r"C:\Temp\example.png") + }; + #[cfg(not(target_os = "linux"))] + let expected = PathBuf::from(r"C:\Temp\example.png"); + assert_eq!(result, expected); } #[test] @@ -291,10 +378,17 @@ mod pasted_paths_tests { fn normalize_unquoted_windows_path_with_spaces() { let input = r"C:\\Users\\Alice\\My Pictures\\example image.png"; let result = normalize_pasted_path(input).expect("should accept unquoted windows path"); - assert_eq!( - result, + #[cfg(target_os = "linux")] + let expected = if is_probably_wsl() + && let Some(converted) = convert_windows_path_to_wsl(input) + { + converted + } else { PathBuf::from(r"C:\\Users\\Alice\\My Pictures\\example image.png") - ); + }; + #[cfg(not(target_os = "linux"))] + let expected = PathBuf::from(r"C:\\Users\\Alice\\My Pictures\\example image.png"); + assert_eq!(result, expected); } #[test] @@ -322,4 +416,16 @@ mod pasted_paths_tests { EncodedImageFormat::Other ); } + + #[cfg(target_os = "linux")] + #[test] + fn normalize_windows_path_in_wsl() { + let _guard = EnvVarGuard::set("WSL_DISTRO_NAME", "Ubuntu-24.04"); + let input = r"C:\\Users\\Alice\\Pictures\\example image.png"; + let result = normalize_pasted_path(input).expect("should convert windows path on wsl"); + assert_eq!( + result, + PathBuf::from("/mnt/c/Users/Alice/Pictures/example image.png") + ); + } } diff --git a/codex-rs/tui/src/exec_cell/model.rs b/codex-rs/tui/src/exec_cell/model.rs index 943fb83654..76316968c6 100644 --- a/codex-rs/tui/src/exec_cell/model.rs +++ b/codex-rs/tui/src/exec_cell/model.rs @@ -28,11 +28,15 @@ pub(crate) struct ExecCall { #[derive(Debug)] pub(crate) struct ExecCell { pub(crate) calls: Vec, + animations_enabled: bool, } impl ExecCell { - pub(crate) fn new(call: ExecCall) -> Self { - Self { calls: vec![call] } + pub(crate) fn new(call: ExecCall, animations_enabled: bool) -> Self { + Self { + calls: vec![call], + animations_enabled, + } } pub(crate) fn with_added_call( @@ -56,6 +60,7 @@ impl ExecCell { if self.is_exploring_cell() && Self::is_exploring_call(&call) { Some(Self { calls: [self.calls.clone(), vec![call]].concat(), + animations_enabled: self.animations_enabled, }) } else { None @@ -112,12 +117,17 @@ impl ExecCell { .and_then(|c| c.start_time) } + pub(crate) fn animations_enabled(&self) -> bool { + self.animations_enabled + } + pub(crate) fn iter_calls(&self) -> impl Iterator { self.calls.iter() } pub(super) fn is_exploring_call(call: &ExecCall) -> bool { - !call.parsed.is_empty() + !matches!(call.source, ExecCommandSource::UserShell) + && !call.parsed.is_empty() && call.parsed.iter().all(|p| { matches!( p, diff --git a/codex-rs/tui/src/exec_cell/render.rs b/codex-rs/tui/src/exec_cell/render.rs index 352a61476e..3e434138d4 100644 --- a/codex-rs/tui/src/exec_cell/render.rs +++ b/codex-rs/tui/src/exec_cell/render.rs @@ -40,17 +40,21 @@ pub(crate) fn new_active_exec_command( parsed: Vec, source: ExecCommandSource, interaction_input: Option, + animations_enabled: bool, ) -> ExecCell { - ExecCell::new(ExecCall { - call_id, - command, - parsed, - output: None, - source, - start_time: Some(Instant::now()), - duration: None, - interaction_input, - }) + ExecCell::new( + ExecCall { + call_id, + command, + parsed, + output: None, + source, + start_time: Some(Instant::now()), + duration: None, + interaction_input, + }, + animations_enabled, + ) } fn format_unified_exec_interaction(command: &[String], input: Option<&str>) -> String { @@ -168,7 +172,10 @@ pub(crate) fn output_lines( } } -pub(crate) fn spinner(start_time: Option) -> Span<'static> { +pub(crate) fn spinner(start_time: Option, animations_enabled: bool) -> Span<'static> { + if !animations_enabled { + return "•".dim(); + } let elapsed = start_time.map(|st| st.elapsed()).unwrap_or_default(); if supports_color::on_cached(supports_color::Stream::Stdout) .map(|level| level.has_16m) @@ -239,7 +246,7 @@ impl ExecCell { let mut out: Vec> = Vec::new(); out.push(Line::from(vec![ if self.is_active() { - spinner(self.active_start_time()) + spinner(self.active_start_time(), self.animations_enabled()) } else { "•".dim() }, @@ -347,7 +354,7 @@ impl ExecCell { let bullet = match success { Some(true) => "•".green().bold(), Some(false) => "•".red().bold(), - None => spinner(call.start_time), + None => spinner(call.start_time, self.animations_enabled()), }; let is_interaction = call.is_unified_exec_interaction(); let title = if is_interaction { diff --git a/codex-rs/tui/src/file_search.rs b/codex-rs/tui/src/file_search.rs index 61327cfd24..af46512640 100644 --- a/codex-rs/tui/src/file_search.rs +++ b/codex-rs/tui/src/file_search.rs @@ -31,7 +31,7 @@ use std::time::Duration; use crate::app_event::AppEvent; use crate::app_event_sender::AppEventSender; -const MAX_FILE_SEARCH_RESULTS: NonZeroUsize = NonZeroUsize::new(8).unwrap(); +const MAX_FILE_SEARCH_RESULTS: NonZeroUsize = NonZeroUsize::new(20).unwrap(); const NUM_FILE_SEARCH_THREADS: NonZeroUsize = NonZeroUsize::new(2).unwrap(); /// How long to wait after a keystroke before firing the first search when none diff --git a/codex-rs/tui/src/history_cell.rs b/codex-rs/tui/src/history_cell.rs index 394dc33919..475eb1db17 100644 --- a/codex-rs/tui/src/history_cell.rs +++ b/codex-rs/tui/src/history_cell.rs @@ -15,6 +15,7 @@ use crate::render::renderable::Renderable; use crate::style::user_message_style; use crate::text_formatting::format_and_truncate_tool_result; use crate::text_formatting::truncate_text; +use crate::tooltips; use crate::ui_consts::LIVE_PREFIX_COLS; use crate::update_action::UpdateAction; use crate::version::CODEX_CLI_VERSION; @@ -559,6 +560,34 @@ pub(crate) fn padded_emoji(emoji: &str) -> String { format!("{emoji}\u{200A}") } +#[derive(Debug)] +struct TooltipHistoryCell { + tip: &'static str, +} + +impl TooltipHistoryCell { + fn new(tip: &'static str) -> Self { + Self { tip } + } +} + +impl HistoryCell for TooltipHistoryCell { + fn display_lines(&self, width: u16) -> Vec> { + let indent: Line<'static> = " ".into(); + let mut lines = Vec::new(); + let tooltip_line: Line<'static> = vec!["Tip: ".cyan(), self.tip.into()].into(); + let wrap_opts = RtOptions::new(usize::from(width.max(1))) + .initial_indent(indent.clone()) + .subsequent_indent(indent.clone()); + lines.extend( + word_wrap_line(&tooltip_line, wrap_opts.clone()) + .into_iter() + .map(|line| line_to_static(&line)), + ); + lines + } +} + #[derive(Debug)] pub struct SessionInfoCell(CompositeHistoryCell); @@ -586,15 +615,16 @@ pub(crate) fn new_session_info( reasoning_effort, .. } = event; - SessionInfoCell(if is_first_event { - // Header box rendered as history (so it appears at the very top) - let header = SessionHeaderHistoryCell::new( - model, - reasoning_effort, - config.cwd.clone(), - crate::version::CODEX_CLI_VERSION, - ); + // Header box rendered as history (so it appears at the very top) + let header = SessionHeaderHistoryCell::new( + model.clone(), + reasoning_effort, + config.cwd.clone(), + CODEX_CLI_VERSION, + ); + let mut parts: Vec> = vec![Box::new(header)]; + if is_first_event { // Help lines below the header (new copy and list) let help_lines: Vec> = vec![ " To get started, describe a task or try one of these commands:" @@ -628,24 +658,24 @@ pub(crate) fn new_session_info( ]), ]; - CompositeHistoryCell { - parts: vec![ - Box::new(header), - Box::new(PlainHistoryCell { lines: help_lines }), - ], - } - } else if config.model == model { - CompositeHistoryCell { parts: vec![] } + parts.push(Box::new(PlainHistoryCell { lines: help_lines })); } else { - let lines = vec![ - "model changed:".magenta().bold().into(), - format!("requested: {}", config.model).into(), - format!("used: {model}").into(), - ]; - CompositeHistoryCell { - parts: vec![Box::new(PlainHistoryCell { lines })], + if config.show_tooltips + && let Some(tooltips) = tooltips::random_tooltip().map(TooltipHistoryCell::new) + { + parts.push(Box::new(tooltips)); } - }) + if config.model != model { + let lines = vec![ + "model changed:".magenta().bold().into(), + format!("requested: {}", config.model).into(), + format!("used: {model}").into(), + ]; + parts.push(Box::new(PlainHistoryCell { lines })); + } + } + + SessionInfoCell(CompositeHistoryCell { parts }) } pub(crate) fn new_user_prompt(message: String) -> UserHistoryCell { @@ -806,16 +836,22 @@ pub(crate) struct McpToolCallCell { start_time: Instant, duration: Option, result: Option>, + animations_enabled: bool, } impl McpToolCallCell { - pub(crate) fn new(call_id: String, invocation: McpInvocation) -> Self { + pub(crate) fn new( + call_id: String, + invocation: McpInvocation, + animations_enabled: bool, + ) -> Self { Self { call_id, invocation, start_time: Instant::now(), duration: None, result: None, + animations_enabled, } } @@ -877,7 +913,7 @@ impl HistoryCell for McpToolCallCell { let bullet = match status { Some(true) => "•".green().bold(), Some(false) => "•".red().bold(), - None => spinner(Some(self.start_time)), + None => spinner(Some(self.start_time), self.animations_enabled), }; let header_text = if status.is_some() { "Called" @@ -965,8 +1001,9 @@ impl HistoryCell for McpToolCallCell { pub(crate) fn new_active_mcp_tool_call( call_id: String, invocation: McpInvocation, + animations_enabled: bool, ) -> McpToolCallCell { - McpToolCallCell::new(call_id, invocation) + McpToolCallCell::new(call_id, invocation, animations_enabled) } pub(crate) fn new_web_search_call(query: String) -> PlainHistoryCell { @@ -1631,7 +1668,7 @@ mod tests { })), }; - let cell = new_active_mcp_tool_call("call-1".into(), invocation); + let cell = new_active_mcp_tool_call("call-1".into(), invocation, true); let rendered = render_lines(&cell.display_lines(80)).join("\n"); insta::assert_snapshot!(rendered); @@ -1658,7 +1695,7 @@ mod tests { structured_content: None, }; - let mut cell = new_active_mcp_tool_call("call-2".into(), invocation); + let mut cell = new_active_mcp_tool_call("call-2".into(), invocation, true); assert!( cell.complete(Duration::from_millis(1420), Ok(result)) .is_none() @@ -1680,7 +1717,7 @@ mod tests { })), }; - let mut cell = new_active_mcp_tool_call("call-3".into(), invocation); + let mut cell = new_active_mcp_tool_call("call-3".into(), invocation, true); assert!( cell.complete(Duration::from_secs(2), Err("network timeout".into())) .is_none() @@ -1724,7 +1761,7 @@ mod tests { structured_content: None, }; - let mut cell = new_active_mcp_tool_call("call-4".into(), invocation); + let mut cell = new_active_mcp_tool_call("call-4".into(), invocation, true); assert!( cell.complete(Duration::from_millis(640), Ok(result)) .is_none() @@ -1756,7 +1793,7 @@ mod tests { structured_content: None, }; - let mut cell = new_active_mcp_tool_call("call-5".into(), invocation); + let mut cell = new_active_mcp_tool_call("call-5".into(), invocation, true); assert!( cell.complete(Duration::from_millis(1280), Ok(result)) .is_none() @@ -1795,7 +1832,7 @@ mod tests { structured_content: None, }; - let mut cell = new_active_mcp_tool_call("call-6".into(), invocation); + let mut cell = new_active_mcp_tool_call("call-6".into(), invocation, true); assert!( cell.complete(Duration::from_millis(320), Ok(result)) .is_none() @@ -1853,32 +1890,35 @@ mod tests { fn coalesces_sequential_reads_within_one_call() { // Build one exec cell with a Search followed by two Reads let call_id = "c1".to_string(); - let mut cell = ExecCell::new(ExecCall { - call_id: call_id.clone(), - command: vec!["bash".into(), "-lc".into(), "echo".into()], - parsed: vec![ - ParsedCommand::Search { - query: Some("shimmer_spans".into()), - path: None, - cmd: "rg shimmer_spans".into(), - }, - ParsedCommand::Read { - name: "shimmer.rs".into(), - cmd: "cat shimmer.rs".into(), - path: "shimmer.rs".into(), - }, - ParsedCommand::Read { - name: "status_indicator_widget.rs".into(), - cmd: "cat status_indicator_widget.rs".into(), - path: "status_indicator_widget.rs".into(), - }, - ], - output: None, - source: ExecCommandSource::Agent, - start_time: Some(Instant::now()), - duration: None, - interaction_input: None, - }); + let mut cell = ExecCell::new( + ExecCall { + call_id: call_id.clone(), + command: vec!["bash".into(), "-lc".into(), "echo".into()], + parsed: vec![ + ParsedCommand::Search { + query: Some("shimmer_spans".into()), + path: None, + cmd: "rg shimmer_spans".into(), + }, + ParsedCommand::Read { + name: "shimmer.rs".into(), + cmd: "cat shimmer.rs".into(), + path: "shimmer.rs".into(), + }, + ParsedCommand::Read { + name: "status_indicator_widget.rs".into(), + cmd: "cat status_indicator_widget.rs".into(), + path: "status_indicator_widget.rs".into(), + }, + ], + output: None, + source: ExecCommandSource::Agent, + start_time: Some(Instant::now()), + duration: None, + interaction_input: None, + }, + true, + ); // Mark call complete so markers are ✓ cell.complete_call(&call_id, CommandOutput::default(), Duration::from_millis(1)); @@ -1889,20 +1929,23 @@ mod tests { #[test] fn coalesces_reads_across_multiple_calls() { - let mut cell = ExecCell::new(ExecCall { - call_id: "c1".to_string(), - command: vec!["bash".into(), "-lc".into(), "echo".into()], - parsed: vec![ParsedCommand::Search { - query: Some("shimmer_spans".into()), - path: None, - cmd: "rg shimmer_spans".into(), - }], - output: None, - source: ExecCommandSource::Agent, - start_time: Some(Instant::now()), - duration: None, - interaction_input: None, - }); + let mut cell = ExecCell::new( + ExecCall { + call_id: "c1".to_string(), + command: vec!["bash".into(), "-lc".into(), "echo".into()], + parsed: vec![ParsedCommand::Search { + query: Some("shimmer_spans".into()), + path: None, + cmd: "rg shimmer_spans".into(), + }], + output: None, + source: ExecCommandSource::Agent, + start_time: Some(Instant::now()), + duration: None, + interaction_input: None, + }, + true, + ); // Call 1: Search only cell.complete_call("c1", CommandOutput::default(), Duration::from_millis(1)); // Call 2: Read A @@ -1943,32 +1986,35 @@ mod tests { #[test] fn coalesced_reads_dedupe_names() { - let mut cell = ExecCell::new(ExecCall { - call_id: "c1".to_string(), - command: vec!["bash".into(), "-lc".into(), "echo".into()], - parsed: vec![ - ParsedCommand::Read { - name: "auth.rs".into(), - cmd: "cat auth.rs".into(), - path: "auth.rs".into(), - }, - ParsedCommand::Read { - name: "auth.rs".into(), - cmd: "cat auth.rs".into(), - path: "auth.rs".into(), - }, - ParsedCommand::Read { - name: "shimmer.rs".into(), - cmd: "cat shimmer.rs".into(), - path: "shimmer.rs".into(), - }, - ], - output: None, - source: ExecCommandSource::Agent, - start_time: Some(Instant::now()), - duration: None, - interaction_input: None, - }); + let mut cell = ExecCell::new( + ExecCall { + call_id: "c1".to_string(), + command: vec!["bash".into(), "-lc".into(), "echo".into()], + parsed: vec![ + ParsedCommand::Read { + name: "auth.rs".into(), + cmd: "cat auth.rs".into(), + path: "auth.rs".into(), + }, + ParsedCommand::Read { + name: "auth.rs".into(), + cmd: "cat auth.rs".into(), + path: "auth.rs".into(), + }, + ParsedCommand::Read { + name: "shimmer.rs".into(), + cmd: "cat shimmer.rs".into(), + path: "shimmer.rs".into(), + }, + ], + output: None, + source: ExecCommandSource::Agent, + start_time: Some(Instant::now()), + duration: None, + interaction_input: None, + }, + true, + ); cell.complete_call("c1", CommandOutput::default(), Duration::from_millis(1)); let lines = cell.display_lines(80); let rendered = render_lines(&lines).join("\n"); @@ -1980,16 +2026,19 @@ mod tests { // Create a completed exec cell with a multiline command let cmd = "set -o pipefail\ncargo test --all-features --quiet".to_string(); let call_id = "c1".to_string(); - let mut cell = ExecCell::new(ExecCall { - call_id: call_id.clone(), - command: vec!["bash".into(), "-lc".into(), cmd], - parsed: Vec::new(), - output: None, - source: ExecCommandSource::Agent, - start_time: Some(Instant::now()), - duration: None, - interaction_input: None, - }); + let mut cell = ExecCell::new( + ExecCall { + call_id: call_id.clone(), + command: vec!["bash".into(), "-lc".into(), cmd], + parsed: Vec::new(), + output: None, + source: ExecCommandSource::Agent, + start_time: Some(Instant::now()), + duration: None, + interaction_input: None, + }, + true, + ); // Mark call complete so it renders as "Ran" cell.complete_call(&call_id, CommandOutput::default(), Duration::from_millis(1)); @@ -2003,16 +2052,19 @@ mod tests { #[test] fn single_line_command_compact_when_fits() { let call_id = "c1".to_string(); - let mut cell = ExecCell::new(ExecCall { - call_id: call_id.clone(), - command: vec!["echo".into(), "ok".into()], - parsed: Vec::new(), - output: None, - source: ExecCommandSource::Agent, - start_time: Some(Instant::now()), - duration: None, - interaction_input: None, - }); + let mut cell = ExecCell::new( + ExecCall { + call_id: call_id.clone(), + command: vec!["echo".into(), "ok".into()], + parsed: Vec::new(), + output: None, + source: ExecCommandSource::Agent, + start_time: Some(Instant::now()), + duration: None, + interaction_input: None, + }, + true, + ); cell.complete_call(&call_id, CommandOutput::default(), Duration::from_millis(1)); // Wide enough that it fits inline let lines = cell.display_lines(80); @@ -2024,16 +2076,19 @@ mod tests { fn single_line_command_wraps_with_four_space_continuation() { let call_id = "c1".to_string(); let long = "a_very_long_token_without_spaces_to_force_wrapping".to_string(); - let mut cell = ExecCell::new(ExecCall { - call_id: call_id.clone(), - command: vec!["bash".into(), "-lc".into(), long], - parsed: Vec::new(), - output: None, - source: ExecCommandSource::Agent, - start_time: Some(Instant::now()), - duration: None, - interaction_input: None, - }); + let mut cell = ExecCell::new( + ExecCall { + call_id: call_id.clone(), + command: vec!["bash".into(), "-lc".into(), long], + parsed: Vec::new(), + output: None, + source: ExecCommandSource::Agent, + start_time: Some(Instant::now()), + duration: None, + interaction_input: None, + }, + true, + ); cell.complete_call(&call_id, CommandOutput::default(), Duration::from_millis(1)); let lines = cell.display_lines(24); let rendered = render_lines(&lines).join("\n"); @@ -2044,16 +2099,19 @@ mod tests { fn multiline_command_without_wrap_uses_branch_then_eight_spaces() { let call_id = "c1".to_string(); let cmd = "echo one\necho two".to_string(); - let mut cell = ExecCell::new(ExecCall { - call_id: call_id.clone(), - command: vec!["bash".into(), "-lc".into(), cmd], - parsed: Vec::new(), - output: None, - source: ExecCommandSource::Agent, - start_time: Some(Instant::now()), - duration: None, - interaction_input: None, - }); + let mut cell = ExecCell::new( + ExecCall { + call_id: call_id.clone(), + command: vec!["bash".into(), "-lc".into(), cmd], + parsed: Vec::new(), + output: None, + source: ExecCommandSource::Agent, + start_time: Some(Instant::now()), + duration: None, + interaction_input: None, + }, + true, + ); cell.complete_call(&call_id, CommandOutput::default(), Duration::from_millis(1)); let lines = cell.display_lines(80); let rendered = render_lines(&lines).join("\n"); @@ -2065,16 +2123,19 @@ mod tests { let call_id = "c1".to_string(); let cmd = "first_token_is_long_enough_to_wrap\nsecond_token_is_also_long_enough_to_wrap" .to_string(); - let mut cell = ExecCell::new(ExecCall { - call_id: call_id.clone(), - command: vec!["bash".into(), "-lc".into(), cmd], - parsed: Vec::new(), - output: None, - source: ExecCommandSource::Agent, - start_time: Some(Instant::now()), - duration: None, - interaction_input: None, - }); + let mut cell = ExecCell::new( + ExecCall { + call_id: call_id.clone(), + command: vec!["bash".into(), "-lc".into(), cmd], + parsed: Vec::new(), + output: None, + source: ExecCommandSource::Agent, + start_time: Some(Instant::now()), + duration: None, + interaction_input: None, + }, + true, + ); cell.complete_call(&call_id, CommandOutput::default(), Duration::from_millis(1)); let lines = cell.display_lines(28); let rendered = render_lines(&lines).join("\n"); @@ -2086,16 +2147,19 @@ mod tests { // Build an exec cell with a non-zero exit and 10 lines on stderr to exercise // the head/tail rendering and gutter prefixes. let call_id = "c_err".to_string(); - let mut cell = ExecCell::new(ExecCall { - call_id: call_id.clone(), - command: vec!["bash".into(), "-lc".into(), "seq 1 10 1>&2 && false".into()], - parsed: Vec::new(), - output: None, - source: ExecCommandSource::Agent, - start_time: Some(Instant::now()), - duration: None, - interaction_input: None, - }); + let mut cell = ExecCell::new( + ExecCall { + call_id: call_id.clone(), + command: vec!["bash".into(), "-lc".into(), "seq 1 10 1>&2 && false".into()], + parsed: Vec::new(), + output: None, + source: ExecCommandSource::Agent, + start_time: Some(Instant::now()), + duration: None, + interaction_input: None, + }, + true, + ); let stderr: String = (1..=10) .map(|n| n.to_string()) .collect::>() @@ -2133,16 +2197,19 @@ mod tests { let call_id = "c_wrap_err".to_string(); let long_cmd = "echo this_is_a_very_long_single_token_that_will_wrap_across_the_available_width"; - let mut cell = ExecCell::new(ExecCall { - call_id: call_id.clone(), - command: vec!["bash".into(), "-lc".into(), long_cmd.to_string()], - parsed: Vec::new(), - output: None, - source: ExecCommandSource::Agent, - start_time: Some(Instant::now()), - duration: None, - interaction_input: None, - }); + let mut cell = ExecCell::new( + ExecCall { + call_id: call_id.clone(), + command: vec!["bash".into(), "-lc".into(), long_cmd.to_string()], + parsed: Vec::new(), + output: None, + source: ExecCommandSource::Agent, + start_time: Some(Instant::now()), + duration: None, + interaction_input: None, + }, + true, + ); let stderr = "error: first line on stderr\nerror: second line on stderr".to_string(); cell.complete_call( diff --git a/codex-rs/tui/src/lib.rs b/codex-rs/tui/src/lib.rs index 1dfa0a292b..ee2e5fc68f 100644 --- a/codex-rs/tui/src/lib.rs +++ b/codex-rs/tui/src/lib.rs @@ -67,6 +67,7 @@ mod resume_picker; mod selection_list; mod session_log; mod shimmer; +mod skill_error_prompt; mod slash_command; mod status; mod status_indicator_widget; @@ -74,6 +75,7 @@ mod streaming; mod style; mod terminal_palette; mod text_formatting; +mod tooltips; mod tui; mod ui_consts; pub mod update_action; @@ -525,6 +527,7 @@ async fn run_ratatui_app( images, resume_selection, feedback, + should_show_trust_screen, // Proxy to: is it a first run in this directory? ) .await; diff --git a/codex-rs/tui/src/markdown_render.rs b/codex-rs/tui/src/markdown_render.rs index 099d0860c2..19cf944928 100644 --- a/codex-rs/tui/src/markdown_render.rs +++ b/codex-rs/tui/src/markdown_render.rs @@ -10,11 +10,50 @@ use pulldown_cmark::Parser; use pulldown_cmark::Tag; use pulldown_cmark::TagEnd; use ratatui::style::Style; -use ratatui::style::Stylize; use ratatui::text::Line; use ratatui::text::Span; use ratatui::text::Text; +struct MarkdownStyles { + h1: Style, + h2: Style, + h3: Style, + h4: Style, + h5: Style, + h6: Style, + code: Style, + emphasis: Style, + strong: Style, + strikethrough: Style, + ordered_list_marker: Style, + unordered_list_marker: Style, + link: Style, + blockquote: Style, +} + +impl Default for MarkdownStyles { + fn default() -> Self { + use ratatui::style::Stylize; + + Self { + h1: Style::new().bold().underlined(), + h2: Style::new().bold(), + h3: Style::new().bold().italic(), + h4: Style::new().italic(), + h5: Style::new().italic(), + h6: Style::new().italic(), + code: Style::new().cyan(), + emphasis: Style::new().italic(), + strong: Style::new().bold(), + strikethrough: Style::new().crossed_out(), + ordered_list_marker: Style::new().light_blue(), + unordered_list_marker: Style::new(), + link: Style::new().cyan().underlined(), + blockquote: Style::new().green(), + } + } +} + #[derive(Clone, Debug)] struct IndentContext { prefix: Vec>, @@ -51,6 +90,7 @@ where { iter: I, text: Text<'static>, + styles: MarkdownStyles, inline_styles: Vec