diff --git a/.codespellignore b/.codespellignore index 835c0e538e..947511b0c3 100644 --- a/.codespellignore +++ b/.codespellignore @@ -1,3 +1,5 @@ iTerm iTerm2 -psuedo \ No newline at end of file +psuedo +te +TE diff --git a/.codespellrc b/.codespellrc index ec1f02408e..4f3067ed76 100644 --- a/.codespellrc +++ b/.codespellrc @@ -3,4 +3,4 @@ skip = .git*,vendor,*-lock.yaml,*.lock,.codespellrc,*test.ts,*.jsonl,frame*.txt,*.snap,*.snap.new,*meriyah.umd.min.js check-hidden = true ignore-regex = ^\s*"image/\S+": ".*|\b(afterAll)\b -ignore-words-list = ratatui,ser,iTerm,iterm2,iterm +ignore-words-list = ratatui,ser,iTerm,iterm2,iterm,te,TE diff --git a/.github/ISSUE_TEMPLATE/1-codex-app.yml b/.github/ISSUE_TEMPLATE/1-codex-app.yml index 569094907f..6e294ee27b 100644 --- a/.github/ISSUE_TEMPLATE/1-codex-app.yml +++ b/.github/ISSUE_TEMPLATE/1-codex-app.yml @@ -21,6 +21,13 @@ body: label: What subscription do you have? validations: required: true + - type: input + id: platform + attributes: + label: What platform is your computer? + description: | + For macOS and Linux: copy the output of `uname -mprs` + For Windows: copy the output of `"$([Environment]::OSVersion | ForEach-Object VersionString) $(if ([Environment]::Is64BitOperatingSystem) { "x64" } else { "x86" })"` in the PowerShell console - type: textarea id: actual attributes: diff --git a/.github/prompts/issue-deduplicator.txt b/.github/prompts/issue-deduplicator.txt deleted file mode 100644 index 9bdd3af9ef..0000000000 --- a/.github/prompts/issue-deduplicator.txt +++ /dev/null @@ -1,18 +0,0 @@ -You are an assistant that triages new GitHub issues by identifying potential duplicates. - -You will receive the following JSON files located in the current working directory: -- `codex-current-issue.json`: JSON object describing the newly created issue (fields: number, title, body). -- `codex-existing-issues.json`: JSON array of recent issues (each element includes number, title, body, createdAt). - -Instructions: -- Load both files as JSON and review their contents carefully. The codex-existing-issues.json file is large, ensure you explore all of it. -- Compare the current issue against the existing issues to find up to five that appear to describe the same underlying problem or request. -- Only consider an issue a potential duplicate if there is a clear overlap in symptoms, feature requests, reproduction steps, or error messages. -- Prioritize newer issues when similarity is comparable. -- Ignore pull requests and issues whose similarity is tenuous. -- When unsure, prefer returning fewer matches. - -Output requirements: -- Respond with a JSON array of issue numbers (integers), ordered from most likely duplicate to least. -- Include at most five numbers. -- If you find no plausible duplicates, respond with `[]`. diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index 292e549b41..59a1e07fa7 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -65,6 +65,11 @@ jobs: - name: Set up Bazel uses: bazelbuild/setup-bazelisk@v3 + - name: Check MODULE.bazel.lock is up to date + if: matrix.os == 'ubuntu-24.04' && matrix.target == 'x86_64-unknown-linux-gnu' + shell: bash + run: ./scripts/check-module-bazel-lock.sh + # TODO(mbolin): Bring this back once we have caching working. Currently, # we never seem to get a cache hit but we still end up paying the cost of # uploading at the end of the build, which takes over a minute! diff --git a/.github/workflows/issue-deduplicator.yml b/.github/workflows/issue-deduplicator.yml index 4b417ae59d..ae855ab90b 100644 --- a/.github/workflows/issue-deduplicator.yml +++ b/.github/workflows/issue-deduplicator.yml @@ -15,34 +15,68 @@ jobs: permissions: contents: read outputs: - codex_output: ${{ steps.codex.outputs.final-message }} + codex_output: ${{ steps.select-final.outputs.codex_output }} steps: - uses: actions/checkout@v6 - name: Prepare Codex inputs env: GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + ISSUE_NUMBER: ${{ github.event.issue.number }} run: | set -eo pipefail CURRENT_ISSUE_FILE=codex-current-issue.json - EXISTING_ISSUES_FILE=codex-existing-issues.json + EXISTING_ALL_FILE=codex-existing-issues-all.json + EXISTING_OPEN_FILE=codex-existing-issues-open.json - gh issue list --repo "${{ github.repository }}" \ - --json number,title,body,createdAt \ + gh issue list --repo "$REPO" \ + --json number,title,body,createdAt,updatedAt,state,labels \ --limit 1000 \ --state all \ --search "sort:created-desc" \ - | jq '.' \ - > "$EXISTING_ISSUES_FILE" + | jq '[.[] | { + number, + title, + body: ((.body // "")[0:4000]), + createdAt, + updatedAt, + state, + labels: ((.labels // []) | map(.name)) + }]' \ + > "$EXISTING_ALL_FILE" - gh issue view "${{ github.event.issue.number }}" \ - --repo "${{ github.repository }}" \ + gh issue list --repo "$REPO" \ + --json number,title,body,createdAt,updatedAt,state,labels \ + --limit 1000 \ + --state open \ + --search "sort:created-desc" \ + | jq '[.[] | { + number, + title, + body: ((.body // "")[0:4000]), + createdAt, + updatedAt, + state, + labels: ((.labels // []) | map(.name)) + }]' \ + > "$EXISTING_OPEN_FILE" + + gh issue view "$ISSUE_NUMBER" \ + --repo "$REPO" \ --json number,title,body \ - | jq '.' \ + | jq '{number, title, body: ((.body // "")[0:4000])}' \ > "$CURRENT_ISSUE_FILE" - - id: codex + echo "Prepared duplicate detection input files." + echo "all_issue_count=$(jq 'length' "$EXISTING_ALL_FILE")" + echo "open_issue_count=$(jq 'length' "$EXISTING_OPEN_FILE")" + + # Prompt instructions are intentionally inline in this workflow. The old + # .github/prompts/issue-deduplicator.txt file is obsolete and removed. + - id: codex-all + name: Find duplicates (pass 1, all issues) uses: openai/codex-action@main with: openai-api-key: ${{ secrets.CODEX_OPENAI_API_KEY }} @@ -52,14 +86,17 @@ jobs: You will receive the following JSON files located in the current working directory: - `codex-current-issue.json`: JSON object describing the newly created issue (fields: number, title, body). - - `codex-existing-issues.json`: JSON array of recent issues (each element includes number, title, body, createdAt). + - `codex-existing-issues-all.json`: JSON array of recent issues with states, timestamps, and labels. Instructions: - Compare the current issue against the existing issues to find up to five that appear to describe the same underlying problem or request. - - Focus on the underlying intent and context of each issue—such as reported symptoms, feature requests, reproduction steps, or error messages—rather than relying solely on string similarity or synthetic metrics. - - After your analysis, validate your results in 1-2 lines explaining your decision to return the selected matches. - - When unsure, prefer returning fewer matches. - - Include at most five numbers. + - Prioritize concrete overlap in symptoms, reproduction details, error signatures, and user intent. + - Prefer active unresolved issues when confidence is similar. + - Closed issues can still be valid duplicates if they clearly match. + - Return fewer matches rather than speculative ones. + - If confidence is low, return an empty list. + - Include at most five issue numbers. + - After analysis, provide a short reason for your decision. output-schema: | { @@ -77,6 +114,179 @@ jobs: "additionalProperties": false } + - id: normalize-all + name: Normalize pass 1 output + env: + CODEX_OUTPUT: ${{ steps.codex-all.outputs.final-message }} + CURRENT_ISSUE_NUMBER: ${{ github.event.issue.number }} + run: | + set -eo pipefail + + raw=${CODEX_OUTPUT//$'\r'/} + parsed=false + issues='[]' + reason='' + + if [ -n "$raw" ] && printf '%s' "$raw" | jq -e 'type == "object" and (.issues | type == "array")' >/dev/null 2>&1; then + parsed=true + issues=$(printf '%s' "$raw" | jq -c '[.issues[] | tostring]') + reason=$(printf '%s' "$raw" | jq -r '.reason // ""') + else + reason='Pass 1 output was empty or invalid JSON.' + fi + + filtered=$(jq -cn --argjson issues "$issues" --arg current "$CURRENT_ISSUE_NUMBER" '[ + $issues[] + | tostring + | select(. != $current) + ] | reduce .[] as $issue ([]; if index($issue) then . else . + [$issue] end) | .[:5]') + + has_matches=false + if [ "$(jq 'length' <<< "$filtered")" -gt 0 ]; then + has_matches=true + fi + + echo "Pass 1 parsed: $parsed" + echo "Pass 1 matches after filtering: $(jq 'length' <<< "$filtered")" + echo "Pass 1 reason: $reason" + + { + echo "issues_json=$filtered" + echo "reason<> "$GITHUB_OUTPUT" + + - id: codex-open + name: Find duplicates (pass 2, open issues) + if: ${{ steps.normalize-all.outputs.has_matches != 'true' }} + uses: openai/codex-action@main + with: + openai-api-key: ${{ secrets.CODEX_OPENAI_API_KEY }} + allow-users: "*" + prompt: | + You are an assistant that triages new GitHub issues by identifying potential duplicates. + + This is a fallback pass because a broad search did not find convincing matches. + + You will receive the following JSON files located in the current working directory: + - `codex-current-issue.json`: JSON object describing the newly created issue (fields: number, title, body). + - `codex-existing-issues-open.json`: JSON array of open issues only. + + Instructions: + - Search only these active unresolved issues for duplicates of the current issue. + - Prioritize concrete overlap in symptoms, reproduction details, error signatures, and user intent. + - Prefer fewer, higher-confidence matches. + - If confidence is low, return an empty list. + - Include at most five issue numbers. + - After analysis, provide a short reason for your decision. + + output-schema: | + { + "type": "object", + "properties": { + "issues": { + "type": "array", + "items": { + "type": "string" + } + }, + "reason": { "type": "string" } + }, + "required": ["issues", "reason"], + "additionalProperties": false + } + + - id: normalize-open + name: Normalize pass 2 output + if: ${{ steps.normalize-all.outputs.has_matches != 'true' }} + env: + CODEX_OUTPUT: ${{ steps.codex-open.outputs.final-message }} + CURRENT_ISSUE_NUMBER: ${{ github.event.issue.number }} + run: | + set -eo pipefail + + raw=${CODEX_OUTPUT//$'\r'/} + parsed=false + issues='[]' + reason='' + + if [ -n "$raw" ] && printf '%s' "$raw" | jq -e 'type == "object" and (.issues | type == "array")' >/dev/null 2>&1; then + parsed=true + issues=$(printf '%s' "$raw" | jq -c '[.issues[] | tostring]') + reason=$(printf '%s' "$raw" | jq -r '.reason // ""') + else + reason='Pass 2 output was empty or invalid JSON.' + fi + + filtered=$(jq -cn --argjson issues "$issues" --arg current "$CURRENT_ISSUE_NUMBER" '[ + $issues[] + | tostring + | select(. != $current) + ] | reduce .[] as $issue ([]; if index($issue) then . else . + [$issue] end) | .[:5]') + + has_matches=false + if [ "$(jq 'length' <<< "$filtered")" -gt 0 ]; then + has_matches=true + fi + + echo "Pass 2 parsed: $parsed" + echo "Pass 2 matches after filtering: $(jq 'length' <<< "$filtered")" + echo "Pass 2 reason: $reason" + + { + echo "issues_json=$filtered" + echo "reason<> "$GITHUB_OUTPUT" + + - id: select-final + name: Select final duplicate set + env: + PASS1_ISSUES: ${{ steps.normalize-all.outputs.issues_json }} + PASS1_REASON: ${{ steps.normalize-all.outputs.reason }} + PASS2_ISSUES: ${{ steps.normalize-open.outputs.issues_json }} + PASS2_REASON: ${{ steps.normalize-open.outputs.reason }} + PASS1_HAS_MATCHES: ${{ steps.normalize-all.outputs.has_matches }} + PASS2_HAS_MATCHES: ${{ steps.normalize-open.outputs.has_matches }} + run: | + set -eo pipefail + + selected_issues='[]' + selected_reason='No plausible duplicates found.' + selected_pass='none' + + if [ "$PASS1_HAS_MATCHES" = "true" ]; then + selected_issues=${PASS1_ISSUES:-'[]'} + selected_reason=${PASS1_REASON:-'Pass 1 found duplicates.'} + selected_pass='all' + fi + + if [ "$PASS2_HAS_MATCHES" = "true" ]; then + selected_issues=${PASS2_ISSUES:-'[]'} + selected_reason=${PASS2_REASON:-'Pass 2 found duplicates.'} + selected_pass='open-fallback' + fi + + final_json=$(jq -cn \ + --argjson issues "$selected_issues" \ + --arg reason "$selected_reason" \ + --arg pass "$selected_pass" \ + '{issues: $issues, reason: $reason, pass: $pass}') + + echo "Final pass used: $selected_pass" + echo "Final duplicate count: $(jq '.issues | length' <<< "$final_json")" + echo "Final reason: $(jq -r '.reason' <<< "$final_json")" + + { + echo "codex_output<> "$GITHUB_OUTPUT" + comment-on-issue: name: Comment with potential duplicates needs: gather-duplicates @@ -105,11 +315,17 @@ jobs: const issues = Array.isArray(parsed?.issues) ? parsed.issues : []; const currentIssueNumber = String(context.payload.issue.number); + const passUsed = typeof parsed?.pass === 'string' ? parsed.pass : 'unknown'; + const reason = typeof parsed?.reason === 'string' ? parsed.reason : ''; console.log(`Current issue number: ${currentIssueNumber}`); + console.log(`Pass used: ${passUsed}`); + if (reason) { + console.log(`Reason: ${reason}`); + } console.log(issues); - const filteredIssues = issues.filter((value) => String(value) !== currentIssueNumber); + const filteredIssues = [...new Set(issues.map((value) => String(value)))].filter((value) => value !== currentIssueNumber).slice(0, 5); if (filteredIssues.length === 0) { core.info('Codex reported no potential duplicates.'); diff --git a/.github/workflows/rust-ci.yml b/.github/workflows/rust-ci.yml index c52013149c..f4664c16a8 100644 --- a/.github/workflows/rust-ci.yml +++ b/.github/workflows/rust-ci.yml @@ -59,7 +59,7 @@ jobs: working-directory: codex-rs steps: - uses: actions/checkout@v6 - - uses: dtolnay/rust-toolchain@1.93 + - uses: dtolnay/rust-toolchain@1.93.0 with: components: rustfmt - name: cargo fmt @@ -75,7 +75,7 @@ jobs: working-directory: codex-rs steps: - uses: actions/checkout@v6 - - uses: dtolnay/rust-toolchain@1.93 + - uses: dtolnay/rust-toolchain@1.93.0 - uses: taiki-e/install-action@44c6d64aa62cd779e873306675c7a58e86d6d532 # v2 with: tool: cargo-shear @@ -196,7 +196,7 @@ jobs: fi sudo DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends "${packages[@]}" fi - - uses: dtolnay/rust-toolchain@1.93 + - uses: dtolnay/rust-toolchain@1.93.0 with: targets: ${{ matrix.target }} components: clippy @@ -513,7 +513,7 @@ jobs: - name: Install DotSlash uses: facebook/install-dotslash@v2 - - uses: dtolnay/rust-toolchain@1.93 + - uses: dtolnay/rust-toolchain@1.93.0 with: targets: ${{ matrix.target }} diff --git a/.github/workflows/rust-release-windows.yml b/.github/workflows/rust-release-windows.yml index fd411715be..85556bc8c9 100644 --- a/.github/workflows/rust-release-windows.yml +++ b/.github/workflows/rust-release-windows.yml @@ -82,7 +82,7 @@ jobs: Write-Host "Total RAM: $ramGiB GiB" Write-Host "Disk usage:" Get-PSDrive -PSProvider FileSystem | Format-Table -AutoSize Name, @{Name='Size(GB)';Expression={[math]::Round(($_.Used + $_.Free) / 1GB, 1)}}, @{Name='Free(GB)';Expression={[math]::Round($_.Free / 1GB, 1)}} - - uses: dtolnay/rust-toolchain@1.93 + - uses: dtolnay/rust-toolchain@1.93.0 with: targets: ${{ matrix.target }} diff --git a/.github/workflows/rust-release.yml b/.github/workflows/rust-release.yml index 3ed2243951..30090089c6 100644 --- a/.github/workflows/rust-release.yml +++ b/.github/workflows/rust-release.yml @@ -123,7 +123,7 @@ jobs: sudo apt-get update -y sudo DEBIAN_FRONTEND=noninteractive apt-get install -y libubsan1 fi - - uses: dtolnay/rust-toolchain@1.93 + - uses: dtolnay/rust-toolchain@1.93.0 with: targets: ${{ matrix.target }} diff --git a/.github/workflows/sdk.yml b/.github/workflows/sdk.yml index ec67cd1871..33e0c080ee 100644 --- a/.github/workflows/sdk.yml +++ b/.github/workflows/sdk.yml @@ -31,7 +31,7 @@ jobs: node-version: 22 cache: pnpm - - uses: dtolnay/rust-toolchain@1.93 + - uses: dtolnay/rust-toolchain@1.93.0 - name: build codex run: cargo build --bin codex diff --git a/.github/workflows/shell-tool-mcp.yml b/.github/workflows/shell-tool-mcp.yml index 983a5b1517..04e127a785 100644 --- a/.github/workflows/shell-tool-mcp.yml +++ b/.github/workflows/shell-tool-mcp.yml @@ -105,7 +105,7 @@ jobs: sudo DEBIAN_FRONTEND=noninteractive apt-get install -y libubsan1 fi - - uses: dtolnay/rust-toolchain@1.93 + - uses: dtolnay/rust-toolchain@1.93.0 with: targets: ${{ matrix.target }} @@ -251,11 +251,11 @@ jobs: set -euo pipefail if command -v apt-get >/dev/null 2>&1; then apt-get update - DEBIAN_FRONTEND=noninteractive apt-get install -y git build-essential bison autoconf gettext + DEBIAN_FRONTEND=noninteractive apt-get install -y git build-essential bison autoconf gettext libncursesw5-dev elif command -v dnf >/dev/null 2>&1; then - dnf install -y git gcc gcc-c++ make bison autoconf gettext + dnf install -y git gcc gcc-c++ make bison autoconf gettext ncurses-devel elif command -v yum >/dev/null 2>&1; then - yum install -y git gcc gcc-c++ make bison autoconf gettext + yum install -y git gcc gcc-c++ make bison autoconf gettext ncurses-devel else echo "Unsupported package manager in container" exit 1 @@ -329,6 +329,212 @@ jobs: path: artifacts/** if-no-files-found: error + zsh-linux: + name: Build zsh (Linux) - ${{ matrix.variant }} - ${{ matrix.target }} + needs: metadata + runs-on: ${{ matrix.runner }} + timeout-minutes: 30 + container: + image: ${{ matrix.image }} + strategy: + fail-fast: false + matrix: + include: + - runner: ubuntu-24.04 + target: x86_64-unknown-linux-musl + variant: ubuntu-24.04 + image: ubuntu:24.04 + - runner: ubuntu-24.04 + target: x86_64-unknown-linux-musl + variant: ubuntu-22.04 + image: ubuntu:22.04 + - runner: ubuntu-24.04 + target: x86_64-unknown-linux-musl + variant: debian-12 + image: debian:12 + - runner: ubuntu-24.04 + target: x86_64-unknown-linux-musl + variant: debian-11 + image: debian:11 + - runner: ubuntu-24.04 + target: x86_64-unknown-linux-musl + variant: centos-9 + image: quay.io/centos/centos:stream9 + - runner: ubuntu-24.04-arm + target: aarch64-unknown-linux-musl + variant: ubuntu-24.04 + image: arm64v8/ubuntu:24.04 + - runner: ubuntu-24.04-arm + target: aarch64-unknown-linux-musl + variant: ubuntu-22.04 + image: arm64v8/ubuntu:22.04 + - runner: ubuntu-24.04-arm + target: aarch64-unknown-linux-musl + variant: ubuntu-20.04 + image: arm64v8/ubuntu:20.04 + - runner: ubuntu-24.04-arm + target: aarch64-unknown-linux-musl + variant: debian-12 + image: arm64v8/debian:12 + - runner: ubuntu-24.04-arm + target: aarch64-unknown-linux-musl + variant: debian-11 + image: arm64v8/debian:11 + - runner: ubuntu-24.04-arm + target: aarch64-unknown-linux-musl + variant: centos-9 + image: quay.io/centos/centos:stream9 + steps: + - name: Install build prerequisites + shell: bash + run: | + set -euo pipefail + if command -v apt-get >/dev/null 2>&1; then + apt-get update + DEBIAN_FRONTEND=noninteractive apt-get install -y git build-essential bison autoconf gettext libncursesw5-dev + elif command -v dnf >/dev/null 2>&1; then + dnf install -y git gcc gcc-c++ make bison autoconf gettext ncurses-devel + elif command -v yum >/dev/null 2>&1; then + yum install -y git gcc gcc-c++ make bison autoconf gettext ncurses-devel + else + echo "Unsupported package manager in container" + exit 1 + fi + + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Build patched zsh + shell: bash + run: | + set -euo pipefail + git clone --depth 1 https://git.code.sf.net/p/zsh/code /tmp/zsh + cd /tmp/zsh + git fetch --depth 1 origin 77045ef899e53b9598bebc5a41db93a548a40ca6 + git checkout 77045ef899e53b9598bebc5a41db93a548a40ca6 + git apply "${GITHUB_WORKSPACE}/shell-tool-mcp/patches/zsh-exec-wrapper.patch" + ./Util/preconfig + ./configure + cores="$(command -v nproc >/dev/null 2>&1 && nproc || getconf _NPROCESSORS_ONLN)" + make -j"${cores}" + + dest="${GITHUB_WORKSPACE}/artifacts/vendor/${{ matrix.target }}/zsh/${{ matrix.variant }}" + mkdir -p "$dest" + cp Src/zsh "$dest/zsh" + + - name: Smoke test zsh exec wrapper + shell: bash + run: | + set -euo pipefail + tmpdir="$(mktemp -d)" + cat > "$tmpdir/exec-wrapper" <<'EOF' + #!/usr/bin/env bash + set -euo pipefail + : "${CODEX_WRAPPER_LOG:?missing CODEX_WRAPPER_LOG}" + printf '%s\n' "$@" > "$CODEX_WRAPPER_LOG" + file="$1" + shift + if [[ "$#" -eq 0 ]]; then + exec "$file" + fi + arg0="$1" + shift + exec -a "$arg0" "$file" "$@" + EOF + chmod +x "$tmpdir/exec-wrapper" + + CODEX_WRAPPER_LOG="$tmpdir/wrapper.log" \ + EXEC_WRAPPER="$tmpdir/exec-wrapper" \ + /tmp/zsh/Src/zsh -fc '/bin/echo smoke-zsh' > "$tmpdir/stdout.txt" + + grep -Fx "smoke-zsh" "$tmpdir/stdout.txt" + grep -Fx "/bin/echo" "$tmpdir/wrapper.log" + + - uses: actions/upload-artifact@v6 + with: + name: shell-tool-mcp-zsh-${{ matrix.target }}-${{ matrix.variant }} + path: artifacts/** + if-no-files-found: error + + zsh-darwin: + name: Build zsh (macOS) - ${{ matrix.variant }} - ${{ matrix.target }} + needs: metadata + runs-on: ${{ matrix.runner }} + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + include: + - runner: macos-15-xlarge + target: aarch64-apple-darwin + variant: macos-15 + - runner: macos-14 + target: aarch64-apple-darwin + variant: macos-14 + steps: + - name: Install build prerequisites + shell: bash + run: | + set -euo pipefail + if ! command -v autoconf >/dev/null 2>&1; then + brew install autoconf + fi + + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Build patched zsh + shell: bash + run: | + set -euo pipefail + git clone --depth 1 https://git.code.sf.net/p/zsh/code /tmp/zsh + cd /tmp/zsh + git fetch --depth 1 origin 77045ef899e53b9598bebc5a41db93a548a40ca6 + git checkout 77045ef899e53b9598bebc5a41db93a548a40ca6 + git apply "${GITHUB_WORKSPACE}/shell-tool-mcp/patches/zsh-exec-wrapper.patch" + ./Util/preconfig + ./configure + cores="$(getconf _NPROCESSORS_ONLN)" + make -j"${cores}" + + dest="${GITHUB_WORKSPACE}/artifacts/vendor/${{ matrix.target }}/zsh/${{ matrix.variant }}" + mkdir -p "$dest" + cp Src/zsh "$dest/zsh" + + - name: Smoke test zsh exec wrapper + shell: bash + run: | + set -euo pipefail + tmpdir="$(mktemp -d)" + cat > "$tmpdir/exec-wrapper" <<'EOF' + #!/usr/bin/env bash + set -euo pipefail + : "${CODEX_WRAPPER_LOG:?missing CODEX_WRAPPER_LOG}" + printf '%s\n' "$@" > "$CODEX_WRAPPER_LOG" + file="$1" + shift + if [[ "$#" -eq 0 ]]; then + exec "$file" + fi + arg0="$1" + shift + exec -a "$arg0" "$file" "$@" + EOF + chmod +x "$tmpdir/exec-wrapper" + + CODEX_WRAPPER_LOG="$tmpdir/wrapper.log" \ + EXEC_WRAPPER="$tmpdir/exec-wrapper" \ + /tmp/zsh/Src/zsh -fc '/bin/echo smoke-zsh' > "$tmpdir/stdout.txt" + + grep -Fx "smoke-zsh" "$tmpdir/stdout.txt" + grep -Fx "/bin/echo" "$tmpdir/wrapper.log" + + - uses: actions/upload-artifact@v6 + with: + name: shell-tool-mcp-zsh-${{ matrix.target }}-${{ matrix.variant }} + path: artifacts/** + if-no-files-found: error + package: name: Package npm module needs: @@ -336,6 +542,8 @@ jobs: - rust-binaries - bash-linux - bash-darwin + - zsh-linux + - zsh-darwin runs-on: ubuntu-latest env: PACKAGE_VERSION: ${{ needs.metadata.outputs.version }} @@ -409,7 +617,8 @@ jobs: chmod +x \ "$staging"/vendor/*/codex-exec-mcp-server \ "$staging"/vendor/*/codex-execve-wrapper \ - "$staging"/vendor/*/bash/*/bash + "$staging"/vendor/*/bash/*/bash \ + "$staging"/vendor/*/zsh/*/zsh - name: Create npm tarball shell: bash diff --git a/AGENTS.md b/AGENTS.md index fc545c5080..fa2a4bbaba 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -15,6 +15,10 @@ In the codex-rs folder where the rust code lives: - When writing tests, prefer comparing the equality of entire objects over fields one by one. - When making a change that adds or changes an API, ensure that the documentation in the `docs/` folder is up to date if applicable. - If you change `ConfigToml` or nested config types, run `just write-config-schema` to update `codex-rs/core/config.schema.json`. +- If you change Rust dependencies (`Cargo.toml` or `Cargo.lock`), run `just bazel-lock-update` from the + repo root to refresh `MODULE.bazel.lock`, and include that lockfile update in the same change. +- After dependency changes, run `just bazel-lock-check` from the repo root so lockfile drift is caught + locally before CI. - Do not create small helper methods that are referenced only once. Run `just fmt` (in `codex-rs` directory) automatically after you have finished making Rust code changes; do not ask for approval to run it. Additionally, run the tests: diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 98efd3cede..6f15063691 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -1154,8 +1154,8 @@ "reqwest_0.12.28": "{\"dependencies\":[{\"name\":\"base64\",\"req\":\"^0.22\"},{\"kind\":\"dev\",\"name\":\"brotli_crate\",\"package\":\"brotli\",\"req\":\"^8\",\"target\":\"cfg(not(target_arch = \\\"wasm32\\\"))\"},{\"name\":\"bytes\",\"req\":\"^1.2\"},{\"name\":\"cookie_crate\",\"optional\":true,\"package\":\"cookie\",\"req\":\"^0.18.0\",\"target\":\"cfg(not(target_arch = \\\"wasm32\\\"))\"},{\"name\":\"cookie_store\",\"optional\":true,\"req\":\"^0.22.0\",\"target\":\"cfg(not(target_arch = \\\"wasm32\\\"))\"},{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3\",\"target\":\"cfg(not(target_arch = \\\"wasm32\\\"))\"},{\"name\":\"encoding_rs\",\"optional\":true,\"req\":\"^0.8\",\"target\":\"cfg(not(target_arch = \\\"wasm32\\\"))\"},{\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"^0.10\",\"target\":\"cfg(not(target_arch = \\\"wasm32\\\"))\"},{\"kind\":\"dev\",\"name\":\"flate2\",\"req\":\"^1.0.13\",\"target\":\"cfg(not(target_arch = \\\"wasm32\\\"))\"},{\"name\":\"futures-channel\",\"optional\":true,\"req\":\"^0.3\",\"target\":\"cfg(not(target_arch = \\\"wasm32\\\"))\"},{\"default_features\":false,\"name\":\"futures-core\",\"req\":\"^0.3.28\"},{\"default_features\":false,\"name\":\"futures-util\",\"optional\":true,\"req\":\"^0.3.28\"},{\"default_features\":false,\"features\":[\"std\",\"alloc\"],\"kind\":\"dev\",\"name\":\"futures-util\",\"req\":\"^0.3.28\",\"target\":\"cfg(not(target_arch = \\\"wasm32\\\"))\"},{\"name\":\"h2\",\"optional\":true,\"req\":\"^0.4\",\"target\":\"cfg(not(target_arch = \\\"wasm32\\\"))\"},{\"name\":\"h3\",\"optional\":true,\"req\":\"^0.0.8\",\"target\":\"cfg(not(target_arch = \\\"wasm32\\\"))\"},{\"name\":\"h3-quinn\",\"optional\":true,\"req\":\"^0.0.10\",\"target\":\"cfg(not(target_arch = \\\"wasm32\\\"))\"},{\"features\":[\"tokio\"],\"name\":\"hickory-resolver\",\"optional\":true,\"req\":\"^0.25\",\"target\":\"cfg(not(target_arch = \\\"wasm32\\\"))\"},{\"name\":\"http\",\"req\":\"^1.1\"},{\"name\":\"http-body\",\"req\":\"^1\",\"target\":\"cfg(not(target_arch = \\\"wasm32\\\"))\"},{\"name\":\"http-body-util\",\"req\":\"^0.1.2\",\"target\":\"cfg(not(target_arch = \\\"wasm32\\\"))\"},{\"features\":[\"http1\",\"client\"],\"name\":\"hyper\",\"req\":\"^1.1\",\"target\":\"cfg(not(target_arch = \\\"wasm32\\\"))\"},{\"default_features\":false,\"features\":[\"http1\",\"http2\",\"client\",\"server\"],\"kind\":\"dev\",\"name\":\"hyper\",\"req\":\"^1.1.0\",\"target\":\"cfg(not(target_arch = \\\"wasm32\\\"))\"},{\"default_features\":false,\"features\":[\"http1\",\"tls12\"],\"name\":\"hyper-rustls\",\"optional\":true,\"req\":\"^0.27.0\",\"target\":\"cfg(not(target_arch = \\\"wasm32\\\"))\"},{\"name\":\"hyper-tls\",\"optional\":true,\"req\":\"^0.6\",\"target\":\"cfg(not(target_arch = \\\"wasm32\\\"))\"},{\"features\":[\"http1\",\"client\",\"client-legacy\",\"client-proxy\",\"tokio\"],\"name\":\"hyper-util\",\"req\":\"^0.1.12\",\"target\":\"cfg(not(target_arch = \\\"wasm32\\\"))\"},{\"features\":[\"http1\",\"http2\",\"client\",\"client-legacy\",\"server-auto\",\"server-graceful\",\"tokio\"],\"kind\":\"dev\",\"name\":\"hyper-util\",\"req\":\"^0.1.12\",\"target\":\"cfg(not(target_arch = \\\"wasm32\\\"))\"},{\"name\":\"js-sys\",\"req\":\"^0.3.77\",\"target\":\"cfg(target_arch = \\\"wasm32\\\")\"},{\"kind\":\"dev\",\"name\":\"libc\",\"req\":\"^0\"},{\"name\":\"log\",\"req\":\"^0.4.17\",\"target\":\"cfg(not(target_arch = \\\"wasm32\\\"))\"},{\"name\":\"mime\",\"optional\":true,\"req\":\"^0.3.16\",\"target\":\"cfg(not(target_arch = \\\"wasm32\\\"))\"},{\"default_features\":false,\"name\":\"mime_guess\",\"optional\":true,\"req\":\"^2.0\"},{\"name\":\"native-tls-crate\",\"optional\":true,\"package\":\"native-tls\",\"req\":\"^0.2.10\",\"target\":\"cfg(not(target_arch = \\\"wasm32\\\"))\"},{\"kind\":\"dev\",\"name\":\"num_cpus\",\"req\":\"^1.0\"},{\"name\":\"once_cell\",\"optional\":true,\"req\":\"^1.18\",\"target\":\"cfg(not(target_arch = \\\"wasm32\\\"))\"},{\"name\":\"percent-encoding\",\"req\":\"^2.3\",\"target\":\"cfg(not(target_arch = \\\"wasm32\\\"))\"},{\"name\":\"pin-project-lite\",\"req\":\"^0.2.11\",\"target\":\"cfg(not(target_arch = \\\"wasm32\\\"))\"},{\"default_features\":false,\"features\":[\"rustls\",\"runtime-tokio\"],\"name\":\"quinn\",\"optional\":true,\"req\":\"^0.11.1\",\"target\":\"cfg(not(target_arch = \\\"wasm32\\\"))\"},{\"default_features\":false,\"features\":[\"std\",\"tls12\"],\"name\":\"rustls\",\"optional\":true,\"req\":\"^0.23.4\",\"target\":\"cfg(not(target_arch = \\\"wasm32\\\"))\"},{\"name\":\"rustls-native-certs\",\"optional\":true,\"req\":\"^0.8.0\",\"target\":\"cfg(not(target_arch = \\\"wasm32\\\"))\"},{\"features\":[\"std\"],\"name\":\"rustls-pki-types\",\"optional\":true,\"req\":\"^1.9.0\",\"target\":\"cfg(not(target_arch = \\\"wasm32\\\"))\"},{\"name\":\"serde\",\"req\":\"^1.0\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0\",\"target\":\"cfg(not(target_arch = \\\"wasm32\\\"))\"},{\"name\":\"serde_json\",\"req\":\"^1.0\",\"target\":\"cfg(target_arch = \\\"wasm32\\\")\"},{\"name\":\"serde_json\",\"optional\":true,\"req\":\"^1.0\"},{\"name\":\"serde_urlencoded\",\"req\":\"^0.7.1\"},{\"features\":[\"futures\"],\"name\":\"sync_wrapper\",\"req\":\"^1.0\"},{\"default_features\":false,\"features\":[\"net\",\"time\"],\"name\":\"tokio\",\"req\":\"^1.0\",\"target\":\"cfg(not(target_arch = \\\"wasm32\\\"))\"},{\"default_features\":false,\"features\":[\"macros\",\"rt-multi-thread\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.0\",\"target\":\"cfg(not(target_arch = \\\"wasm32\\\"))\"},{\"name\":\"tokio-native-tls\",\"optional\":true,\"req\":\"^0.3.0\",\"target\":\"cfg(not(target_arch = \\\"wasm32\\\"))\"},{\"default_features\":false,\"features\":[\"tls12\"],\"name\":\"tokio-rustls\",\"optional\":true,\"req\":\"^0.26\",\"target\":\"cfg(not(target_arch = \\\"wasm32\\\"))\"},{\"default_features\":false,\"features\":[\"io\"],\"name\":\"tokio-util\",\"optional\":true,\"req\":\"^0.7.9\",\"target\":\"cfg(not(target_arch = \\\"wasm32\\\"))\"},{\"default_features\":false,\"features\":[\"retry\",\"timeout\",\"util\"],\"name\":\"tower\",\"req\":\"^0.5.2\",\"target\":\"cfg(not(target_arch = \\\"wasm32\\\"))\"},{\"default_features\":false,\"features\":[\"limit\"],\"kind\":\"dev\",\"name\":\"tower\",\"req\":\"^0.5.2\"},{\"default_features\":false,\"features\":[\"follow-redirect\"],\"name\":\"tower-http\",\"req\":\"^0.6.8\",\"target\":\"cfg(not(target_arch = \\\"wasm32\\\"))\"},{\"name\":\"tower-service\",\"req\":\"^0.3\",\"target\":\"cfg(not(target_arch = \\\"wasm32\\\"))\"},{\"name\":\"url\",\"req\":\"^2.4\"},{\"name\":\"wasm-bindgen\",\"req\":\"^0.2.89\",\"target\":\"cfg(target_arch = \\\"wasm32\\\")\"},{\"features\":[\"serde-serialize\"],\"kind\":\"dev\",\"name\":\"wasm-bindgen\",\"req\":\"^0.2.89\",\"target\":\"cfg(target_arch = \\\"wasm32\\\")\"},{\"name\":\"wasm-bindgen-futures\",\"req\":\"^0.4.18\",\"target\":\"cfg(target_arch = \\\"wasm32\\\")\"},{\"kind\":\"dev\",\"name\":\"wasm-bindgen-test\",\"req\":\"^0.3\",\"target\":\"cfg(target_arch = \\\"wasm32\\\")\"},{\"name\":\"wasm-streams\",\"optional\":true,\"req\":\"^0.4\",\"target\":\"cfg(target_arch = \\\"wasm32\\\")\"},{\"features\":[\"AbortController\",\"AbortSignal\",\"Headers\",\"Request\",\"RequestInit\",\"RequestMode\",\"Response\",\"Window\",\"FormData\",\"Blob\",\"BlobPropertyBag\",\"ServiceWorkerGlobalScope\",\"RequestCredentials\",\"File\",\"ReadableStream\",\"RequestCache\"],\"name\":\"web-sys\",\"req\":\"^0.3.28\",\"target\":\"cfg(target_arch = \\\"wasm32\\\")\"},{\"name\":\"webpki-roots\",\"optional\":true,\"req\":\"^1\",\"target\":\"cfg(not(target_arch = \\\"wasm32\\\"))\"},{\"kind\":\"dev\",\"name\":\"zstd_crate\",\"package\":\"zstd\",\"req\":\"^0.13\",\"target\":\"cfg(not(target_arch = \\\"wasm32\\\"))\"}],\"features\":{\"__rustls\":[\"dep:hyper-rustls\",\"dep:tokio-rustls\",\"dep:rustls\",\"__tls\"],\"__rustls-ring\":[\"hyper-rustls?/ring\",\"tokio-rustls?/ring\",\"rustls?/ring\",\"quinn?/ring\"],\"__tls\":[\"dep:rustls-pki-types\",\"tokio/io-util\"],\"blocking\":[\"dep:futures-channel\",\"futures-channel?/sink\",\"dep:futures-util\",\"futures-util?/io\",\"futures-util?/sink\",\"tokio/sync\"],\"brotli\":[\"tower-http/decompression-br\"],\"charset\":[\"dep:encoding_rs\",\"dep:mime\"],\"cookies\":[\"dep:cookie_crate\",\"dep:cookie_store\"],\"default\":[\"default-tls\",\"charset\",\"http2\",\"system-proxy\"],\"default-tls\":[\"dep:hyper-tls\",\"dep:native-tls-crate\",\"__tls\",\"dep:tokio-native-tls\"],\"deflate\":[\"tower-http/decompression-deflate\"],\"gzip\":[\"tower-http/decompression-gzip\"],\"hickory-dns\":[\"dep:hickory-resolver\",\"dep:once_cell\"],\"http2\":[\"h2\",\"hyper/http2\",\"hyper-util/http2\",\"hyper-rustls?/http2\"],\"http3\":[\"rustls-tls-manual-roots\",\"dep:h3\",\"dep:h3-quinn\",\"dep:quinn\",\"tokio/macros\"],\"json\":[\"dep:serde_json\"],\"macos-system-configuration\":[\"system-proxy\"],\"multipart\":[\"dep:mime_guess\",\"dep:futures-util\"],\"native-tls\":[\"default-tls\"],\"native-tls-alpn\":[\"native-tls\",\"native-tls-crate?/alpn\",\"hyper-tls?/alpn\"],\"native-tls-vendored\":[\"native-tls\",\"native-tls-crate?/vendored\"],\"rustls-tls\":[\"rustls-tls-webpki-roots\"],\"rustls-tls-manual-roots\":[\"rustls-tls-manual-roots-no-provider\",\"__rustls-ring\"],\"rustls-tls-manual-roots-no-provider\":[\"__rustls\"],\"rustls-tls-native-roots\":[\"rustls-tls-native-roots-no-provider\",\"__rustls-ring\"],\"rustls-tls-native-roots-no-provider\":[\"dep:rustls-native-certs\",\"hyper-rustls?/native-tokio\",\"__rustls\"],\"rustls-tls-no-provider\":[\"rustls-tls-manual-roots-no-provider\"],\"rustls-tls-webpki-roots\":[\"rustls-tls-webpki-roots-no-provider\",\"__rustls-ring\"],\"rustls-tls-webpki-roots-no-provider\":[\"dep:webpki-roots\",\"hyper-rustls?/webpki-tokio\",\"__rustls\"],\"socks\":[],\"stream\":[\"tokio/fs\",\"dep:futures-util\",\"dep:tokio-util\",\"dep:wasm-streams\"],\"system-proxy\":[\"hyper-util/client-proxy-system\"],\"trust-dns\":[],\"zstd\":[\"tower-http/decompression-zstd\"]}}", "resolv-conf_0.7.6": "{\"dependencies\":[],\"features\":{\"system\":[]}}", "ring_0.17.14": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"build\",\"name\":\"cc\",\"req\":\"^1.2.8\"},{\"default_features\":false,\"name\":\"cfg-if\",\"req\":\"^1.0.0\"},{\"name\":\"getrandom\",\"req\":\"^0.2.10\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.148\",\"target\":\"cfg(all(any(all(target_arch = \\\"aarch64\\\", target_endian = \\\"little\\\"), all(target_arch = \\\"arm\\\", target_endian = \\\"little\\\")), any(target_os = \\\"android\\\", target_os = \\\"linux\\\")))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.155\",\"target\":\"cfg(all(all(target_arch = \\\"aarch64\\\", target_endian = \\\"little\\\"), target_vendor = \\\"apple\\\", any(target_os = \\\"ios\\\", target_os = \\\"macos\\\", target_os = \\\"tvos\\\", target_os = \\\"visionos\\\", target_os = \\\"watchos\\\")))\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"libc\",\"req\":\"^0.2.148\",\"target\":\"cfg(any(unix, windows, target_os = \\\"wasi\\\"))\"},{\"name\":\"untrusted\",\"req\":\"^0.9\"},{\"default_features\":false,\"features\":[\"std\"],\"kind\":\"dev\",\"name\":\"wasm-bindgen-test\",\"req\":\"^0.3.37\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", target_os = \\\"unknown\\\"))\"},{\"features\":[\"Win32_Foundation\",\"Win32_System_Threading\"],\"name\":\"windows-sys\",\"req\":\"^0.52\",\"target\":\"cfg(all(all(target_arch = \\\"aarch64\\\", target_endian = \\\"little\\\"), target_os = \\\"windows\\\"))\"}],\"features\":{\"alloc\":[],\"default\":[\"alloc\",\"dev_urandom_fallback\"],\"dev_urandom_fallback\":[],\"less-safe-getrandom-custom-or-rdrand\":[],\"less-safe-getrandom-espidf\":[],\"slow_tests\":[],\"std\":[\"alloc\"],\"test_logging\":[],\"unstable-testing-arm-no-hw\":[],\"unstable-testing-arm-no-neon\":[],\"wasm32_unknown_unknown_js\":[\"getrandom/js\"]}}", - "rmcp-macros_0.14.0": "{\"dependencies\":[{\"name\":\"darling\",\"req\":\"^0.23\"},{\"name\":\"proc-macro2\",\"req\":\"^1\"},{\"name\":\"quote\",\"req\":\"^1\"},{\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"features\":[\"full\"],\"name\":\"syn\",\"req\":\"^2\"}],\"features\":{}}", - "rmcp_0.14.0": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1.0\"},{\"name\":\"async-trait\",\"req\":\"^0.1.89\"},{\"kind\":\"dev\",\"name\":\"async-trait\",\"req\":\"^0.1\"},{\"name\":\"axum\",\"optional\":true,\"req\":\"^0.8\"},{\"name\":\"base64\",\"optional\":true,\"req\":\"^0.22\"},{\"name\":\"bytes\",\"optional\":true,\"req\":\"^1\"},{\"default_features\":false,\"features\":[\"serde\",\"clock\",\"std\",\"oldtime\"],\"name\":\"chrono\",\"req\":\"^0.4.38\",\"target\":\"cfg(all(target_family = \\\"wasm\\\", target_os = \\\"unknown\\\"))\"},{\"features\":[\"serde\"],\"name\":\"chrono\",\"req\":\"^0.4.38\",\"target\":\"cfg(not(all(target_family = \\\"wasm\\\", target_os = \\\"unknown\\\")))\"},{\"name\":\"futures\",\"req\":\"^0.3\"},{\"name\":\"http\",\"optional\":true,\"req\":\"^1\"},{\"name\":\"http-body\",\"optional\":true,\"req\":\"^1\"},{\"name\":\"http-body-util\",\"optional\":true,\"req\":\"^0.1\"},{\"default_features\":false,\"features\":[\"reqwest\"],\"name\":\"oauth2\",\"optional\":true,\"req\":\"^5.0\"},{\"name\":\"pastey\",\"optional\":true,\"req\":\"^0.2.0\"},{\"name\":\"pin-project-lite\",\"req\":\"^0.2\"},{\"features\":[\"tokio1\"],\"name\":\"process-wrap\",\"optional\":true,\"req\":\"^9.0\"},{\"name\":\"rand\",\"optional\":true,\"req\":\"^0.9\"},{\"default_features\":false,\"features\":[\"json\",\"stream\"],\"name\":\"reqwest\",\"optional\":true,\"req\":\"^0.12\"},{\"name\":\"rmcp-macros\",\"optional\":true,\"req\":\"^0.14.0\"},{\"features\":[\"chrono04\"],\"name\":\"schemars\",\"optional\":true,\"req\":\"^1.0\"},{\"features\":[\"chrono04\"],\"kind\":\"dev\",\"name\":\"schemars\",\"req\":\"^1.1.0\"},{\"features\":[\"derive\",\"rc\"],\"name\":\"serde\",\"req\":\"^1.0\"},{\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"name\":\"sse-stream\",\"optional\":true,\"req\":\"^0.2\"},{\"name\":\"thiserror\",\"req\":\"^2\"},{\"features\":[\"sync\",\"macros\",\"rt\",\"time\"],\"name\":\"tokio\",\"req\":\"^1\"},{\"features\":[\"full\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1\"},{\"name\":\"tokio-stream\",\"optional\":true,\"req\":\"^0.1\"},{\"name\":\"tokio-util\",\"req\":\"^0.7\"},{\"name\":\"tower-service\",\"optional\":true,\"req\":\"^0.3\"},{\"name\":\"tracing\",\"req\":\"^0.1\"},{\"features\":[\"env-filter\",\"std\",\"fmt\"],\"kind\":\"dev\",\"name\":\"tracing-subscriber\",\"req\":\"^0.3\"},{\"name\":\"url\",\"optional\":true,\"req\":\"^2.4\"},{\"features\":[\"v4\"],\"name\":\"uuid\",\"optional\":true,\"req\":\"^1\"}],\"features\":{\"__reqwest\":[\"dep:reqwest\"],\"auth\":[\"dep:oauth2\",\"__reqwest\",\"dep:url\"],\"client\":[\"dep:tokio-stream\"],\"client-side-sse\":[\"dep:sse-stream\",\"dep:http\"],\"default\":[\"base64\",\"macros\",\"server\"],\"elicitation\":[],\"macros\":[\"dep:rmcp-macros\",\"dep:pastey\"],\"reqwest\":[\"__reqwest\",\"reqwest?/rustls-tls\"],\"reqwest-tls-no-provider\":[\"__reqwest\",\"reqwest?/rustls-tls-no-provider\"],\"schemars\":[\"dep:schemars\"],\"server\":[\"transport-async-rw\",\"dep:schemars\"],\"server-side-http\":[\"uuid\",\"dep:rand\",\"dep:tokio-stream\",\"dep:http\",\"dep:http-body\",\"dep:http-body-util\",\"dep:bytes\",\"dep:sse-stream\",\"dep:axum\",\"tower\"],\"tower\":[\"dep:tower-service\"],\"transport-async-rw\":[\"tokio/io-util\",\"tokio-util/codec\"],\"transport-child-process\":[\"transport-async-rw\",\"tokio/process\",\"dep:process-wrap\"],\"transport-io\":[\"transport-async-rw\",\"tokio/io-std\"],\"transport-streamable-http-client\":[\"client-side-sse\",\"transport-worker\"],\"transport-streamable-http-client-reqwest\":[\"transport-streamable-http-client\",\"__reqwest\"],\"transport-streamable-http-server\":[\"transport-streamable-http-server-session\",\"server-side-http\",\"transport-worker\"],\"transport-streamable-http-server-session\":[\"transport-async-rw\",\"dep:tokio-stream\"],\"transport-worker\":[\"dep:tokio-stream\"]}}", + "rmcp-macros_0.15.0": "{\"dependencies\":[{\"name\":\"darling\",\"req\":\"^0.23\"},{\"name\":\"proc-macro2\",\"req\":\"^1\"},{\"name\":\"quote\",\"req\":\"^1\"},{\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"features\":[\"full\"],\"name\":\"syn\",\"req\":\"^2\"}],\"features\":{}}", + "rmcp_0.15.0": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1.0\"},{\"name\":\"async-trait\",\"req\":\"^0.1.89\"},{\"kind\":\"dev\",\"name\":\"async-trait\",\"req\":\"^0.1\"},{\"name\":\"axum\",\"optional\":true,\"req\":\"^0.8\"},{\"name\":\"base64\",\"optional\":true,\"req\":\"^0.22\"},{\"name\":\"bytes\",\"optional\":true,\"req\":\"^1\"},{\"default_features\":false,\"features\":[\"serde\",\"clock\",\"std\",\"oldtime\"],\"name\":\"chrono\",\"req\":\"^0.4.38\",\"target\":\"cfg(all(target_family = \\\"wasm\\\", target_os = \\\"unknown\\\"))\"},{\"features\":[\"serde\"],\"name\":\"chrono\",\"req\":\"^0.4.38\",\"target\":\"cfg(not(all(target_family = \\\"wasm\\\", target_os = \\\"unknown\\\")))\"},{\"name\":\"futures\",\"req\":\"^0.3\"},{\"name\":\"http\",\"optional\":true,\"req\":\"^1\"},{\"name\":\"http-body\",\"optional\":true,\"req\":\"^1\"},{\"name\":\"http-body-util\",\"optional\":true,\"req\":\"^0.1\"},{\"default_features\":false,\"features\":[\"reqwest\"],\"name\":\"oauth2\",\"optional\":true,\"req\":\"^5.0\"},{\"name\":\"pastey\",\"optional\":true,\"req\":\"^0.2.0\"},{\"name\":\"pin-project-lite\",\"req\":\"^0.2\"},{\"features\":[\"tokio1\"],\"name\":\"process-wrap\",\"optional\":true,\"req\":\"^9.0\"},{\"name\":\"rand\",\"optional\":true,\"req\":\"^0.9\"},{\"default_features\":false,\"features\":[\"json\",\"stream\"],\"name\":\"reqwest\",\"optional\":true,\"req\":\"^0.12\"},{\"name\":\"rmcp-macros\",\"optional\":true,\"req\":\"^0.15.0\"},{\"features\":[\"chrono04\"],\"name\":\"schemars\",\"optional\":true,\"req\":\"^1.0\"},{\"features\":[\"chrono04\"],\"kind\":\"dev\",\"name\":\"schemars\",\"req\":\"^1.1.0\"},{\"features\":[\"derive\",\"rc\"],\"name\":\"serde\",\"req\":\"^1.0\"},{\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"name\":\"sse-stream\",\"optional\":true,\"req\":\"^0.2\"},{\"name\":\"thiserror\",\"req\":\"^2\"},{\"features\":[\"sync\",\"macros\",\"rt\",\"time\"],\"name\":\"tokio\",\"req\":\"^1\"},{\"features\":[\"full\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1\"},{\"name\":\"tokio-stream\",\"optional\":true,\"req\":\"^0.1\"},{\"name\":\"tokio-util\",\"req\":\"^0.7\"},{\"name\":\"tower-service\",\"optional\":true,\"req\":\"^0.3\"},{\"name\":\"tracing\",\"req\":\"^0.1\"},{\"features\":[\"env-filter\",\"std\",\"fmt\"],\"kind\":\"dev\",\"name\":\"tracing-subscriber\",\"req\":\"^0.3\"},{\"name\":\"url\",\"optional\":true,\"req\":\"^2.4\"},{\"features\":[\"v4\"],\"name\":\"uuid\",\"optional\":true,\"req\":\"^1\"}],\"features\":{\"__reqwest\":[\"dep:reqwest\"],\"auth\":[\"dep:oauth2\",\"__reqwest\",\"dep:url\"],\"client\":[\"dep:tokio-stream\"],\"client-side-sse\":[\"dep:sse-stream\",\"dep:http\"],\"default\":[\"base64\",\"macros\",\"server\"],\"elicitation\":[\"dep:url\"],\"macros\":[\"dep:rmcp-macros\",\"dep:pastey\"],\"reqwest\":[\"__reqwest\",\"reqwest?/rustls-tls\"],\"reqwest-native-tls\":[\"__reqwest\",\"reqwest?/native-tls\"],\"reqwest-tls-no-provider\":[\"__reqwest\",\"reqwest?/rustls-tls-no-provider\"],\"schemars\":[\"dep:schemars\"],\"server\":[\"transport-async-rw\",\"dep:schemars\",\"dep:pastey\"],\"server-side-http\":[\"uuid\",\"dep:rand\",\"dep:tokio-stream\",\"dep:http\",\"dep:http-body\",\"dep:http-body-util\",\"dep:bytes\",\"dep:sse-stream\",\"dep:axum\",\"tower\"],\"tower\":[\"dep:tower-service\"],\"transport-async-rw\":[\"tokio/io-util\",\"tokio-util/codec\"],\"transport-child-process\":[\"transport-async-rw\",\"tokio/process\",\"dep:process-wrap\"],\"transport-io\":[\"transport-async-rw\",\"tokio/io-std\"],\"transport-streamable-http-client\":[\"client-side-sse\",\"transport-worker\"],\"transport-streamable-http-client-reqwest\":[\"transport-streamable-http-client\",\"__reqwest\"],\"transport-streamable-http-server\":[\"transport-streamable-http-server-session\",\"server-side-http\",\"transport-worker\"],\"transport-streamable-http-server-session\":[\"transport-async-rw\",\"dep:tokio-stream\"],\"transport-worker\":[\"dep:tokio-stream\"]}}", "rsa_0.9.10": "{\"dependencies\":[{\"features\":[\"alloc\"],\"kind\":\"dev\",\"name\":\"base64ct\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"const-oid\",\"req\":\"^0.9\"},{\"default_features\":false,\"features\":[\"alloc\",\"oid\"],\"name\":\"digest\",\"req\":\"^0.10.5\"},{\"kind\":\"dev\",\"name\":\"hex-literal\",\"req\":\"^0.4.1\"},{\"default_features\":false,\"features\":[\"i128\",\"prime\",\"zeroize\"],\"name\":\"num-bigint\",\"package\":\"num-bigint-dig\",\"req\":\"^0.8.6\"},{\"default_features\":false,\"name\":\"num-integer\",\"req\":\"^0.1.39\"},{\"default_features\":false,\"features\":[\"libm\"],\"name\":\"num-traits\",\"req\":\"^0.2.9\"},{\"default_features\":false,\"features\":[\"alloc\",\"pkcs8\"],\"name\":\"pkcs1\",\"req\":\"^0.7.5\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"pkcs8\",\"req\":\"^0.10.2\"},{\"kind\":\"dev\",\"name\":\"proptest\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"},{\"kind\":\"dev\",\"name\":\"rand_chacha\",\"req\":\"^0.3\"},{\"default_features\":false,\"name\":\"rand_core\",\"req\":\"^0.6.4\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"rand_core\",\"req\":\"^0.6\"},{\"kind\":\"dev\",\"name\":\"rand_xorshift\",\"req\":\"^0.3\"},{\"default_features\":false,\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.184\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1.0.89\"},{\"default_features\":false,\"features\":[\"oid\"],\"name\":\"sha1\",\"optional\":true,\"req\":\"^0.10.5\"},{\"default_features\":false,\"features\":[\"oid\"],\"kind\":\"dev\",\"name\":\"sha1\",\"req\":\"^0.10.5\"},{\"default_features\":false,\"features\":[\"oid\"],\"name\":\"sha2\",\"optional\":true,\"req\":\"^0.10.6\"},{\"default_features\":false,\"features\":[\"oid\"],\"kind\":\"dev\",\"name\":\"sha2\",\"req\":\"^0.10.6\"},{\"default_features\":false,\"features\":[\"oid\"],\"kind\":\"dev\",\"name\":\"sha3\",\"req\":\"^0.10.7\"},{\"default_features\":false,\"features\":[\"alloc\",\"digest\",\"rand_core\"],\"name\":\"signature\",\"req\":\">2.0, <2.3\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"spki\",\"req\":\"^0.7.3\"},{\"default_features\":false,\"name\":\"subtle\",\"req\":\"^2.1.1\"},{\"features\":[\"alloc\"],\"name\":\"zeroize\",\"req\":\"^1.5\"}],\"features\":{\"default\":[\"std\",\"pem\",\"u64_digit\"],\"getrandom\":[\"rand_core/getrandom\"],\"hazmat\":[],\"nightly\":[\"num-bigint/nightly\"],\"pem\":[\"pkcs1/pem\",\"pkcs8/pem\"],\"pkcs5\":[\"pkcs8/encryption\"],\"serde\":[\"dep:serde\",\"num-bigint/serde\"],\"std\":[\"digest/std\",\"pkcs1/std\",\"pkcs8/std\",\"rand_core/std\",\"signature/std\"],\"u64_digit\":[\"num-bigint/u64_digit\"]}}", "rust-embed-impl_8.11.0": "{\"dependencies\":[{\"name\":\"proc-macro2\",\"req\":\"^1\"},{\"name\":\"quote\",\"req\":\"^1\"},{\"name\":\"rust-embed-utils\",\"req\":\"^8.11.0\"},{\"name\":\"shellexpand\",\"optional\":true,\"req\":\"^3\"},{\"default_features\":false,\"features\":[\"derive\",\"parsing\",\"proc-macro\",\"printing\"],\"name\":\"syn\",\"req\":\"^2\"},{\"name\":\"walkdir\",\"req\":\"^2.3.1\"}],\"features\":{\"compression\":[],\"debug-embed\":[],\"deterministic-timestamps\":[],\"include-exclude\":[\"rust-embed-utils/include-exclude\"],\"interpolate-folder-path\":[\"shellexpand\"],\"mime-guess\":[\"rust-embed-utils/mime-guess\"]}}", "rust-embed-utils_8.11.0": "{\"dependencies\":[{\"name\":\"globset\",\"optional\":true,\"req\":\"^0.4.8\"},{\"name\":\"mime_guess\",\"optional\":true,\"req\":\"^2.0.4\"},{\"name\":\"sha2\",\"req\":\"^0.10.5\"},{\"name\":\"walkdir\",\"req\":\"^2.3.1\"}],\"features\":{\"debug-embed\":[],\"include-exclude\":[\"globset\"],\"mime-guess\":[\"mime_guess\"]}}", diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index ad6ba09107..3456f9d731 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -1420,6 +1420,8 @@ dependencies = [ "codex-protocol", "serde", "serde_json", + "tungstenite", + "url", "uuid", ] @@ -1710,6 +1712,7 @@ dependencies = [ "include_dir", "indexmap 2.13.0", "indoc", + "insta", "keyring", "landlock", "libc", @@ -2035,6 +2038,7 @@ version = "0.0.0" dependencies = [ "anyhow", "async-trait", + "base64 0.22.1", "clap", "codex-utils-absolute-path", "codex-utils-rustls-provider", @@ -2294,6 +2298,7 @@ dependencies = [ "codex-utils-oss", "codex-utils-pty", "codex-utils-sandbox-summary", + "codex-utils-sleep-inhibitor", "codex-windows-sandbox", "color-eyre", "crossterm", @@ -2493,6 +2498,15 @@ dependencies = [ "regex", ] +[[package]] +name = "codex-utils-sleep-inhibitor" +version = "0.0.0" +dependencies = [ + "core-foundation 0.9.4", + "libc", + "tracing", +] + [[package]] name = "codex-utils-string" version = "0.0.0" diff --git a/codex-rs/Cargo.toml b/codex-rs/Cargo.toml index 8f76c64ed9..75b3d18d3e 100644 --- a/codex-rs/Cargo.toml +++ b/codex-rs/Cargo.toml @@ -54,6 +54,7 @@ members = [ "utils/elapsed", "utils/sandbox-summary", "utils/sanitizer", + "utils/sleep-inhibitor", "utils/approval-presets", "utils/oss", "utils/fuzzy-match", @@ -131,6 +132,7 @@ codex-utils-readiness = { path = "utils/readiness" } codex-utils-rustls-provider = { path = "utils/rustls-provider" } codex-utils-sandbox-summary = { path = "utils/sandbox-summary" } codex-utils-sanitizer = { path = "utils/sanitizer" } +codex-utils-sleep-inhibitor = { path = "utils/sleep-inhibitor" } codex-utils-string = { path = "utils/string" } codex-windows-sandbox = { path = "windows-sandbox-rs" } core_test_support = { path = "core/tests/common" } diff --git a/codex-rs/app-server-protocol/schema/json/ClientRequest.json b/codex-rs/app-server-protocol/schema/json/ClientRequest.json index c7a3504fba..a8b66a202b 100644 --- a/codex-rs/app-server-protocol/schema/json/ClientRequest.json +++ b/codex-rs/app-server-protocol/schema/json/ClientRequest.json @@ -1128,6 +1128,13 @@ "null" ] }, + "includeHidden": { + "description": "When true, include models that are hidden from the default picker list.", + "type": [ + "boolean", + "null" + ] + }, "limit": { "description": "Optional page size; defaults to a reasonable server-side value.", "format": "uint32", @@ -2565,6 +2572,13 @@ "null" ] }, + "cwd": { + "description": "Optional cwd filter; when set, only threads whose session cwd exactly matches this path are returned.", + "type": [ + "string", + "null" + ] + }, "limit": { "description": "Optional page size; defaults to a reasonable server-side value.", "format": "uint32", diff --git a/codex-rs/app-server-protocol/schema/json/EventMsg.json b/codex-rs/app-server-protocol/schema/json/EventMsg.json index 6af66294c8..21d8784c34 100644 --- a/codex-rs/app-server-protocol/schema/json/EventMsg.json +++ b/codex-rs/app-server-protocol/schema/json/EventMsg.json @@ -1438,6 +1438,17 @@ "description": "The command's working directory.", "type": "string" }, + "network_approval_context": { + "anyOf": [ + { + "$ref": "#/definitions/NetworkApprovalContext" + }, + { + "type": "null" + } + ], + "description": "Optional network context for a blocked request that can be approved." + }, "parsed_cmd": { "items": { "$ref": "#/definitions/ParsedCommand" @@ -3315,6 +3326,30 @@ ], "type": "string" }, + "NetworkApprovalContext": { + "properties": { + "host": { + "type": "string" + }, + "protocol": { + "$ref": "#/definitions/NetworkApprovalProtocol" + } + }, + "required": [ + "host", + "protocol" + ], + "type": "object" + }, + "NetworkApprovalProtocol": { + "enum": [ + "http", + "https", + "socks5_tcp", + "socks5_udp" + ], + "type": "string" + }, "ParsedCommand": { "oneOf": [ { @@ -6308,6 +6343,17 @@ "description": "The command's working directory.", "type": "string" }, + "network_approval_context": { + "anyOf": [ + { + "$ref": "#/definitions/NetworkApprovalContext" + }, + { + "type": "null" + } + ], + "description": "Optional network context for a blocked request that can be approved." + }, "parsed_cmd": { "items": { "$ref": "#/definitions/ParsedCommand" diff --git a/codex-rs/app-server-protocol/schema/json/FuzzyFileSearchSessionCompletedNotification.json b/codex-rs/app-server-protocol/schema/json/FuzzyFileSearchSessionCompletedNotification.json new file mode 100644 index 0000000000..c8924e77ca --- /dev/null +++ b/codex-rs/app-server-protocol/schema/json/FuzzyFileSearchSessionCompletedNotification.json @@ -0,0 +1,13 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "properties": { + "sessionId": { + "type": "string" + } + }, + "required": [ + "sessionId" + ], + "title": "FuzzyFileSearchSessionCompletedNotification", + "type": "object" +} \ No newline at end of file diff --git a/codex-rs/app-server-protocol/schema/json/ServerNotification.json b/codex-rs/app-server-protocol/schema/json/ServerNotification.json index 05667704d7..6b12cee9cf 100644 --- a/codex-rs/app-server-protocol/schema/json/ServerNotification.json +++ b/codex-rs/app-server-protocol/schema/json/ServerNotification.json @@ -193,6 +193,11 @@ "default": false, "type": "boolean" }, + "isEnabled": { + "default": true, + "description": "Whether this app is enabled in config.toml. Example: ```toml [apps.bad_app] enabled = false ```", + "type": "boolean" + }, "logoUrl": { "type": [ "string", @@ -2054,6 +2059,17 @@ "description": "The command's working directory.", "type": "string" }, + "network_approval_context": { + "anyOf": [ + { + "$ref": "#/definitions/NetworkApprovalContext" + }, + { + "type": "null" + } + ], + "description": "Optional network context for a blocked request that can be approved." + }, "parsed_cmd": { "items": { "$ref": "#/definitions/ParsedCommand" @@ -3747,6 +3763,17 @@ ], "type": "object" }, + "FuzzyFileSearchSessionCompletedNotification": { + "properties": { + "sessionId": { + "type": "string" + } + }, + "required": [ + "sessionId" + ], + "type": "object" + }, "FuzzyFileSearchSessionUpdatedNotification": { "properties": { "files": { @@ -4191,6 +4218,30 @@ ], "type": "string" }, + "NetworkApprovalContext": { + "properties": { + "host": { + "type": "string" + }, + "protocol": { + "$ref": "#/definitions/NetworkApprovalProtocol" + } + }, + "required": [ + "host", + "protocol" + ], + "type": "object" + }, + "NetworkApprovalProtocol": { + "enum": [ + "http", + "https", + "socks5_tcp", + "socks5_udp" + ], + "type": "string" + }, "ParsedCommand": { "oneOf": [ { @@ -5992,7 +6043,8 @@ { "enum": [ "review", - "compact" + "compact", + "memory_consolidation" ], "type": "string" }, @@ -8284,6 +8336,26 @@ "title": "FuzzyFileSearch/sessionUpdatedNotification", "type": "object" }, + { + "properties": { + "method": { + "enum": [ + "fuzzyFileSearch/sessionCompleted" + ], + "title": "FuzzyFileSearch/sessionCompletedNotificationMethod", + "type": "string" + }, + "params": { + "$ref": "#/definitions/FuzzyFileSearchSessionCompletedNotification" + } + }, + "required": [ + "method", + "params" + ], + "title": "FuzzyFileSearch/sessionCompletedNotification", + "type": "object" + }, { "description": "Notifies the user of world-writable directories on Windows, which cannot be protected by the sandbox.", "properties": { diff --git a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json index 5820997fb6..5f352004cb 100644 --- a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json +++ b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json @@ -3451,6 +3451,17 @@ "description": "The command's working directory.", "type": "string" }, + "network_approval_context": { + "anyOf": [ + { + "$ref": "#/definitions/NetworkApprovalContext" + }, + { + "type": "null" + } + ], + "description": "Optional network context for a blocked request that can be approved." + }, "parsed_cmd": { "items": { "$ref": "#/definitions/ParsedCommand" @@ -5412,6 +5423,19 @@ ], "type": "object" }, + "FuzzyFileSearchSessionCompletedNotification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "properties": { + "sessionId": { + "type": "string" + } + }, + "required": [ + "sessionId" + ], + "title": "FuzzyFileSearchSessionCompletedNotification", + "type": "object" + }, "FuzzyFileSearchSessionUpdatedNotification": { "$schema": "http://json-schema.org/draft-07/schema#", "properties": { @@ -6250,6 +6274,30 @@ ], "type": "string" }, + "NetworkApprovalContext": { + "properties": { + "host": { + "type": "string" + }, + "protocol": { + "$ref": "#/definitions/NetworkApprovalProtocol" + } + }, + "required": [ + "host", + "protocol" + ], + "type": "object" + }, + "NetworkApprovalProtocol": { + "enum": [ + "http", + "https", + "socks5_tcp", + "socks5_udp" + ], + "type": "string" + }, "NewConversationParams": { "properties": { "approvalPolicy": { @@ -8503,6 +8551,26 @@ "title": "FuzzyFileSearch/sessionUpdatedNotification", "type": "object" }, + { + "properties": { + "method": { + "enum": [ + "fuzzyFileSearch/sessionCompleted" + ], + "title": "FuzzyFileSearch/sessionCompletedNotificationMethod", + "type": "string" + }, + "params": { + "$ref": "#/definitions/FuzzyFileSearchSessionCompletedNotification" + } + }, + "required": [ + "method", + "params" + ], + "title": "FuzzyFileSearch/sessionCompletedNotification", + "type": "object" + }, { "description": "Notifies the user of world-writable directories on Windows, which cannot be protected by the sandbox.", "properties": { @@ -9126,7 +9194,8 @@ { "enum": [ "review", - "compact" + "compact", + "memory_consolidation" ], "type": "string" }, @@ -10201,6 +10270,11 @@ "default": false, "type": "boolean" }, + "isEnabled": { + "default": true, + "description": "Whether this app is enabled in config.toml. Example: ```toml [apps.bad_app] enabled = false ```", + "type": "boolean" + }, "logoUrl": { "type": [ "string", @@ -12572,6 +12646,9 @@ "displayName": { "type": "string" }, + "hidden": { + "type": "boolean" + }, "id": { "type": "string" }, @@ -12612,6 +12689,7 @@ "defaultReasoningEffort", "description", "displayName", + "hidden", "id", "isDefault", "model", @@ -12629,6 +12707,13 @@ "null" ] }, + "includeHidden": { + "description": "When true, include models that are hidden from the default picker list.", + "type": [ + "boolean", + "null" + ] + }, "limit": { "description": "Optional page size; defaults to a reasonable server-side value.", "format": "uint32", @@ -14475,7 +14560,8 @@ { "enum": [ "review", - "compact" + "compact", + "memory_consolidation" ], "type": "string" }, @@ -15329,6 +15415,13 @@ "null" ] }, + "cwd": { + "description": "Optional cwd filter; when set, only threads whose session cwd exactly matches this path are returned.", + "type": [ + "string", + "null" + ] + }, "limit": { "description": "Optional page size; defaults to a reasonable server-side value.", "format": "uint32", diff --git a/codex-rs/app-server-protocol/schema/json/v1/ForkConversationResponse.json b/codex-rs/app-server-protocol/schema/json/v1/ForkConversationResponse.json index 12db241e1a..225ea3f42a 100644 --- a/codex-rs/app-server-protocol/schema/json/v1/ForkConversationResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v1/ForkConversationResponse.json @@ -1438,6 +1438,17 @@ "description": "The command's working directory.", "type": "string" }, + "network_approval_context": { + "anyOf": [ + { + "$ref": "#/definitions/NetworkApprovalContext" + }, + { + "type": "null" + } + ], + "description": "Optional network context for a blocked request that can be approved." + }, "parsed_cmd": { "items": { "$ref": "#/definitions/ParsedCommand" @@ -3315,6 +3326,30 @@ ], "type": "string" }, + "NetworkApprovalContext": { + "properties": { + "host": { + "type": "string" + }, + "protocol": { + "$ref": "#/definitions/NetworkApprovalProtocol" + } + }, + "required": [ + "host", + "protocol" + ], + "type": "object" + }, + "NetworkApprovalProtocol": { + "enum": [ + "http", + "https", + "socks5_tcp", + "socks5_udp" + ], + "type": "string" + }, "ParsedCommand": { "oneOf": [ { diff --git a/codex-rs/app-server-protocol/schema/json/v1/GetConversationSummaryResponse.json b/codex-rs/app-server-protocol/schema/json/v1/GetConversationSummaryResponse.json index 954ac28ed1..a54b7ddac1 100644 --- a/codex-rs/app-server-protocol/schema/json/v1/GetConversationSummaryResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v1/GetConversationSummaryResponse.json @@ -113,7 +113,8 @@ { "enum": [ "review", - "compact" + "compact", + "memory_consolidation" ], "type": "string" }, diff --git a/codex-rs/app-server-protocol/schema/json/v1/ListConversationsResponse.json b/codex-rs/app-server-protocol/schema/json/v1/ListConversationsResponse.json index b7e3b8f8f1..bb5a0f4861 100644 --- a/codex-rs/app-server-protocol/schema/json/v1/ListConversationsResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v1/ListConversationsResponse.json @@ -113,7 +113,8 @@ { "enum": [ "review", - "compact" + "compact", + "memory_consolidation" ], "type": "string" }, diff --git a/codex-rs/app-server-protocol/schema/json/v1/ResumeConversationResponse.json b/codex-rs/app-server-protocol/schema/json/v1/ResumeConversationResponse.json index dd7606b5e0..2b305281c8 100644 --- a/codex-rs/app-server-protocol/schema/json/v1/ResumeConversationResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v1/ResumeConversationResponse.json @@ -1438,6 +1438,17 @@ "description": "The command's working directory.", "type": "string" }, + "network_approval_context": { + "anyOf": [ + { + "$ref": "#/definitions/NetworkApprovalContext" + }, + { + "type": "null" + } + ], + "description": "Optional network context for a blocked request that can be approved." + }, "parsed_cmd": { "items": { "$ref": "#/definitions/ParsedCommand" @@ -3315,6 +3326,30 @@ ], "type": "string" }, + "NetworkApprovalContext": { + "properties": { + "host": { + "type": "string" + }, + "protocol": { + "$ref": "#/definitions/NetworkApprovalProtocol" + } + }, + "required": [ + "host", + "protocol" + ], + "type": "object" + }, + "NetworkApprovalProtocol": { + "enum": [ + "http", + "https", + "socks5_tcp", + "socks5_udp" + ], + "type": "string" + }, "ParsedCommand": { "oneOf": [ { diff --git a/codex-rs/app-server-protocol/schema/json/v1/SessionConfiguredNotification.json b/codex-rs/app-server-protocol/schema/json/v1/SessionConfiguredNotification.json index 89ab825d32..d4bbf908d5 100644 --- a/codex-rs/app-server-protocol/schema/json/v1/SessionConfiguredNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v1/SessionConfiguredNotification.json @@ -1438,6 +1438,17 @@ "description": "The command's working directory.", "type": "string" }, + "network_approval_context": { + "anyOf": [ + { + "$ref": "#/definitions/NetworkApprovalContext" + }, + { + "type": "null" + } + ], + "description": "Optional network context for a blocked request that can be approved." + }, "parsed_cmd": { "items": { "$ref": "#/definitions/ParsedCommand" @@ -3315,6 +3326,30 @@ ], "type": "string" }, + "NetworkApprovalContext": { + "properties": { + "host": { + "type": "string" + }, + "protocol": { + "$ref": "#/definitions/NetworkApprovalProtocol" + } + }, + "required": [ + "host", + "protocol" + ], + "type": "object" + }, + "NetworkApprovalProtocol": { + "enum": [ + "http", + "https", + "socks5_tcp", + "socks5_udp" + ], + "type": "string" + }, "ParsedCommand": { "oneOf": [ { diff --git a/codex-rs/app-server-protocol/schema/json/v2/AppListUpdatedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/AppListUpdatedNotification.json index cf75f10ce0..1638472c4e 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/AppListUpdatedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/AppListUpdatedNotification.json @@ -29,6 +29,11 @@ "default": false, "type": "boolean" }, + "isEnabled": { + "default": true, + "description": "Whether this app is enabled in config.toml. Example: ```toml [apps.bad_app] enabled = false ```", + "type": "boolean" + }, "logoUrl": { "type": [ "string", diff --git a/codex-rs/app-server-protocol/schema/json/v2/AppsListResponse.json b/codex-rs/app-server-protocol/schema/json/v2/AppsListResponse.json index d2b5cc5ef9..cec3da5ccc 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/AppsListResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/AppsListResponse.json @@ -29,6 +29,11 @@ "default": false, "type": "boolean" }, + "isEnabled": { + "default": true, + "description": "Whether this app is enabled in config.toml. Example: ```toml [apps.bad_app] enabled = false ```", + "type": "boolean" + }, "logoUrl": { "type": [ "string", diff --git a/codex-rs/app-server-protocol/schema/json/v2/ModelListParams.json b/codex-rs/app-server-protocol/schema/json/v2/ModelListParams.json index 13bb29977e..11a3476240 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ModelListParams.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ModelListParams.json @@ -8,6 +8,13 @@ "null" ] }, + "includeHidden": { + "description": "When true, include models that are hidden from the default picker list.", + "type": [ + "boolean", + "null" + ] + }, "limit": { "description": "Optional page size; defaults to a reasonable server-side value.", "format": "uint32", diff --git a/codex-rs/app-server-protocol/schema/json/v2/ModelListResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ModelListResponse.json index a1b3f7114c..2afa018dc5 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ModelListResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ModelListResponse.json @@ -31,6 +31,9 @@ "displayName": { "type": "string" }, + "hidden": { + "type": "boolean" + }, "id": { "type": "string" }, @@ -71,6 +74,7 @@ "defaultReasoningEffort", "description", "displayName", + "hidden", "id", "isDefault", "model", diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json index 10b9d50007..de1823e9de 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json @@ -666,7 +666,8 @@ { "enum": [ "review", - "compact" + "compact", + "memory_consolidation" ], "type": "string" }, diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadListParams.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadListParams.json index dd4c7a4f12..fa88ea3f7c 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadListParams.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadListParams.json @@ -39,6 +39,13 @@ "null" ] }, + "cwd": { + "description": "Optional cwd filter; when set, only threads whose session cwd exactly matches this path are returned.", + "type": [ + "string", + "null" + ] + }, "limit": { "description": "Optional page size; defaults to a reasonable server-side value.", "format": "uint32", diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json index 6fe793cc19..96e3761978 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json @@ -472,7 +472,8 @@ { "enum": [ "review", - "compact" + "compact", + "memory_consolidation" ], "type": "string" }, diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json index 444db89431..e0d4cc4f7a 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json @@ -472,7 +472,8 @@ { "enum": [ "review", - "compact" + "compact", + "memory_consolidation" ], "type": "string" }, diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json index 9b2fb74003..4385565f96 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json @@ -666,7 +666,8 @@ { "enum": [ "review", - "compact" + "compact", + "memory_consolidation" ], "type": "string" }, diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadRollbackResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadRollbackResponse.json index b6ec97be10..9f81c5ea7b 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadRollbackResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadRollbackResponse.json @@ -472,7 +472,8 @@ { "enum": [ "review", - "compact" + "compact", + "memory_consolidation" ], "type": "string" }, diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json index 0d22e08810..fd75b56a28 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json @@ -666,7 +666,8 @@ { "enum": [ "review", - "compact" + "compact", + "memory_consolidation" ], "type": "string" }, diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json index 6a9324409e..3ad16408ee 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json @@ -472,7 +472,8 @@ { "enum": [ "review", - "compact" + "compact", + "memory_consolidation" ], "type": "string" }, diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json index 2c1c9950fe..3f8bd66861 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json @@ -472,7 +472,8 @@ { "enum": [ "review", - "compact" + "compact", + "memory_consolidation" ], "type": "string" }, diff --git a/codex-rs/app-server-protocol/schema/typescript/ExecApprovalRequestEvent.ts b/codex-rs/app-server-protocol/schema/typescript/ExecApprovalRequestEvent.ts index 66db8fd40a..542ed879b0 100644 --- a/codex-rs/app-server-protocol/schema/typescript/ExecApprovalRequestEvent.ts +++ b/codex-rs/app-server-protocol/schema/typescript/ExecApprovalRequestEvent.ts @@ -2,6 +2,7 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. import type { ExecPolicyAmendment } from "./ExecPolicyAmendment"; +import type { NetworkApprovalContext } from "./NetworkApprovalContext"; import type { ParsedCommand } from "./ParsedCommand"; export type ExecApprovalRequestEvent = { @@ -26,6 +27,10 @@ cwd: string, * Optional human-readable reason for the approval (e.g. retry without sandbox). */ reason: string | null, +/** + * Optional network context for a blocked request that can be approved. + */ +network_approval_context?: NetworkApprovalContext, /** * Proposed execpolicy amendment that can be applied to allow future runs. */ diff --git a/codex-rs/app-server-protocol/schema/typescript/FuzzyFileSearchSessionCompletedNotification.ts b/codex-rs/app-server-protocol/schema/typescript/FuzzyFileSearchSessionCompletedNotification.ts new file mode 100644 index 0000000000..f4dc7fac11 --- /dev/null +++ b/codex-rs/app-server-protocol/schema/typescript/FuzzyFileSearchSessionCompletedNotification.ts @@ -0,0 +1,5 @@ +// GENERATED CODE! DO NOT MODIFY BY HAND! + +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +export type FuzzyFileSearchSessionCompletedNotification = { sessionId: string, }; diff --git a/codex-rs/app-server-protocol/schema/typescript/NetworkApprovalContext.ts b/codex-rs/app-server-protocol/schema/typescript/NetworkApprovalContext.ts new file mode 100644 index 0000000000..b4b78e473c --- /dev/null +++ b/codex-rs/app-server-protocol/schema/typescript/NetworkApprovalContext.ts @@ -0,0 +1,6 @@ +// GENERATED CODE! DO NOT MODIFY BY HAND! + +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { NetworkApprovalProtocol } from "./NetworkApprovalProtocol"; + +export type NetworkApprovalContext = { host: string, protocol: NetworkApprovalProtocol, }; diff --git a/codex-rs/app-server-protocol/schema/typescript/NetworkApprovalProtocol.ts b/codex-rs/app-server-protocol/schema/typescript/NetworkApprovalProtocol.ts new file mode 100644 index 0000000000..a33eab566f --- /dev/null +++ b/codex-rs/app-server-protocol/schema/typescript/NetworkApprovalProtocol.ts @@ -0,0 +1,5 @@ +// GENERATED CODE! DO NOT MODIFY BY HAND! + +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +export type NetworkApprovalProtocol = "http" | "https" | "socks5_tcp" | "socks5_udp"; diff --git a/codex-rs/app-server-protocol/schema/typescript/ServerNotification.ts b/codex-rs/app-server-protocol/schema/typescript/ServerNotification.ts index 0e49c060c0..ffc39598b6 100644 --- a/codex-rs/app-server-protocol/schema/typescript/ServerNotification.ts +++ b/codex-rs/app-server-protocol/schema/typescript/ServerNotification.ts @@ -2,6 +2,7 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. import type { AuthStatusChangeNotification } from "./AuthStatusChangeNotification"; +import type { FuzzyFileSearchSessionCompletedNotification } from "./FuzzyFileSearchSessionCompletedNotification"; import type { FuzzyFileSearchSessionUpdatedNotification } from "./FuzzyFileSearchSessionUpdatedNotification"; import type { LoginChatGptCompleteNotification } from "./LoginChatGptCompleteNotification"; import type { SessionConfiguredNotification } from "./SessionConfiguredNotification"; @@ -38,4 +39,4 @@ import type { WindowsWorldWritableWarningNotification } from "./v2/WindowsWorldW /** * Notification sent from the server to the client. */ -export type ServerNotification = { "method": "error", "params": ErrorNotification } | { "method": "thread/started", "params": ThreadStartedNotification } | { "method": "thread/name/updated", "params": ThreadNameUpdatedNotification } | { "method": "thread/tokenUsage/updated", "params": ThreadTokenUsageUpdatedNotification } | { "method": "turn/started", "params": TurnStartedNotification } | { "method": "turn/completed", "params": TurnCompletedNotification } | { "method": "turn/diff/updated", "params": TurnDiffUpdatedNotification } | { "method": "turn/plan/updated", "params": TurnPlanUpdatedNotification } | { "method": "item/started", "params": ItemStartedNotification } | { "method": "item/completed", "params": ItemCompletedNotification } | { "method": "rawResponseItem/completed", "params": RawResponseItemCompletedNotification } | { "method": "item/agentMessage/delta", "params": AgentMessageDeltaNotification } | { "method": "item/plan/delta", "params": PlanDeltaNotification } | { "method": "item/commandExecution/outputDelta", "params": CommandExecutionOutputDeltaNotification } | { "method": "item/commandExecution/terminalInteraction", "params": TerminalInteractionNotification } | { "method": "item/fileChange/outputDelta", "params": FileChangeOutputDeltaNotification } | { "method": "item/mcpToolCall/progress", "params": McpToolCallProgressNotification } | { "method": "mcpServer/oauthLogin/completed", "params": McpServerOauthLoginCompletedNotification } | { "method": "account/updated", "params": AccountUpdatedNotification } | { "method": "account/rateLimits/updated", "params": AccountRateLimitsUpdatedNotification } | { "method": "app/list/updated", "params": AppListUpdatedNotification } | { "method": "item/reasoning/summaryTextDelta", "params": ReasoningSummaryTextDeltaNotification } | { "method": "item/reasoning/summaryPartAdded", "params": ReasoningSummaryPartAddedNotification } | { "method": "item/reasoning/textDelta", "params": ReasoningTextDeltaNotification } | { "method": "thread/compacted", "params": ContextCompactedNotification } | { "method": "deprecationNotice", "params": DeprecationNoticeNotification } | { "method": "configWarning", "params": ConfigWarningNotification } | { "method": "fuzzyFileSearch/sessionUpdated", "params": FuzzyFileSearchSessionUpdatedNotification } | { "method": "windows/worldWritableWarning", "params": WindowsWorldWritableWarningNotification } | { "method": "account/login/completed", "params": AccountLoginCompletedNotification } | { "method": "authStatusChange", "params": AuthStatusChangeNotification } | { "method": "loginChatGptComplete", "params": LoginChatGptCompleteNotification } | { "method": "sessionConfigured", "params": SessionConfiguredNotification }; +export type ServerNotification = { "method": "error", "params": ErrorNotification } | { "method": "thread/started", "params": ThreadStartedNotification } | { "method": "thread/name/updated", "params": ThreadNameUpdatedNotification } | { "method": "thread/tokenUsage/updated", "params": ThreadTokenUsageUpdatedNotification } | { "method": "turn/started", "params": TurnStartedNotification } | { "method": "turn/completed", "params": TurnCompletedNotification } | { "method": "turn/diff/updated", "params": TurnDiffUpdatedNotification } | { "method": "turn/plan/updated", "params": TurnPlanUpdatedNotification } | { "method": "item/started", "params": ItemStartedNotification } | { "method": "item/completed", "params": ItemCompletedNotification } | { "method": "rawResponseItem/completed", "params": RawResponseItemCompletedNotification } | { "method": "item/agentMessage/delta", "params": AgentMessageDeltaNotification } | { "method": "item/plan/delta", "params": PlanDeltaNotification } | { "method": "item/commandExecution/outputDelta", "params": CommandExecutionOutputDeltaNotification } | { "method": "item/commandExecution/terminalInteraction", "params": TerminalInteractionNotification } | { "method": "item/fileChange/outputDelta", "params": FileChangeOutputDeltaNotification } | { "method": "item/mcpToolCall/progress", "params": McpToolCallProgressNotification } | { "method": "mcpServer/oauthLogin/completed", "params": McpServerOauthLoginCompletedNotification } | { "method": "account/updated", "params": AccountUpdatedNotification } | { "method": "account/rateLimits/updated", "params": AccountRateLimitsUpdatedNotification } | { "method": "app/list/updated", "params": AppListUpdatedNotification } | { "method": "item/reasoning/summaryTextDelta", "params": ReasoningSummaryTextDeltaNotification } | { "method": "item/reasoning/summaryPartAdded", "params": ReasoningSummaryPartAddedNotification } | { "method": "item/reasoning/textDelta", "params": ReasoningTextDeltaNotification } | { "method": "thread/compacted", "params": ContextCompactedNotification } | { "method": "deprecationNotice", "params": DeprecationNoticeNotification } | { "method": "configWarning", "params": ConfigWarningNotification } | { "method": "fuzzyFileSearch/sessionUpdated", "params": FuzzyFileSearchSessionUpdatedNotification } | { "method": "fuzzyFileSearch/sessionCompleted", "params": FuzzyFileSearchSessionCompletedNotification } | { "method": "windows/worldWritableWarning", "params": WindowsWorldWritableWarningNotification } | { "method": "account/login/completed", "params": AccountLoginCompletedNotification } | { "method": "authStatusChange", "params": AuthStatusChangeNotification } | { "method": "loginChatGptComplete", "params": LoginChatGptCompleteNotification } | { "method": "sessionConfigured", "params": SessionConfiguredNotification }; diff --git a/codex-rs/app-server-protocol/schema/typescript/SubAgentSource.ts b/codex-rs/app-server-protocol/schema/typescript/SubAgentSource.ts index d6da7a466b..269a411be9 100644 --- a/codex-rs/app-server-protocol/schema/typescript/SubAgentSource.ts +++ b/codex-rs/app-server-protocol/schema/typescript/SubAgentSource.ts @@ -3,4 +3,4 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. import type { ThreadId } from "./ThreadId"; -export type SubAgentSource = "review" | "compact" | { "thread_spawn": { parent_thread_id: ThreadId, depth: number, } } | { "other": string }; +export type SubAgentSource = "review" | "compact" | { "thread_spawn": { parent_thread_id: ThreadId, depth: number, } } | "memory_consolidation" | { "other": string }; diff --git a/codex-rs/app-server-protocol/schema/typescript/index.ts b/codex-rs/app-server-protocol/schema/typescript/index.ts index 2e7446a3db..3c0826712a 100644 --- a/codex-rs/app-server-protocol/schema/typescript/index.ts +++ b/codex-rs/app-server-protocol/schema/typescript/index.ts @@ -78,6 +78,7 @@ export type { FunctionCallOutputPayload } from "./FunctionCallOutputPayload"; export type { FuzzyFileSearchParams } from "./FuzzyFileSearchParams"; export type { FuzzyFileSearchResponse } from "./FuzzyFileSearchResponse"; export type { FuzzyFileSearchResult } from "./FuzzyFileSearchResult"; +export type { FuzzyFileSearchSessionCompletedNotification } from "./FuzzyFileSearchSessionCompletedNotification"; export type { FuzzyFileSearchSessionUpdatedNotification } from "./FuzzyFileSearchSessionUpdatedNotification"; export type { GetAuthStatusParams } from "./GetAuthStatusParams"; export type { GetAuthStatusResponse } from "./GetAuthStatusResponse"; @@ -125,6 +126,8 @@ export type { McpToolCallEndEvent } from "./McpToolCallEndEvent"; export type { MessagePhase } from "./MessagePhase"; export type { ModeKind } from "./ModeKind"; export type { NetworkAccess } from "./NetworkAccess"; +export type { NetworkApprovalContext } from "./NetworkApprovalContext"; +export type { NetworkApprovalProtocol } from "./NetworkApprovalProtocol"; export type { NewConversationParams } from "./NewConversationParams"; export type { NewConversationResponse } from "./NewConversationResponse"; export type { ParsedCommand } from "./ParsedCommand"; diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/AppInfo.ts b/codex-rs/app-server-protocol/schema/typescript/v2/AppInfo.ts index 0957c0dd4f..2f65c18327 100644 --- a/codex-rs/app-server-protocol/schema/typescript/v2/AppInfo.ts +++ b/codex-rs/app-server-protocol/schema/typescript/v2/AppInfo.ts @@ -5,4 +5,13 @@ /** * EXPERIMENTAL - app metadata returned by app-list APIs. */ -export type AppInfo = { id: string, name: string, description: string | null, logoUrl: string | null, logoUrlDark: string | null, distributionChannel: string | null, installUrl: string | null, isAccessible: boolean, }; +export type AppInfo = { id: string, name: string, description: string | null, logoUrl: string | null, logoUrlDark: string | null, distributionChannel: string | null, installUrl: string | null, isAccessible: boolean, +/** + * Whether this app is enabled in config.toml. + * Example: + * ```toml + * [apps.bad_app] + * enabled = false + * ``` + */ +isEnabled: boolean, }; diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/Model.ts b/codex-rs/app-server-protocol/schema/typescript/v2/Model.ts index 7528a8fad3..de7b240173 100644 --- a/codex-rs/app-server-protocol/schema/typescript/v2/Model.ts +++ b/codex-rs/app-server-protocol/schema/typescript/v2/Model.ts @@ -5,4 +5,4 @@ import type { InputModality } from "../InputModality"; import type { ReasoningEffort } from "../ReasoningEffort"; import type { ReasoningEffortOption } from "./ReasoningEffortOption"; -export type Model = { id: string, model: string, upgrade: string | null, displayName: string, description: string, supportedReasoningEfforts: Array, defaultReasoningEffort: ReasoningEffort, inputModalities: Array, supportsPersonality: boolean, isDefault: boolean, }; +export type Model = { id: string, model: string, upgrade: string | null, displayName: string, description: string, hidden: boolean, supportedReasoningEfforts: Array, defaultReasoningEffort: ReasoningEffort, inputModalities: Array, supportsPersonality: boolean, isDefault: boolean, }; diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/ModelListParams.ts b/codex-rs/app-server-protocol/schema/typescript/v2/ModelListParams.ts index b0bc5326c1..a1e7b8fa74 100644 --- a/codex-rs/app-server-protocol/schema/typescript/v2/ModelListParams.ts +++ b/codex-rs/app-server-protocol/schema/typescript/v2/ModelListParams.ts @@ -10,4 +10,8 @@ cursor?: string | null, /** * Optional page size; defaults to a reasonable server-side value. */ -limit?: number | null, }; +limit?: number | null, +/** + * When true, include models that are hidden from the default picker list. + */ +includeHidden?: boolean | null, }; diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/ThreadListParams.ts b/codex-rs/app-server-protocol/schema/typescript/v2/ThreadListParams.ts index c54f323f55..7fdadbad67 100644 --- a/codex-rs/app-server-protocol/schema/typescript/v2/ThreadListParams.ts +++ b/codex-rs/app-server-protocol/schema/typescript/v2/ThreadListParams.ts @@ -31,4 +31,9 @@ sourceKinds?: Array | null, * Optional archived filter; when set to true, only archived threads are returned. * If false or null, only non-archived threads are returned. */ -archived?: boolean | null, }; +archived?: boolean | null, +/** + * Optional cwd filter; when set, only threads whose session cwd exactly + * matches this path are returned. + */ +cwd?: string | null, }; diff --git a/codex-rs/app-server-protocol/src/protocol/common.rs b/codex-rs/app-server-protocol/src/protocol/common.rs index bc91341166..6c58d3b90c 100644 --- a/codex-rs/app-server-protocol/src/protocol/common.rs +++ b/codex-rs/app-server-protocol/src/protocol/common.rs @@ -758,6 +758,13 @@ pub struct FuzzyFileSearchSessionUpdatedNotification { pub files: Vec, } +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(rename_all = "camelCase")] +pub struct FuzzyFileSearchSessionCompletedNotification { + pub session_id: String, +} + server_notification_definitions! { /// NEW NOTIFICATIONS Error => "error" (v2::ErrorNotification), @@ -791,6 +798,7 @@ server_notification_definitions! { DeprecationNotice => "deprecationNotice" (v2::DeprecationNoticeNotification), ConfigWarning => "configWarning" (v2::ConfigWarningNotification), FuzzyFileSearchSessionUpdated => "fuzzyFileSearch/sessionUpdated" (FuzzyFileSearchSessionUpdatedNotification), + FuzzyFileSearchSessionCompleted => "fuzzyFileSearch/sessionCompleted" (FuzzyFileSearchSessionCompletedNotification), /// Notifies the user of world-writable directories on Windows, which cannot be protected by the sandbox. WindowsWorldWritableWarning => "windows/worldWritableWarning" (v2::WindowsWorldWritableWarningNotification), @@ -1227,7 +1235,8 @@ mod tests { "id": 6, "params": { "limit": null, - "cursor": null + "cursor": null, + "includeHidden": null } }), serde_json::to_value(&request)?, diff --git a/codex-rs/app-server-protocol/src/protocol/v2.rs b/codex-rs/app-server-protocol/src/protocol/v2.rs index 7cff3e2def..9a184d1754 100644 --- a/codex-rs/app-server-protocol/src/protocol/v2.rs +++ b/codex-rs/app-server-protocol/src/protocol/v2.rs @@ -1110,6 +1110,9 @@ pub struct ModelListParams { /// Optional page size; defaults to a reasonable server-side value. #[ts(optional = nullable)] pub limit: Option, + /// When true, include models that are hidden from the default picker list. + #[ts(optional = nullable)] + pub include_hidden: Option, } #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] @@ -1121,6 +1124,7 @@ pub struct Model { pub upgrade: Option, pub display_name: String, pub description: String, + pub hidden: bool, pub supported_reasoning_efforts: Vec, pub default_reasoning_effort: ReasoningEffort, #[serde(default = "default_input_modalities")] @@ -1290,6 +1294,14 @@ pub struct AppInfo { pub install_url: Option, #[serde(default)] pub is_accessible: bool, + /// Whether this app is enabled in config.toml. + /// Example: + /// ```toml + /// [apps.bad_app] + /// enabled = false + /// ``` + #[serde(default = "default_enabled")] + pub is_enabled: bool, } #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] @@ -1700,6 +1712,10 @@ pub struct ThreadListParams { /// If false or null, only non-archived threads are returned. #[ts(optional = nullable)] pub archived: Option, + /// Optional cwd filter; when set, only threads whose session cwd exactly + /// matches this path are returned. + #[ts(optional = nullable)] + pub cwd: Option, } #[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, JsonSchema, TS)] diff --git a/codex-rs/app-server-test-client/Cargo.toml b/codex-rs/app-server-test-client/Cargo.toml index 25a8813645..05c8938f05 100644 --- a/codex-rs/app-server-test-client/Cargo.toml +++ b/codex-rs/app-server-test-client/Cargo.toml @@ -14,4 +14,6 @@ codex-app-server-protocol = { workspace = true } codex-protocol = { workspace = true } serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } +tungstenite = { workspace = true } +url = { workspace = true } uuid = { workspace = true, features = ["v4"] } diff --git a/codex-rs/app-server-test-client/README.md b/codex-rs/app-server-test-client/README.md index ee75348c98..ab11f12353 100644 --- a/codex-rs/app-server-test-client/README.md +++ b/codex-rs/app-server-test-client/README.md @@ -1,2 +1,49 @@ # App Server Test Client -Exercises simple `codex app-server` flows end-to-end, logging JSON-RPC messages sent between client and server to stdout. +Quickstart for running and hitting `codex app-server`. + +## Quickstart + +Run from `/codex-rs`. + +```bash +# 1) Build debug codex binary +cargo build -p codex-cli --bin codex + +# 2) Start websocket app-server in background +cargo run -p codex-app-server-test-client -- \ + --codex-bin ./target/debug/codex \ + serve --listen ws://127.0.0.1:4222 --kill + +# 3) Call app-server (defaults to ws://127.0.0.1:4222) +cargo run -p codex-app-server-test-client -- model-list +``` + +## Testing Thread Rejoin Behavior + +Build and start an app server using commands above. The app-server log is written to `/tmp/codex-app-server-test-client/app-server.log` + +### 1) Get a thread id + +Create at least one thread, then list threads: + +```bash +cargo run -p codex-app-server-test-client -- send-message-v2 "seed thread for rejoin test" +cargo run -p codex-app-server-test-client -- thread-list --limit 5 +``` + +Copy a thread id from the `thread-list` output. + +### 2) Rejoin while a turn is in progress (two terminals) + +Terminal A: + +```bash +cargo run --bin codex-app-server-test-client -- \ + resume-message-v2 "respond with thorough docs on the rust core" +``` + +Terminal B (while Terminal A is still streaming): + +```bash +cargo run --bin codex-app-server-test-client -- thread-resume +``` diff --git a/codex-rs/app-server-test-client/src/lib.rs b/codex-rs/app-server-test-client/src/lib.rs index 91012572ac..7b55c30a34 100644 --- a/codex-rs/app-server-test-client/src/lib.rs +++ b/codex-rs/app-server-test-client/src/lib.rs @@ -1,8 +1,10 @@ use std::collections::VecDeque; use std::fs; +use std::fs::OpenOptions; use std::io::BufRead; use std::io::BufReader; use std::io::Write; +use std::net::TcpStream; use std::path::Path; use std::path::PathBuf; use std::process::Child; @@ -53,6 +55,8 @@ use codex_app_server_protocol::SendUserMessageParams; use codex_app_server_protocol::SendUserMessageResponse; use codex_app_server_protocol::ServerNotification; use codex_app_server_protocol::ServerRequest; +use codex_app_server_protocol::ThreadListParams; +use codex_app_server_protocol::ThreadListResponse; use codex_app_server_protocol::ThreadResumeParams; use codex_app_server_protocol::ThreadResumeResponse; use codex_app_server_protocol::ThreadStartParams; @@ -67,15 +71,28 @@ use codex_protocol::protocol::EventMsg; use serde::Serialize; use serde::de::DeserializeOwned; use serde_json::Value; +use tungstenite::Message; +use tungstenite::WebSocket; +use tungstenite::connect; +use tungstenite::stream::MaybeTlsStream; +use url::Url; use uuid::Uuid; /// Minimal launcher that initializes the Codex app-server and logs the handshake. #[derive(Parser)] #[command(author = "Codex", version, about = "Bootstrap Codex app-server", long_about = None)] struct Cli { - /// Path to the `codex` CLI binary. - #[arg(long, env = "CODEX_BIN", default_value = "codex")] - codex_bin: PathBuf, + /// Path to the `codex` CLI binary. When set, requests use stdio by + /// spawning `codex app-server` as a child process. + #[arg(long, env = "CODEX_BIN", global = true)] + codex_bin: Option, + + /// Existing websocket server URL to connect to. + /// + /// If neither `--codex-bin` nor `--url` is provided, defaults to + /// `ws://127.0.0.1:4222`. + #[arg(long, env = "CODEX_APP_SERVER_URL", global = true)] + url: Option, /// Forwarded to the `codex` CLI as `--config key=value`. Repeatable. /// @@ -105,6 +122,18 @@ struct Cli { #[derive(Subcommand)] enum CliCommand { + /// Start `codex app-server` on a websocket endpoint in the background. + /// + /// Logs are written to: + /// `/tmp/codex-app-server-test-client/` + Serve { + /// WebSocket listen URL passed to `codex app-server --listen`. + #[arg(long, default_value = "ws://127.0.0.1:4222")] + listen: String, + /// Kill any process listening on the same port before starting. + #[arg(long, default_value_t = false)] + kill: bool, + }, /// Send a user message through the Codex app-server. SendMessage { /// User message to send to Codex. @@ -122,6 +151,13 @@ enum CliCommand { /// User message to send to Codex. user_message: String, }, + /// Resume a V2 thread and continuously stream notifications/events. + /// + /// This command does not auto-exit; stop it with SIGINT/SIGTERM/SIGKILL. + ThreadResume { + /// Existing thread id to resume. + thread_id: String, + }, /// Start a V2 turn that elicits an ExecCommand approval. #[command(name = "trigger-cmd-approval")] TriggerCmdApproval { @@ -151,11 +187,19 @@ enum CliCommand { /// List the available models from the Codex app-server. #[command(name = "model-list")] ModelList, + /// List stored threads from the Codex app-server. + #[command(name = "thread-list")] + ThreadList { + /// Number of threads to return. + #[arg(long, default_value_t = 20)] + limit: u32, + }, } pub fn run() -> Result<()> { let Cli { codex_bin, + url, config_overrides, dynamic_tools, command, @@ -164,59 +208,222 @@ pub fn run() -> Result<()> { let dynamic_tools = parse_dynamic_tools_arg(&dynamic_tools)?; match command { + CliCommand::Serve { listen, kill } => { + ensure_dynamic_tools_unused(&dynamic_tools, "serve")?; + let codex_bin = codex_bin.unwrap_or_else(|| PathBuf::from("codex")); + serve(&codex_bin, &config_overrides, &listen, kill) + } CliCommand::SendMessage { user_message } => { ensure_dynamic_tools_unused(&dynamic_tools, "send-message")?; - send_message(&codex_bin, &config_overrides, user_message) + let endpoint = resolve_endpoint(codex_bin, url)?; + send_message(&endpoint, &config_overrides, user_message) } CliCommand::SendMessageV2 { user_message } => { - send_message_v2(&codex_bin, &config_overrides, user_message, &dynamic_tools) + let endpoint = resolve_endpoint(codex_bin, url)?; + send_message_v2_endpoint(&endpoint, &config_overrides, user_message, &dynamic_tools) } CliCommand::ResumeMessageV2 { thread_id, user_message, - } => resume_message_v2( - &codex_bin, - &config_overrides, - thread_id, - user_message, - &dynamic_tools, - ), + } => { + let endpoint = resolve_endpoint(codex_bin, url)?; + resume_message_v2( + &endpoint, + &config_overrides, + thread_id, + user_message, + &dynamic_tools, + ) + } + CliCommand::ThreadResume { thread_id } => { + ensure_dynamic_tools_unused(&dynamic_tools, "thread-resume")?; + let endpoint = resolve_endpoint(codex_bin, url)?; + thread_resume_follow(&endpoint, &config_overrides, thread_id) + } CliCommand::TriggerCmdApproval { user_message } => { - trigger_cmd_approval(&codex_bin, &config_overrides, user_message, &dynamic_tools) + let endpoint = resolve_endpoint(codex_bin, url)?; + trigger_cmd_approval(&endpoint, &config_overrides, user_message, &dynamic_tools) } CliCommand::TriggerPatchApproval { user_message } => { - trigger_patch_approval(&codex_bin, &config_overrides, user_message, &dynamic_tools) + let endpoint = resolve_endpoint(codex_bin, url)?; + trigger_patch_approval(&endpoint, &config_overrides, user_message, &dynamic_tools) } CliCommand::NoTriggerCmdApproval => { - no_trigger_cmd_approval(&codex_bin, &config_overrides, &dynamic_tools) + let endpoint = resolve_endpoint(codex_bin, url)?; + no_trigger_cmd_approval(&endpoint, &config_overrides, &dynamic_tools) } CliCommand::SendFollowUpV2 { first_message, follow_up_message, - } => send_follow_up_v2( - &codex_bin, - &config_overrides, - first_message, - follow_up_message, - &dynamic_tools, - ), + } => { + let endpoint = resolve_endpoint(codex_bin, url)?; + send_follow_up_v2( + &endpoint, + &config_overrides, + first_message, + follow_up_message, + &dynamic_tools, + ) + } CliCommand::TestLogin => { ensure_dynamic_tools_unused(&dynamic_tools, "test-login")?; - test_login(&codex_bin, &config_overrides) + let endpoint = resolve_endpoint(codex_bin, url)?; + test_login(&endpoint, &config_overrides) } CliCommand::GetAccountRateLimits => { ensure_dynamic_tools_unused(&dynamic_tools, "get-account-rate-limits")?; - get_account_rate_limits(&codex_bin, &config_overrides) + let endpoint = resolve_endpoint(codex_bin, url)?; + get_account_rate_limits(&endpoint, &config_overrides) } CliCommand::ModelList => { ensure_dynamic_tools_unused(&dynamic_tools, "model-list")?; - model_list(&codex_bin, &config_overrides) + let endpoint = resolve_endpoint(codex_bin, url)?; + model_list(&endpoint, &config_overrides) + } + CliCommand::ThreadList { limit } => { + ensure_dynamic_tools_unused(&dynamic_tools, "thread-list")?; + let endpoint = resolve_endpoint(codex_bin, url)?; + thread_list(&endpoint, &config_overrides, limit) } } } -fn send_message(codex_bin: &Path, config_overrides: &[String], user_message: String) -> Result<()> { - let mut client = CodexClient::spawn(codex_bin, config_overrides)?; +enum Endpoint { + SpawnCodex(PathBuf), + ConnectWs(String), +} + +fn resolve_endpoint(codex_bin: Option, url: Option) -> Result { + if codex_bin.is_some() && url.is_some() { + bail!("--codex-bin and --url are mutually exclusive"); + } + if let Some(codex_bin) = codex_bin { + return Ok(Endpoint::SpawnCodex(codex_bin)); + } + if let Some(url) = url { + return Ok(Endpoint::ConnectWs(url)); + } + Ok(Endpoint::ConnectWs("ws://127.0.0.1:4222".to_string())) +} + +fn serve(codex_bin: &Path, config_overrides: &[String], listen: &str, kill: bool) -> Result<()> { + let runtime_dir = PathBuf::from("/tmp/codex-app-server-test-client"); + fs::create_dir_all(&runtime_dir) + .with_context(|| format!("failed to create runtime dir {}", runtime_dir.display()))?; + let log_path = runtime_dir.join("app-server.log"); + if kill { + kill_listeners_on_same_port(listen)?; + } + + let log_file = OpenOptions::new() + .create(true) + .append(true) + .open(&log_path) + .with_context(|| format!("failed to open log file {}", log_path.display()))?; + let log_file_stderr = log_file + .try_clone() + .with_context(|| format!("failed to clone log file handle {}", log_path.display()))?; + + let mut cmdline = format!( + "tail -f /dev/null | RUST_BACKTRACE=full RUST_LOG=warn,codex_=trace {}", + shell_quote(&codex_bin.display().to_string()) + ); + for override_kv in config_overrides { + cmdline.push_str(&format!(" --config {}", shell_quote(override_kv))); + } + cmdline.push_str(&format!(" app-server --listen {}", shell_quote(listen))); + + let child = Command::new("nohup") + .arg("sh") + .arg("-c") + .arg(cmdline) + .stdin(Stdio::null()) + .stdout(Stdio::from(log_file)) + .stderr(Stdio::from(log_file_stderr)) + .spawn() + .with_context(|| format!("failed to start `{}` app-server", codex_bin.display()))?; + + let pid = child.id(); + + println!("started codex app-server"); + println!("listen: {listen}"); + println!("pid: {pid} (launcher process)"); + println!("log: {}", log_path.display()); + + Ok(()) +} + +fn kill_listeners_on_same_port(listen: &str) -> Result<()> { + let url = Url::parse(listen).with_context(|| format!("invalid --listen URL `{listen}`"))?; + let port = url + .port_or_known_default() + .with_context(|| format!("unable to infer port from --listen URL `{listen}`"))?; + + let output = Command::new("lsof") + .arg("-nP") + .arg(format!("-tiTCP:{port}")) + .arg("-sTCP:LISTEN") + .output() + .with_context(|| format!("failed to run lsof for port {port}"))?; + + if !output.status.success() { + return Ok(()); + } + + let pids: Vec = String::from_utf8_lossy(&output.stdout) + .lines() + .filter_map(|line| line.trim().parse::().ok()) + .collect(); + + if pids.is_empty() { + return Ok(()); + } + + for pid in pids { + println!("killing listener pid {pid} on port {port}"); + let pid_str = pid.to_string(); + let term_status = Command::new("kill") + .arg(&pid_str) + .status() + .with_context(|| format!("failed to send SIGTERM to pid {pid}"))?; + if !term_status.success() { + continue; + } + } + + thread::sleep(Duration::from_millis(300)); + + let output = Command::new("lsof") + .arg("-nP") + .arg(format!("-tiTCP:{port}")) + .arg("-sTCP:LISTEN") + .output() + .with_context(|| format!("failed to re-check listeners on port {port}"))?; + if !output.status.success() { + return Ok(()); + } + let remaining: Vec = String::from_utf8_lossy(&output.stdout) + .lines() + .filter_map(|line| line.trim().parse::().ok()) + .collect(); + for pid in remaining { + println!("force killing remaining listener pid {pid} on port {port}"); + let _ = Command::new("kill").arg("-9").arg(pid.to_string()).status(); + } + + Ok(()) +} + +fn shell_quote(input: &str) -> String { + format!("'{}'", input.replace('\'', "'\\''")) +} + +fn send_message( + endpoint: &Endpoint, + config_overrides: &[String], + user_message: String, +) -> Result<()> { + let mut client = CodexClient::connect(endpoint, config_overrides)?; let initialize = client.initialize()?; println!("< initialize response: {initialize:?}"); @@ -242,9 +449,19 @@ pub fn send_message_v2( config_overrides: &[String], user_message: String, dynamic_tools: &Option>, +) -> Result<()> { + let endpoint = Endpoint::SpawnCodex(codex_bin.to_path_buf()); + send_message_v2_endpoint(&endpoint, config_overrides, user_message, dynamic_tools) +} + +fn send_message_v2_endpoint( + endpoint: &Endpoint, + config_overrides: &[String], + user_message: String, + dynamic_tools: &Option>, ) -> Result<()> { send_message_v2_with_policies( - codex_bin, + endpoint, config_overrides, user_message, None, @@ -254,7 +471,7 @@ pub fn send_message_v2( } fn resume_message_v2( - codex_bin: &Path, + endpoint: &Endpoint, config_overrides: &[String], thread_id: String, user_message: String, @@ -262,7 +479,7 @@ fn resume_message_v2( ) -> Result<()> { ensure_dynamic_tools_unused(dynamic_tools, "resume-message-v2")?; - let mut client = CodexClient::spawn(codex_bin, config_overrides)?; + let mut client = CodexClient::connect(endpoint, config_overrides)?; let initialize = client.initialize()?; println!("< initialize response: {initialize:?}"); @@ -288,8 +505,28 @@ fn resume_message_v2( Ok(()) } +fn thread_resume_follow( + endpoint: &Endpoint, + config_overrides: &[String], + thread_id: String, +) -> Result<()> { + let mut client = CodexClient::connect(endpoint, config_overrides)?; + + let initialize = client.initialize()?; + println!("< initialize response: {initialize:?}"); + + let resume_response = client.thread_resume(ThreadResumeParams { + thread_id, + ..Default::default() + })?; + println!("< thread/resume response: {resume_response:?}"); + println!("< streaming notifications until process is terminated"); + + client.stream_notifications_forever() +} + fn trigger_cmd_approval( - codex_bin: &Path, + endpoint: &Endpoint, config_overrides: &[String], user_message: Option, dynamic_tools: &Option>, @@ -298,7 +535,7 @@ fn trigger_cmd_approval( "Run `touch /tmp/should-trigger-approval` so I can confirm the file exists."; let message = user_message.unwrap_or_else(|| default_prompt.to_string()); send_message_v2_with_policies( - codex_bin, + endpoint, config_overrides, message, Some(AskForApproval::OnRequest), @@ -310,7 +547,7 @@ fn trigger_cmd_approval( } fn trigger_patch_approval( - codex_bin: &Path, + endpoint: &Endpoint, config_overrides: &[String], user_message: Option, dynamic_tools: &Option>, @@ -319,7 +556,7 @@ fn trigger_patch_approval( "Create a file named APPROVAL_DEMO.txt containing a short hello message using apply_patch."; let message = user_message.unwrap_or_else(|| default_prompt.to_string()); send_message_v2_with_policies( - codex_bin, + endpoint, config_overrides, message, Some(AskForApproval::OnRequest), @@ -331,13 +568,13 @@ fn trigger_patch_approval( } fn no_trigger_cmd_approval( - codex_bin: &Path, + endpoint: &Endpoint, config_overrides: &[String], dynamic_tools: &Option>, ) -> Result<()> { let prompt = "Run `touch should_not_trigger_approval.txt`"; send_message_v2_with_policies( - codex_bin, + endpoint, config_overrides, prompt.to_string(), None, @@ -347,14 +584,14 @@ fn no_trigger_cmd_approval( } fn send_message_v2_with_policies( - codex_bin: &Path, + endpoint: &Endpoint, config_overrides: &[String], user_message: String, approval_policy: Option, sandbox_policy: Option, dynamic_tools: &Option>, ) -> Result<()> { - let mut client = CodexClient::spawn(codex_bin, config_overrides)?; + let mut client = CodexClient::connect(endpoint, config_overrides)?; let initialize = client.initialize()?; println!("< initialize response: {initialize:?}"); @@ -385,13 +622,13 @@ fn send_message_v2_with_policies( } fn send_follow_up_v2( - codex_bin: &Path, + endpoint: &Endpoint, config_overrides: &[String], first_message: String, follow_up_message: String, dynamic_tools: &Option>, ) -> Result<()> { - let mut client = CodexClient::spawn(codex_bin, config_overrides)?; + let mut client = CodexClient::connect(endpoint, config_overrides)?; let initialize = client.initialize()?; println!("< initialize response: {initialize:?}"); @@ -431,8 +668,8 @@ fn send_follow_up_v2( Ok(()) } -fn test_login(codex_bin: &Path, config_overrides: &[String]) -> Result<()> { - let mut client = CodexClient::spawn(codex_bin, config_overrides)?; +fn test_login(endpoint: &Endpoint, config_overrides: &[String]) -> Result<()> { + let mut client = CodexClient::connect(endpoint, config_overrides)?; let initialize = client.initialize()?; println!("< initialize response: {initialize:?}"); @@ -461,8 +698,8 @@ fn test_login(codex_bin: &Path, config_overrides: &[String]) -> Result<()> { } } -fn get_account_rate_limits(codex_bin: &Path, config_overrides: &[String]) -> Result<()> { - let mut client = CodexClient::spawn(codex_bin, config_overrides)?; +fn get_account_rate_limits(endpoint: &Endpoint, config_overrides: &[String]) -> Result<()> { + let mut client = CodexClient::connect(endpoint, config_overrides)?; let initialize = client.initialize()?; println!("< initialize response: {initialize:?}"); @@ -473,8 +710,8 @@ fn get_account_rate_limits(codex_bin: &Path, config_overrides: &[String]) -> Res Ok(()) } -fn model_list(codex_bin: &Path, config_overrides: &[String]) -> Result<()> { - let mut client = CodexClient::spawn(codex_bin, config_overrides)?; +fn model_list(endpoint: &Endpoint, config_overrides: &[String]) -> Result<()> { + let mut client = CodexClient::connect(endpoint, config_overrides)?; let initialize = client.initialize()?; println!("< initialize response: {initialize:?}"); @@ -485,6 +722,26 @@ fn model_list(codex_bin: &Path, config_overrides: &[String]) -> Result<()> { Ok(()) } +fn thread_list(endpoint: &Endpoint, config_overrides: &[String], limit: u32) -> Result<()> { + let mut client = CodexClient::connect(endpoint, config_overrides)?; + + let initialize = client.initialize()?; + println!("< initialize response: {initialize:?}"); + + let response = client.thread_list(ThreadListParams { + cursor: None, + limit: Some(limit), + sort_key: None, + model_providers: None, + source_kinds: None, + archived: None, + cwd: None, + })?; + println!("< thread/list response: {response:?}"); + + Ok(()) +} + fn ensure_dynamic_tools_unused( dynamic_tools: &Option>, command: &str, @@ -519,15 +776,32 @@ fn parse_dynamic_tools_arg(dynamic_tools: &Option) -> Result, + stdout: BufReader, + }, + WebSocket { + url: String, + socket: Box>>, + }, +} + struct CodexClient { - child: Child, - stdin: Option, - stdout: BufReader, + transport: ClientTransport, pending_notifications: VecDeque, } impl CodexClient { - fn spawn(codex_bin: &Path, config_overrides: &[String]) -> Result { + fn connect(endpoint: &Endpoint, config_overrides: &[String]) -> Result { + match endpoint { + Endpoint::SpawnCodex(codex_bin) => Self::spawn_stdio(codex_bin, config_overrides), + Endpoint::ConnectWs(url) => Self::connect_websocket(url), + } + } + + fn spawn_stdio(codex_bin: &Path, config_overrides: &[String]) -> Result { let codex_bin_display = codex_bin.display(); let mut cmd = Command::new(codex_bin); for override_kv in config_overrides { @@ -551,9 +825,27 @@ impl CodexClient { .context("codex app-server stdout unavailable")?; Ok(Self { - child: codex_app_server, - stdin: Some(stdin), - stdout: BufReader::new(stdout), + transport: ClientTransport::Stdio { + child: codex_app_server, + stdin: Some(stdin), + stdout: BufReader::new(stdout), + }, + pending_notifications: VecDeque::new(), + }) + } + + fn connect_websocket(url: &str) -> Result { + let parsed = Url::parse(url).with_context(|| format!("invalid websocket URL `{url}`"))?; + let (socket, _response) = connect(parsed.as_str()).with_context(|| { + format!( + "failed to connect to websocket app-server at `{url}`; if no server is running, start one with `codex-app-server-test-client serve --listen {url}`" + ) + })?; + Ok(Self { + transport: ClientTransport::WebSocket { + url: url.to_string(), + socket: Box::new(socket), + }, pending_notifications: VecDeque::new(), }) } @@ -575,7 +867,16 @@ impl CodexClient { }, }; - self.send_request(request, request_id, "initialize") + let response: InitializeResponse = self.send_request(request, request_id, "initialize")?; + + // Complete the initialize handshake. + let initialized = JSONRPCMessage::Notification(JSONRPCNotification { + method: "initialized".to_string(), + params: None, + }); + self.write_jsonrpc_message(initialized)?; + + Ok(response) } fn start_thread(&mut self) -> Result { @@ -701,6 +1002,16 @@ impl CodexClient { self.send_request(request, request_id, "model/list") } + fn thread_list(&mut self, params: ThreadListParams) -> Result { + let request_id = self.request_id(); + let request = ClientRequest::ThreadList { + request_id: request_id.clone(), + params, + }; + + self.send_request(request, request_id, "thread/list") + } + fn stream_conversation(&mut self, conversation_id: &ThreadId) -> Result<()> { loop { let notification = self.next_notification()?; @@ -835,6 +1146,12 @@ impl CodexClient { Ok(()) } + fn stream_notifications_forever(&mut self) -> Result<()> { + loop { + let _ = self.next_notification()?; + } + } + fn extract_event( &self, notification: JSONRPCNotification, @@ -882,17 +1199,7 @@ impl CodexClient { let request_json = serde_json::to_string(request)?; let request_pretty = serde_json::to_string_pretty(request)?; print_multiline_with_prefix("> ", &request_pretty); - - if let Some(stdin) = self.stdin.as_mut() { - writeln!(stdin, "{request_json}")?; - stdin - .flush() - .context("failed to flush request to codex app-server")?; - } else { - bail!("codex app-server stdin closed"); - } - - Ok(()) + self.write_payload(&request_json) } fn wait_for_response(&mut self, request_id: RequestId, method: &str) -> Result @@ -947,17 +1254,8 @@ impl CodexClient { fn read_jsonrpc_message(&mut self) -> Result { loop { - let mut response_line = String::new(); - let bytes = self - .stdout - .read_line(&mut response_line) - .context("failed to read from codex app-server")?; - - if bytes == 0 { - bail!("codex app-server closed stdout"); - } - - let trimmed = response_line.trim(); + let raw = self.read_payload()?; + let trimmed = raw.trim(); if trimmed.is_empty() { continue; } @@ -1086,16 +1384,56 @@ impl CodexClient { let payload = serde_json::to_string(&message)?; let pretty = serde_json::to_string_pretty(&message)?; print_multiline_with_prefix("> ", &pretty); + self.write_payload(&payload) + } - if let Some(stdin) = self.stdin.as_mut() { - writeln!(stdin, "{payload}")?; - stdin - .flush() - .context("failed to flush response to codex app-server")?; - return Ok(()); + fn write_payload(&mut self, payload: &str) -> Result<()> { + match &mut self.transport { + ClientTransport::Stdio { stdin, .. } => { + if let Some(stdin) = stdin.as_mut() { + writeln!(stdin, "{payload}")?; + stdin + .flush() + .context("failed to flush payload to codex app-server")?; + return Ok(()); + } + bail!("codex app-server stdin closed") + } + ClientTransport::WebSocket { socket, url } => { + socket + .send(Message::Text(payload.to_string().into())) + .with_context(|| format!("failed to write websocket message to `{url}`"))?; + Ok(()) + } } + } - bail!("codex app-server stdin closed") + fn read_payload(&mut self) -> Result { + match &mut self.transport { + ClientTransport::Stdio { stdout, .. } => { + let mut response_line = String::new(); + let bytes = stdout + .read_line(&mut response_line) + .context("failed to read from codex app-server")?; + if bytes == 0 { + bail!("codex app-server closed stdout"); + } + Ok(response_line) + } + ClientTransport::WebSocket { socket, url } => loop { + let frame = socket + .read() + .with_context(|| format!("failed to read websocket message from `{url}`"))?; + match frame { + Message::Text(text) => return Ok(text.to_string()), + Message::Binary(_) | Message::Ping(_) | Message::Pong(_) => continue, + Message::Close(_) => { + bail!("websocket app-server at `{url}` closed the connection") + } + Message::Frame(_) => continue, + } + }, + } } } @@ -1107,21 +1445,25 @@ fn print_multiline_with_prefix(prefix: &str, payload: &str) { impl Drop for CodexClient { fn drop(&mut self) { - let _ = self.stdin.take(); + let ClientTransport::Stdio { child, stdin, .. } = &mut self.transport else { + return; + }; - if let Ok(Some(status)) = self.child.try_wait() { + let _ = stdin.take(); + + if let Ok(Some(status)) = child.try_wait() { println!("[codex app-server exited: {status}]"); return; } thread::sleep(Duration::from_millis(100)); - if let Ok(Some(status)) = self.child.try_wait() { + if let Ok(Some(status)) = child.try_wait() { println!("[codex app-server exited: {status}]"); return; } - let _ = self.child.kill(); - let _ = self.child.wait(); + let _ = child.kill(); + let _ = child.wait(); } } diff --git a/codex-rs/app-server/README.md b/codex-rs/app-server/README.md index 13c3fda1e4..aece5fa3cb 100644 --- a/codex-rs/app-server/README.md +++ b/codex-rs/app-server/README.md @@ -117,7 +117,7 @@ Example with notification opt-out: - `thread/start` — create a new thread; emits `thread/started` and auto-subscribes you to turn/item events for that thread. - `thread/resume` — reopen an existing thread by id so subsequent `turn/start` calls append to it. - `thread/fork` — fork an existing thread into a new thread id by copying the stored history; emits `thread/started` and auto-subscribes you to turn/item events for the new thread. -- `thread/list` — page through stored rollouts; supports cursor-based pagination and optional `modelProviders` filtering. +- `thread/list` — page through stored rollouts; supports cursor-based pagination and optional `modelProviders`, `sourceKinds`, `archived`, and `cwd` filters. - `thread/loaded/list` — list the thread ids currently loaded in memory. - `thread/read` — read a stored thread by id without resuming it; optionally include turns via `includeTurns`. - `thread/archive` — move a thread’s rollout file into the archived directory; returns `{}` on success. @@ -131,7 +131,7 @@ Example with notification opt-out: - `turn/interrupt` — request cancellation of an in-flight turn by `(thread_id, turn_id)`; success is an empty `{}` response and the turn finishes with `status: "interrupted"`. - `review/start` — kick off Codex’s automated reviewer for a thread; responds like `turn/start` and emits `item/started`/`item/completed` notifications with `enteredReviewMode` and `exitedReviewMode` items, plus a final assistant `agentMessage` containing the review. - `command/exec` — run a single command under the server sandbox without starting a thread/turn (handy for utilities and validation). -- `model/list` — list available models (with reasoning effort options and optional `upgrade` model ids). +- `model/list` — list available models (set `includeHidden: true` to include entries with `hidden: true`), with reasoning effort options and optional `upgrade` model ids. - `experimentalFeature/list` — list feature flags with stage metadata (`beta`, `underDevelopment`, `stable`, etc.), enabled/default-enabled state, and cursor pagination. For non-beta flags, `displayName`/`description`/`announcement` are `null`. - `collaborationMode/list` — list available collaboration mode presets (experimental, no pagination). - `skills/list` — list skills for one or more `cwd` values (optional `forceReload`). @@ -221,6 +221,7 @@ Experimental API: `thread/start`, `thread/resume`, and `thread/fork` accept `per - `modelProviders` — restrict results to specific providers; unset, null, or an empty array will include all providers. - `sourceKinds` — restrict results to specific sources; omit or pass `[]` for interactive sessions only (`cli`, `vscode`). - `archived` — when `true`, list archived threads only. When `false` or `null`, list non-archived threads (default). +- `cwd` — restrict results to threads whose session cwd exactly matches this path. Example: @@ -534,6 +535,13 @@ Examples: - Opt out of legacy session setup event: `codex/event/session_configured` - Opt out of streamed agent text deltas: `item/agentMessage/delta` +### Fuzzy file search events (experimental) + +The fuzzy file search session API emits per-query notifications: + +- `fuzzyFileSearch/sessionUpdated` — `{ sessionId, query, files }` with the current matching files for the active query. +- `fuzzyFileSearch/sessionCompleted` — `{ sessionId, query }` once indexing/matching for that query has completed. + ### Turn events The app-server streams JSON-RPC notifications while a turn is running. Each turn starts with `turn/started` (initial `turn`) and ends with `turn/completed` (final `turn` status). Token usage events stream separately via `thread/tokenUsage/updated`. Clients subscribe to the events they care about, rendering each item incrementally as updates arrive. The per-item lifecycle is always: `item/started` → zero or more item-specific deltas → `item/completed`. @@ -763,7 +771,7 @@ To enable or disable a skill by path: ## Apps -Use `app/list` to fetch available apps (connectors). Each entry includes metadata like the app `id`, display `name`, `installUrl`, and whether it is currently accessible. +Use `app/list` to fetch available apps (connectors). Each entry includes metadata like the app `id`, display `name`, `installUrl`, whether it is currently accessible, and whether it is enabled in config. ```json { "method": "app/list", "id": 50, "params": { @@ -782,7 +790,8 @@ Use `app/list` to fetch available apps (connectors). Each entry includes metadat "logoUrlDark": null, "distributionChannel": null, "installUrl": "https://chatgpt.com/apps/demo-app/demo-app", - "isAccessible": true + "isAccessible": true, + "isEnabled": true } ], "nextCursor": null @@ -808,7 +817,8 @@ The server also emits `app/list/updated` notifications whenever either source (a "logoUrlDark": null, "distributionChannel": null, "installUrl": "https://chatgpt.com/apps/demo-app/demo-app", - "isAccessible": true + "isAccessible": true, + "isEnabled": true } ] } diff --git a/codex-rs/app-server/src/bespoke_event_handling.rs b/codex-rs/app-server/src/bespoke_event_handling.rs index c1f36e1ef3..da7abe0593 100644 --- a/codex-rs/app-server/src/bespoke_event_handling.rs +++ b/codex-rs/app-server/src/bespoke_event_handling.rs @@ -206,6 +206,7 @@ pub(crate) async fn apply_bespoke_event_handling( reason, proposed_execpolicy_amendment, parsed_cmd, + .. }) => match api_version { ApiVersion::V1 => { let params = ExecCommandApprovalParams { diff --git a/codex-rs/app-server/src/codex_message_processor.rs b/codex-rs/app-server/src/codex_message_processor.rs index f1e4b83d2c..0439068db9 100644 --- a/codex-rs/app-server/src/codex_message_processor.rs +++ b/codex-rs/app-server/src/codex_message_processor.rs @@ -266,6 +266,13 @@ use crate::thread_state::ThreadStateManager; const THREAD_LIST_DEFAULT_LIMIT: usize = 25; const THREAD_LIST_MAX_LIMIT: usize = 100; +struct ThreadListFilters { + model_providers: Option>, + source_kinds: Option>, + archived: bool, + cwd: Option, +} + // Duration before a ChatGPT login attempt is abandoned. const LOGIN_CHATGPT_TIMEOUT: Duration = Duration::from_secs(10 * 60); const APP_LIST_LOAD_TIMEOUT: Duration = Duration::from_secs(90); @@ -1681,8 +1688,18 @@ impl CodexMessageProcessor { let timeout_ms = params .timeout_ms .and_then(|timeout_ms| u64::try_from(timeout_ms).ok()); + let managed_network_requirements_enabled = + self.config.managed_network_requirements_enabled(); let started_network_proxy = match self.config.permissions.network.as_ref() { - Some(spec) => match spec.start_proxy().await { + Some(spec) => match spec + .start_proxy( + self.config.permissions.sandbox_policy.get(), + None, + None, + managed_network_requirements_enabled, + ) + .await + { Ok(started) => Some(started), Err(err) => { let error = JSONRPCErrorError { @@ -1705,6 +1722,7 @@ impl CodexMessageProcessor { network: started_network_proxy .as_ref() .map(codex_core::config::StartedNetworkProxy::proxy), + network_attempt_id: None, sandbox_permissions: SandboxPermissions::UseDefault, windows_sandbox_level, justification: None, @@ -2430,6 +2448,7 @@ impl CodexMessageProcessor { model_providers, source_kinds, archived, + cwd, } = params; let requested_page_size = limit @@ -2444,10 +2463,13 @@ impl CodexMessageProcessor { .list_threads_common( requested_page_size, cursor, - model_providers, - source_kinds, core_sort_key, - archived.unwrap_or(false), + ThreadListFilters { + model_providers, + source_kinds, + archived: archived.unwrap_or(false), + cwd: cwd.map(PathBuf::from), + }, ) .await { @@ -2678,6 +2700,13 @@ impl CodexMessageProcessor { } async fn thread_resume(&mut self, request_id: ConnectionRequestId, params: ThreadResumeParams) { + if self + .resume_running_thread(request_id.clone(), ¶ms) + .await + { + return; + } + let ThreadResumeParams { thread_id, history, @@ -2695,77 +2724,21 @@ impl CodexMessageProcessor { } = params; let thread_history = if let Some(history) = history { - if history.is_empty() { - self.send_invalid_request_error( - request_id, - "history must not be empty".to_string(), - ) - .await; + let Some(thread_history) = self + .resume_thread_from_history(request_id.clone(), history.as_slice()) + .await + else { return; - } - InitialHistory::Forked(history.into_iter().map(RolloutItem::ResponseItem).collect()) - } else if let Some(path) = path { - match RolloutRecorder::get_rollout_history(&path).await { - Ok(initial_history) => initial_history, - Err(err) => { - self.send_invalid_request_error( - request_id, - format!("failed to load rollout `{}`: {err}", path.display()), - ) - .await; - return; - } - } + }; + thread_history } else { - let existing_thread_id = match ThreadId::from_string(&thread_id) { - Ok(id) => id, - Err(err) => { - let error = JSONRPCErrorError { - code: INVALID_REQUEST_ERROR_CODE, - message: format!("invalid thread id: {err}"), - data: None, - }; - self.outgoing.send_error(request_id, error).await; - return; - } + let Some(thread_history) = self + .resume_thread_from_rollout(request_id.clone(), &thread_id, path.as_ref()) + .await + else { + return; }; - - let path = match find_thread_path_by_id_str( - &self.config.codex_home, - &existing_thread_id.to_string(), - ) - .await - { - Ok(Some(p)) => p, - Ok(None) => { - self.send_invalid_request_error( - request_id, - format!("no rollout found for thread id {existing_thread_id}"), - ) - .await; - return; - } - Err(err) => { - self.send_invalid_request_error( - request_id, - format!("failed to locate thread id {existing_thread_id}: {err}"), - ) - .await; - return; - } - }; - - match RolloutRecorder::get_rollout_history(&path).await { - Ok(initial_history) => initial_history, - Err(err) => { - self.send_invalid_request_error( - request_id, - format!("failed to load rollout `{}`: {err}", path.display()), - ) - .await; - return; - } - } + thread_history }; let history_cwd = thread_history.session_cwd(); @@ -2846,41 +2819,17 @@ impl CodexMessageProcessor { ); } - let mut thread = match read_summary_from_rollout( - rollout_path.as_path(), - fallback_model_provider.as_str(), - ) - .await - { - Ok(summary) => summary_to_thread(summary), - Err(err) => { - self.send_internal_error( - request_id, - format!( - "failed to load rollout `{}` for thread {thread_id}: {err}", - rollout_path.display() - ), - ) - .await; - return; - } + let Some(thread) = self + .load_thread_from_rollout_or_send_internal( + request_id.clone(), + thread_id, + rollout_path.as_path(), + fallback_model_provider.as_str(), + ) + .await + else { + return; }; - match read_rollout_items_from_rollout(rollout_path.as_path()).await { - Ok(items) => { - thread.turns = build_turns_from_rollout_items(&items); - } - Err(err) => { - self.send_internal_error( - request_id, - format!( - "failed to load rollout `{}` for thread {thread_id}: {err}", - rollout_path.display() - ), - ) - .await; - return; - } - } let response = ThreadResumeResponse { thread, @@ -2905,6 +2854,278 @@ impl CodexMessageProcessor { } } + async fn resume_running_thread( + &mut self, + request_id: ConnectionRequestId, + params: &ThreadResumeParams, + ) -> bool { + if let Ok(existing_thread_id) = ThreadId::from_string(¶ms.thread_id) + && let Ok(existing_thread) = self.thread_manager.get_thread(existing_thread_id).await + { + if params.history.is_some() { + self.send_invalid_request_error( + request_id, + format!( + "cannot resume thread {existing_thread_id} with history while it is already running" + ), + ) + .await; + return true; + } + + let rollout_path = if let Some(path) = existing_thread.rollout_path() { + if path.exists() { + path + } else { + match find_thread_path_by_id_str( + &self.config.codex_home, + &existing_thread_id.to_string(), + ) + .await + { + Ok(Some(path)) => path, + Ok(None) => { + self.send_invalid_request_error( + request_id, + format!("no rollout found for thread id {existing_thread_id}"), + ) + .await; + return true; + } + Err(err) => { + self.send_invalid_request_error( + request_id, + format!("failed to locate thread id {existing_thread_id}: {err}"), + ) + .await; + return true; + } + } + } + } else { + match find_thread_path_by_id_str( + &self.config.codex_home, + &existing_thread_id.to_string(), + ) + .await + { + Ok(Some(path)) => path, + Ok(None) => { + self.send_invalid_request_error( + request_id, + format!("no rollout found for thread id {existing_thread_id}"), + ) + .await; + return true; + } + Err(err) => { + self.send_invalid_request_error( + request_id, + format!("failed to locate thread id {existing_thread_id}: {err}"), + ) + .await; + return true; + } + } + }; + + if let Some(requested_path) = params.path.as_ref() + && requested_path != &rollout_path + { + self.send_invalid_request_error( + request_id, + format!( + "cannot resume running thread {existing_thread_id} with mismatched path: requested `{}`, active `{}`", + requested_path.display(), + rollout_path.display() + ), + ) + .await; + return true; + } + + if let Err(err) = self + .ensure_conversation_listener( + existing_thread_id, + request_id.connection_id, + false, + ApiVersion::V2, + ) + .await + { + tracing::warn!( + "failed to attach listener for thread {}: {}", + existing_thread_id, + err.message + ); + } + + let config_snapshot = existing_thread.config_snapshot().await; + let mismatch_details = collect_resume_override_mismatches(params, &config_snapshot); + if !mismatch_details.is_empty() { + tracing::warn!( + "thread/resume overrides ignored for running thread {}: {}", + existing_thread_id, + mismatch_details.join("; ") + ); + } + + let Some(thread) = self + .load_thread_from_rollout_or_send_internal( + request_id.clone(), + existing_thread_id, + rollout_path.as_path(), + config_snapshot.model_provider_id.as_str(), + ) + .await + else { + return true; + }; + + let ThreadConfigSnapshot { + model, + model_provider_id, + approval_policy, + sandbox_policy, + cwd, + reasoning_effort, + .. + } = config_snapshot; + let response = ThreadResumeResponse { + thread, + model, + model_provider: model_provider_id, + cwd, + approval_policy: approval_policy.into(), + sandbox: sandbox_policy.into(), + reasoning_effort, + }; + self.outgoing.send_response(request_id, response).await; + return true; + } + false + } + + async fn resume_thread_from_history( + &self, + request_id: ConnectionRequestId, + history: &[ResponseItem], + ) -> Option { + if history.is_empty() { + self.send_invalid_request_error(request_id, "history must not be empty".to_string()) + .await; + return None; + } + Some(InitialHistory::Forked( + history + .iter() + .cloned() + .map(RolloutItem::ResponseItem) + .collect(), + )) + } + + async fn resume_thread_from_rollout( + &self, + request_id: ConnectionRequestId, + thread_id: &str, + path: Option<&PathBuf>, + ) -> Option { + let rollout_path = if let Some(path) = path { + path.clone() + } else { + let existing_thread_id = match ThreadId::from_string(thread_id) { + Ok(id) => id, + Err(err) => { + let error = JSONRPCErrorError { + code: INVALID_REQUEST_ERROR_CODE, + message: format!("invalid thread id: {err}"), + data: None, + }; + self.outgoing.send_error(request_id, error).await; + return None; + } + }; + + match find_thread_path_by_id_str( + &self.config.codex_home, + &existing_thread_id.to_string(), + ) + .await + { + Ok(Some(path)) => path, + Ok(None) => { + self.send_invalid_request_error( + request_id, + format!("no rollout found for thread id {existing_thread_id}"), + ) + .await; + return None; + } + Err(err) => { + self.send_invalid_request_error( + request_id, + format!("failed to locate thread id {existing_thread_id}: {err}"), + ) + .await; + return None; + } + } + }; + + match RolloutRecorder::get_rollout_history(&rollout_path).await { + Ok(initial_history) => Some(initial_history), + Err(err) => { + self.send_invalid_request_error( + request_id, + format!("failed to load rollout `{}`: {err}", rollout_path.display()), + ) + .await; + None + } + } + } + + async fn load_thread_from_rollout_or_send_internal( + &self, + request_id: ConnectionRequestId, + thread_id: ThreadId, + rollout_path: &Path, + fallback_provider: &str, + ) -> Option { + let mut thread = match read_summary_from_rollout(rollout_path, fallback_provider).await { + Ok(summary) => summary_to_thread(summary), + Err(err) => { + self.send_internal_error( + request_id, + format!( + "failed to load rollout `{}` for thread {thread_id}: {err}", + rollout_path.display() + ), + ) + .await; + return None; + } + }; + match read_rollout_items_from_rollout(rollout_path).await { + Ok(items) => { + thread.turns = build_turns_from_rollout_items(&items); + Some(thread) + } + Err(err) => { + self.send_internal_error( + request_id, + format!( + "failed to load rollout `{}` for thread {thread_id}: {err}", + rollout_path.display() + ), + ) + .await; + None + } + } + } + async fn thread_fork(&mut self, request_id: ConnectionRequestId, params: ThreadForkParams) { let ThreadForkParams { thread_id, @@ -3221,10 +3442,13 @@ impl CodexMessageProcessor { .list_threads_common( requested_page_size, cursor, - model_providers, - None, CoreThreadSortKey::UpdatedAt, - false, + ThreadListFilters { + model_providers, + source_kinds: None, + archived: false, + cwd: None, + }, ) .await { @@ -3242,11 +3466,15 @@ impl CodexMessageProcessor { &self, requested_page_size: usize, cursor: Option, - model_providers: Option>, - source_kinds: Option>, sort_key: CoreThreadSortKey, - archived: bool, + filters: ThreadListFilters, ) -> Result<(Vec, Option), JSONRPCErrorError> { + let ThreadListFilters { + model_providers, + source_kinds, + archived, + cwd, + } = filters; let mut cursor_obj: Option = match cursor.as_ref() { Some(cursor_str) => { Some(parse_cursor(cursor_str).ok_or_else(|| JSONRPCErrorError { @@ -3327,6 +3555,9 @@ impl CodexMessageProcessor { if source_kind_filter .as_ref() .is_none_or(|filter| source_kind_matches(&summary.source, filter)) + && cwd + .as_ref() + .is_none_or(|expected_cwd| &summary.cwd == expected_cwd) { filtered.push(summary); if filtered.len() >= remaining { @@ -3372,10 +3603,15 @@ impl CodexMessageProcessor { request_id: ConnectionRequestId, params: ModelListParams, ) { - let ModelListParams { limit, cursor } = params; + let ModelListParams { + limit, + cursor, + include_hidden, + } = params; let mut config = (*config).clone(); config.features.enable(Feature::RemoteModels); - let models = supported_models(thread_manager, &config).await; + let models = + supported_models(thread_manager, &config, include_hidden.unwrap_or(false)).await; let total = models.len(); if total == 0 { @@ -4589,6 +4825,11 @@ impl CodexMessageProcessor { None => 0, }; + let (mut accessible_connectors, mut all_connectors) = tokio::join!( + connectors::list_cached_accessible_connectors_from_mcp_tools(&config), + connectors::list_cached_all_connectors(&config) + ); + let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel(); let accessible_config = config.clone(); @@ -4603,16 +4844,17 @@ impl CodexMessageProcessor { let _ = accessible_tx.send(AppListLoadResult::Accessible(result)); }); + let all_config = config.clone(); tokio::spawn(async move { - let result = connectors::list_all_connectors_with_options(&config, force_refetch) + let result = connectors::list_all_connectors_with_options(&all_config, force_refetch) .await .map_err(|err| format!("failed to list apps: {err}")); let _ = tx.send(AppListLoadResult::Directory(result)); }); - let mut accessible_connectors: Option> = None; - let mut all_connectors: Option> = None; let app_list_deadline = tokio::time::Instant::now() + APP_LIST_LOAD_TIMEOUT; + let mut accessible_loaded = false; + let mut all_loaded = false; loop { let result = match tokio::time::timeout_at(app_list_deadline, rx.recv()).await { @@ -4643,6 +4885,7 @@ impl CodexMessageProcessor { match result { AppListLoadResult::Accessible(Ok(connectors)) => { accessible_connectors = Some(connectors); + accessible_loaded = true; } AppListLoadResult::Accessible(Err(err)) => { let error = JSONRPCErrorError { @@ -4655,6 +4898,7 @@ impl CodexMessageProcessor { } AppListLoadResult::Directory(Ok(connectors)) => { all_connectors = Some(connectors); + all_loaded = true; } AppListLoadResult::Directory(Err(err)) => { let error = JSONRPCErrorError { @@ -4667,13 +4911,16 @@ impl CodexMessageProcessor { } } - let merged = Self::merge_loaded_apps( - all_connectors.as_deref(), - accessible_connectors.as_deref(), + let merged = connectors::with_app_enabled_state( + Self::merge_loaded_apps( + all_connectors.as_deref(), + accessible_connectors.as_deref(), + ), + &config, ); Self::send_app_list_updated_notification(&outgoing, merged.clone()).await; - if accessible_connectors.is_some() && all_connectors.is_some() { + if accessible_loaded && all_loaded { match Self::paginate_apps(merged.as_slice(), start, limit) { Ok(response) => { outgoing.send_response(request_id, response).await; @@ -4692,9 +4939,10 @@ impl CodexMessageProcessor { all_connectors: Option<&[AppInfo]>, accessible_connectors: Option<&[AppInfo]>, ) -> Vec { + let all_connectors_loaded = all_connectors.is_some(); let all = all_connectors.map_or_else(Vec::new, <[AppInfo]>::to_vec); let accessible = accessible_connectors.map_or_else(Vec::new, <[AppInfo]>::to_vec); - connectors::merge_connectors_with_accessible(all, accessible) + connectors::merge_connectors_with_accessible(all, accessible, all_connectors_loaded) } fn paginate_apps( @@ -5775,6 +6023,101 @@ impl CodexMessageProcessor { } } +fn collect_resume_override_mismatches( + request: &ThreadResumeParams, + config_snapshot: &ThreadConfigSnapshot, +) -> Vec { + let mut mismatch_details = Vec::new(); + + if let Some(requested_model) = request.model.as_deref() + && requested_model != config_snapshot.model + { + mismatch_details.push(format!( + "model requested={requested_model} active={}", + config_snapshot.model + )); + } + if let Some(requested_provider) = request.model_provider.as_deref() + && requested_provider != config_snapshot.model_provider_id + { + mismatch_details.push(format!( + "model_provider requested={requested_provider} active={}", + config_snapshot.model_provider_id + )); + } + if let Some(requested_cwd) = request.cwd.as_deref() { + let requested_cwd_path = std::path::PathBuf::from(requested_cwd); + if requested_cwd_path != config_snapshot.cwd { + mismatch_details.push(format!( + "cwd requested={} active={}", + requested_cwd_path.display(), + config_snapshot.cwd.display() + )); + } + } + if let Some(requested_approval) = request.approval_policy.as_ref() { + let active_approval: AskForApproval = config_snapshot.approval_policy.into(); + if requested_approval != &active_approval { + mismatch_details.push(format!( + "approval_policy requested={requested_approval:?} active={active_approval:?}" + )); + } + } + if let Some(requested_sandbox) = request.sandbox.as_ref() { + let sandbox_matches = matches!( + (requested_sandbox, &config_snapshot.sandbox_policy), + ( + SandboxMode::ReadOnly, + codex_protocol::protocol::SandboxPolicy::ReadOnly { .. } + ) | ( + SandboxMode::WorkspaceWrite, + codex_protocol::protocol::SandboxPolicy::WorkspaceWrite { .. } + ) | ( + SandboxMode::DangerFullAccess, + codex_protocol::protocol::SandboxPolicy::DangerFullAccess + ) | ( + SandboxMode::DangerFullAccess, + codex_protocol::protocol::SandboxPolicy::ExternalSandbox { .. } + ) + ); + if !sandbox_matches { + mismatch_details.push(format!( + "sandbox requested={requested_sandbox:?} active={:?}", + config_snapshot.sandbox_policy + )); + } + } + if let Some(requested_personality) = request.personality.as_ref() + && config_snapshot.personality.as_ref() != Some(requested_personality) + { + mismatch_details.push(format!( + "personality requested={requested_personality:?} active={:?}", + config_snapshot.personality + )); + } + + if request.config.is_some() { + mismatch_details + .push("config overrides were provided and ignored while running".to_string()); + } + if request.base_instructions.is_some() { + mismatch_details + .push("baseInstructions override was provided and ignored while running".to_string()); + } + if request.developer_instructions.is_some() { + mismatch_details.push( + "developerInstructions override was provided and ignored while running".to_string(), + ); + } + if request.persist_extended_history { + mismatch_details.push( + "persistExtendedHistory override was provided and ignored while running".to_string(), + ); + } + + mismatch_details +} + fn skills_to_info( skills: &[codex_core::skills::SkillMetadata], disabled_paths: &std::collections::HashSet, diff --git a/codex-rs/app-server/src/fuzzy_file_search.rs b/codex-rs/app-server/src/fuzzy_file_search.rs index cb1f61c362..d40d3fc242 100644 --- a/codex-rs/app-server/src/fuzzy_file_search.rs +++ b/codex-rs/app-server/src/fuzzy_file_search.rs @@ -6,6 +6,7 @@ use std::sync::atomic::AtomicBool; use std::sync::atomic::Ordering; use codex_app_server_protocol::FuzzyFileSearchResult; +use codex_app_server_protocol::FuzzyFileSearchSessionCompletedNotification; use codex_app_server_protocol::FuzzyFileSearchSessionUpdatedNotification; use codex_app_server_protocol::ServerNotification; use codex_file_search as file_search; @@ -195,6 +196,20 @@ impl SessionReporterImpl { outgoing.send_server_notification(notification).await; }); } + + fn send_complete(&self) { + if self.shared.canceled.load(Ordering::Relaxed) { + return; + } + let session_id = self.shared.session_id.clone(); + let outgoing = self.shared.outgoing.clone(); + self.shared.runtime.spawn(async move { + let notification = ServerNotification::FuzzyFileSearchSessionCompleted( + FuzzyFileSearchSessionCompletedNotification { session_id }, + ); + outgoing.send_server_notification(notification).await; + }); + } } impl file_search::SessionReporter for SessionReporterImpl { @@ -202,7 +217,9 @@ impl file_search::SessionReporter for SessionReporterImpl { self.send_snapshot(snapshot); } - fn on_complete(&self) {} + fn on_complete(&self) { + self.send_complete(); + } } fn collect_files(snapshot: &file_search::FileSearchSnapshot) -> Vec { diff --git a/codex-rs/app-server/src/models.rs b/codex-rs/app-server/src/models.rs index 350b86f92a..d31e005210 100644 --- a/codex-rs/app-server/src/models.rs +++ b/codex-rs/app-server/src/models.rs @@ -8,12 +8,16 @@ use codex_core::models_manager::manager::RefreshStrategy; use codex_protocol::openai_models::ModelPreset; use codex_protocol::openai_models::ReasoningEffortPreset; -pub async fn supported_models(thread_manager: Arc, config: &Config) -> Vec { +pub async fn supported_models( + thread_manager: Arc, + config: &Config, + include_hidden: bool, +) -> Vec { thread_manager .list_models(config, RefreshStrategy::OnlineIfUncached) .await .into_iter() - .filter(|preset| preset.show_in_picker) + .filter(|preset| include_hidden || preset.show_in_picker) .map(model_from_preset) .collect() } @@ -25,6 +29,7 @@ fn model_from_preset(preset: ModelPreset) -> Model { upgrade: preset.upgrade.map(|upgrade| upgrade.id), display_name: preset.display_name.to_string(), description: preset.description.to_string(), + hidden: !preset.show_in_picker, supported_reasoning_efforts: reasoning_efforts_from_preset( preset.supported_reasoning_efforts, ), diff --git a/codex-rs/app-server/tests/common/models_cache.rs b/codex-rs/app-server/tests/common/models_cache.rs index 242a71f98d..5095bf0f3d 100644 --- a/codex-rs/app-server/tests/common/models_cache.rs +++ b/codex-rs/app-server/tests/common/models_cache.rs @@ -41,6 +41,7 @@ fn preset_to_info(preset: &ModelPreset, priority: i32) -> ModelInfo { experimental_supported_tools: Vec::new(), input_modalities: default_input_modalities(), prefer_websockets: false, + used_fallback_model_metadata: false, } } diff --git a/codex-rs/app-server/tests/suite/fuzzy_file_search.rs b/codex-rs/app-server/tests/suite/fuzzy_file_search.rs index ffa4fa2cca..b6932560f0 100644 --- a/codex-rs/app-server/tests/suite/fuzzy_file_search.rs +++ b/codex-rs/app-server/tests/suite/fuzzy_file_search.rs @@ -1,6 +1,7 @@ use anyhow::Result; use anyhow::anyhow; use app_test_support::McpProcess; +use codex_app_server_protocol::FuzzyFileSearchSessionCompletedNotification; use codex_app_server_protocol::FuzzyFileSearchSessionUpdatedNotification; use codex_app_server_protocol::JSONRPCResponse; use codex_app_server_protocol::RequestId; @@ -13,6 +14,7 @@ const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs const SHORT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_millis(500); const STOP_GRACE_PERIOD: std::time::Duration = std::time::Duration::from_millis(250); const SESSION_UPDATED_METHOD: &str = "fuzzyFileSearch/sessionUpdated"; +const SESSION_COMPLETED_METHOD: &str = "fuzzyFileSearch/sessionCompleted"; #[derive(Clone, Copy, Debug, PartialEq, Eq)] enum FileExpectation { @@ -60,6 +62,29 @@ async fn wait_for_session_updated( ); } +async fn wait_for_session_completed( + mcp: &mut McpProcess, + session_id: &str, +) -> Result { + for _ in 0..20 { + let notification = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_notification_message(SESSION_COMPLETED_METHOD), + ) + .await??; + let params = notification + .params + .ok_or_else(|| anyhow!("missing notification params"))?; + let payload = + serde_json::from_value::(params)?; + if payload.session_id == session_id { + return Ok(payload); + } + } + + anyhow::bail!("did not receive expected session completion for sessionId={session_id}"); +} + async fn assert_update_request_fails_for_missing_session( mcp: &mut McpProcess, session_id: &str, @@ -271,12 +296,41 @@ async fn test_fuzzy_file_search_session_streams_updates() -> Result<()> { assert_eq!(payload.files.len(), 1); assert_eq!(payload.files[0].root, root_path); assert_eq!(payload.files[0].path, "alpha.txt"); + let completed = wait_for_session_completed(&mut mcp, session_id).await?; + assert_eq!(completed.session_id, session_id); mcp.stop_fuzzy_file_search_session(session_id).await?; Ok(()) } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_fuzzy_file_search_session_no_updates_after_complete_until_query_edited() -> Result<()> +{ + let codex_home = TempDir::new()?; + let root = TempDir::new()?; + std::fs::write(root.path().join("alpha.txt"), "contents")?; + let mut mcp = initialized_mcp(&codex_home).await?; + + let root_path = root.path().to_string_lossy().to_string(); + let session_id = "session-complete-invariant"; + mcp.start_fuzzy_file_search_session(session_id, vec![root_path]) + .await?; + + mcp.update_fuzzy_file_search_session(session_id, "alp") + .await?; + wait_for_session_updated(&mut mcp, session_id, "alp", FileExpectation::NonEmpty).await?; + wait_for_session_completed(&mut mcp, session_id).await?; + assert_no_session_updates_for(&mut mcp, session_id, STOP_GRACE_PERIOD, SHORT_READ_TIMEOUT) + .await?; + + mcp.update_fuzzy_file_search_session(session_id, "alpha") + .await?; + wait_for_session_updated(&mut mcp, session_id, "alpha", FileExpectation::NonEmpty).await?; + + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_fuzzy_file_search_session_update_before_start_errors() -> Result<()> { let codex_home = TempDir::new()?; @@ -345,6 +399,7 @@ async fn test_fuzzy_file_search_session_multiple_query_updates_work() -> Result< alp_payload.files.iter().all(|file| file.root == root_path), true ); + wait_for_session_completed(&mut mcp, session_id).await?; mcp.update_fuzzy_file_search_session(session_id, "zzzz") .await?; @@ -352,6 +407,7 @@ async fn test_fuzzy_file_search_session_multiple_query_updates_work() -> Result< wait_for_session_updated(&mut mcp, session_id, "zzzz", FileExpectation::Any).await?; assert_eq!(zzzz_payload.query, "zzzz"); assert_eq!(zzzz_payload.files.is_empty(), true); + wait_for_session_completed(&mut mcp, session_id).await?; Ok(()) } diff --git a/codex-rs/app-server/tests/suite/send_message.rs b/codex-rs/app-server/tests/suite/send_message.rs index 66032da20b..d5e0154121 100644 --- a/codex-rs/app-server/tests/suite/send_message.rs +++ b/codex-rs/app-server/tests/suite/send_message.rs @@ -564,6 +564,7 @@ fn append_rollout_turn_context(path: &Path, timestamp: &str, model: &str) -> std cwd: PathBuf::from("/"), approval_policy: AskForApproval::Never, sandbox_policy: SandboxPolicy::DangerFullAccess, + network: None, model: model.to_string(), personality: None, collaboration_mode: None, diff --git a/codex-rs/app-server/tests/suite/v2/app_list.rs b/codex-rs/app-server/tests/suite/v2/app_list.rs index 6fce729084..079eba4794 100644 --- a/codex-rs/app-server/tests/suite/v2/app_list.rs +++ b/codex-rs/app-server/tests/suite/v2/app_list.rs @@ -1,5 +1,6 @@ use std::borrow::Cow; use std::sync::Arc; +use std::sync::Mutex as StdMutex; use std::time::Duration; use anyhow::Result; @@ -86,6 +87,7 @@ async fn list_apps_uses_thread_feature_flag_when_thread_id_is_provided() -> Resu distribution_channel: None, install_url: None, is_accessible: false, + is_enabled: true, }]; let tools = vec![connector_tool("beta", "Beta App")?]; let (server_url, server_handle) = @@ -173,6 +175,78 @@ connectors = false Ok(()) } +#[tokio::test] +async fn list_apps_reports_is_enabled_from_config() -> Result<()> { + let connectors = vec![AppInfo { + id: "beta".to_string(), + name: "Beta".to_string(), + description: Some("Beta connector".to_string()), + logo_url: None, + logo_url_dark: None, + distribution_channel: None, + install_url: None, + is_accessible: false, + is_enabled: true, + }]; + let tools = vec![connector_tool("beta", "Beta App")?]; + let (server_url, server_handle) = + start_apps_server_with_delays(connectors, tools, Duration::ZERO, Duration::ZERO).await?; + + let codex_home = TempDir::new()?; + std::fs::write( + codex_home.path().join("config.toml"), + format!( + r#" +chatgpt_base_url = "{server_url}" + +[features] +connectors = true + +[apps.beta] +enabled = false +"# + ), + )?; + write_chatgpt_auth( + codex_home.path(), + ChatGptAuthFixture::new("chatgpt-token") + .account_id("account-123") + .chatgpt_user_id("user-123") + .chatgpt_account_id("account-123"), + AuthCredentialsStoreMode::File, + )?; + + let mut mcp = McpProcess::new(codex_home.path()).await?; + timeout(DEFAULT_TIMEOUT, mcp.initialize()).await??; + + let request_id = mcp + .send_apps_list_request(AppsListParams { + limit: None, + cursor: None, + thread_id: None, + force_refetch: false, + }) + .await?; + + let response: JSONRPCResponse = timeout( + DEFAULT_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(request_id)), + ) + .await??; + let AppsListResponse { + data: response_data, + next_cursor, + } = to_response(response)?; + assert!(next_cursor.is_none()); + assert_eq!(response_data.len(), 1); + assert_eq!(response_data[0].id, "beta"); + assert!(!response_data[0].is_enabled); + + server_handle.abort(); + let _ = server_handle.await; + Ok(()) +} + #[tokio::test] async fn list_apps_emits_updates_and_returns_after_both_lists_load() -> Result<()> { let connectors = vec![ @@ -185,6 +259,7 @@ async fn list_apps_emits_updates_and_returns_after_both_lists_load() -> Result<( distribution_channel: None, install_url: None, is_accessible: false, + is_enabled: true, }, AppInfo { id: "beta".to_string(), @@ -195,6 +270,7 @@ async fn list_apps_emits_updates_and_returns_after_both_lists_load() -> Result<( distribution_channel: None, install_url: None, is_accessible: false, + is_enabled: true, }, ]; @@ -239,6 +315,7 @@ async fn list_apps_emits_updates_and_returns_after_both_lists_load() -> Result<( distribution_channel: None, install_url: Some("https://chatgpt.com/apps/beta-app/beta".to_string()), is_accessible: true, + is_enabled: true, }]; let first_update = read_app_list_updated_notification(&mut mcp).await?; @@ -254,6 +331,7 @@ async fn list_apps_emits_updates_and_returns_after_both_lists_load() -> Result<( distribution_channel: None, install_url: Some("https://chatgpt.com/apps/beta/beta".to_string()), is_accessible: true, + is_enabled: true, }, AppInfo { id: "alpha".to_string(), @@ -264,6 +342,7 @@ async fn list_apps_emits_updates_and_returns_after_both_lists_load() -> Result<( distribution_channel: None, install_url: Some("https://chatgpt.com/apps/alpha/alpha".to_string()), is_accessible: false, + is_enabled: true, }, ]; @@ -300,6 +379,7 @@ async fn list_apps_returns_connectors_with_accessible_flags() -> Result<()> { distribution_channel: None, install_url: None, is_accessible: false, + is_enabled: true, }, AppInfo { id: "beta".to_string(), @@ -310,6 +390,7 @@ async fn list_apps_returns_connectors_with_accessible_flags() -> Result<()> { distribution_channel: None, install_url: None, is_accessible: false, + is_enabled: true, }, ]; @@ -358,6 +439,7 @@ async fn list_apps_returns_connectors_with_accessible_flags() -> Result<()> { distribution_channel: None, install_url: Some("https://chatgpt.com/apps/alpha/alpha".to_string()), is_accessible: false, + is_enabled: true, }, AppInfo { id: "beta".to_string(), @@ -368,6 +450,7 @@ async fn list_apps_returns_connectors_with_accessible_flags() -> Result<()> { distribution_channel: None, install_url: Some("https://chatgpt.com/apps/beta/beta".to_string()), is_accessible: false, + is_enabled: true, }, ] ); @@ -382,6 +465,7 @@ async fn list_apps_returns_connectors_with_accessible_flags() -> Result<()> { distribution_channel: None, install_url: Some("https://chatgpt.com/apps/beta/beta".to_string()), is_accessible: true, + is_enabled: true, }, AppInfo { id: "alpha".to_string(), @@ -392,6 +476,7 @@ async fn list_apps_returns_connectors_with_accessible_flags() -> Result<()> { distribution_channel: None, install_url: Some("https://chatgpt.com/apps/alpha/alpha".to_string()), is_accessible: false, + is_enabled: true, }, ]; @@ -423,6 +508,7 @@ async fn list_apps_paginates_results() -> Result<()> { distribution_channel: None, install_url: None, is_accessible: false, + is_enabled: true, }, AppInfo { id: "beta".to_string(), @@ -433,6 +519,7 @@ async fn list_apps_paginates_results() -> Result<()> { distribution_channel: None, install_url: None, is_accessible: false, + is_enabled: true, }, ]; @@ -486,6 +573,7 @@ async fn list_apps_paginates_results() -> Result<()> { distribution_channel: None, install_url: Some("https://chatgpt.com/apps/beta/beta".to_string()), is_accessible: true, + is_enabled: true, }]; assert_eq!(first_page, expected_first); @@ -525,6 +613,7 @@ async fn list_apps_paginates_results() -> Result<()> { distribution_channel: None, install_url: Some("https://chatgpt.com/apps/alpha/alpha".to_string()), is_accessible: false, + is_enabled: true, }]; assert_eq!(second_page, expected_second); @@ -545,6 +634,7 @@ async fn list_apps_force_refetch_preserves_previous_cache_on_failure() -> Result distribution_channel: None, install_url: None, is_accessible: false, + is_enabled: true, }]; let tools = vec![connector_tool("beta", "Beta App")?]; let (server_url, server_handle) = @@ -633,6 +723,201 @@ async fn list_apps_force_refetch_preserves_previous_cache_on_failure() -> Result Ok(()) } +#[tokio::test] +async fn list_apps_force_refetch_patches_updates_from_cached_snapshots() -> Result<()> { + let initial_connectors = vec![ + AppInfo { + id: "alpha".to_string(), + name: "Alpha".to_string(), + description: Some("Alpha v1".to_string()), + logo_url: None, + logo_url_dark: None, + distribution_channel: None, + install_url: None, + is_accessible: false, + is_enabled: true, + }, + AppInfo { + id: "beta".to_string(), + name: "Beta App".to_string(), + description: Some("Beta v1".to_string()), + logo_url: None, + logo_url_dark: None, + distribution_channel: None, + install_url: None, + is_accessible: false, + is_enabled: true, + }, + ]; + let initial_tools = vec![connector_tool("beta", "Beta App")?]; + let (server_url, server_handle, server_control) = start_apps_server_with_delays_and_control( + initial_connectors, + initial_tools, + Duration::from_millis(300), + Duration::ZERO, + ) + .await?; + + let codex_home = TempDir::new()?; + write_connectors_config(codex_home.path(), &server_url)?; + write_chatgpt_auth( + codex_home.path(), + ChatGptAuthFixture::new("chatgpt-token") + .account_id("account-123") + .chatgpt_user_id("user-123") + .chatgpt_account_id("account-123"), + AuthCredentialsStoreMode::File, + )?; + + let mut mcp = McpProcess::new(codex_home.path()).await?; + timeout(DEFAULT_TIMEOUT, mcp.initialize()).await??; + + let warm_request = mcp + .send_apps_list_request(AppsListParams { + limit: None, + cursor: None, + thread_id: None, + force_refetch: false, + }) + .await?; + let warm_first_update = read_app_list_updated_notification(&mut mcp).await?; + assert_eq!( + warm_first_update.data, + vec![AppInfo { + id: "beta".to_string(), + name: "Beta App".to_string(), + description: None, + logo_url: None, + logo_url_dark: None, + distribution_channel: None, + install_url: Some("https://chatgpt.com/apps/beta-app/beta".to_string()), + is_accessible: true, + is_enabled: true, + }] + ); + + let warm_second_update = read_app_list_updated_notification(&mut mcp).await?; + assert_eq!( + warm_second_update.data, + vec![ + AppInfo { + id: "beta".to_string(), + name: "Beta App".to_string(), + description: Some("Beta v1".to_string()), + logo_url: None, + logo_url_dark: None, + distribution_channel: None, + install_url: Some("https://chatgpt.com/apps/beta-app/beta".to_string()), + is_accessible: true, + is_enabled: true, + }, + AppInfo { + id: "alpha".to_string(), + name: "Alpha".to_string(), + description: Some("Alpha v1".to_string()), + logo_url: None, + logo_url_dark: None, + distribution_channel: None, + install_url: Some("https://chatgpt.com/apps/alpha/alpha".to_string()), + is_accessible: false, + is_enabled: true, + }, + ] + ); + + let warm_response: JSONRPCResponse = timeout( + DEFAULT_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(warm_request)), + ) + .await??; + let AppsListResponse { + data: warm_data, + next_cursor: warm_next_cursor, + } = to_response(warm_response)?; + assert_eq!(warm_data, warm_second_update.data); + assert!(warm_next_cursor.is_none()); + + server_control.set_connectors(vec![AppInfo { + id: "alpha".to_string(), + name: "Alpha".to_string(), + description: Some("Alpha v2".to_string()), + logo_url: None, + logo_url_dark: None, + distribution_channel: None, + install_url: None, + is_accessible: false, + is_enabled: true, + }]); + server_control.set_tools(Vec::new()); + + let refetch_request = mcp + .send_apps_list_request(AppsListParams { + limit: None, + cursor: None, + thread_id: None, + force_refetch: true, + }) + .await?; + + let first_update = read_app_list_updated_notification(&mut mcp).await?; + assert_eq!( + first_update.data, + vec![ + AppInfo { + id: "alpha".to_string(), + name: "Alpha".to_string(), + description: Some("Alpha v1".to_string()), + logo_url: None, + logo_url_dark: None, + distribution_channel: None, + install_url: Some("https://chatgpt.com/apps/alpha/alpha".to_string()), + is_accessible: false, + is_enabled: true, + }, + AppInfo { + id: "beta".to_string(), + name: "Beta App".to_string(), + description: Some("Beta v1".to_string()), + logo_url: None, + logo_url_dark: None, + distribution_channel: None, + install_url: Some("https://chatgpt.com/apps/beta-app/beta".to_string()), + is_accessible: false, + is_enabled: true, + }, + ] + ); + + let expected_final = vec![AppInfo { + id: "alpha".to_string(), + name: "Alpha".to_string(), + description: Some("Alpha v2".to_string()), + logo_url: None, + logo_url_dark: None, + distribution_channel: None, + install_url: Some("https://chatgpt.com/apps/alpha/alpha".to_string()), + is_accessible: false, + is_enabled: true, + }]; + let second_update = read_app_list_updated_notification(&mut mcp).await?; + assert_eq!(second_update.data, expected_final); + + let refetch_response: JSONRPCResponse = timeout( + DEFAULT_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(refetch_request)), + ) + .await??; + let AppsListResponse { + data: refetch_data, + next_cursor: refetch_next_cursor, + } = to_response(refetch_response)?; + assert_eq!(refetch_data, expected_final); + assert!(refetch_next_cursor.is_none()); + + server_handle.abort(); + Ok(()) +} + async fn read_app_list_updated_notification( mcp: &mut McpProcess, ) -> Result { @@ -652,22 +937,46 @@ async fn read_app_list_updated_notification( struct AppsServerState { expected_bearer: String, expected_account_id: String, - response: serde_json::Value, + response: Arc>, directory_delay: Duration, } #[derive(Clone)] struct AppListMcpServer { - tools: Arc>, + tools: Arc>>, tools_delay: Duration, } impl AppListMcpServer { - fn new(tools: Arc>, tools_delay: Duration) -> Self { + fn new(tools: Arc>>, tools_delay: Duration) -> Self { Self { tools, tools_delay } } } +#[derive(Clone)] +struct AppsServerControl { + response: Arc>, + tools: Arc>>, +} + +impl AppsServerControl { + fn set_connectors(&self, connectors: Vec) { + let mut response_guard = self + .response + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + *response_guard = json!({ "apps": connectors, "next_token": null }); + } + + fn set_tools(&self, tools: Vec) { + let mut tools_guard = self + .tools + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + *tools_guard = tools; + } +} + impl ServerHandler for AppListMcpServer { fn get_info(&self) -> ServerInfo { ServerInfo { @@ -688,8 +997,12 @@ impl ServerHandler for AppListMcpServer { if tools_delay > Duration::ZERO { tokio::time::sleep(tools_delay).await; } + let tools = tools + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clone(); Ok(ListToolsResult { - tools: (*tools).clone(), + tools, next_cursor: None, meta: None, }) @@ -703,14 +1016,33 @@ async fn start_apps_server_with_delays( directory_delay: Duration, tools_delay: Duration, ) -> Result<(String, JoinHandle<()>)> { + let (server_url, server_handle, _server_control) = + start_apps_server_with_delays_and_control(connectors, tools, directory_delay, tools_delay) + .await?; + Ok((server_url, server_handle)) +} + +async fn start_apps_server_with_delays_and_control( + connectors: Vec, + tools: Vec, + directory_delay: Duration, + tools_delay: Duration, +) -> Result<(String, JoinHandle<()>, AppsServerControl)> { + let response = Arc::new(StdMutex::new( + json!({ "apps": connectors, "next_token": null }), + )); + let tools = Arc::new(StdMutex::new(tools)); let state = AppsServerState { expected_bearer: "Bearer chatgpt-token".to_string(), expected_account_id: "account-123".to_string(), - response: json!({ "apps": connectors, "next_token": null }), + response: response.clone(), directory_delay, }; let state = Arc::new(state); - let tools = Arc::new(tools); + let server_control = AppsServerControl { + response, + tools: tools.clone(), + }; let listener = TcpListener::bind("127.0.0.1:0").await?; let addr = listener.local_addr()?; @@ -737,7 +1069,7 @@ async fn start_apps_server_with_delays( let _ = axum::serve(listener, router).await; }); - Ok((format!("http://{addr}"), handle)) + Ok((format!("http://{addr}"), handle, server_control)) } async fn list_directory_connectors( @@ -758,7 +1090,12 @@ async fn list_directory_connectors( .is_some_and(|value| value == state.expected_account_id); if bearer_ok && account_ok { - Ok(Json(state.response.clone())) + let response = state + .response + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clone(); + Ok(Json(response)) } else { Err(StatusCode::UNAUTHORIZED) } diff --git a/codex-rs/app-server/tests/suite/v2/model_list.rs b/codex-rs/app-server/tests/suite/v2/model_list.rs index c2431d24d0..ebcbdf99e7 100644 --- a/codex-rs/app-server/tests/suite/v2/model_list.rs +++ b/codex-rs/app-server/tests/suite/v2/model_list.rs @@ -33,6 +33,7 @@ async fn list_models_returns_all_models_with_large_limit() -> Result<()> { .send_list_models_request(ModelListParams { limit: Some(100), cursor: None, + include_hidden: None, }) .await?; @@ -54,6 +55,7 @@ async fn list_models_returns_all_models_with_large_limit() -> Result<()> { upgrade: None, display_name: "gpt-5.2-codex".to_string(), description: "Latest frontier agentic coding model.".to_string(), + hidden: false, supported_reasoning_efforts: vec![ ReasoningEffortOption { reasoning_effort: ReasoningEffort::Low, @@ -84,6 +86,7 @@ async fn list_models_returns_all_models_with_large_limit() -> Result<()> { upgrade: Some("gpt-5.2-codex".to_string()), display_name: "gpt-5.1-codex-max".to_string(), description: "Codex-optimized flagship for deep and fast reasoning.".to_string(), + hidden: false, supported_reasoning_efforts: vec![ ReasoningEffortOption { reasoning_effort: ReasoningEffort::Low, @@ -114,6 +117,7 @@ async fn list_models_returns_all_models_with_large_limit() -> Result<()> { upgrade: Some("gpt-5.2-codex".to_string()), display_name: "gpt-5.1-codex-mini".to_string(), description: "Optimized for codex. Cheaper, faster, but less capable.".to_string(), + hidden: false, supported_reasoning_efforts: vec![ ReasoningEffortOption { reasoning_effort: ReasoningEffort::Medium, @@ -138,6 +142,7 @@ async fn list_models_returns_all_models_with_large_limit() -> Result<()> { description: "Latest frontier model with improvements across knowledge, reasoning and coding" .to_string(), + hidden: false, supported_reasoning_efforts: vec![ ReasoningEffortOption { reasoning_effort: ReasoningEffort::Low, @@ -173,6 +178,38 @@ async fn list_models_returns_all_models_with_large_limit() -> Result<()> { Ok(()) } +#[tokio::test] +async fn list_models_includes_hidden_models() -> Result<()> { + let codex_home = TempDir::new()?; + write_models_cache(codex_home.path())?; + let mut mcp = McpProcess::new(codex_home.path()).await?; + + timeout(DEFAULT_TIMEOUT, mcp.initialize()).await??; + + let request_id = mcp + .send_list_models_request(ModelListParams { + limit: Some(100), + cursor: None, + include_hidden: Some(true), + }) + .await?; + + let response: JSONRPCResponse = timeout( + DEFAULT_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(request_id)), + ) + .await??; + + let ModelListResponse { + data: items, + next_cursor, + } = to_response::(response)?; + + assert!(items.iter().any(|item| item.hidden)); + assert!(next_cursor.is_none()); + Ok(()) +} + #[tokio::test] async fn list_models_pagination_works() -> Result<()> { let codex_home = TempDir::new()?; @@ -185,6 +222,7 @@ async fn list_models_pagination_works() -> Result<()> { .send_list_models_request(ModelListParams { limit: Some(1), cursor: None, + include_hidden: None, }) .await?; @@ -207,6 +245,7 @@ async fn list_models_pagination_works() -> Result<()> { .send_list_models_request(ModelListParams { limit: Some(1), cursor: Some(next_cursor.clone()), + include_hidden: None, }) .await?; @@ -229,6 +268,7 @@ async fn list_models_pagination_works() -> Result<()> { .send_list_models_request(ModelListParams { limit: Some(1), cursor: Some(third_cursor.clone()), + include_hidden: None, }) .await?; @@ -251,6 +291,7 @@ async fn list_models_pagination_works() -> Result<()> { .send_list_models_request(ModelListParams { limit: Some(1), cursor: Some(fourth_cursor.clone()), + include_hidden: None, }) .await?; @@ -283,6 +324,7 @@ async fn list_models_rejects_invalid_cursor() -> Result<()> { .send_list_models_request(ModelListParams { limit: None, cursor: Some("invalid".to_string()), + include_hidden: None, }) .await?; diff --git a/codex-rs/app-server/tests/suite/v2/thread_list.rs b/codex-rs/app-server/tests/suite/v2/thread_list.rs index 1e415600a0..cee88a18d6 100644 --- a/codex-rs/app-server/tests/suite/v2/thread_list.rs +++ b/codex-rs/app-server/tests/suite/v2/thread_list.rs @@ -17,6 +17,8 @@ use codex_app_server_protocol::ThreadSourceKind; use codex_core::ARCHIVED_SESSIONS_SUBDIR; use codex_protocol::ThreadId; use codex_protocol::protocol::GitInfo as CoreGitInfo; +use codex_protocol::protocol::RolloutItem; +use codex_protocol::protocol::RolloutLine; use codex_protocol::protocol::SessionSource as CoreSessionSource; use codex_protocol::protocol::SubAgentSource; use pretty_assertions::assert_eq; @@ -66,6 +68,7 @@ async fn list_threads_with_sort( model_providers: providers, source_kinds, archived, + cwd: None, }) .await?; let resp: JSONRPCResponse = timeout( @@ -127,6 +130,26 @@ fn set_rollout_mtime(path: &Path, updated_at_rfc3339: &str) -> Result<()> { Ok(()) } +fn set_rollout_cwd(path: &Path, cwd: &Path) -> Result<()> { + let content = fs::read_to_string(path)?; + let mut lines: Vec = content.lines().map(str::to_string).collect(); + let first_line = lines + .first_mut() + .ok_or_else(|| anyhow::anyhow!("rollout at {} is empty", path.display()))?; + let mut rollout_line: RolloutLine = serde_json::from_str(first_line)?; + let RolloutItem::SessionMeta(mut session_meta_line) = rollout_line.item else { + return Err(anyhow::anyhow!( + "rollout at {} does not start with session metadata", + path.display() + )); + }; + session_meta_line.meta.cwd = cwd.to_path_buf(); + rollout_line.item = RolloutItem::SessionMeta(session_meta_line); + *first_line = serde_json::to_string(&rollout_line)?; + fs::write(path, lines.join("\n") + "\n")?; + Ok(()) +} + #[tokio::test] async fn thread_list_basic_empty() -> Result<()> { let codex_home = TempDir::new()?; @@ -300,6 +323,63 @@ async fn thread_list_respects_provider_filter() -> Result<()> { Ok(()) } +#[tokio::test] +async fn thread_list_respects_cwd_filter() -> Result<()> { + let codex_home = TempDir::new()?; + create_minimal_config(codex_home.path())?; + + let filtered_id = create_fake_rollout( + codex_home.path(), + "2025-01-02T10-00-00", + "2025-01-02T10:00:00Z", + "filtered", + Some("mock_provider"), + None, + )?; + let unfiltered_id = create_fake_rollout( + codex_home.path(), + "2025-01-02T11-00-00", + "2025-01-02T11:00:00Z", + "unfiltered", + Some("mock_provider"), + None, + )?; + + let target_cwd = codex_home.path().join("target-cwd"); + fs::create_dir_all(&target_cwd)?; + set_rollout_cwd( + rollout_path(codex_home.path(), "2025-01-02T10-00-00", &filtered_id).as_path(), + &target_cwd, + )?; + + let mut mcp = init_mcp(codex_home.path()).await?; + let request_id = mcp + .send_thread_list_request(codex_app_server_protocol::ThreadListParams { + cursor: None, + limit: Some(10), + sort_key: None, + model_providers: Some(vec!["mock_provider".to_string()]), + source_kinds: None, + archived: None, + cwd: Some(target_cwd.to_string_lossy().into_owned()), + }) + .await?; + let resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(request_id)), + ) + .await??; + let ThreadListResponse { data, next_cursor } = to_response::(resp)?; + + assert_eq!(next_cursor, None); + assert_eq!(data.len(), 1); + assert_eq!(data[0].id, filtered_id); + assert_ne!(data[0].id, unfiltered_id); + assert_eq!(data[0].cwd, target_cwd); + + Ok(()) +} + #[tokio::test] async fn thread_list_empty_source_kinds_defaults_to_interactive_only() -> Result<()> { let codex_home = TempDir::new()?; @@ -1107,6 +1187,7 @@ async fn thread_list_invalid_cursor_returns_error() -> Result<()> { model_providers: Some(vec!["mock_provider".to_string()]), source_kinds: None, archived: None, + cwd: None, }) .await?; let error: JSONRPCError = timeout( diff --git a/codex-rs/app-server/tests/suite/v2/thread_resume.rs b/codex-rs/app-server/tests/suite/v2/thread_resume.rs index a92d24ed08..e001d4a7fd 100644 --- a/codex-rs/app-server/tests/suite/v2/thread_resume.rs +++ b/codex-rs/app-server/tests/suite/v2/thread_resume.rs @@ -209,18 +209,412 @@ async fn thread_resume_without_overrides_does_not_change_updated_at_or_mtime() - Ok(()) } +#[tokio::test] +async fn thread_resume_keeps_in_flight_turn_streaming() -> Result<()> { + let server = create_mock_responses_server_repeating_assistant("Done").await; + let codex_home = TempDir::new()?; + create_config_toml(codex_home.path(), &server.uri())?; + + let mut primary = McpProcess::new(codex_home.path()).await?; + timeout(DEFAULT_READ_TIMEOUT, primary.initialize()).await??; + + let start_id = primary + .send_thread_start_request(ThreadStartParams { + model: Some("gpt-5.1-codex-max".to_string()), + ..Default::default() + }) + .await?; + let start_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + primary.read_stream_until_response_message(RequestId::Integer(start_id)), + ) + .await??; + let ThreadStartResponse { thread, .. } = to_response::(start_resp)?; + + let seed_turn_id = primary + .send_turn_start_request(TurnStartParams { + thread_id: thread.id.clone(), + input: vec![UserInput::Text { + text: "seed history".to_string(), + text_elements: Vec::new(), + }], + ..Default::default() + }) + .await?; + timeout( + DEFAULT_READ_TIMEOUT, + primary.read_stream_until_response_message(RequestId::Integer(seed_turn_id)), + ) + .await??; + timeout( + DEFAULT_READ_TIMEOUT, + primary.read_stream_until_notification_message("turn/completed"), + ) + .await??; + primary.clear_message_buffer(); + + let mut secondary = McpProcess::new(codex_home.path()).await?; + timeout(DEFAULT_READ_TIMEOUT, secondary.initialize()).await??; + + let turn_id = primary + .send_turn_start_request(TurnStartParams { + thread_id: thread.id.clone(), + input: vec![UserInput::Text { + text: "respond with docs".to_string(), + text_elements: Vec::new(), + }], + ..Default::default() + }) + .await?; + timeout( + DEFAULT_READ_TIMEOUT, + primary.read_stream_until_response_message(RequestId::Integer(turn_id)), + ) + .await??; + timeout( + DEFAULT_READ_TIMEOUT, + primary.read_stream_until_notification_message("turn/started"), + ) + .await??; + + let resume_id = secondary + .send_thread_resume_request(ThreadResumeParams { + thread_id: thread.id, + ..Default::default() + }) + .await?; + timeout( + DEFAULT_READ_TIMEOUT, + secondary.read_stream_until_response_message(RequestId::Integer(resume_id)), + ) + .await??; + + timeout( + DEFAULT_READ_TIMEOUT, + primary.read_stream_until_notification_message("turn/completed"), + ) + .await??; + + Ok(()) +} + +#[tokio::test] +async fn thread_resume_rejects_history_when_thread_is_running() -> Result<()> { + let server = responses::start_mock_server().await; + let first_body = responses::sse(vec![ + responses::ev_response_created("resp-1"), + responses::ev_assistant_message("msg-1", "Done"), + responses::ev_completed("resp-1"), + ]); + let second_body = responses::sse(vec![responses::ev_response_created("resp-2")]); + let _first_response_mock = responses::mount_sse_once(&server, first_body).await; + let _second_response_mock = responses::mount_sse_once(&server, second_body).await; + let codex_home = TempDir::new()?; + create_config_toml(codex_home.path(), &server.uri())?; + + let mut primary = McpProcess::new(codex_home.path()).await?; + timeout(DEFAULT_READ_TIMEOUT, primary.initialize()).await??; + + let start_id = primary + .send_thread_start_request(ThreadStartParams { + model: Some("gpt-5.1-codex-max".to_string()), + ..Default::default() + }) + .await?; + let start_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + primary.read_stream_until_response_message(RequestId::Integer(start_id)), + ) + .await??; + let ThreadStartResponse { thread, .. } = to_response::(start_resp)?; + + let seed_turn_id = primary + .send_turn_start_request(TurnStartParams { + thread_id: thread.id.clone(), + input: vec![UserInput::Text { + text: "seed history".to_string(), + text_elements: Vec::new(), + }], + ..Default::default() + }) + .await?; + timeout( + DEFAULT_READ_TIMEOUT, + primary.read_stream_until_response_message(RequestId::Integer(seed_turn_id)), + ) + .await??; + timeout( + DEFAULT_READ_TIMEOUT, + primary.read_stream_until_notification_message("turn/completed"), + ) + .await??; + primary.clear_message_buffer(); + + let running_turn_id = primary + .send_turn_start_request(TurnStartParams { + thread_id: thread.id.clone(), + input: vec![UserInput::Text { + text: "keep running".to_string(), + text_elements: Vec::new(), + }], + ..Default::default() + }) + .await?; + timeout( + DEFAULT_READ_TIMEOUT, + primary.read_stream_until_response_message(RequestId::Integer(running_turn_id)), + ) + .await??; + timeout( + DEFAULT_READ_TIMEOUT, + primary.read_stream_until_notification_message("turn/started"), + ) + .await??; + + let resume_id = primary + .send_thread_resume_request(ThreadResumeParams { + thread_id: thread.id, + history: Some(vec![ResponseItem::Message { + id: None, + role: "user".to_string(), + content: vec![ContentItem::InputText { + text: "history override".to_string(), + }], + end_turn: None, + phase: None, + }]), + ..Default::default() + }) + .await?; + let resume_err: JSONRPCError = timeout( + DEFAULT_READ_TIMEOUT, + primary.read_stream_until_error_message(RequestId::Integer(resume_id)), + ) + .await??; + assert!( + resume_err.error.message.contains("cannot resume thread") + && resume_err.error.message.contains("with history") + && resume_err.error.message.contains("running"), + "unexpected resume error: {}", + resume_err.error.message + ); + + Ok(()) +} + +#[tokio::test] +async fn thread_resume_rejects_mismatched_path_when_thread_is_running() -> Result<()> { + let server = responses::start_mock_server().await; + let first_body = responses::sse(vec![ + responses::ev_response_created("resp-1"), + responses::ev_assistant_message("msg-1", "Done"), + responses::ev_completed("resp-1"), + ]); + let second_body = responses::sse(vec![responses::ev_response_created("resp-2")]); + let _first_response_mock = responses::mount_sse_once(&server, first_body).await; + let _second_response_mock = responses::mount_sse_once(&server, second_body).await; + let codex_home = TempDir::new()?; + create_config_toml(codex_home.path(), &server.uri())?; + + let mut primary = McpProcess::new(codex_home.path()).await?; + timeout(DEFAULT_READ_TIMEOUT, primary.initialize()).await??; + + let start_id = primary + .send_thread_start_request(ThreadStartParams { + model: Some("gpt-5.1-codex-max".to_string()), + ..Default::default() + }) + .await?; + let start_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + primary.read_stream_until_response_message(RequestId::Integer(start_id)), + ) + .await??; + let ThreadStartResponse { thread, .. } = to_response::(start_resp)?; + + let seed_turn_id = primary + .send_turn_start_request(TurnStartParams { + thread_id: thread.id.clone(), + input: vec![UserInput::Text { + text: "seed history".to_string(), + text_elements: Vec::new(), + }], + ..Default::default() + }) + .await?; + timeout( + DEFAULT_READ_TIMEOUT, + primary.read_stream_until_response_message(RequestId::Integer(seed_turn_id)), + ) + .await??; + timeout( + DEFAULT_READ_TIMEOUT, + primary.read_stream_until_notification_message("turn/completed"), + ) + .await??; + primary.clear_message_buffer(); + + let running_turn_id = primary + .send_turn_start_request(TurnStartParams { + thread_id: thread.id.clone(), + input: vec![UserInput::Text { + text: "keep running".to_string(), + text_elements: Vec::new(), + }], + ..Default::default() + }) + .await?; + timeout( + DEFAULT_READ_TIMEOUT, + primary.read_stream_until_response_message(RequestId::Integer(running_turn_id)), + ) + .await??; + timeout( + DEFAULT_READ_TIMEOUT, + primary.read_stream_until_notification_message("turn/started"), + ) + .await??; + + let resume_id = primary + .send_thread_resume_request(ThreadResumeParams { + thread_id: thread.id, + path: Some(PathBuf::from("/tmp/does-not-match-running-rollout.jsonl")), + ..Default::default() + }) + .await?; + let resume_err: JSONRPCError = timeout( + DEFAULT_READ_TIMEOUT, + primary.read_stream_until_error_message(RequestId::Integer(resume_id)), + ) + .await??; + assert!( + resume_err.error.message.contains("mismatched path"), + "unexpected resume error: {}", + resume_err.error.message + ); + + Ok(()) +} + +#[tokio::test] +async fn thread_resume_rejoins_running_thread_even_with_override_mismatch() -> Result<()> { + let server = responses::start_mock_server().await; + let first_body = responses::sse(vec![ + responses::ev_response_created("resp-1"), + responses::ev_assistant_message("msg-1", "Done"), + responses::ev_completed("resp-1"), + ]); + let second_body = responses::sse(vec![responses::ev_response_created("resp-2")]); + let _response_mock = + responses::mount_sse_sequence(&server, vec![first_body, second_body]).await; + let codex_home = TempDir::new()?; + create_config_toml(codex_home.path(), &server.uri())?; + + let mut primary = McpProcess::new(codex_home.path()).await?; + timeout(DEFAULT_READ_TIMEOUT, primary.initialize()).await??; + + let start_id = primary + .send_thread_start_request(ThreadStartParams { + model: Some("gpt-5.1-codex-max".to_string()), + ..Default::default() + }) + .await?; + let start_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + primary.read_stream_until_response_message(RequestId::Integer(start_id)), + ) + .await??; + let ThreadStartResponse { thread, .. } = to_response::(start_resp)?; + + let seed_turn_id = primary + .send_turn_start_request(TurnStartParams { + thread_id: thread.id.clone(), + input: vec![UserInput::Text { + text: "seed history".to_string(), + text_elements: Vec::new(), + }], + ..Default::default() + }) + .await?; + timeout( + DEFAULT_READ_TIMEOUT, + primary.read_stream_until_response_message(RequestId::Integer(seed_turn_id)), + ) + .await??; + timeout( + DEFAULT_READ_TIMEOUT, + primary.read_stream_until_notification_message("turn/completed"), + ) + .await??; + primary.clear_message_buffer(); + + let running_turn_id = primary + .send_turn_start_request(TurnStartParams { + thread_id: thread.id.clone(), + input: vec![UserInput::Text { + text: "keep running".to_string(), + text_elements: Vec::new(), + }], + ..Default::default() + }) + .await?; + timeout( + DEFAULT_READ_TIMEOUT, + primary.read_stream_until_response_message(RequestId::Integer(running_turn_id)), + ) + .await??; + timeout( + DEFAULT_READ_TIMEOUT, + primary.read_stream_until_notification_message("turn/started"), + ) + .await??; + + let resume_id = primary + .send_thread_resume_request(ThreadResumeParams { + thread_id: thread.id.clone(), + model: Some("not-the-running-model".to_string()), + cwd: Some("/tmp".to_string()), + ..Default::default() + }) + .await?; + let resume_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + primary.read_stream_until_response_message(RequestId::Integer(resume_id)), + ) + .await??; + let ThreadResumeResponse { model, .. } = to_response::(resume_resp)?; + assert_eq!(model, "gpt-5.1-codex-max"); + + timeout( + DEFAULT_READ_TIMEOUT, + primary.read_stream_until_notification_message("turn/completed"), + ) + .await??; + + Ok(()) +} + #[tokio::test] async fn thread_resume_with_overrides_defers_updated_at_until_turn_start() -> Result<()> { let server = create_mock_responses_server_repeating_assistant("Done").await; let codex_home = TempDir::new()?; - let rollout = setup_rollout_fixture(codex_home.path(), &server.uri())?; + create_config_toml(codex_home.path(), &server.uri())?; - let mut mcp = McpProcess::new(codex_home.path()).await?; - timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; + let RestartedThreadFixture { + mut mcp, + thread_id, + rollout_file_path, + } = start_materialized_thread_and_restart(codex_home.path(), "materialize").await?; + let expected_updated_at_rfc3339 = "2025-01-07T00:00:00Z"; + set_rollout_mtime(rollout_file_path.as_path(), expected_updated_at_rfc3339)?; + let before_modified = std::fs::metadata(&rollout_file_path)?.modified()?; + let expected_updated_at = chrono::DateTime::parse_from_rfc3339(expected_updated_at_rfc3339)? + .with_timezone(&Utc) + .timestamp(); let resume_id = mcp .send_thread_resume_request(ThreadResumeParams { - thread_id: rollout.conversation_id.clone(), + thread_id, model: Some("mock-model".to_string()), ..Default::default() }) @@ -230,16 +624,19 @@ async fn thread_resume_with_overrides_defers_updated_at_until_turn_start() -> Re mcp.read_stream_until_response_message(RequestId::Integer(resume_id)), ) .await??; - let ThreadResumeResponse { thread, .. } = to_response::(resume_resp)?; + let ThreadResumeResponse { + thread: resumed_thread, + .. + } = to_response::(resume_resp)?; - assert_eq!(thread.updated_at, rollout.expected_updated_at); + assert_eq!(resumed_thread.updated_at, expected_updated_at); - let after_resume_modified = std::fs::metadata(&rollout.rollout_file_path)?.modified()?; - assert_eq!(after_resume_modified, rollout.before_modified); + let after_resume_modified = std::fs::metadata(&rollout_file_path)?.modified()?; + assert_eq!(after_resume_modified, before_modified); let turn_id = mcp .send_turn_start_request(TurnStartParams { - thread_id: rollout.conversation_id, + thread_id: resumed_thread.id, input: vec![UserInput::Text { text: "Hello".to_string(), text_elements: Vec::new(), @@ -258,8 +655,8 @@ async fn thread_resume_with_overrides_defers_updated_at_until_turn_start() -> Re ) .await??; - let after_turn_modified = std::fs::metadata(&rollout.rollout_file_path)?.modified()?; - assert!(after_turn_modified > rollout.before_modified); + let after_turn_modified = std::fs::metadata(&rollout_file_path)?.modified()?; + assert!(after_turn_modified > before_modified); Ok(()) } @@ -374,22 +771,9 @@ async fn thread_resume_supports_history_and_overrides() -> Result<()> { let codex_home = TempDir::new()?; create_config_toml(codex_home.path(), &server.uri())?; - let mut mcp = McpProcess::new(codex_home.path()).await?; - timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; - - // Start a thread. - let start_id = mcp - .send_thread_start_request(ThreadStartParams { - model: Some("gpt-5.1-codex-max".to_string()), - ..Default::default() - }) - .await?; - let start_resp: JSONRPCResponse = timeout( - DEFAULT_READ_TIMEOUT, - mcp.read_stream_until_response_message(RequestId::Integer(start_id)), - ) - .await??; - let ThreadStartResponse { thread, .. } = to_response::(start_resp)?; + let RestartedThreadFixture { + mut mcp, thread_id, .. + } = start_materialized_thread_and_restart(codex_home.path(), "seed history").await?; let history_text = "Hello from history"; let history = vec![ResponseItem::Message { @@ -405,7 +789,7 @@ async fn thread_resume_supports_history_and_overrides() -> Result<()> { // Resume with explicit history and override the model. let resume_id = mcp .send_thread_resume_request(ThreadResumeParams { - thread_id: thread.id, + thread_id, history: Some(history), model: Some("mock-model".to_string()), model_provider: Some("mock_provider".to_string()), @@ -429,6 +813,70 @@ async fn thread_resume_supports_history_and_overrides() -> Result<()> { Ok(()) } +struct RestartedThreadFixture { + mcp: McpProcess, + thread_id: String, + rollout_file_path: PathBuf, +} + +async fn start_materialized_thread_and_restart( + codex_home: &Path, + seed_text: &str, +) -> Result { + let mut first_mcp = McpProcess::new(codex_home).await?; + timeout(DEFAULT_READ_TIMEOUT, first_mcp.initialize()).await??; + + let start_id = first_mcp + .send_thread_start_request(ThreadStartParams { + model: Some("gpt-5.1-codex-max".to_string()), + ..Default::default() + }) + .await?; + let start_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + first_mcp.read_stream_until_response_message(RequestId::Integer(start_id)), + ) + .await??; + let ThreadStartResponse { thread, .. } = to_response::(start_resp)?; + + let materialize_turn_id = first_mcp + .send_turn_start_request(TurnStartParams { + thread_id: thread.id.clone(), + input: vec![UserInput::Text { + text: seed_text.to_string(), + text_elements: Vec::new(), + }], + ..Default::default() + }) + .await?; + timeout( + DEFAULT_READ_TIMEOUT, + first_mcp.read_stream_until_response_message(RequestId::Integer(materialize_turn_id)), + ) + .await??; + timeout( + DEFAULT_READ_TIMEOUT, + first_mcp.read_stream_until_notification_message("turn/completed"), + ) + .await??; + + let thread_id = thread.id; + let rollout_file_path = thread + .path + .ok_or_else(|| anyhow::anyhow!("thread path missing from thread/start response"))?; + + drop(first_mcp); + + let mut second_mcp = McpProcess::new(codex_home).await?; + timeout(DEFAULT_READ_TIMEOUT, second_mcp.initialize()).await??; + + Ok(RestartedThreadFixture { + mcp: second_mcp, + thread_id, + rollout_file_path, + }) +} + #[tokio::test] async fn thread_resume_accepts_personality_override() -> Result<()> { skip_if_no_network!(Ok(())); @@ -449,10 +897,10 @@ async fn thread_resume_accepts_personality_override() -> Result<()> { let codex_home = TempDir::new()?; create_config_toml(codex_home.path(), &server.uri())?; - let mut mcp = McpProcess::new(codex_home.path()).await?; - timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; + let mut primary = McpProcess::new(codex_home.path()).await?; + timeout(DEFAULT_READ_TIMEOUT, primary.initialize()).await??; - let start_id = mcp + let start_id = primary .send_thread_start_request(ThreadStartParams { model: Some("gpt-5.2-codex".to_string()), ..Default::default() @@ -460,12 +908,12 @@ async fn thread_resume_accepts_personality_override() -> Result<()> { .await?; let start_resp: JSONRPCResponse = timeout( DEFAULT_READ_TIMEOUT, - mcp.read_stream_until_response_message(RequestId::Integer(start_id)), + primary.read_stream_until_response_message(RequestId::Integer(start_id)), ) .await??; let ThreadStartResponse { thread, .. } = to_response::(start_resp)?; - let materialize_id = mcp + let materialize_id = primary .send_turn_start_request(TurnStartParams { thread_id: thread.id.clone(), input: vec![UserInput::Text { @@ -477,16 +925,19 @@ async fn thread_resume_accepts_personality_override() -> Result<()> { .await?; timeout( DEFAULT_READ_TIMEOUT, - mcp.read_stream_until_response_message(RequestId::Integer(materialize_id)), + primary.read_stream_until_response_message(RequestId::Integer(materialize_id)), ) .await??; timeout( DEFAULT_READ_TIMEOUT, - mcp.read_stream_until_notification_message("turn/completed"), + primary.read_stream_until_notification_message("turn/completed"), ) .await??; - let resume_id = mcp + let mut secondary = McpProcess::new(codex_home.path()).await?; + timeout(DEFAULT_READ_TIMEOUT, secondary.initialize()).await??; + + let resume_id = secondary .send_thread_resume_request(ThreadResumeParams { thread_id: thread.id, model: Some("gpt-5.2-codex".to_string()), @@ -496,12 +947,12 @@ async fn thread_resume_accepts_personality_override() -> Result<()> { .await?; let resume_resp: JSONRPCResponse = timeout( DEFAULT_READ_TIMEOUT, - mcp.read_stream_until_response_message(RequestId::Integer(resume_id)), + secondary.read_stream_until_response_message(RequestId::Integer(resume_id)), ) .await??; let resume: ThreadResumeResponse = to_response::(resume_resp)?; - let turn_id = mcp + let turn_id = secondary .send_turn_start_request(TurnStartParams { thread_id: resume.thread.id, input: vec![UserInput::Text { @@ -513,13 +964,13 @@ async fn thread_resume_accepts_personality_override() -> Result<()> { .await?; timeout( DEFAULT_READ_TIMEOUT, - mcp.read_stream_until_response_message(RequestId::Integer(turn_id)), + secondary.read_stream_until_response_message(RequestId::Integer(turn_id)), ) .await??; timeout( DEFAULT_READ_TIMEOUT, - mcp.read_stream_until_notification_message("turn/completed"), + secondary.read_stream_until_notification_message("turn/completed"), ) .await??; diff --git a/codex-rs/arg0/src/lib.rs b/codex-rs/arg0/src/lib.rs index 9c455ddbba..9563737839 100644 --- a/codex-rs/arg0/src/lib.rs +++ b/codex-rs/arg0/src/lib.rs @@ -12,6 +12,8 @@ const LINUX_SANDBOX_ARG0: &str = "codex-linux-sandbox"; const APPLY_PATCH_ARG0: &str = "apply_patch"; const MISSPELLED_APPLY_PATCH_ARG0: &str = "applypatch"; const LOCK_FILENAME: &str = ".lock"; +#[cfg(target_os = "windows")] +const WINDOWS_TOKIO_WORKER_STACK_SIZE_BYTES: usize = 16 * 1024 * 1024; /// Keeps the per-session PATH entry alive and locked for the process lifetime. pub struct Arg0PathEntryGuard { @@ -112,7 +114,7 @@ where // Regular invocation – create a Tokio runtime and execute the provided // async entry-point. - let runtime = tokio::runtime::Runtime::new()?; + let runtime = build_runtime()?; runtime.block_on(async move { let codex_linux_sandbox_exe: Option = if cfg!(target_os = "linux") { std::env::current_exe().ok() @@ -124,6 +126,18 @@ where }) } +fn build_runtime() -> anyhow::Result { + let mut builder = tokio::runtime::Builder::new_multi_thread(); + builder.enable_all(); + #[cfg(target_os = "windows")] + { + // Defensive hardening: Windows worker threads have lower effective + // stack headroom, so use a larger stack for runtime workers. + builder.thread_stack_size(WINDOWS_TOKIO_WORKER_STACK_SIZE_BYTES); + } + Ok(builder.build()?) +} + const ILLEGAL_ENV_VAR_PREFIX: &str = "CODEX_"; /// Load env vars from ~/.codex/.env. diff --git a/codex-rs/chatgpt/src/connectors.rs b/codex-rs/chatgpt/src/connectors.rs index 6398d13d17..bafb0273a4 100644 --- a/codex-rs/chatgpt/src/connectors.rs +++ b/codex-rs/chatgpt/src/connectors.rs @@ -1,4 +1,5 @@ use std::collections::HashMap; +use std::collections::HashSet; use std::sync::LazyLock; use std::sync::Mutex as StdMutex; @@ -19,7 +20,9 @@ pub use codex_core::connectors::connector_display_label; use codex_core::connectors::connector_install_url; pub use codex_core::connectors::list_accessible_connectors_from_mcp_tools; pub use codex_core::connectors::list_accessible_connectors_from_mcp_tools_with_options; +pub use codex_core::connectors::list_cached_accessible_connectors_from_mcp_tools; use codex_core::connectors::merge_connectors; +pub use codex_core::connectors::with_app_enabled_state; #[derive(Debug, Deserialize)] struct DirectoryListResponse { @@ -72,13 +75,32 @@ pub async fn list_connectors(config: &Config) -> anyhow::Result> { ); let connectors = connectors_result?; let accessible = accessible_result?; - Ok(merge_connectors_with_accessible(connectors, accessible)) + Ok(with_app_enabled_state( + merge_connectors_with_accessible(connectors, accessible, true), + config, + )) } pub async fn list_all_connectors(config: &Config) -> anyhow::Result> { list_all_connectors_with_options(config, false).await } +pub async fn list_cached_all_connectors(config: &Config) -> Option> { + if !config.features.enabled(Feature::Apps) { + return Some(Vec::new()); + } + + if init_chatgpt_token_from_auth(&config.codex_home, config.cli_auth_credentials_store_mode) + .await + .is_err() + { + return None; + } + let token_data = get_chatgpt_token_data()?; + let cache_key = all_connectors_cache_key(config, &token_data); + read_cached_all_connectors(&cache_key) +} + pub async fn list_all_connectors_with_options( config: &Config, force_refetch: bool, @@ -164,7 +186,20 @@ fn write_cached_all_connectors(cache_key: AllConnectorsCacheKey, connectors: &[A pub fn merge_connectors_with_accessible( connectors: Vec, accessible_connectors: Vec, + all_connectors_loaded: bool, ) -> Vec { + let accessible_connectors = if all_connectors_loaded { + let connector_ids: HashSet<&str> = connectors + .iter() + .map(|connector| connector.id.as_str()) + .collect(); + accessible_connectors + .into_iter() + .filter(|connector| connector_ids.contains(connector.id.as_str())) + .collect() + } else { + accessible_connectors + }; let merged = merge_connectors(connectors, accessible_connectors); filter_disallowed_connectors(merged) } @@ -283,6 +318,7 @@ fn directory_app_to_app_info(app: DirectoryApp) -> AppInfo { distribution_channel: app.distribution_channel, install_url: None, is_accessible: false, + is_enabled: true, } } @@ -341,6 +377,7 @@ mod tests { distribution_channel: None, install_url: None, is_accessible: false, + is_enabled: true, } } @@ -383,4 +420,41 @@ mod tests { ]); assert_eq!(filtered, vec![app("delta")]); } + + fn merged_app(id: &str, is_accessible: bool) -> AppInfo { + AppInfo { + id: id.to_string(), + name: id.to_string(), + description: None, + logo_url: None, + logo_url_dark: None, + distribution_channel: None, + install_url: Some(connector_install_url(id, id)), + is_accessible, + is_enabled: true, + } + } + + #[test] + fn excludes_accessible_connectors_not_in_all_when_all_loaded() { + let merged = merge_connectors_with_accessible( + vec![app("alpha")], + vec![app("alpha"), app("beta")], + true, + ); + assert_eq!(merged, vec![merged_app("alpha", true)]); + } + + #[test] + fn keeps_accessible_connectors_not_in_all_while_all_loading() { + let merged = merge_connectors_with_accessible( + vec![app("alpha")], + vec![app("alpha"), app("beta")], + false, + ); + assert_eq!( + merged, + vec![merged_app("alpha", true), merged_app("beta", true)] + ); + } } diff --git a/codex-rs/cli/src/debug_sandbox.rs b/codex-rs/cli/src/debug_sandbox.rs index c19d58189f..a65fa486e4 100644 --- a/codex-rs/cli/src/debug_sandbox.rs +++ b/codex-rs/cli/src/debug_sandbox.rs @@ -213,12 +213,19 @@ async fn run_command_under_sandbox( #[cfg(not(target_os = "macos"))] let _ = log_denials; + let managed_network_requirements_enabled = config.managed_network_requirements_enabled(); + // This proxy should only live for the lifetime of the child process. let network_proxy = match config.permissions.network.as_ref() { Some(spec) => Some( - spec.start_proxy() - .await - .map_err(|err| anyhow::anyhow!("failed to start managed network proxy: {err}"))?, + spec.start_proxy( + config.permissions.sandbox_policy.get(), + None, + None, + managed_network_requirements_enabled, + ) + .await + .map_err(|err| anyhow::anyhow!("failed to start managed network proxy: {err}"))?, ), None => None, }; diff --git a/codex-rs/cli/src/main.rs b/codex-rs/cli/src/main.rs index 494f8d8935..cae322c77e 100644 --- a/codex-rs/cli/src/main.rs +++ b/codex-rs/cli/src/main.rs @@ -93,10 +93,10 @@ enum Subcommand { /// Remove stored authentication credentials. Logout(LogoutCommand), - /// [experimental] Run Codex as an MCP server and manage MCP servers. + /// Manage external MCP servers for Codex. Mcp(McpCli), - /// [experimental] Run the Codex MCP server (stdio transport). + /// Start Codex as an MCP server (stdio). McpServer, /// [experimental] Run the app server or related tooling. diff --git a/codex-rs/codex-api/src/requests/headers.rs b/codex-rs/codex-api/src/requests/headers.rs index 02f8c61c31..d1ab834109 100644 --- a/codex-rs/codex-api/src/requests/headers.rs +++ b/codex-rs/codex-api/src/requests/headers.rs @@ -17,6 +17,9 @@ pub(crate) fn subagent_header(source: &Option) -> Option match sub { codex_protocol::protocol::SubAgentSource::Review => Some("review".to_string()), codex_protocol::protocol::SubAgentSource::Compact => Some("compact".to_string()), + codex_protocol::protocol::SubAgentSource::MemoryConsolidation => { + Some("memory_consolidation".to_string()) + } codex_protocol::protocol::SubAgentSource::ThreadSpawn { .. } => { Some("collab_spawn".to_string()) } diff --git a/codex-rs/codex-api/tests/models_integration.rs b/codex-rs/codex-api/tests/models_integration.rs index b33f8b308b..9f7e380665 100644 --- a/codex-rs/codex-api/tests/models_integration.rs +++ b/codex-rs/codex-api/tests/models_integration.rs @@ -89,6 +89,7 @@ async fn models_client_hits_models_endpoint() { experimental_supported_tools: Vec::new(), input_modalities: default_input_modalities(), prefer_websockets: false, + used_fallback_model_metadata: false, }], }; diff --git a/codex-rs/core/Cargo.toml b/codex-rs/core/Cargo.toml index 1b3f62c5aa..208ba1c3a6 100644 --- a/codex-rs/core/Cargo.toml +++ b/codex-rs/core/Cargo.toml @@ -152,6 +152,7 @@ codex-utils-cargo-bin = { workspace = true } core_test_support = { workspace = true } ctor = { workspace = true } image = { workspace = true, features = ["jpeg", "png"] } +insta = { workspace = true } maplit = { workspace = true } predicates = { workspace = true } pretty_assertions = { workspace = true } diff --git a/codex-rs/core/config.schema.json b/codex-rs/core/config.schema.json index 1bd7c1a938..7a1c40a0a1 100644 --- a/codex-rs/core/config.schema.json +++ b/codex-rs/core/config.schema.json @@ -188,6 +188,9 @@ "apps": { "type": "boolean" }, + "apps_mcp_gateway": { + "type": "boolean" + }, "child_agents_md": { "type": "boolean" }, @@ -236,6 +239,9 @@ "powershell_utf8": { "type": "boolean" }, + "prevent_idle_sleep": { + "type": "boolean" + }, "remote_models": { "type": "boolean" }, @@ -431,6 +437,43 @@ } ] }, + "MemoriesToml": { + "additionalProperties": false, + "description": "Memories settings loaded from config.toml.", + "properties": { + "max_raw_memories_for_global": { + "description": "Maximum number of recent raw memories retained for global consolidation.", + "format": "uint", + "minimum": 0.0, + "type": "integer" + }, + "max_rollout_age_days": { + "description": "Maximum age of the threads used for memories.", + "format": "int64", + "type": "integer" + }, + "max_rollouts_per_startup": { + "description": "Maximum number of rollout candidates processed per pass.", + "format": "uint", + "minimum": 0.0, + "type": "integer" + }, + "min_rollout_idle_hours": { + "description": "Minimum idle time between last thread activity and memory creation (hours). > 12h recommended.", + "format": "int64", + "type": "integer" + }, + "phase_1_model": { + "description": "Model used for thread summarisation.", + "type": "string" + }, + "phase_2_model": { + "description": "Model used for memory consolidation.", + "type": "string" + } + }, + "type": "object" + }, "ModeKind": { "description": "Initial collaboration mode to use when the TUI starts.", "enum": [ @@ -1275,6 +1318,9 @@ "apps": { "type": "boolean" }, + "apps_mcp_gateway": { + "type": "boolean" + }, "child_agents_md": { "type": "boolean" }, @@ -1323,6 +1369,9 @@ "powershell_utf8": { "type": "boolean" }, + "prevent_idle_sleep": { + "type": "boolean" + }, "remote_models": { "type": "boolean" }, @@ -1475,6 +1524,14 @@ "description": "Definition for MCP servers that Codex can reach out to for tool calls.", "type": "object" }, + "memories": { + "allOf": [ + { + "$ref": "#/definitions/MemoriesToml" + } + ], + "description": "Memories subsystem settings." + }, "model": { "description": "Optional override of model selection.", "type": "string" diff --git a/codex-rs/core/src/analytics_client.rs b/codex-rs/core/src/analytics_client.rs index d625166b09..922a25aca8 100644 --- a/codex-rs/core/src/analytics_client.rs +++ b/codex-rs/core/src/analytics_client.rs @@ -7,9 +7,11 @@ use codex_protocol::protocol::SkillScope; use serde::Serialize; use sha1::Digest; use sha1::Sha1; +use std::collections::HashSet; use std::path::Path; use std::path::PathBuf; use std::sync::Arc; +use std::sync::Mutex; use std::time::Duration; use tokio::sync::mpsc; @@ -17,15 +19,18 @@ use tokio::sync::mpsc; pub(crate) struct TrackEventsContext { pub(crate) model_slug: String, pub(crate) thread_id: String, + pub(crate) turn_id: String, } pub(crate) fn build_track_events_context( model_slug: String, thread_id: String, + turn_id: String, ) -> TrackEventsContext { TrackEventsContext { model_slug, thread_id, + turn_id, } } @@ -35,9 +40,16 @@ pub(crate) struct SkillInvocation { pub(crate) skill_path: PathBuf, } +pub(crate) struct AppInvocation { + pub(crate) connector_id: Option, + pub(crate) app_name: Option, + pub(crate) invoke_type: Option, +} + #[derive(Clone)] pub(crate) struct AnalyticsEventsQueue { sender: mpsc::Sender, + app_used_emitted_keys: Arc>>, } pub(crate) struct AnalyticsEventsClient { @@ -50,18 +62,45 @@ impl AnalyticsEventsQueue { let (sender, mut receiver) = mpsc::channel(ANALYTICS_EVENTS_QUEUE_SIZE); tokio::spawn(async move { while let Some(job) = receiver.recv().await { - send_track_skill_invocations(&auth_manager, job).await; + match job { + TrackEventsJob::SkillInvocations(job) => { + send_track_skill_invocations(&auth_manager, job).await; + } + TrackEventsJob::AppMentioned(job) => { + send_track_app_mentioned(&auth_manager, job).await; + } + TrackEventsJob::AppUsed(job) => { + send_track_app_used(&auth_manager, job).await; + } + } } }); - Self { sender } + Self { + sender, + app_used_emitted_keys: Arc::new(Mutex::new(HashSet::new())), + } } fn try_send(&self, job: TrackEventsJob) { if self.sender.try_send(job).is_err() { //TODO: add a metric for this - tracing::warn!("dropping skill analytics events: queue is full"); + tracing::warn!("dropping analytics events: queue is full"); } } + + fn should_enqueue_app_used(&self, tracking: &TrackEventsContext, app: &AppInvocation) -> bool { + let Some(connector_id) = app.connector_id.as_ref() else { + return true; + }; + let mut emitted = self + .app_used_emitted_keys + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if emitted.len() >= ANALYTICS_APP_USED_DEDUPE_MAX_KEYS { + emitted.clear(); + } + emitted.insert((tracking.turn_id.clone(), connector_id.clone())) + } } impl AnalyticsEventsClient { @@ -84,32 +123,76 @@ impl AnalyticsEventsClient { invocations, ); } + + pub(crate) fn track_app_mentioned( + &self, + tracking: TrackEventsContext, + mentions: Vec, + ) { + track_app_mentioned( + &self.queue, + Arc::clone(&self.config), + Some(tracking), + mentions, + ); + } + + pub(crate) fn track_app_used(&self, tracking: TrackEventsContext, app: AppInvocation) { + track_app_used(&self.queue, Arc::clone(&self.config), Some(tracking), app); + } } -struct TrackEventsJob { +enum TrackEventsJob { + SkillInvocations(TrackSkillInvocationsJob), + AppMentioned(TrackAppMentionedJob), + AppUsed(TrackAppUsedJob), +} + +struct TrackSkillInvocationsJob { config: Arc, tracking: TrackEventsContext, invocations: Vec, } +struct TrackAppMentionedJob { + config: Arc, + tracking: TrackEventsContext, + mentions: Vec, +} + +struct TrackAppUsedJob { + config: Arc, + tracking: TrackEventsContext, + app: AppInvocation, +} + const ANALYTICS_EVENTS_QUEUE_SIZE: usize = 256; const ANALYTICS_EVENTS_TIMEOUT: Duration = Duration::from_secs(10); +const ANALYTICS_APP_USED_DEDUPE_MAX_KEYS: usize = 4096; #[derive(Serialize)] struct TrackEventsRequest { - events: Vec, + events: Vec, } #[derive(Serialize)] -struct TrackEvent { +#[serde(untagged)] +enum TrackEventRequest { + SkillInvocation(SkillInvocationEventRequest), + AppMentioned(CodexAppMentionedEventRequest), + AppUsed(CodexAppUsedEventRequest), +} + +#[derive(Serialize)] +struct SkillInvocationEventRequest { event_type: &'static str, skill_id: String, skill_name: String, - event_params: TrackEventParams, + event_params: SkillInvocationEventParams, } #[derive(Serialize)] -struct TrackEventParams { +struct SkillInvocationEventParams { product_client_id: Option, skill_scope: Option, repo_url: Option, @@ -118,6 +201,29 @@ struct TrackEventParams { model_slug: Option, } +#[derive(Serialize)] +struct CodexAppMetadata { + connector_id: Option, + thread_id: Option, + turn_id: Option, + app_name: Option, + product_client_id: Option, + invoke_type: Option, + model_slug: Option, +} + +#[derive(Serialize)] +struct CodexAppMentionedEventRequest { + event_type: &'static str, + event_params: CodexAppMetadata, +} + +#[derive(Serialize)] +struct CodexAppUsedEventRequest { + event_type: &'static str, + event_params: CodexAppMetadata, +} + pub(crate) fn track_skill_invocations( queue: &AnalyticsEventsQueue, config: Arc, @@ -133,34 +239,66 @@ pub(crate) fn track_skill_invocations( if invocations.is_empty() { return; } - let job = TrackEventsJob { + let job = TrackEventsJob::SkillInvocations(TrackSkillInvocationsJob { config, tracking, invocations, - }; + }); queue.try_send(job); } -async fn send_track_skill_invocations(auth_manager: &AuthManager, job: TrackEventsJob) { - let TrackEventsJob { +pub(crate) fn track_app_mentioned( + queue: &AnalyticsEventsQueue, + config: Arc, + tracking: Option, + mentions: Vec, +) { + if config.analytics_enabled == Some(false) { + return; + } + let Some(tracking) = tracking else { + return; + }; + if mentions.is_empty() { + return; + } + let job = TrackEventsJob::AppMentioned(TrackAppMentionedJob { + config, + tracking, + mentions, + }); + queue.try_send(job); +} + +pub(crate) fn track_app_used( + queue: &AnalyticsEventsQueue, + config: Arc, + tracking: Option, + app: AppInvocation, +) { + if config.analytics_enabled == Some(false) { + return; + } + let Some(tracking) = tracking else { + return; + }; + if !queue.should_enqueue_app_used(&tracking, &app) { + return; + } + let job = TrackEventsJob::AppUsed(TrackAppUsedJob { + config, + tracking, + app, + }); + queue.try_send(job); +} + +async fn send_track_skill_invocations(auth_manager: &AuthManager, job: TrackSkillInvocationsJob) { + let TrackSkillInvocationsJob { config, tracking, invocations, } = job; - let Some(auth) = auth_manager.auth().await else { - return; - }; - if !auth.is_chatgpt_auth() { - return; - } - let access_token = match auth.get_token() { - Ok(token) => token, - Err(_) => return, - }; - let Some(account_id) = auth.get_account_id() else { - return; - }; - let mut events = Vec::with_capacity(invocations.len()); for invocation in invocations { let skill_scope = match invocation.skill_scope { @@ -183,21 +321,95 @@ async fn send_track_skill_invocations(auth_manager: &AuthManager, job: TrackEven invocation.skill_path.as_path(), invocation.skill_name.as_str(), ); - events.push(TrackEvent { - event_type: "skill_invocation", - skill_id, - skill_name: invocation.skill_name.clone(), - event_params: TrackEventParams { - thread_id: Some(tracking.thread_id.clone()), - invoke_type: Some("explicit".to_string()), - model_slug: Some(tracking.model_slug.clone()), - product_client_id: Some(crate::default_client::originator().value), - repo_url, - skill_scope: Some(skill_scope.to_string()), + events.push(TrackEventRequest::SkillInvocation( + SkillInvocationEventRequest { + event_type: "skill_invocation", + skill_id, + skill_name: invocation.skill_name.clone(), + event_params: SkillInvocationEventParams { + thread_id: Some(tracking.thread_id.clone()), + invoke_type: Some("explicit".to_string()), + model_slug: Some(tracking.model_slug.clone()), + product_client_id: Some(crate::default_client::originator().value), + repo_url, + skill_scope: Some(skill_scope.to_string()), + }, }, - }); + )); } + send_track_events(auth_manager, config, events).await; +} + +async fn send_track_app_mentioned(auth_manager: &AuthManager, job: TrackAppMentionedJob) { + let TrackAppMentionedJob { + config, + tracking, + mentions, + } = job; + let events = mentions + .into_iter() + .map(|mention| { + let event_params = codex_app_metadata(&tracking, mention); + TrackEventRequest::AppMentioned(CodexAppMentionedEventRequest { + event_type: "codex_app_mentioned", + event_params, + }) + }) + .collect::>(); + + send_track_events(auth_manager, config, events).await; +} + +async fn send_track_app_used(auth_manager: &AuthManager, job: TrackAppUsedJob) { + let TrackAppUsedJob { + config, + tracking, + app, + } = job; + let event_params = codex_app_metadata(&tracking, app); + let events = vec![TrackEventRequest::AppUsed(CodexAppUsedEventRequest { + event_type: "codex_app_used", + event_params, + })]; + + send_track_events(auth_manager, config, events).await; +} + +fn codex_app_metadata(tracking: &TrackEventsContext, app: AppInvocation) -> CodexAppMetadata { + CodexAppMetadata { + connector_id: app.connector_id, + thread_id: Some(tracking.thread_id.clone()), + turn_id: Some(tracking.turn_id.clone()), + app_name: app.app_name, + product_client_id: Some(crate::default_client::originator().value), + invoke_type: app.invoke_type, + model_slug: Some(tracking.model_slug.clone()), + } +} + +async fn send_track_events( + auth_manager: &AuthManager, + config: Arc, + events: Vec, +) { + if events.is_empty() { + return; + } + let Some(auth) = auth_manager.auth().await else { + return; + }; + if !auth.is_chatgpt_auth() { + return; + } + let access_token = match auth.get_token() { + Ok(token) => token, + Err(_) => return, + }; + let Some(account_id) = auth.get_account_id() else { + return; + }; + let base_url = config.chatgpt_base_url.trim_end_matches('/'); let url = format!("{base_url}/codex/analytics-events/events"); let payload = TrackEventsRequest { events }; @@ -269,9 +481,21 @@ fn normalize_path_for_skill_id( #[cfg(test)] mod tests { + use super::AnalyticsEventsQueue; + use super::AppInvocation; + use super::CodexAppMentionedEventRequest; + use super::CodexAppUsedEventRequest; + use super::TrackEventRequest; + use super::TrackEventsContext; + use super::codex_app_metadata; use super::normalize_path_for_skill_id; use pretty_assertions::assert_eq; + use serde_json::json; + use std::collections::HashSet; use std::path::PathBuf; + use std::sync::Arc; + use std::sync::Mutex; + use tokio::sync::mpsc; fn expected_absolute_path(path: &PathBuf) -> String { std::fs::canonicalize(path) @@ -328,4 +552,109 @@ mod tests { assert_eq!(path, expected); } + + #[test] + fn app_mentioned_event_serializes_expected_shape() { + let tracking = TrackEventsContext { + model_slug: "gpt-5".to_string(), + thread_id: "thread-1".to_string(), + turn_id: "turn-1".to_string(), + }; + let event = TrackEventRequest::AppMentioned(CodexAppMentionedEventRequest { + event_type: "codex_app_mentioned", + event_params: codex_app_metadata( + &tracking, + AppInvocation { + connector_id: Some("calendar".to_string()), + app_name: Some("Calendar".to_string()), + invoke_type: Some("explicit".to_string()), + }, + ), + }); + + let payload = serde_json::to_value(&event).expect("serialize app mentioned event"); + + assert_eq!( + payload, + json!({ + "event_type": "codex_app_mentioned", + "event_params": { + "connector_id": "calendar", + "thread_id": "thread-1", + "turn_id": "turn-1", + "app_name": "Calendar", + "product_client_id": crate::default_client::originator().value, + "invoke_type": "explicit", + "model_slug": "gpt-5" + } + }) + ); + } + + #[test] + fn app_used_event_serializes_expected_shape() { + let tracking = TrackEventsContext { + model_slug: "gpt-5".to_string(), + thread_id: "thread-2".to_string(), + turn_id: "turn-2".to_string(), + }; + let event = TrackEventRequest::AppUsed(CodexAppUsedEventRequest { + event_type: "codex_app_used", + event_params: codex_app_metadata( + &tracking, + AppInvocation { + connector_id: Some("drive".to_string()), + app_name: Some("Google Drive".to_string()), + invoke_type: Some("implicit".to_string()), + }, + ), + }); + + let payload = serde_json::to_value(&event).expect("serialize app used event"); + + assert_eq!( + payload, + json!({ + "event_type": "codex_app_used", + "event_params": { + "connector_id": "drive", + "thread_id": "thread-2", + "turn_id": "turn-2", + "app_name": "Google Drive", + "product_client_id": crate::default_client::originator().value, + "invoke_type": "implicit", + "model_slug": "gpt-5" + } + }) + ); + } + + #[test] + fn app_used_dedupe_is_keyed_by_turn_and_connector() { + let (sender, _receiver) = mpsc::channel(1); + let queue = AnalyticsEventsQueue { + sender, + app_used_emitted_keys: Arc::new(Mutex::new(HashSet::new())), + }; + let app = AppInvocation { + connector_id: Some("calendar".to_string()), + app_name: Some("Calendar".to_string()), + invoke_type: Some("implicit".to_string()), + }; + + let turn_1 = TrackEventsContext { + model_slug: "gpt-5".to_string(), + thread_id: "thread-1".to_string(), + turn_id: "turn-1".to_string(), + }; + let turn_2 = TrackEventsContext { + model_slug: "gpt-5".to_string(), + thread_id: "thread-1".to_string(), + turn_id: "turn-2".to_string(), + }; + + assert_eq!(queue.should_enqueue_app_used(&turn_1, &app), true); + assert_eq!(queue.should_enqueue_app_used(&turn_1, &app), false); + assert_eq!(queue.should_enqueue_app_used(&turn_2, &app), true); + } } diff --git a/codex-rs/core/src/client.rs b/codex-rs/core/src/client.rs index bbf9b26c98..99f8464acd 100644 --- a/codex-rs/core/src/client.rs +++ b/codex-rs/core/src/client.rs @@ -325,6 +325,9 @@ impl ModelClient { let subagent = match sub { crate::protocol::SubAgentSource::Review => "review".to_string(), crate::protocol::SubAgentSource::Compact => "compact".to_string(), + crate::protocol::SubAgentSource::MemoryConsolidation => { + "memory_consolidation".to_string() + } crate::protocol::SubAgentSource::ThreadSpawn { .. } => "collab_spawn".to_string(), crate::protocol::SubAgentSource::Other(label) => label.clone(), }; @@ -630,7 +633,6 @@ impl ModelClientSession { &mut self, otel_manager: &OtelManager, model_info: &ModelInfo, - turn_metadata_header: Option<&str>, ) -> std::result::Result<(), ApiError> { if !self.client.responses_websocket_enabled(model_info) || self.client.websockets_disabled() { @@ -653,7 +655,7 @@ impl ModelClientSession { client_setup.api_provider, client_setup.api_auth, Some(Arc::clone(&self.turn_state)), - turn_metadata_header, + None, ) .await?; self.connection = Some(connection); diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index 4405635ce3..437fd9e803 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -14,6 +14,7 @@ use crate::agent::AgentStatus; use crate::agent::MAX_THREAD_SPAWN_DEPTH; use crate::agent::agent_status_from_event; use crate::analytics_client::AnalyticsEventsClient; +use crate::analytics_client::AppInvocation; use crate::analytics_client::build_track_events_context; use crate::apps::render_apps_section; use crate::compact; @@ -30,15 +31,13 @@ use crate::models_manager::manager::ModelsManager; use crate::parse_command::parse_command; use crate::parse_turn_item; use crate::rollout::session_index; -use crate::sandbox_tags::sandbox_tag; use crate::stream_events_utils::HandleOutputCtx; use crate::stream_events_utils::handle_non_tool_response_item; use crate::stream_events_utils::handle_output_item_done; use crate::stream_events_utils::last_assistant_message_from_item; use crate::terminal; use crate::truncate::TruncationPolicy; -use crate::turn_metadata::build_turn_metadata_header; -use crate::turn_metadata::resolve_turn_metadata_header_with_timeout; +use crate::turn_metadata::TurnMetadataState; use crate::util::error_or_panic; use async_channel::Receiver; use async_channel::Sender; @@ -73,6 +72,7 @@ use codex_protocol::protocol::SessionSource; use codex_protocol::protocol::SubAgentSource; use codex_protocol::protocol::TurnAbortReason; use codex_protocol::protocol::TurnContextItem; +use codex_protocol::protocol::TurnContextNetworkItem; use codex_protocol::protocol::TurnStartedEvent; use codex_protocol::request_user_input::RequestUserInputArgs; use codex_protocol::request_user_input::RequestUserInputResponse; @@ -90,7 +90,6 @@ use rmcp::model::RequestId; use serde_json; use serde_json::Value; use tokio::sync::Mutex; -use tokio::sync::OnceCell; use tokio::sync::RwLock; use tokio::sync::oneshot; use tokio::sync::watch; @@ -114,6 +113,7 @@ use crate::client_common::Prompt; use crate::client_common::ResponseEvent; use crate::codex_thread::ThreadConfigSnapshot; use crate::compact::collect_user_messages; +use crate::config::CONFIG_TOML_FILE; use crate::config::Config; use crate::config::Constrained; use crate::config::ConstraintResult; @@ -150,6 +150,7 @@ use crate::mcp::with_codex_apps_mcp; use crate::mcp_connection_manager::McpConnectionManager; use crate::mcp_connection_manager::filter_codex_apps_mcp_tools_only; use crate::mcp_connection_manager::filter_mcp_tools_by_name; +use crate::mcp_connection_manager::filter_non_codex_apps_mcp_tools_only; use crate::memories; use crate::mentions::build_connector_slug_counts; use crate::mentions::build_skill_name_counts; @@ -170,6 +171,7 @@ use crate::protocol::Event; use crate::protocol::EventMsg; use crate::protocol::ExecApprovalRequestEvent; use crate::protocol::McpServerRefreshConfig; +use crate::protocol::NetworkApprovalContext; use crate::protocol::Op; use crate::protocol::PlanDeltaEvent; use crate::protocol::RateLimitSnapshot; @@ -222,7 +224,11 @@ use crate::tasks::SessionTask; use crate::tasks::SessionTaskContext; use crate::tools::ToolRouter; use crate::tools::context::SharedTurnDiffTracker; +use crate::tools::handlers::SEARCH_TOOL_BM25_TOOL_NAME; use crate::tools::js_repl::JsReplHandle; +use crate::tools::network_approval::NetworkApprovalService; +use crate::tools::network_approval::build_blocked_request_observer; +use crate::tools::network_approval::build_network_policy_decider; use crate::tools::parallel::ToolCallRuntime; use crate::tools::sandboxing::ApprovalStore; use crate::tools::spec::ToolsConfig; @@ -246,6 +252,7 @@ use codex_protocol::openai_models::ReasoningEffort as ReasoningEffortConfig; use codex_protocol::protocol::CodexErrorInfo; use codex_protocol::protocol::InitialHistory; use codex_protocol::user_input::UserInput; +use codex_utils_absolute_path::AbsolutePathBuf; use codex_utils_readiness::Readiness; use codex_utils_readiness::ReadinessFlag; @@ -518,10 +525,6 @@ pub(crate) struct Session { js_repl: Arc, next_internal_sub_id: AtomicU64, } - -const SEARCH_TOOL_DEVELOPER_INSTRUCTIONS: &str = - include_str!("../templates/search_tool/developer_instructions.md"); - /// The context needed for a single turn of the thread. #[derive(Debug)] pub(crate) struct TurnContext { @@ -557,7 +560,7 @@ pub(crate) struct TurnContext { pub(crate) truncation_policy: TruncationPolicy, pub(crate) js_repl: Arc, pub(crate) dynamic_tools: Vec, - turn_metadata_header: OnceCell>, + pub(crate) turn_metadata_state: Arc, } impl TurnContext { pub(crate) fn model_context_window(&self) -> Option { @@ -637,7 +640,7 @@ impl TurnContext { truncation_policy, js_repl: Arc::clone(&self.js_repl), dynamic_tools: self.dynamic_tools.clone(), - turn_metadata_header: self.turn_metadata_header.clone(), + turn_metadata_state: self.turn_metadata_state.clone(), } } @@ -653,39 +656,39 @@ impl TurnContext { .unwrap_or(compact::SUMMARIZATION_PROMPT) } - async fn build_turn_metadata_header(&self) -> Option { - let sandbox = sandbox_tag(&self.sandbox_policy, self.windows_sandbox_level); - self.turn_metadata_header - .get_or_init(|| async { - build_turn_metadata_header(self.cwd.as_path(), Some(sandbox)).await - }) - .await - .clone() + pub(crate) fn to_turn_context_item( + &self, + collaboration_mode: CollaborationMode, + ) -> TurnContextItem { + TurnContextItem { + turn_id: Some(self.sub_id.clone()), + cwd: self.cwd.clone(), + approval_policy: self.approval_policy, + sandbox_policy: self.sandbox_policy.clone(), + network: self.turn_context_network_item(), + model: self.model_info.slug.clone(), + personality: self.personality, + collaboration_mode: Some(collaboration_mode), + effort: self.reasoning_effort, + summary: self.reasoning_summary, + user_instructions: self.user_instructions.clone(), + developer_instructions: self.developer_instructions.clone(), + final_output_json_schema: self.final_output_json_schema.clone(), + truncation_policy: Some(self.truncation_policy.into()), + } } - /// Resolves the per-turn metadata header under a shared timeout policy. - /// - /// This uses the same timeout helper as websocket startup prewarm so both turn execution and - /// background prewarm observe identical "timeout means best-effort fallback" behavior. - pub async fn resolve_turn_metadata_header(&self) -> Option { - resolve_turn_metadata_header_with_timeout( - self.build_turn_metadata_header(), - self.turn_metadata_header.get().cloned().flatten(), - ) - .await - } - - /// Starts best-effort background computation of turn metadata. - /// - /// This warms the cached value used by [`TurnContext::resolve_turn_metadata_header`] so turns - /// and websocket prewarm are less likely to pay metadata construction latency on demand. - pub fn spawn_turn_metadata_header_task(self: &Arc) { - let context = Arc::clone(self); - tokio::spawn(async move { - trace!("Spawning turn metadata calculation task"); - context.build_turn_metadata_header().await; - trace!("Turn metadata calculation task completed"); - }); + fn turn_context_network_item(&self) -> Option { + let network = self + .config + .config_layer_stack + .requirements() + .network + .as_ref()?; + Some(TurnContextNetworkItem { + allowed_domains: network.allowed_domains.clone().unwrap_or_default(), + denied_domains: network.denied_domains.clone().unwrap_or_default(), + }) } } @@ -824,6 +827,33 @@ impl Session { } } + async fn start_managed_network_proxy( + spec: &crate::config::NetworkProxySpec, + sandbox_policy: &SandboxPolicy, + network_policy_decider: Option>, + blocked_request_observer: Option>, + managed_network_requirements_enabled: bool, + ) -> anyhow::Result<(StartedNetworkProxy, SessionNetworkProxyRuntime)> { + let network_proxy = spec + .start_proxy( + sandbox_policy, + network_policy_decider, + blocked_request_observer, + managed_network_requirements_enabled, + ) + .await + .map_err(|err| anyhow::anyhow!("failed to start managed network proxy: {err}"))?; + let session_network_proxy = { + let proxy = network_proxy.proxy(); + SessionNetworkProxyRuntime { + http_addr: proxy.http_addr().to_string(), + socks_addr: proxy.socks_addr().to_string(), + admin_addr: proxy.admin_addr().to_string(), + } + }; + Ok((network_proxy, session_network_proxy)) + } + /// Don't expand the number of mutated arguments on config. We are in the process of getting rid of it. pub(crate) fn build_per_turn_config(session_configuration: &SessionConfiguration) -> Config { // todo(aibrahim): store this state somewhere else so we don't need to mut config @@ -912,6 +942,15 @@ impl Session { }); let cwd = session_configuration.cwd.clone(); + let turn_metadata_state = Arc::new(TurnMetadataState::new( + sub_id.clone(), + cwd.clone(), + session_configuration.sandbox_policy.get(), + session_configuration.windows_sandbox_level, + per_turn_config + .features + .enabled(Feature::UseLinuxSandboxBwrap), + )); TurnContext { sub_id, config: per_turn_config.clone(), @@ -942,7 +981,7 @@ impl Session { truncation_policy: model_info.truncation_policy.into(), js_repl, dynamic_tools: session_configuration.dynamic_tools.clone(), - turn_metadata_header: OnceCell::new(), + turn_metadata_state, } } @@ -1174,21 +1213,50 @@ impl Session { }; session_configuration.thread_name = thread_name.clone(); let mut state = SessionState::new(session_configuration.clone()); - let network_proxy = - match config.permissions.network.as_ref() { - Some(spec) => Some(spec.start_proxy().await.map_err(|err| { - anyhow::anyhow!("failed to start managed network proxy: {err}") - })?), - None => None, + let managed_network_requirements_enabled = config.managed_network_requirements_enabled(); + let network_approval = Arc::new(NetworkApprovalService::default()); + // The managed proxy can call back into core for allowlist-miss decisions. + let network_policy_decider_session = if managed_network_requirements_enabled { + config + .permissions + .network + .as_ref() + .map(|_| Arc::new(RwLock::new(std::sync::Weak::::new()))) + } else { + None + }; + let blocked_request_observer = if managed_network_requirements_enabled { + config + .permissions + .network + .as_ref() + .map(|_| build_blocked_request_observer(Arc::clone(&network_approval))) + } else { + None + }; + let network_policy_decider = + network_policy_decider_session + .as_ref() + .map(|network_policy_decider_session| { + build_network_policy_decider( + Arc::clone(&network_approval), + Arc::clone(network_policy_decider_session), + ) + }); + let (network_proxy, session_network_proxy) = + if let Some(spec) = config.permissions.network.as_ref() { + let (network_proxy, session_network_proxy) = Self::start_managed_network_proxy( + spec, + config.permissions.sandbox_policy.get(), + network_policy_decider.as_ref().map(Arc::clone), + blocked_request_observer.as_ref().map(Arc::clone), + managed_network_requirements_enabled, + ) + .await?; + (Some(network_proxy), Some(session_network_proxy)) + } else { + (None, None) }; - let session_network_proxy = network_proxy.as_ref().map(|started| { - let proxy = started.proxy(); - SessionNetworkProxyRuntime { - http_addr: proxy.http_addr().to_string(), - socks_addr: proxy.socks_addr().to_string(), - admin_addr: proxy.admin_addr().to_string(), - } - }); let services = SessionServices { mcp_connection_manager: Arc::new(RwLock::new(McpConnectionManager::default())), @@ -1214,6 +1282,7 @@ impl Session { file_watcher, agent_control, network_proxy, + network_approval: Arc::clone(&network_approval), state_db: state_db_ctx.clone(), model_client: ModelClient::new( Some(Arc::clone(&auth_manager)), @@ -1237,17 +1306,10 @@ impl Session { let prewarm_model_info = models_manager .get_model_info(session_configuration.collaboration_mode.model(), &config) .await; - let prewarm_cwd = session_configuration.cwd.clone(); - let turn_metadata_header = resolve_turn_metadata_header_with_timeout( - async move { build_turn_metadata_header(prewarm_cwd.as_path(), None).await }, - None, - ) - .boxed(); let startup_regular_task = RegularTask::with_startup_prewarm( services.model_client.clone(), services.otel_manager.clone(), prewarm_model_info, - turn_metadata_header, ); state.set_startup_regular_task(startup_regular_task); @@ -1263,6 +1325,10 @@ impl Session { js_repl, next_internal_sub_id: AtomicU64::new(0), }); + if let Some(network_policy_decider_session) = network_policy_decider_session { + let mut guard = network_policy_decider_session.write().await; + *guard = Arc::downgrade(&sess); + } // Dispatch the SessionConfiguredEvent first and then report any errors. // If resuming, include converted initial messages in the payload so UIs can render them immediately. @@ -1425,6 +1491,11 @@ impl Session { state.merge_mcp_tool_selection(tool_names) } + pub(crate) async fn set_mcp_tool_selection(&self, tool_names: Vec) { + let mut state = self.state.lock().await; + state.set_mcp_tool_selection(tool_names); + } + pub(crate) async fn get_mcp_tool_selection(&self) -> Option> { let state = self.state.lock().await; state.get_mcp_tool_selection() @@ -1458,6 +1529,7 @@ impl Session { async fn record_initial_history(&self, conversation_history: InitialHistory) { let turn_context = self.new_default_turn().await; + self.clear_mcp_tool_selection().await; match conversation_history { InitialHistory::New => { // Build and record initial items (user instructions + environment context) @@ -1473,6 +1545,8 @@ impl Session { } InitialHistory::Resumed(resumed_history) => { let rollout_items = resumed_history.history; + let restored_tool_selection = + Self::extract_mcp_tool_selection_from_rollout(&rollout_items); let previous_model = Self::last_rollout_model_name(&rollout_items) .map(std::string::ToString::to_string); { @@ -1514,12 +1588,17 @@ impl Session { let mut state = self.state.lock().await; state.set_token_info(Some(info)); } + if let Some(selected_tools) = restored_tool_selection { + self.set_mcp_tool_selection(selected_tools).await; + } // Defer seeding the session's initial context until the first turn starts so // turn/start overrides can be merged before we write to the rollout. self.flush_rollout().await; } InitialHistory::Forked(rollout_items) => { + let restored_tool_selection = + Self::extract_mcp_tool_selection_from_rollout(&rollout_items); let previous_model = Self::last_rollout_model_name(&rollout_items) .map(std::string::ToString::to_string); self.set_previous_model(previous_model).await; @@ -1539,6 +1618,9 @@ impl Session { let mut state = self.state.lock().await; state.set_token_info(Some(info)); } + if let Some(selected_tools) = restored_tool_selection { + self.set_mcp_tool_selection(selected_tools).await; + } // If persisting, persist all rollout items as-is (recorder filters) if !rollout_items.is_empty() { @@ -1580,6 +1662,54 @@ impl Session { }) } + fn extract_mcp_tool_selection_from_rollout( + rollout_items: &[RolloutItem], + ) -> Option> { + let mut search_call_ids = HashSet::new(); + let mut active_selected_tools: Option> = None; + + for item in rollout_items { + let RolloutItem::ResponseItem(response_item) = item else { + continue; + }; + match response_item { + ResponseItem::FunctionCall { name, call_id, .. } => { + if name == SEARCH_TOOL_BM25_TOOL_NAME { + search_call_ids.insert(call_id.clone()); + } + } + ResponseItem::FunctionCallOutput { call_id, output } => { + if !search_call_ids.contains(call_id) { + continue; + } + let Some(content) = output.body.to_text() else { + continue; + }; + let Ok(payload) = serde_json::from_str::(&content) else { + continue; + }; + let Some(selected_tools) = payload + .get("active_selected_tools") + .and_then(Value::as_array) + else { + continue; + }; + let Some(selected_tools) = selected_tools + .iter() + .map(|value| value.as_str().map(str::to_string)) + .collect::>>() + else { + continue; + }; + active_selected_tools = Some(selected_tools); + } + _ => {} + } + } + + active_selected_tools + } + async fn previous_model(&self) -> Option { let state = self.state.lock().await; state.previous_model() @@ -1716,13 +1846,11 @@ impl Session { } } + let resolved_model_slug = session_configuration.collaboration_mode.model().to_string(); let model_info = self .services .models_manager - .get_model_info( - session_configuration.collaboration_mode.model(), - &per_turn_config, - ) + .get_model_info(resolved_model_slug.as_str(), &per_turn_config) .await; let mut turn_context: TurnContext = Self::make_turn_context( Some(Arc::clone(&self.services.auth_manager)), @@ -1730,7 +1858,7 @@ impl Session { session_configuration.provider.clone(), &session_configuration, per_turn_config, - model_info, + model_info.clone(), self.services .network_proxy .as_ref() @@ -1743,7 +1871,18 @@ impl Session { turn_context.final_output_json_schema = final_schema; } let turn_context = Arc::new(turn_context); - turn_context.spawn_turn_metadata_header_task(); + if model_info.used_fallback_model_metadata { + self.send_event( + turn_context.as_ref(), + EventMsg::Warning(WarningEvent { + message: format!( + "Model metadata for `{resolved_model_slug}` not found. Defaulting to fallback metadata; this can degrade performance and cause issues." + ), + }), + ) + .await; + } + turn_context.turn_metadata_state.spawn_git_enrichment_task(); turn_context } @@ -1765,6 +1904,48 @@ impl Session { .clone() } + pub(crate) async fn reload_user_config_layer(&self) { + let config_toml_path = { + let state = self.state.lock().await; + state + .session_configuration + .codex_home + .join(CONFIG_TOML_FILE) + }; + + let user_config = match std::fs::read_to_string(&config_toml_path) { + Ok(contents) => match toml::from_str::(&contents) { + Ok(config) => config, + Err(err) => { + warn!("failed to parse user config while reloading layer: {err}"); + return; + } + }, + Err(err) if err.kind() == std::io::ErrorKind::NotFound => { + toml::Value::Table(Default::default()) + } + Err(err) => { + warn!("failed to read user config while reloading layer: {err}"); + return; + } + }; + + let config_toml_path = match AbsolutePathBuf::try_from(config_toml_path) { + Ok(path) => path, + Err(err) => { + warn!("failed to resolve user config path while reloading layer: {err}"); + return; + } + }; + + let mut state = self.state.lock().await; + let mut config = (*state.session_configuration.original_config_do_not_use).clone(); + config.config_layer_stack = config + .config_layer_stack + .with_user_config(&config_toml_path, user_config); + state.session_configuration.original_config_do_not_use = Arc::new(config); + } + pub(crate) async fn new_default_turn_with_sub_id(&self, sub_id: String) -> Arc { let session_configuration = { let state = self.state.lock().await; @@ -1893,6 +2074,19 @@ impl Session { Some(DeveloperInstructions::model_switch_message(model_instructions).into()) } + pub(crate) fn is_model_switch_developer_message(item: &ResponseItem) -> bool { + let ResponseItem::Message { role, content, .. } = item else { + return false; + }; + role == "developer" + && content.iter().any(|content_item| { + matches!( + content_item, + ContentItem::InputText { text } if text.starts_with("") + ) + }) + } + fn build_settings_update_items( &self, previous_context: Option<&Arc>, @@ -2030,7 +2224,7 @@ impl Session { Ok(()) } - async fn turn_context_for_sub_id(&self, sub_id: &str) -> Option> { + pub(crate) async fn turn_context_for_sub_id(&self, sub_id: &str) -> Option> { let active = self.active_turn.lock().await; active .as_ref() @@ -2092,6 +2286,7 @@ impl Session { command: Vec, cwd: PathBuf, reason: Option, + network_approval_context: Option, proposed_execpolicy_amendment: Option, ) -> ReviewDecision { // Add the tx_approve callback to the map before sending the request. @@ -2118,6 +2313,7 @@ impl Session { command, cwd, reason, + network_approval_context, proposed_execpolicy_amendment, parsed_cmd, }); @@ -2429,11 +2625,6 @@ impl Session { if let Some(developer_instructions) = turn_context.developer_instructions.as_deref() { items.push(DeveloperInstructions::new(developer_instructions.to_string()).into()); } - if turn_context.tools_config.search_tool { - items.push( - DeveloperInstructions::new(SEARCH_TOOL_DEVELOPER_INSTRUCTIONS.to_string()).into(), - ); - } // Add developer instructions for memories. if let Some(memory_prompt) = build_memory_tool_developer_instructions(&turn_context.config.codex_home).await @@ -3050,6 +3241,9 @@ async fn submission_loop(sess: Arc, config: Arc, rx_sub: Receiv Op::RefreshMcpServers { config } => { handlers::refresh_mcp_servers(&sess, config).await; } + Op::ReloadUserConfig => { + handlers::reload_user_config(&sess).await; + } Op::ListCustomPrompts => { handlers::list_custom_prompts(&sess, sub.id.clone()).await; } @@ -3474,6 +3668,10 @@ mod handlers { *guard = Some(refresh_config); } + pub async fn reload_user_config(sess: &Arc) { + sess.reload_user_config_layer().await; + } + pub async fn list_mcp_tools(sess: &Session, config: &Arc, sub_id: String) { let mcp_connection_manager = sess.services.mcp_connection_manager.read().await; let auth = sess.services.auth_manager.auth().await; @@ -3946,9 +4144,19 @@ async fn spawn_review_thread( let session_source = parent_turn_context.session_source.clone(); let per_turn_config = Arc::new(per_turn_config); + let review_turn_id = sub_id.to_string(); + let turn_metadata_state = Arc::new(TurnMetadataState::new( + review_turn_id.clone(), + parent_turn_context.cwd.clone(), + &parent_turn_context.sandbox_policy, + parent_turn_context.windows_sandbox_level, + parent_turn_context + .features + .enabled(Feature::UseLinuxSandboxBwrap), + )); let review_turn_context = TurnContext { - sub_id: sub_id.to_string(), + sub_id: review_turn_id, config: per_turn_config, auth_manager: auth_manager_for_context, model_info: model_info.clone(), @@ -3977,7 +4185,7 @@ async fn spawn_review_thread( js_repl: Arc::clone(&sess.js_repl), dynamic_tools: parent_turn_context.dynamic_tools.clone(), truncation_policy: model_info.truncation_policy.into(), - turn_metadata_header: parent_turn_context.turn_metadata_header.clone(), + turn_metadata_state, }; // Seed the child task with the review prompt as the initial user message. @@ -3987,6 +4195,7 @@ async fn spawn_review_thread( text_elements: Vec::new(), }]; let tc = Arc::new(review_turn_context); + tc.turn_metadata_state.spawn_git_enrichment_task(); sess.spawn_task(tc.clone(), input, ReviewTask::new()).await; // Announce entering review mode so UIs can switch modes. @@ -4114,7 +4323,10 @@ pub(crate) async fn run_turn( Ok(mcp_tools) => mcp_tools, Err(_) => return None, }; - connectors::accessible_connectors_from_mcp_tools(&mcp_tools) + connectors::with_app_enabled_state( + connectors::accessible_connectors_from_mcp_tools(&mcp_tools), + &turn_context.config, + ) } else { Vec::new() }; @@ -4151,7 +4363,11 @@ pub(crate) async fn run_turn( let otel_manager = turn_context.otel_manager.clone(); let thread_id = sess.conversation_id.to_string(); - let tracking = build_track_events_context(turn_context.model_info.slug.clone(), thread_id); + let tracking = build_track_events_context( + turn_context.model_info.slug.clone(), + thread_id, + turn_context.sub_id.clone(), + ); let SkillInjections { items: skill_items, warnings: skill_warnings, @@ -4174,6 +4390,23 @@ pub(crate) async fn run_turn( &available_connectors, &skill_name_counts_lower, )); + let connector_names_by_id = available_connectors + .iter() + .map(|connector| (connector.id.as_str(), connector.name.as_str())) + .collect::>(); + let mentioned_app_invocations = explicitly_enabled_connectors + .iter() + .map(|connector_id| AppInvocation { + connector_id: Some(connector_id.clone()), + app_name: connector_names_by_id + .get(connector_id.as_str()) + .map(|name| (*name).to_string()), + invoke_type: Some("explicit".to_string()), + }) + .collect::>(); + sess.services + .analytics_events_client + .track_app_mentioned(tracking.clone(), mentioned_app_invocations); sess.merge_connector_selection(explicitly_enabled_connectors.clone()) .await; @@ -4194,7 +4427,6 @@ pub(crate) async fn run_turn( // many turns, from the perspective of the user, it is a single turn. let turn_diff_tracker = Arc::new(tokio::sync::Mutex::new(TurnDiffTracker::new())); - let turn_metadata_header = turn_context.resolve_turn_metadata_header().await; // `ModelClientSession` is turn-scoped and caches WebSocket + sticky routing state, so we reuse // one instance across retries within this turn. let mut client_session = @@ -4246,6 +4478,7 @@ pub(crate) async fn run_turn( }) .map(|user_message| user_message.message()) .collect::>(); + let turn_metadata_header = turn_context.turn_metadata_state.current_header_value(); match run_sampling_request( Arc::clone(&sess), Arc::clone(&turn_context), @@ -4270,7 +4503,7 @@ pub(crate) async fn run_turn( let estimated_token_count = sess.get_estimated_token_count(turn_context.as_ref()).await; - info!( + trace!( turn_id = %turn_context.sub_id, total_usage_tokens, estimated_token_count = ?estimated_token_count, @@ -4365,6 +4598,12 @@ async fn run_pre_sampling_compact( Ok(()) } +/// Runs pre-sampling compaction against the previous model when switching to a smaller +/// context-window model. +/// +/// Returns `Ok(())` when compaction either completed successfully or was skipped because the +/// model/context-window preconditions were not met. Returns `Err(_)` only when compaction was +/// attempted and failed. async fn maybe_run_previous_model_inline_compact( sess: &Arc, turn_context: &Arc, @@ -4459,11 +4698,20 @@ fn collect_explicit_app_ids_from_skill_items( } fn filter_connectors_for_input( - connectors: Vec, + connectors: &[connectors::AppInfo], input: &[ResponseItem], explicitly_enabled_connectors: &HashSet, skill_name_counts_lower: &HashMap, ) -> Vec { + let connectors: Vec = connectors + .iter() + .filter(|connector| connector.is_enabled) + .cloned() + .collect::>(); + if connectors.is_empty() { + return Vec::new(); + } + let user_messages = collect_user_messages(input); if user_messages.is_empty() && explicitly_enabled_connectors.is_empty() { return Vec::new(); @@ -4526,7 +4774,7 @@ fn connector_inserted_in_messages( } fn filter_codex_apps_mcp_tools( - mut mcp_tools: HashMap, + mcp_tools: &HashMap, connectors: &[connectors::AppInfo], ) -> HashMap { let allowed: HashSet<&str> = connectors @@ -4534,17 +4782,19 @@ fn filter_codex_apps_mcp_tools( .map(|connector| connector.id.as_str()) .collect(); - mcp_tools.retain(|_, tool| { - if tool.server_name != CODEX_APPS_MCP_SERVER_NAME { - return true; - } - let Some(connector_id) = codex_apps_connector_id(tool) else { - return false; - }; - allowed.contains(connector_id) - }); - mcp_tools + .iter() + .filter(|(_, tool)| { + if tool.server_name != CODEX_APPS_MCP_SERVER_NAME { + return true; + } + let Some(connector_id) = codex_apps_connector_id(tool) else { + return false; + }; + allowed.contains(connector_id) + }) + .map(|(name, tool)| (name.clone(), tool.clone())) + .collect() } fn codex_apps_connector_id(tool: &crate::mcp_connection_manager::ToolInfo) -> Option<&str> { @@ -4706,37 +4956,43 @@ async fn built_tools( let mut effective_explicitly_enabled_connectors = explicitly_enabled_connectors.clone(); effective_explicitly_enabled_connectors.extend(sess.get_connector_selection().await); - let connectors_for_tools = if turn_context.config.features.enabled(Feature::Apps) { - let skill_name_counts_lower = skills_outcome.map_or_else(HashMap::new, |outcome| { - build_skill_name_counts(&outcome.skills, &outcome.disabled_paths).1 - }); - let connectors = connectors::accessible_connectors_from_mcp_tools(&mcp_tools); - Some(filter_connectors_for_input( - connectors, - input, - &effective_explicitly_enabled_connectors, - &skill_name_counts_lower, + let connectors = if turn_context.features.enabled(Feature::Apps) { + Some(connectors::with_app_enabled_state( + connectors::accessible_connectors_from_mcp_tools(&mcp_tools), + &turn_context.config, )) } else { None }; - if turn_context.config.features.enabled(Feature::Apps) { - let mut selected_mcp_tools = - if let Some(selected_tools) = sess.get_mcp_tool_selection().await { - filter_mcp_tools_by_name(mcp_tools.clone(), &selected_tools) - } else { - HashMap::new() - }; + let app_tools = connectors + .as_ref() + .map(|connectors| filter_codex_apps_mcp_tools(&mcp_tools, connectors)); - if let Some(connectors) = connectors_for_tools.as_ref() { - let apps_mcp_tools = filter_codex_apps_mcp_tools_only(mcp_tools, connectors); - selected_mcp_tools.extend(apps_mcp_tools); + if let Some(connectors) = connectors.as_ref() { + let skill_name_counts_lower = skills_outcome.map_or_else(HashMap::new, |outcome| { + build_skill_name_counts(&outcome.skills, &outcome.disabled_paths).1 + }); + + let explicitly_enabled = filter_connectors_for_input( + connectors, + input, + &effective_explicitly_enabled_connectors, + &skill_name_counts_lower, + ); + + let mut selected_mcp_tools = filter_non_codex_apps_mcp_tools_only(&mcp_tools); + + if let Some(selected_tools) = sess.get_mcp_tool_selection().await { + selected_mcp_tools.extend(filter_mcp_tools_by_name(&mcp_tools, &selected_tools)); } + selected_mcp_tools.extend(filter_codex_apps_mcp_tools_only( + &mcp_tools, + explicitly_enabled.as_ref(), + )); + mcp_tools = selected_mcp_tools; - } else if let Some(connectors) = connectors_for_tools.as_ref() { - mcp_tools = filter_codex_apps_mcp_tools(mcp_tools, connectors); } Ok(Arc::new(ToolRouter::from_config( @@ -4747,6 +5003,7 @@ async fn built_tools( .map(|(name, tool)| (name, tool.tool)) .collect(), ), + app_tools, turn_context.dynamic_tools.as_slice(), ))) } @@ -5171,21 +5428,8 @@ async fn try_run_sampling_request( cancellation_token: CancellationToken, ) -> CodexResult { let collaboration_mode = sess.current_collaboration_mode().await; - let rollout_item = RolloutItem::TurnContext(TurnContextItem { - turn_id: Some(turn_context.sub_id.clone()), - cwd: turn_context.cwd.clone(), - approval_policy: turn_context.approval_policy, - sandbox_policy: turn_context.sandbox_policy.clone(), - model: turn_context.model_info.slug.clone(), - personality: turn_context.personality, - collaboration_mode: Some(collaboration_mode), - effort: turn_context.reasoning_effort, - summary: turn_context.reasoning_summary, - user_instructions: turn_context.user_instructions.clone(), - developer_instructions: turn_context.developer_instructions.clone(), - final_output_json_schema: turn_context.final_output_json_schema.clone(), - truncation_policy: Some(turn_context.truncation_policy.into()), - }); + let rollout_item = + RolloutItem::TurnContext(turn_context.to_turn_context_item(collaboration_mode)); feedback_tags!( model = turn_context.model_info.slug.clone(), @@ -5485,6 +5729,11 @@ mod tests { use crate::CodexAuth; use crate::config::ConfigBuilder; use crate::config::test_config; + use crate::config_loader::ConfigLayerStack; + use crate::config_loader::ConfigLayerStackOrdering; + use crate::config_loader::NetworkConstraints; + use crate::config_loader::RequirementSource; + use crate::config_loader::Sourced; use crate::exec::ExecToolCallOutput; use crate::function_tool::FunctionCallError; use crate::mcp_connection_manager::ToolInfo; @@ -5577,6 +5826,7 @@ mod tests { distribution_channel: None, install_url: None, is_accessible: true, + is_enabled: true, } } @@ -5605,6 +5855,22 @@ mod tests { } } + fn function_call_rollout_item(name: &str, call_id: &str) -> RolloutItem { + RolloutItem::ResponseItem(ResponseItem::FunctionCall { + id: None, + name: name.to_string(), + arguments: "{}".to_string(), + call_id: call_id.to_string(), + }) + } + + fn function_call_output_rollout_item(call_id: &str, output: &str) -> RolloutItem { + RolloutItem::ResponseItem(ResponseItem::FunctionCallOutput { + call_id: call_id.to_string(), + output: FunctionCallOutputPayload::from_text(output.to_string()), + }) + } + #[tokio::test] async fn get_base_instructions_no_user_content() { let prompt_with_apply_patch_instructions = @@ -5662,6 +5928,42 @@ mod tests { } } + #[tokio::test] + async fn reload_user_config_layer_updates_effective_apps_config() { + let (session, _turn_context) = make_session_and_context().await; + let codex_home = session.codex_home().await; + std::fs::create_dir_all(&codex_home).expect("create codex home"); + let config_toml_path = codex_home.join(CONFIG_TOML_FILE); + std::fs::write( + &config_toml_path, + "[apps.calendar]\nenabled = false\ndisabled_reason = \"user\"\n", + ) + .expect("write user config"); + + session.reload_user_config_layer().await; + + let config = session.get_config().await; + let apps_toml = config + .config_layer_stack + .effective_config() + .as_table() + .and_then(|table| table.get("apps")) + .cloned() + .expect("apps table"); + let apps = crate::config::types::AppsConfigToml::deserialize(apps_toml) + .expect("deserialize apps config"); + let app = apps + .apps + .get("calendar") + .expect("calendar app config exists"); + + assert!(!app.enabled); + assert_eq!( + app.disabled_reason, + Some(crate::config::types::AppDisabledReason::User) + ); + } + #[test] fn filter_connectors_for_input_skips_duplicate_slug_mentions() { let connectors = vec![ @@ -5673,7 +5975,7 @@ mod tests { let skill_name_counts_lower = HashMap::new(); let selected = filter_connectors_for_input( - connectors, + &connectors, &input, &explicitly_enabled_connectors, &skill_name_counts_lower, @@ -5690,7 +5992,7 @@ mod tests { let skill_name_counts_lower = HashMap::from([("todoist".to_string(), 1)]); let selected = filter_connectors_for_input( - connectors, + &connectors, &input, &explicitly_enabled_connectors, &skill_name_counts_lower, @@ -5699,6 +6001,22 @@ mod tests { assert_eq!(selected, Vec::new()); } + #[test] + fn filter_connectors_for_input_skips_disabled_connectors() { + let mut connector = make_connector("calendar", "Calendar"); + connector.is_enabled = false; + let input = vec![user_message("use $calendar")]; + let explicitly_enabled_connectors = HashSet::new(); + let selected = filter_connectors_for_input( + &[connector], + &input, + &explicitly_enabled_connectors, + &HashMap::new(), + ); + + assert_eq!(selected, Vec::new()); + } + #[test] fn collect_explicit_app_ids_from_skill_items_includes_linked_mentions() { let connectors = vec![make_connector("calendar", "Calendar")]; @@ -5742,6 +6060,46 @@ mod tests { assert_eq!(connector_ids, HashSet::::new()); } + #[test] + fn non_app_mcp_tools_remain_visible_without_search_selection() { + let mcp_tools = HashMap::from([ + ( + "mcp__codex_apps__calendar_create_event".to_string(), + make_mcp_tool( + CODEX_APPS_MCP_SERVER_NAME, + "calendar_create_event", + Some("calendar"), + Some("Calendar"), + ), + ), + ( + "mcp__rmcp__echo".to_string(), + make_mcp_tool("rmcp", "echo", None, None), + ), + ]); + + let mut selected_mcp_tools = mcp_tools + .iter() + .filter(|(_, tool)| tool.server_name != CODEX_APPS_MCP_SERVER_NAME) + .map(|(name, tool)| (name.clone(), tool.clone())) + .collect::>(); + + let connectors = connectors::accessible_connectors_from_mcp_tools(&mcp_tools); + let explicitly_enabled_connectors = HashSet::new(); + let connectors = filter_connectors_for_input( + &connectors, + &[user_message("run echo")], + &explicitly_enabled_connectors, + &HashMap::new(), + ); + let apps_mcp_tools = filter_codex_apps_mcp_tools_only(&mcp_tools, &connectors); + selected_mcp_tools.extend(apps_mcp_tools); + + let mut tool_names: Vec = selected_mcp_tools.into_keys().collect(); + tool_names.sort(); + assert_eq!(tool_names, vec!["mcp__rmcp__echo".to_string()]); + } + #[test] fn search_tool_selection_keeps_codex_apps_tools_without_mentions() { let selected_tool_names = vec![ @@ -5764,17 +6122,16 @@ mod tests { ), ]); - let mut selected_mcp_tools = - filter_mcp_tools_by_name(mcp_tools.clone(), &selected_tool_names); + let mut selected_mcp_tools = filter_mcp_tools_by_name(&mcp_tools, &selected_tool_names); let connectors = connectors::accessible_connectors_from_mcp_tools(&mcp_tools); let explicitly_enabled_connectors = HashSet::new(); let connectors = filter_connectors_for_input( - connectors, + &connectors, &[user_message("run the selected tools")], &explicitly_enabled_connectors, &HashMap::new(), ); - let apps_mcp_tools = filter_codex_apps_mcp_tools_only(mcp_tools, &connectors); + let apps_mcp_tools = filter_codex_apps_mcp_tools_only(&mcp_tools, &connectors); selected_mcp_tools.extend(apps_mcp_tools); let mut tool_names: Vec = selected_mcp_tools.into_keys().collect(); @@ -5807,17 +6164,16 @@ mod tests { ), ]); - let mut selected_mcp_tools = - filter_mcp_tools_by_name(mcp_tools.clone(), &selected_tool_names); + let mut selected_mcp_tools = filter_mcp_tools_by_name(&mcp_tools, &selected_tool_names); let connectors = connectors::accessible_connectors_from_mcp_tools(&mcp_tools); let explicitly_enabled_connectors = HashSet::new(); let connectors = filter_connectors_for_input( - connectors, + &connectors, &[user_message("use $calendar and then echo the response")], &explicitly_enabled_connectors, &HashMap::new(), ); - let apps_mcp_tools = filter_codex_apps_mcp_tools_only(mcp_tools, &connectors); + let apps_mcp_tools = filter_codex_apps_mcp_tools_only(&mcp_tools, &connectors); selected_mcp_tools.extend(apps_mcp_tools); let mut tool_names: Vec = selected_mcp_tools.into_keys().collect(); @@ -5831,6 +6187,106 @@ mod tests { ); } + #[test] + fn extract_mcp_tool_selection_from_rollout_reads_search_tool_output() { + let rollout_items = vec![ + function_call_rollout_item(SEARCH_TOOL_BM25_TOOL_NAME, "search-1"), + function_call_output_rollout_item( + "search-1", + &json!({ + "active_selected_tools": [ + "mcp__codex_apps__calendar_create_event", + "mcp__codex_apps__calendar_list_events", + ], + }) + .to_string(), + ), + ]; + + let selected = Session::extract_mcp_tool_selection_from_rollout(&rollout_items); + assert_eq!( + selected, + Some(vec![ + "mcp__codex_apps__calendar_create_event".to_string(), + "mcp__codex_apps__calendar_list_events".to_string(), + ]) + ); + } + + #[test] + fn extract_mcp_tool_selection_from_rollout_latest_valid_payload_wins() { + let rollout_items = vec![ + function_call_rollout_item(SEARCH_TOOL_BM25_TOOL_NAME, "search-1"), + function_call_output_rollout_item( + "search-1", + &json!({ + "active_selected_tools": ["mcp__codex_apps__calendar_create_event"], + }) + .to_string(), + ), + function_call_rollout_item(SEARCH_TOOL_BM25_TOOL_NAME, "search-2"), + function_call_output_rollout_item( + "search-2", + &json!({ + "active_selected_tools": ["mcp__codex_apps__calendar_delete_event"], + }) + .to_string(), + ), + ]; + + let selected = Session::extract_mcp_tool_selection_from_rollout(&rollout_items); + assert_eq!( + selected, + Some(vec!["mcp__codex_apps__calendar_delete_event".to_string(),]) + ); + } + + #[test] + fn extract_mcp_tool_selection_from_rollout_ignores_non_search_and_malformed_payloads() { + let rollout_items = vec![ + function_call_rollout_item("shell", "shell-1"), + function_call_output_rollout_item( + "shell-1", + &json!({ + "active_selected_tools": ["mcp__codex_apps__should_be_ignored"], + }) + .to_string(), + ), + function_call_rollout_item(SEARCH_TOOL_BM25_TOOL_NAME, "search-1"), + function_call_output_rollout_item("search-1", "{not-json"), + function_call_output_rollout_item( + "unknown-search-call", + &json!({ + "active_selected_tools": ["mcp__codex_apps__also_ignored"], + }) + .to_string(), + ), + function_call_output_rollout_item( + "search-1", + &json!({ + "active_selected_tools": ["mcp__codex_apps__calendar_list_events"], + }) + .to_string(), + ), + ]; + + let selected = Session::extract_mcp_tool_selection_from_rollout(&rollout_items); + assert_eq!( + selected, + Some(vec!["mcp__codex_apps__calendar_list_events".to_string(),]) + ); + } + + #[test] + fn extract_mcp_tool_selection_from_rollout_returns_none_without_valid_search_output() { + let rollout_items = vec![function_call_rollout_item( + SEARCH_TOOL_BM25_TOOL_NAME, + "search-1", + )]; + let selected = Session::extract_mcp_tool_selection_from_rollout(&rollout_items); + assert_eq!(selected, None); + } + #[tokio::test] async fn reconstruct_history_matches_live_compactions() { let (session, turn_context) = make_session_and_context().await; @@ -5906,6 +6362,7 @@ mod tests { cwd: turn_context.cwd.clone(), approval_policy: turn_context.approval_policy, sandbox_policy: turn_context.sandbox_policy.clone(), + network: None, model: previous_model.to_string(), personality: turn_context.personality, collaboration_mode: Some(turn_context.collaboration_mode.clone()), @@ -6124,6 +6581,7 @@ mod tests { cwd: turn_context.cwd.clone(), approval_policy: turn_context.approval_policy, sandbox_policy: turn_context.sandbox_policy.clone(), + network: None, model: previous_model.to_string(), personality: turn_context.personality, collaboration_mode: Some(turn_context.collaboration_mode.clone()), @@ -6793,6 +7251,7 @@ mod tests { let mut state = SessionState::new(session_configuration.clone()); mark_state_initial_context_seeded(&mut state); let skills_manager = Arc::new(SkillsManager::new(config.codex_home.clone())); + let network_approval = Arc::new(NetworkApprovalService::default()); let file_watcher = Arc::new(FileWatcher::noop()); let services = SessionServices { @@ -6819,6 +7278,7 @@ mod tests { file_watcher, agent_control, network_proxy: None, + network_approval: Arc::clone(&network_approval), state_db: None, model_client: ModelClient::new( Some(auth_manager.clone()), @@ -6939,6 +7399,7 @@ mod tests { let mut state = SessionState::new(session_configuration.clone()); mark_state_initial_context_seeded(&mut state); let skills_manager = Arc::new(SkillsManager::new(config.codex_home.clone())); + let network_approval = Arc::new(NetworkApprovalService::default()); let file_watcher = Arc::new(FileWatcher::noop()); let services = SessionServices { @@ -6965,6 +7426,7 @@ mod tests { file_watcher, agent_control, network_proxy: None, + network_approval: Arc::clone(&network_approval), state_db: None, model_client: ModelClient::new( Some(Arc::clone(&auth_manager)), @@ -7114,6 +7576,61 @@ mod tests { ); } + #[tokio::test] + async fn build_settings_update_items_emits_environment_item_for_network_changes() { + let (session, previous_context) = make_session_and_context().await; + let previous_context = Arc::new(previous_context); + let mut current_context = previous_context + .with_model( + previous_context.model_info.slug.clone(), + &session.services.models_manager, + ) + .await; + + let mut config = (*current_context.config).clone(); + let mut requirements = config.config_layer_stack.requirements().clone(); + requirements.network = Some(Sourced::new( + NetworkConstraints { + allowed_domains: Some(vec!["api.example.com".to_string()]), + denied_domains: Some(vec!["blocked.example.com".to_string()]), + ..Default::default() + }, + RequirementSource::CloudRequirements, + )); + let layers = config + .config_layer_stack + .get_layers(ConfigLayerStackOrdering::LowestPrecedenceFirst, true) + .into_iter() + .cloned() + .collect(); + config.config_layer_stack = ConfigLayerStack::new( + layers, + requirements, + config.config_layer_stack.requirements_toml().clone(), + ) + .expect("rebuild config layer stack with network requirements"); + current_context.config = Arc::new(config); + + let update_items = + session.build_settings_update_items(Some(&previous_context), None, ¤t_context); + + let environment_update = update_items + .iter() + .find_map(|item| match item { + ResponseItem::Message { role, content, .. } if role == "user" => { + let [ContentItem::InputText { text }] = content.as_slice() else { + return None; + }; + text.contains("").then_some(text) + } + _ => None, + }) + .expect("environment update item should be emitted"); + assert!(environment_update.contains("")); + assert!(environment_update.contains("api.example.com")); + assert!(environment_update.contains("blocked.example.com")); + } + #[derive(Clone, Copy)] struct NeverEndingTask { kind: TaskKind, @@ -7424,6 +7941,7 @@ mod tests { .list_all_tools() .await }; + let app_tools = Some(tools.clone()); let router = ToolRouter::from_config( &turn_context.tools_config, Some( @@ -7432,6 +7950,7 @@ mod tests { .map(|(name, tool)| (name, tool.tool)) .collect(), ), + app_tools, turn_context.dynamic_tools.as_slice(), ); let item = ResponseItem::CustomToolCall { @@ -7670,6 +8189,7 @@ mod tests { expiration: timeout_ms.into(), env: HashMap::new(), network: None, + network_attempt_id: None, sandbox_permissions, windows_sandbox_level: turn_context.windows_sandbox_level, justification: Some("test".to_string()), @@ -7683,6 +8203,7 @@ mod tests { expiration: timeout_ms.into(), env: HashMap::new(), network: None, + network_attempt_id: None, windows_sandbox_level: turn_context.windows_sandbox_level, justification: params.justification.clone(), arg0: None, diff --git a/codex-rs/core/src/codex_delegate.rs b/codex-rs/core/src/codex_delegate.rs index 1f4240f683..69efeefeca 100644 --- a/codex-rs/core/src/codex_delegate.rs +++ b/codex-rs/core/src/codex_delegate.rs @@ -316,6 +316,7 @@ async fn handle_exec_approval( command, cwd, reason, + network_approval_context, proposed_execpolicy_amendment, .. } = event; @@ -327,6 +328,7 @@ async fn handle_exec_approval( command, cwd, reason, + network_approval_context, proposed_execpolicy_amendment, ); let decision = diff --git a/codex-rs/core/src/compact.rs b/codex-rs/core/src/compact.rs index f7de93dfe5..a40f1a0b8e 100644 --- a/codex-rs/core/src/compact.rs +++ b/codex-rs/core/src/compact.rs @@ -7,11 +7,11 @@ use crate::client_common::ResponseEvent; use crate::codex::Session; use crate::codex::TurnContext; use crate::codex::get_last_assistant_message_from_turn; +use crate::context_manager::ContextManager; use crate::error::CodexErr; use crate::error::Result as CodexResult; use crate::protocol::CompactedItem; use crate::protocol::EventMsg; -use crate::protocol::TurnContextItem; use crate::protocol::TurnStartedEvent; use crate::protocol::WarningEvent; use crate::truncate::TruncationPolicy; @@ -36,6 +36,31 @@ pub(crate) fn should_use_remote_compact_task(provider: &ModelProviderInfo) -> bo provider.is_openai() } +pub(crate) fn extract_trailing_model_switch_update_for_compaction_request( + history: &mut ContextManager, +) -> Option { + let history_items = history.raw_items(); + let last_user_turn_boundary_index = history_items + .iter() + .rposition(crate::context_manager::is_user_turn_boundary); + let model_switch_index = history_items + .iter() + .enumerate() + .rev() + .find_map(|(i, item)| { + let is_trailing = last_user_turn_boundary_index.is_none_or(|boundary| i > boundary); + if is_trailing && Session::is_model_switch_developer_message(item) { + Some(i) + } else { + None + } + })?; + let mut replacement = history_items.to_vec(); + let model_switch_item = replacement.remove(model_switch_index); + history.replace(replacement); + Some(model_switch_item) +} + pub(crate) async fn run_inline_auto_compact_task( sess: Arc, turn_context: Arc, @@ -76,6 +101,10 @@ async fn run_compact_task_inner( let initial_input_for_turn: ResponseInputItem = ResponseInputItem::from(input); let mut history = sess.clone_history().await; + // Keep compaction prompts in-distribution: if a model-switch update was injected at the + // tail of history (between turns), exclude it from the compaction request payload. + let stripped_model_switch_item = + extract_trailing_model_switch_update_for_compaction_request(&mut history); history.record_items( &[initial_input_for_turn.into()], turn_context.truncation_policy, @@ -85,7 +114,6 @@ async fn run_compact_task_inner( let max_retries = turn_context.provider.stream_max_retries(); let mut retries = 0; - let turn_metadata_header = turn_context.resolve_turn_metadata_header().await; let mut client_session = sess.services.model_client.new_session(); // Reuse one client session so turn-scoped state (sticky routing, websocket append tracking) // survives retries within this compact turn. @@ -95,21 +123,8 @@ async fn run_compact_task_inner( // duplicating model settings on TurnContext, but an Op after turn start could update the // session config before this write occurs. let collaboration_mode = sess.current_collaboration_mode().await; - let rollout_item = RolloutItem::TurnContext(TurnContextItem { - turn_id: Some(turn_context.sub_id.clone()), - cwd: turn_context.cwd.clone(), - approval_policy: turn_context.approval_policy, - sandbox_policy: turn_context.sandbox_policy.clone(), - model: turn_context.model_info.slug.clone(), - personality: turn_context.personality, - collaboration_mode: Some(collaboration_mode), - effort: turn_context.reasoning_effort, - summary: turn_context.reasoning_summary, - user_instructions: turn_context.user_instructions.clone(), - developer_instructions: turn_context.developer_instructions.clone(), - final_output_json_schema: turn_context.final_output_json_schema.clone(), - truncation_policy: Some(turn_context.truncation_policy.into()), - }); + let rollout_item = + RolloutItem::TurnContext(turn_context.to_turn_context_item(collaboration_mode)); sess.persist_rollout_items(&[rollout_item]).await; loop { @@ -124,6 +139,7 @@ async fn run_compact_task_inner( personality: turn_context.personality, ..Default::default() }; + let turn_metadata_header = turn_context.turn_metadata_state.current_header_value(); let attempt_result = drain_to_completed( &sess, turn_context.as_ref(), @@ -194,6 +210,11 @@ async fn run_compact_task_inner( let initial_context = sess.build_initial_context(turn_context.as_ref()).await; let mut new_history = build_compacted_history(initial_context, &user_messages, &summary_text); + // Reattach the stripped model-switch update only after successful compaction so the model + // still sees the switch instructions on the next real sampling request. + if let Some(model_switch_item) = stripped_model_switch_item { + new_history.push(model_switch_item); + } let ghost_snapshots: Vec = history_items .iter() .filter(|item| matches!(item, ResponseItem::GhostSnapshot { .. })) @@ -457,6 +478,107 @@ mod tests { assert_eq!(None, joined); } + #[test] + fn extract_trailing_model_switch_update_for_compaction_request_removes_trailing_item() { + let mut history = ContextManager::new(); + history.replace(vec![ + ResponseItem::Message { + id: None, + role: "user".to_string(), + content: vec![ContentItem::InputText { + text: "USER_MESSAGE".to_string(), + }], + end_turn: None, + phase: None, + }, + ResponseItem::Message { + id: None, + role: "assistant".to_string(), + content: vec![ContentItem::OutputText { + text: "ASSISTANT_REPLY".to_string(), + }], + end_turn: None, + phase: None, + }, + ResponseItem::Message { + id: None, + role: "developer".to_string(), + content: vec![ContentItem::InputText { + text: "\nNEW_MODEL_INSTRUCTIONS".to_string(), + }], + end_turn: None, + phase: None, + }, + ]); + + let model_switch_item = + extract_trailing_model_switch_update_for_compaction_request(&mut history); + + assert_eq!(history.raw_items().len(), 2); + assert!(model_switch_item.is_some()); + assert!( + history + .raw_items() + .iter() + .all(|item| !Session::is_model_switch_developer_message(item)) + ); + } + + #[test] + fn extract_trailing_model_switch_update_for_compaction_request_keeps_historical_item() { + let mut history = ContextManager::new(); + history.replace(vec![ + ResponseItem::Message { + id: None, + role: "user".to_string(), + content: vec![ContentItem::InputText { + text: "FIRST_USER_MESSAGE".to_string(), + }], + end_turn: None, + phase: None, + }, + ResponseItem::Message { + id: None, + role: "developer".to_string(), + content: vec![ContentItem::InputText { + text: "\nOLDER_MODEL_INSTRUCTIONS".to_string(), + }], + end_turn: None, + phase: None, + }, + ResponseItem::Message { + id: None, + role: "assistant".to_string(), + content: vec![ContentItem::OutputText { + text: "ASSISTANT_REPLY".to_string(), + }], + end_turn: None, + phase: None, + }, + ResponseItem::Message { + id: None, + role: "user".to_string(), + content: vec![ContentItem::InputText { + text: "SECOND_USER_MESSAGE".to_string(), + }], + end_turn: None, + phase: None, + }, + ]); + + let model_switch_item = + extract_trailing_model_switch_update_for_compaction_request(&mut history); + + assert_eq!(history.raw_items().len(), 4); + assert!(model_switch_item.is_none()); + assert!( + history + .raw_items() + .iter() + .any(Session::is_model_switch_developer_message) + ); + } + #[test] fn collect_user_messages_extracts_user_text_only() { let items = vec![ diff --git a/codex-rs/core/src/compact_remote.rs b/codex-rs/core/src/compact_remote.rs index 4fe07560b9..3d181f8855 100644 --- a/codex-rs/core/src/compact_remote.rs +++ b/codex-rs/core/src/compact_remote.rs @@ -3,6 +3,7 @@ use std::sync::Arc; use crate::Prompt; use crate::codex::Session; use crate::codex::TurnContext; +use crate::compact::extract_trailing_model_switch_update_for_compaction_request; use crate::context_manager::ContextManager; use crate::context_manager::TotalTokenUsageBreakdown; use crate::context_manager::estimate_response_item_model_visible_bytes; @@ -65,6 +66,10 @@ async fn run_remote_compact_task_inner_impl( sess.emit_turn_item_started(turn_context, &compaction_item) .await; let mut history = sess.clone_history().await; + // Keep compaction prompts in-distribution: if a model-switch update was injected at the + // tail of history (between turns), exclude it from the compaction request payload. + let stripped_model_switch_item = + extract_trailing_model_switch_update_for_compaction_request(&mut history); let base_instructions = sess.get_base_instructions().await; let deleted_items = trim_function_call_history_to_fit_context_window( &mut history, @@ -120,6 +125,11 @@ async fn run_remote_compact_task_inner_impl( new_history = sess .process_compacted_history(turn_context, new_history) .await; + // Reattach the stripped model-switch update only after successful compaction so the model + // still sees the switch instructions on the next real sampling request. + if let Some(model_switch_item) = stripped_model_switch_item { + new_history.push(model_switch_item); + } if !ghost_snapshots.is_empty() { new_history.extend(ghost_snapshots); diff --git a/codex-rs/core/src/config/mod.rs b/codex-rs/core/src/config/mod.rs index d5de27a40d..5d887a6a59 100644 --- a/codex-rs/core/src/config/mod.rs +++ b/codex-rs/core/src/config/mod.rs @@ -7,6 +7,8 @@ use crate::config::types::History; use crate::config::types::McpServerConfig; use crate::config::types::McpServerDisabledReason; use crate::config::types::McpServerTransportConfig; +use crate::config::types::MemoriesConfig; +use crate::config::types::MemoriesToml; use crate::config::types::Notice; use crate::config::types::NotificationMethod; use crate::config::types::Notifications; @@ -47,6 +49,8 @@ use crate::project_doc::LOCAL_PROJECT_DOC_FILENAME; use crate::protocol::AskForApproval; use crate::protocol::ReadOnlyAccess; use crate::protocol::SandboxPolicy; +#[cfg(target_os = "macos")] +use crate::seatbelt_permissions::MacOsSeatbeltProfileExtensions; use crate::windows_sandbox::WindowsSandboxLevelExt; use crate::windows_sandbox::resolve_windows_sandbox_mode; use codex_app_server_protocol::Tools; @@ -76,6 +80,8 @@ use std::path::Path; use std::path::PathBuf; #[cfg(test)] use tempfile::tempdir; +#[cfg(not(target_os = "macos"))] +type MacOsSeatbeltProfileExtensions = (); use crate::config::profile::ConfigProfile; use toml::Value as TomlValue; @@ -131,6 +137,9 @@ pub struct Permissions { /// Effective Windows sandbox mode derived from `[windows].sandbox` or /// legacy feature keys. pub windows_sandbox_mode: Option, + /// Optional macOS seatbelt extension profile used to extend default + /// seatbelt permissions when running under seatbelt. + pub macos_seatbelt_profile_extensions: Option, } /// Application configuration loaded from disk and merged with overrides. @@ -289,6 +298,9 @@ pub struct Config { /// Maximum number of agent threads that can be open concurrently. pub agent_max_threads: Option, + /// Memories subsystem settings. + pub memories: MemoriesConfig, + /// Directory containing all Codex state (defaults to `~/.codex` but can be /// overridden by the `CODEX_HOME` environment variable). pub codex_home: PathBuf, @@ -1006,6 +1018,9 @@ pub struct ConfigToml { /// Agent-related settings (thread limits, etc.). pub agents: Option, + /// Memories subsystem settings. + pub memories: Option, + /// User-level skill config entries keyed by SKILL.md path. pub skills: Option, @@ -1737,6 +1752,7 @@ impl Config { network, shell_environment_policy, windows_sandbox_mode, + macos_seatbelt_profile_extensions: None, }, enforce_residency: enforce_residency.value, did_user_set_custom_approval_policy_or_sandbox_mode, @@ -1771,6 +1787,7 @@ impl Config { .collect(), tool_output_token_limit: cfg.tool_output_token_limit, agent_max_threads, + memories: cfg.memories.unwrap_or_default().into(), codex_home, log_dir, config_layer_stack, @@ -1933,6 +1950,13 @@ impl Config { self.permissions.windows_sandbox_mode }; } + + pub fn managed_network_requirements_enabled(&self) -> bool { + self.config_layer_stack + .requirements_toml() + .network + .is_some() + } } pub(crate) fn uses_deprecated_instructions_file(config_layer_stack: &ConfigLayerStack) -> bool { @@ -1985,6 +2009,8 @@ mod tests { use crate::config::types::FeedbackConfigToml; use crate::config::types::HistoryPersistence; use crate::config::types::McpServerTransportConfig; + use crate::config::types::MemoriesConfig; + use crate::config::types::MemoriesToml; use crate::config::types::NotificationMethod; use crate::config::types::Notifications; use crate::config_loader::RequirementSource; @@ -2068,6 +2094,47 @@ persistence = "none" }), history_no_persistence_cfg.history ); + + let memories = r#" +[memories] +max_raw_memories_for_global = 512 +max_rollout_age_days = 42 +max_rollouts_per_startup = 9 +min_rollout_idle_hours = 24 +phase_1_model = "gpt-5-mini" +phase_2_model = "gpt-5" +"#; + let memories_cfg = + toml::from_str::(memories).expect("TOML deserialization should succeed"); + assert_eq!( + Some(MemoriesToml { + max_raw_memories_for_global: Some(512), + max_rollout_age_days: Some(42), + max_rollouts_per_startup: Some(9), + min_rollout_idle_hours: Some(24), + phase_1_model: Some("gpt-5-mini".to_string()), + phase_2_model: Some("gpt-5".to_string()), + }), + memories_cfg.memories + ); + + let config = Config::load_from_base_config_with_overrides( + memories_cfg, + ConfigOverrides::default(), + tempdir().expect("tempdir").path().to_path_buf(), + ) + .expect("load config from memories settings"); + assert_eq!( + config.memories, + MemoriesConfig { + max_raw_memories_for_global: 512, + max_rollout_age_days: 42, + max_rollouts_per_startup: 9, + min_rollout_idle_hours: 24, + phase_1_model: Some("gpt-5-mini".to_string()), + phase_2_model: Some("gpt-5".to_string()), + } + ); } #[test] @@ -4032,6 +4099,7 @@ model_verbosity = "high" network: None, shell_environment_policy: ShellEnvironmentPolicy::default(), windows_sandbox_mode: None, + macos_seatbelt_profile_extensions: None, }, enforce_residency: Constrained::allow_any(None), did_user_set_custom_approval_policy_or_sandbox_mode: true, @@ -4047,6 +4115,7 @@ model_verbosity = "high" project_doc_fallback_filenames: Vec::new(), tool_output_token_limit: None, agent_max_threads: DEFAULT_AGENT_MAX_THREADS, + memories: MemoriesConfig::default(), codex_home: fixture.codex_home(), log_dir: fixture.codex_home().join("log"), config_layer_stack: Default::default(), @@ -4141,6 +4210,7 @@ model_verbosity = "high" network: None, shell_environment_policy: ShellEnvironmentPolicy::default(), windows_sandbox_mode: None, + macos_seatbelt_profile_extensions: None, }, enforce_residency: Constrained::allow_any(None), did_user_set_custom_approval_policy_or_sandbox_mode: true, @@ -4156,6 +4226,7 @@ model_verbosity = "high" project_doc_fallback_filenames: Vec::new(), tool_output_token_limit: None, agent_max_threads: DEFAULT_AGENT_MAX_THREADS, + memories: MemoriesConfig::default(), codex_home: fixture.codex_home(), log_dir: fixture.codex_home().join("log"), config_layer_stack: Default::default(), @@ -4248,6 +4319,7 @@ model_verbosity = "high" network: None, shell_environment_policy: ShellEnvironmentPolicy::default(), windows_sandbox_mode: None, + macos_seatbelt_profile_extensions: None, }, enforce_residency: Constrained::allow_any(None), did_user_set_custom_approval_policy_or_sandbox_mode: true, @@ -4263,6 +4335,7 @@ model_verbosity = "high" project_doc_fallback_filenames: Vec::new(), tool_output_token_limit: None, agent_max_threads: DEFAULT_AGENT_MAX_THREADS, + memories: MemoriesConfig::default(), codex_home: fixture.codex_home(), log_dir: fixture.codex_home().join("log"), config_layer_stack: Default::default(), @@ -4341,6 +4414,7 @@ model_verbosity = "high" network: None, shell_environment_policy: ShellEnvironmentPolicy::default(), windows_sandbox_mode: None, + macos_seatbelt_profile_extensions: None, }, enforce_residency: Constrained::allow_any(None), did_user_set_custom_approval_policy_or_sandbox_mode: true, @@ -4356,6 +4430,7 @@ model_verbosity = "high" project_doc_fallback_filenames: Vec::new(), tool_output_token_limit: None, agent_max_threads: DEFAULT_AGENT_MAX_THREADS, + memories: MemoriesConfig::default(), codex_home: fixture.codex_home(), log_dir: fixture.codex_home().join("log"), config_layer_stack: Default::default(), diff --git a/codex-rs/core/src/config/network_proxy_spec.rs b/codex-rs/core/src/config/network_proxy_spec.rs index ee3e8ebed1..183eafcb17 100644 --- a/codex-rs/core/src/config/network_proxy_spec.rs +++ b/codex-rs/core/src/config/network_proxy_spec.rs @@ -1,8 +1,11 @@ use crate::config; use crate::config_loader::NetworkConstraints; use async_trait::async_trait; +use codex_network_proxy::BlockedRequestObserver; use codex_network_proxy::ConfigReloader; use codex_network_proxy::ConfigState; +use codex_network_proxy::NetworkDecision; +use codex_network_proxy::NetworkPolicyDecider; use codex_network_proxy::NetworkProxy; use codex_network_proxy::NetworkProxyConfig; use codex_network_proxy::NetworkProxyConstraints; @@ -11,6 +14,7 @@ use codex_network_proxy::NetworkProxyState; use codex_network_proxy::build_config_state; use codex_network_proxy::host_and_port_from_network_addr; use codex_network_proxy::validate_policy_against_constraints; +use codex_protocol::protocol::SandboxPolicy; use std::sync::Arc; #[derive(Debug, Clone, PartialEq, Eq)] @@ -92,20 +96,41 @@ impl NetworkProxySpec { }) } - pub async fn start_proxy(&self) -> std::io::Result { + pub async fn start_proxy( + &self, + sandbox_policy: &SandboxPolicy, + policy_decider: Option>, + blocked_request_observer: Option>, + enable_network_approval_flow: bool, + ) -> std::io::Result { let state = build_config_state(self.config.clone(), self.constraints.clone()).map_err(|err| { std::io::Error::other(format!("failed to build network proxy state: {err}")) })?; let reloader = Arc::new(StaticNetworkProxyReloader::new(state.clone())); let state = NetworkProxyState::with_reloader(state, reloader); - let proxy = NetworkProxy::builder() - .state(Arc::new(state)) - .build() - .await - .map_err(|err| { - std::io::Error::other(format!("failed to build network proxy: {err}")) - })?; + let mut builder = NetworkProxy::builder().state(Arc::new(state)); + if enable_network_approval_flow + && matches!( + sandbox_policy, + SandboxPolicy::ReadOnly { .. } | SandboxPolicy::WorkspaceWrite { .. } + ) + { + builder = match policy_decider { + Some(policy_decider) => builder.policy_decider_arc(policy_decider), + None => builder.policy_decider(|_request| async { + // In restricted sandbox modes, allowlist misses should ask for + // explicit network approval instead of hard-denying. + NetworkDecision::ask("not_allowed") + }), + }; + } + if let Some(blocked_request_observer) = blocked_request_observer { + builder = builder.blocked_request_observer_arc(blocked_request_observer); + } + let proxy = builder.build().await.map_err(|err| { + std::io::Error::other(format!("failed to build network proxy: {err}")) + })?; let handle = proxy .run() .await diff --git a/codex-rs/core/src/config/types.rs b/codex-rs/core/src/config/types.rs index 5f54c1df28..80c16281b0 100644 --- a/codex-rs/core/src/config/types.rs +++ b/codex-rs/core/src/config/types.rs @@ -23,6 +23,10 @@ use serde::Serialize; use serde::de::Error as SerdeError; pub const DEFAULT_OTEL_ENVIRONMENT: &str = "dev"; +pub const DEFAULT_MEMORIES_MAX_ROLLOUTS_PER_STARTUP: usize = 8; +pub const DEFAULT_MEMORIES_MAX_ROLLOUT_AGE_DAYS: i64 = 30; +pub const DEFAULT_MEMORIES_MIN_ROLLOUT_IDLE_HOURS: i64 = 12; +pub const DEFAULT_MEMORIES_MAX_RAW_MEMORIES_FOR_GLOBAL: usize = 1_024; #[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, JsonSchema)] #[serde(rename_all = "kebab-case")] @@ -353,6 +357,74 @@ pub struct FeedbackConfigToml { pub enabled: Option, } +/// Memories settings loaded from config.toml. +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Default, JsonSchema)] +#[schemars(deny_unknown_fields)] +pub struct MemoriesToml { + /// Maximum number of recent raw memories retained for global consolidation. + pub max_raw_memories_for_global: Option, + /// Maximum age of the threads used for memories. + pub max_rollout_age_days: Option, + /// Maximum number of rollout candidates processed per pass. + pub max_rollouts_per_startup: Option, + /// Minimum idle time between last thread activity and memory creation (hours). > 12h recommended. + pub min_rollout_idle_hours: Option, + /// Model used for thread summarisation. + pub phase_1_model: Option, + /// Model used for memory consolidation. + pub phase_2_model: Option, +} + +/// Effective memories settings after defaults are applied. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct MemoriesConfig { + pub max_raw_memories_for_global: usize, + pub max_rollout_age_days: i64, + pub max_rollouts_per_startup: usize, + pub min_rollout_idle_hours: i64, + pub phase_1_model: Option, + pub phase_2_model: Option, +} + +impl Default for MemoriesConfig { + fn default() -> Self { + Self { + max_raw_memories_for_global: DEFAULT_MEMORIES_MAX_RAW_MEMORIES_FOR_GLOBAL, + max_rollout_age_days: DEFAULT_MEMORIES_MAX_ROLLOUT_AGE_DAYS, + max_rollouts_per_startup: DEFAULT_MEMORIES_MAX_ROLLOUTS_PER_STARTUP, + min_rollout_idle_hours: DEFAULT_MEMORIES_MIN_ROLLOUT_IDLE_HOURS, + phase_1_model: None, + phase_2_model: None, + } + } +} + +impl From for MemoriesConfig { + fn from(toml: MemoriesToml) -> Self { + let defaults = Self::default(); + Self { + max_raw_memories_for_global: toml + .max_raw_memories_for_global + .unwrap_or(defaults.max_raw_memories_for_global) + .min(4096), + max_rollout_age_days: toml + .max_rollout_age_days + .unwrap_or(defaults.max_rollout_age_days) + .clamp(0, 90), + max_rollouts_per_startup: toml + .max_rollouts_per_startup + .unwrap_or(defaults.max_rollouts_per_startup) + .min(128), + min_rollout_idle_hours: toml + .min_rollout_idle_hours + .unwrap_or(defaults.min_rollout_idle_hours) + .clamp(1, 48), + phase_1_model: toml.phase_1_model, + phase_2_model: toml.phase_2_model, + } + } +} + #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, JsonSchema)] #[serde(rename_all = "snake_case")] pub enum AppDisabledReason { diff --git a/codex-rs/core/src/connectors.rs b/codex-rs/core/src/connectors.rs index 1769446b66..35c2dc5ceb 100644 --- a/codex-rs/core/src/connectors.rs +++ b/codex-rs/core/src/connectors.rs @@ -9,6 +9,7 @@ use std::time::Instant; use async_channel::unbounded; pub use codex_app_server_protocol::AppInfo; use codex_protocol::protocol::SandboxPolicy; +use serde::Deserialize; use tokio_util::sync::CancellationToken; use tracing::warn; @@ -16,6 +17,7 @@ use crate::AuthManager; use crate::CodexAuth; use crate::SandboxState; use crate::config::Config; +use crate::config::types::AppsConfigToml; use crate::features::Feature; use crate::mcp::CODEX_APPS_MCP_SERVER_NAME; use crate::mcp::auth::compute_auth_statuses; @@ -50,6 +52,19 @@ pub async fn list_accessible_connectors_from_mcp_tools( list_accessible_connectors_from_mcp_tools_with_options(config, false).await } +pub async fn list_cached_accessible_connectors_from_mcp_tools( + config: &Config, +) -> Option> { + if !config.features.enabled(Feature::Apps) { + return Some(Vec::new()); + } + + let auth_manager = auth_manager_from_config(config); + let auth = auth_manager.auth().await; + let cache_key = accessible_connectors_cache_key(config, auth.as_ref()); + read_cached_accessible_connectors(&cache_key) +} + pub async fn list_accessible_connectors_from_mcp_tools_with_options( config: &Config, force_refetch: bool, @@ -265,6 +280,22 @@ pub fn merge_connectors( merged } +pub fn with_app_enabled_state(mut connectors: Vec, config: &Config) -> Vec { + let apps = read_apps_config(config).map(|apps_config| apps_config.apps); + for connector in &mut connectors { + if let Some(app) = apps.as_ref().and_then(|apps| apps.get(&connector.id)) { + connector.is_enabled = app.enabled; + } + } + connectors +} + +fn read_apps_config(config: &Config) -> Option { + let effective_config = config.config_layer_stack.effective_config(); + let apps_config = effective_config.as_table()?.get("apps")?.clone(); + AppsConfigToml::deserialize(apps_config).ok() +} + fn collect_accessible_connectors(tools: I) -> Vec where I: IntoIterator)>, @@ -291,6 +322,7 @@ where distribution_channel: None, install_url: Some(connector_install_url(&connector_name, &connector_id)), is_accessible: true, + is_enabled: true, }) .collect(); accessible.sort_by(|left, right| { diff --git a/codex-rs/core/src/error.rs b/codex-rs/core/src/error.rs index 404fbc5a7a..ad49c611fc 100644 --- a/codex-rs/core/src/error.rs +++ b/codex-rs/core/src/error.rs @@ -1,4 +1,5 @@ use crate::exec::ExecToolCallOutput; +use crate::network_policy_decision::NetworkPolicyDecisionPayload; use crate::token_data::KnownPlan; use crate::token_data::PlanType; use crate::truncate::TruncationPolicy; @@ -31,7 +32,10 @@ pub enum SandboxErr { "sandbox denied exec error, exit code: {}, stdout: {}, stderr: {}", .output.exit_code, .output.stdout.text, .output.stderr.text )] - Denied { output: Box }, + Denied { + output: Box, + network_policy_decision: Option, + }, /// Error from linux seccomp filter setup #[cfg(target_os = "linux")] @@ -616,7 +620,7 @@ impl CodexErr { pub fn get_error_message_ui(e: &CodexErr) -> String { let message = match e { - CodexErr::Sandbox(SandboxErr::Denied { output }) => { + CodexErr::Sandbox(SandboxErr::Denied { output, .. }) => { let aggregated = output.aggregated_output.text.trim(); if !aggregated.is_empty() { output.aggregated_output.text.clone() @@ -736,6 +740,7 @@ mod tests { }; let err = CodexErr::Sandbox(SandboxErr::Denied { output: Box::new(output), + network_policy_decision: None, }); assert_eq!(get_error_message_ui(&err), "aggregate detail"); } @@ -752,6 +757,7 @@ mod tests { }; let err = CodexErr::Sandbox(SandboxErr::Denied { output: Box::new(output), + network_policy_decision: None, }); assert_eq!(get_error_message_ui(&err), "stderr detail\nstdout detail"); } @@ -768,6 +774,7 @@ mod tests { }; let err = CodexErr::Sandbox(SandboxErr::Denied { output: Box::new(output), + network_policy_decision: None, }); assert_eq!(get_error_message_ui(&err), "stdout only"); } @@ -811,6 +818,7 @@ mod tests { }; let err = CodexErr::Sandbox(SandboxErr::Denied { output: Box::new(output), + network_policy_decision: None, }); assert_eq!( get_error_message_ui(&err), diff --git a/codex-rs/core/src/exec.rs b/codex-rs/core/src/exec.rs index 398a767ce0..0f0f0609a2 100644 --- a/codex-rs/core/src/exec.rs +++ b/codex-rs/core/src/exec.rs @@ -15,6 +15,7 @@ use tokio::io::AsyncReadExt; use tokio::io::BufReader; use tokio::process::Child; use tokio_util::sync::CancellationToken; +use uuid::Uuid; use crate::error::CodexErr; use crate::error::Result; @@ -66,6 +67,7 @@ pub struct ExecParams { pub expiration: ExecExpiration, pub env: HashMap, pub network: Option, + pub network_attempt_id: Option, pub sandbox_permissions: SandboxPermissions, pub windows_sandbox_level: codex_protocol::config_types::WindowsSandboxLevel, pub justification: Option, @@ -184,13 +186,15 @@ pub async fn process_exec_tool_call( mut env, expiration, network, + network_attempt_id, sandbox_permissions, windows_sandbox_level, justification, arg0: _, } = params; + let network_attempt_id = network_attempt_id.map(|attempt_id| attempt_id.to_string()); if let Some(network) = network.as_ref() { - network.apply_to_env(&mut env); + network.apply_to_env_for_attempt(&mut env, network_attempt_id.as_deref()); } let (program, args) = command.split_first().ok_or_else(|| { CodexErr::Io(io::Error::new( @@ -238,6 +242,7 @@ pub(crate) async fn execute_exec_env( cwd, env, network, + network_attempt_id, expiration, sandbox, windows_sandbox_level, @@ -246,12 +251,18 @@ pub(crate) async fn execute_exec_env( arg0, } = env; + let network_attempt_id = match network_attempt_id.as_deref() { + Some(attempt_id) => Uuid::parse_str(attempt_id).ok(), + None => network.as_ref().map(|_| Uuid::new_v4()), + }; + let params = ExecParams { command, cwd, expiration, env, - network, + network: network.clone(), + network_attempt_id, sandbox_permissions, windows_sandbox_level, justification, @@ -345,12 +356,14 @@ async fn exec_windows_sandbox( cwd, mut env, network, + network_attempt_id, expiration, windows_sandbox_level, .. } = params; + let network_attempt_id = network_attempt_id.map(|attempt_id| attempt_id.to_string()); if let Some(network) = network.as_ref() { - network.apply_to_env(&mut env); + network.apply_to_env_for_attempt(&mut env, network_attempt_id.as_deref()); } // TODO(iceweasel-oai): run_windows_sandbox_capture should support all @@ -490,6 +503,7 @@ fn finalize_exec_result( if is_likely_sandbox_denied(sandbox_type, &exec_output) { return Err(CodexErr::Sandbox(SandboxErr::Denied { output: Box::new(exec_output), + network_policy_decision: None, })); } @@ -701,13 +715,19 @@ async fn exec( let ExecParams { command, cwd, - env, + mut env, network, + network_attempt_id, arg0, expiration, windows_sandbox_level: _, .. } = params; + let network_attempt_id = network_attempt_id.map(|attempt_id| attempt_id.to_string()); + + if let Some(network) = network.as_ref() { + network.apply_to_env_for_attempt(&mut env, network_attempt_id.as_deref()); + } let (program, args) = command.split_first().ok_or_else(|| { CodexErr::Io(io::Error::new( @@ -722,7 +742,10 @@ async fn exec( arg0: arg0_ref, cwd, sandbox_policy, - network: network.as_ref(), + // The environment already has attempt-scoped proxy settings from + // apply_to_env_for_attempt above. Passing network here would reapply + // non-attempt proxy vars and drop attempt correlation metadata. + network: None, stdio_policy: StdioPolicy::RedirectForShellTool, env, }) @@ -951,6 +974,17 @@ mod tests { assert!(!is_likely_sandbox_denied(SandboxType::None, &output)); } + #[test] + fn sandbox_detection_ignores_network_policy_text_in_non_sandbox_mode() { + let output = make_exec_output( + 0, + "", + "", + r#"CODEX_NETWORK_POLICY_DECISION {"decision":"ask","reason":"not_allowed","source":"decider","protocol":"http","host":"google.com","port":80}"#, + ); + assert!(!is_likely_sandbox_denied(SandboxType::None, &output)); + } + #[test] fn sandbox_detection_uses_aggregated_output() { let output = make_exec_output( @@ -965,6 +999,21 @@ mod tests { )); } + #[test] + fn sandbox_detection_ignores_network_policy_text_with_zero_exit_code() { + let output = make_exec_output( + 0, + "", + "", + r#"CODEX_NETWORK_POLICY_DECISION {"decision":"ask","source":"decider","protocol":"http","host":"google.com","port":80}"#, + ); + + assert!(!is_likely_sandbox_denied( + SandboxType::LinuxSeccomp, + &output + )); + } + #[tokio::test] async fn read_capped_limits_retained_bytes() { let (mut writer, reader) = tokio::io::duplex(1024); @@ -1088,6 +1137,7 @@ mod tests { expiration: 500.into(), env, network: None, + network_attempt_id: None, sandbox_permissions: SandboxPermissions::UseDefault, windows_sandbox_level: codex_protocol::config_types::WindowsSandboxLevel::Disabled, justification: None, @@ -1141,6 +1191,7 @@ mod tests { expiration: ExecExpiration::Cancellation(cancel_token), env, network: None, + network_attempt_id: None, sandbox_permissions: SandboxPermissions::UseDefault, windows_sandbox_level: codex_protocol::config_types::WindowsSandboxLevel::Disabled, justification: None, diff --git a/codex-rs/core/src/exec_policy.rs b/codex-rs/core/src/exec_policy.rs index 60740d8884..298bc2679e 100644 --- a/codex-rs/core/src/exec_policy.rs +++ b/codex-rs/core/src/exec_policy.rs @@ -35,6 +35,54 @@ const PROMPT_CONFLICT_REASON: &str = const RULES_DIR_NAME: &str = "rules"; const RULE_EXTENSION: &str = "rules"; const DEFAULT_POLICY_FILE: &str = "default.rules"; +static BANNED_PREFIX_SUGGESTIONS: &[&[&str]] = &[ + &["python3"], + &["python3", "-"], + &["python3", "-c"], + &["python"], + &["python", "-"], + &["python", "-c"], + &["py"], + &["py", "-3"], + &["pythonw"], + &["pyw"], + &["pypy"], + &["pypy3"], + &["git"], + &["bash"], + &["bash", "-lc"], + &["sh"], + &["sh", "-c"], + &["sh", "-lc"], + &["zsh"], + &["zsh", "-lc"], + &["/bin/zsh"], + &["/bin/zsh", "-lc"], + &["/bin/bash"], + &["/bin/bash", "-lc"], + &["pwsh"], + &["pwsh", "-Command"], + &["pwsh", "-c"], + &["powershell"], + &["powershell", "-Command"], + &["powershell", "-c"], + &["powershell.exe"], + &["powershell.exe", "-Command"], + &["powershell.exe", "-c"], + &["env"], + &["sudo"], + &["node"], + &["node", "-e"], + &["perl"], + &["perl", "-e"], + &["ruby"], + &["ruby", "-e"], + &["php"], + &["php", "-r"], + &["lua"], + &["lua", "-e"], + &["osascript"], +]; fn is_policy_match(rule_match: &RuleMatch) -> bool { match rule_match { @@ -217,6 +265,77 @@ pub async fn check_execpolicy_for_warnings( Ok(warning) } +fn exec_policy_message_for_display(source: &codex_execpolicy::Error) -> String { + let message = source.to_string(); + if let Some(line) = message + .lines() + .find(|line| line.trim_start().starts_with("error: ")) + { + return line.to_owned(); + } + if let Some(first_line) = message.lines().next() + && let Some((_, detail)) = first_line.rsplit_once(": starlark error: ") + { + return detail.trim().to_string(); + } + + message + .lines() + .next() + .unwrap_or_default() + .trim() + .to_string() +} + +fn parse_starlark_line_from_message(message: &str) -> Option<(PathBuf, usize)> { + let first_line = message.lines().next()?.trim(); + let (path_and_position, _) = first_line.rsplit_once(": starlark error:")?; + + let mut parts = path_and_position.rsplitn(3, ':'); + let _column = parts.next()?.parse::().ok()?; + let line = parts.next()?.parse::().ok()?; + let path = PathBuf::from(parts.next()?); + + if line == 0 { + return None; + } + + Some((path, line)) +} + +pub fn format_exec_policy_error_with_source(error: &ExecPolicyError) -> String { + match error { + ExecPolicyError::ParsePolicy { path, source } => { + let rendered_source = source.to_string(); + let structured_location = source + .location() + .map(|location| (PathBuf::from(location.path), location.range.start.line)); + let parsed_location = parse_starlark_line_from_message(&rendered_source); + let location = match (structured_location, parsed_location) { + (Some((_, 1)), Some((parsed_path, parsed_line))) if parsed_line > 1 => { + Some((parsed_path, parsed_line)) + } + (Some(structured), _) => Some(structured), + (None, parsed) => parsed, + }; + let message = exec_policy_message_for_display(source); + match location { + Some((path, line)) => { + format!( + "{}:{}: {} (problem is on or around line {})", + path.display(), + line, + message, + line + ) + } + None => format!("{path}: {message}"), + } + } + _ => error.to_string(), + } +} + async fn load_exec_policy_with_warning( config_stack: &ConfigLayerStack, ) -> Result<(Policy, Option), ExecPolicyError> { @@ -240,6 +359,10 @@ pub async fn load_exec_policy(config_stack: &ConfigLayerStack) -> Result Result Option { } } +pub async fn get_has_changes(cwd: &Path) -> Option { + let output = run_git_command_with_timeout(&["status", "--porcelain"], cwd).await?; + if !output.status.success() { + return None; + } + + Some(!output.stdout.is_empty()) +} + fn parse_git_remote_urls(stdout: &str) -> Option> { let mut remotes = BTreeMap::new(); for line in stdout.lines() { @@ -254,7 +263,11 @@ pub async fn git_diff_to_remote(cwd: &Path) -> Option { /// Run a git command with a timeout to prevent blocking on large repositories async fn run_git_command_with_timeout(args: &[&str], cwd: &Path) -> Option { let mut command = Command::new("git"); - command.args(args).current_dir(cwd).kill_on_drop(true); + command + .env("GIT_OPTIONAL_LOCKS", "0") + .args(args) + .current_dir(cwd) + .kill_on_drop(true); let result = timeout(GIT_COMMAND_TIMEOUT, command.output()).await; match result { @@ -962,6 +975,37 @@ mod tests { assert_eq!(git_info.branch, Some("feature-branch".to_string())); } + #[tokio::test] + async fn test_get_has_changes_non_git_directory_returns_none() { + let temp_dir = TempDir::new().expect("Failed to create temp dir"); + assert_eq!(get_has_changes(temp_dir.path()).await, None); + } + + #[tokio::test] + async fn test_get_has_changes_clean_repo_returns_false() { + let temp_dir = TempDir::new().expect("Failed to create temp dir"); + let repo_path = create_test_git_repo(&temp_dir).await; + assert_eq!(get_has_changes(&repo_path).await, Some(false)); + } + + #[tokio::test] + async fn test_get_has_changes_with_tracked_change_returns_true() { + let temp_dir = TempDir::new().expect("Failed to create temp dir"); + let repo_path = create_test_git_repo(&temp_dir).await; + + fs::write(repo_path.join("test.txt"), "updated tracked file").expect("write tracked file"); + assert_eq!(get_has_changes(&repo_path).await, Some(true)); + } + + #[tokio::test] + async fn test_get_has_changes_with_untracked_change_returns_true() { + let temp_dir = TempDir::new().expect("Failed to create temp dir"); + let repo_path = create_test_git_repo(&temp_dir).await; + + fs::write(repo_path.join("new_file.txt"), "untracked").expect("write untracked file"); + assert_eq!(get_has_changes(&repo_path).await, Some(true)); + } + #[tokio::test] async fn test_get_git_working_tree_state_clean_repo() { let temp_dir = TempDir::new().expect("Failed to create temp dir"); diff --git a/codex-rs/core/src/lib.rs b/codex-rs/core/src/lib.rs index e3888ecf86..91bbe042ef 100644 --- a/codex-rs/core/src/lib.rs +++ b/codex-rs/core/src/lib.rs @@ -41,6 +41,7 @@ pub mod landlock; pub mod mcp; mod mcp_connection_manager; pub mod models_manager; +mod network_policy_decision; pub mod network_proxy_loader; pub use mcp_connection_manager::MCP_SANDBOX_STATE_CAPABILITY; pub use mcp_connection_manager::MCP_SANDBOX_STATE_METHOD; @@ -144,6 +145,7 @@ pub use apply_patch::CODEX_APPLY_PATCH_ARG1; pub use client::X_CODEX_TURN_METADATA_HEADER; pub use exec_policy::ExecPolicyError; pub use exec_policy::check_execpolicy_for_warnings; +pub use exec_policy::format_exec_policy_error_with_source; pub use exec_policy::load_exec_policy; pub use file_watcher::FileWatcherEvent; pub use safety::get_platform_sandbox; diff --git a/codex-rs/core/src/mcp/mod.rs b/codex-rs/core/src/mcp/mod.rs index 0ed0d221d2..1365b5da8b 100644 --- a/codex-rs/core/src/mcp/mod.rs +++ b/codex-rs/core/src/mcp/mod.rs @@ -30,6 +30,15 @@ const MCP_TOOL_NAME_PREFIX: &str = "mcp"; const MCP_TOOL_NAME_DELIMITER: &str = "__"; pub(crate) const CODEX_APPS_MCP_SERVER_NAME: &str = "codex_apps"; const CODEX_CONNECTORS_TOKEN_ENV_VAR: &str = "CODEX_CONNECTORS_TOKEN"; +const OPENAI_CONNECTORS_MCP_BASE_URL: &str = "https://api.openai.com"; +const OPENAI_CONNECTORS_MCP_PATH: &str = "/v1/connectors/mcp/"; + +// Legacy vs new MCP gateway +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum CodexAppsMcpGateway { + LegacyMCPGateway, + MCPGateway, +} fn codex_apps_mcp_bearer_token_env_var() -> Option { match env::var(CODEX_CONNECTORS_TOKEN_ENV_VAR) { @@ -65,7 +74,15 @@ fn codex_apps_mcp_http_headers(auth: Option<&CodexAuth>) -> Option String { +fn selected_config_codex_apps_mcp_gateway(config: &Config) -> CodexAppsMcpGateway { + if config.features.enabled(Feature::AppsMcpGateway) { + CodexAppsMcpGateway::MCPGateway + } else { + CodexAppsMcpGateway::LegacyMCPGateway + } +} + +fn normalize_codex_apps_base_url(base_url: &str) -> String { let mut base_url = base_url.trim_end_matches('/').to_string(); if (base_url.starts_with("https://chatgpt.com") || base_url.starts_with("https://chat.openai.com")) @@ -73,6 +90,15 @@ fn codex_apps_mcp_url(base_url: &str) -> String { { base_url = format!("{base_url}/backend-api"); } + base_url +} + +fn codex_apps_mcp_url_for_gateway(base_url: &str, gateway: CodexAppsMcpGateway) -> String { + if gateway == CodexAppsMcpGateway::MCPGateway { + return format!("{OPENAI_CONNECTORS_MCP_BASE_URL}{OPENAI_CONNECTORS_MCP_PATH}"); + } + + let base_url = normalize_codex_apps_base_url(base_url); if base_url.contains("/backend-api") { format!("{base_url}/wham/apps") } else if base_url.contains("/api/codex") { @@ -82,6 +108,13 @@ fn codex_apps_mcp_url(base_url: &str) -> String { } } +pub(crate) fn codex_apps_mcp_url(config: &Config) -> String { + codex_apps_mcp_url_for_gateway( + &config.chatgpt_base_url, + selected_config_codex_apps_mcp_gateway(config), + ) +} + fn codex_apps_mcp_server_config(config: &Config, auth: Option<&CodexAuth>) -> McpServerConfig { let bearer_token_env_var = codex_apps_mcp_bearer_token_env_var(); let http_headers = if bearer_token_env_var.is_some() { @@ -89,7 +122,7 @@ fn codex_apps_mcp_server_config(config: &Config, auth: Option<&CodexAuth>) -> Mc } else { codex_apps_mcp_http_headers(auth) }; - let url = codex_apps_mcp_url(&config.chatgpt_base_url); + let url = codex_apps_mcp_url(config); McpServerConfig { transport: McpServerTransportConfig::StreamableHttp { @@ -385,4 +418,128 @@ mod tests { assert_eq!(group_tools_by_server(&tools), expected); } + + #[test] + fn codex_apps_mcp_url_for_default_gateway_keeps_existing_paths() { + assert_eq!( + codex_apps_mcp_url_for_gateway( + "https://chatgpt.com/backend-api", + CodexAppsMcpGateway::LegacyMCPGateway + ), + "https://chatgpt.com/backend-api/wham/apps" + ); + assert_eq!( + codex_apps_mcp_url_for_gateway( + "https://chat.openai.com", + CodexAppsMcpGateway::LegacyMCPGateway + ), + "https://chat.openai.com/backend-api/wham/apps" + ); + assert_eq!( + codex_apps_mcp_url_for_gateway( + "http://localhost:8080/api/codex", + CodexAppsMcpGateway::LegacyMCPGateway + ), + "http://localhost:8080/api/codex/apps" + ); + assert_eq!( + codex_apps_mcp_url_for_gateway( + "http://localhost:8080", + CodexAppsMcpGateway::LegacyMCPGateway + ), + "http://localhost:8080/api/codex/apps" + ); + } + + #[test] + fn codex_apps_mcp_url_for_gateway_uses_openai_connectors_gateway() { + let expected_url = format!("{OPENAI_CONNECTORS_MCP_BASE_URL}{OPENAI_CONNECTORS_MCP_PATH}"); + + assert_eq!( + codex_apps_mcp_url_for_gateway( + "https://chatgpt.com/backend-api", + CodexAppsMcpGateway::MCPGateway + ), + expected_url.as_str() + ); + assert_eq!( + codex_apps_mcp_url_for_gateway( + "https://chat.openai.com", + CodexAppsMcpGateway::MCPGateway + ), + expected_url.as_str() + ); + assert_eq!( + codex_apps_mcp_url_for_gateway( + "http://localhost:8080/api/codex", + CodexAppsMcpGateway::MCPGateway + ), + expected_url.as_str() + ); + assert_eq!( + codex_apps_mcp_url_for_gateway( + "http://localhost:8080", + CodexAppsMcpGateway::MCPGateway + ), + expected_url.as_str() + ); + } + + #[test] + fn codex_apps_mcp_url_uses_default_gateway_when_feature_is_disabled() { + let mut config = crate::config::test_config(); + config.chatgpt_base_url = "https://chatgpt.com".to_string(); + + assert_eq!( + codex_apps_mcp_url(&config), + "https://chatgpt.com/backend-api/wham/apps" + ); + } + + #[test] + fn codex_apps_mcp_url_uses_openai_connectors_gateway_when_feature_is_enabled() { + let mut config = crate::config::test_config(); + config.chatgpt_base_url = "https://chatgpt.com".to_string(); + config.features.enable(Feature::AppsMcpGateway); + + assert_eq!( + codex_apps_mcp_url(&config), + format!("{OPENAI_CONNECTORS_MCP_BASE_URL}{OPENAI_CONNECTORS_MCP_PATH}") + ); + } + + #[test] + fn codex_apps_server_config_switches_gateway_with_flags() { + let mut config = crate::config::test_config(); + config.chatgpt_base_url = "https://chatgpt.com".to_string(); + + let mut servers = with_codex_apps_mcp(HashMap::new(), false, None, &config); + assert!(!servers.contains_key(CODEX_APPS_MCP_SERVER_NAME)); + + config.features.enable(Feature::Apps); + + servers = with_codex_apps_mcp(servers, true, None, &config); + let server = servers + .get(CODEX_APPS_MCP_SERVER_NAME) + .expect("codex apps should be present when apps is enabled"); + let url = match &server.transport { + McpServerTransportConfig::StreamableHttp { url, .. } => url, + _ => panic!("expected streamable http transport for codex apps"), + }; + + assert_eq!(url, "https://chatgpt.com/backend-api/wham/apps"); + + config.features.enable(Feature::AppsMcpGateway); + servers = with_codex_apps_mcp(servers, true, None, &config); + let server = servers + .get(CODEX_APPS_MCP_SERVER_NAME) + .expect("codex apps should remain present when apps stays enabled"); + let url = match &server.transport { + McpServerTransportConfig::StreamableHttp { url, .. } => url, + _ => panic!("expected streamable http transport for codex apps"), + }; + + let expected_url = format!("{OPENAI_CONNECTORS_MCP_BASE_URL}{OPENAI_CONNECTORS_MCP_PATH}"); + assert_eq!(url, &expected_url); + } } diff --git a/codex-rs/core/src/mcp/skill_dependencies.rs b/codex-rs/core/src/mcp/skill_dependencies.rs index e2817d5a4d..0e6d61055e 100644 --- a/codex-rs/core/src/mcp/skill_dependencies.rs +++ b/codex-rs/core/src/mcp/skill_dependencies.rs @@ -432,6 +432,7 @@ mod tests { interface: None, dependencies: Some(SkillDependencies { tools }), policy: None, + permissions: None, path: PathBuf::from("skill"), scope: SkillScope::User, } diff --git a/codex-rs/core/src/mcp_connection_manager.rs b/codex-rs/core/src/mcp_connection_manager.rs index edd1e13f67..5a3c4d6e5c 100644 --- a/codex-rs/core/src/mcp_connection_manager.rs +++ b/codex-rs/core/src/mcp_connection_manager.rs @@ -89,6 +89,9 @@ pub const DEFAULT_STARTUP_TIMEOUT: Duration = Duration::from_secs(10); const DEFAULT_TOOL_TIMEOUT: Duration = Duration::from_secs(60); const CODEX_APPS_TOOLS_CACHE_TTL: Duration = Duration::from_secs(3600); +const MCP_TOOLS_LIST_DURATION_METRIC: &str = "codex.mcp.tools.list.duration_ms"; +const MCP_TOOLS_FETCH_UNCACHED_DURATION_METRIC: &str = "codex.mcp.tools.fetch_uncached.duration_ms"; +const MCP_TOOLS_CACHE_WRITE_DURATION_METRIC: &str = "codex.mcp.tools.cache_write.duration_ms"; /// The Responses API requires tool names to match `^[a-zA-Z0-9_-]+$`. /// MCP server/tool names are user-controlled, so sanitize the fully-qualified @@ -882,7 +885,7 @@ fn filter_tools(tools: Vec, filter: ToolFilter) -> Vec { } pub(crate) fn filter_codex_apps_mcp_tools_only( - mut mcp_tools: HashMap, + mcp_tools: &HashMap, connectors: &[crate::connectors::AppInfo], ) -> HashMap { let allowed: HashSet<&str> = connectors @@ -890,26 +893,42 @@ pub(crate) fn filter_codex_apps_mcp_tools_only( .map(|connector| connector.id.as_str()) .collect(); - mcp_tools.retain(|_, tool| { - if tool.server_name != CODEX_APPS_MCP_SERVER_NAME { - return false; - } - let Some(connector_id) = tool.connector_id.as_deref() else { - return false; - }; - allowed.contains(connector_id) - }); - mcp_tools + .iter() + .filter(|(_, tool)| { + if tool.server_name != CODEX_APPS_MCP_SERVER_NAME { + return false; + } + let Some(connector_id) = tool.connector_id.as_deref() else { + return false; + }; + allowed.contains(connector_id) + }) + .map(|(name, tool)| (name.clone(), tool.clone())) + .collect() +} + +pub(crate) fn filter_non_codex_apps_mcp_tools_only( + mcp_tools: &HashMap, +) -> HashMap { + mcp_tools + .iter() + .filter(|(_, tool)| tool.server_name != CODEX_APPS_MCP_SERVER_NAME) + .map(|(name, tool)| (name.clone(), tool.clone())) + .collect() } pub(crate) fn filter_mcp_tools_by_name( - mut mcp_tools: HashMap, + mcp_tools: &HashMap, selected_tools: &[String], ) -> HashMap { let allowed: HashSet<&str> = selected_tools.iter().map(String::as_str).collect(); - mcp_tools.retain(|name, _| allowed.contains(name.as_str())); + mcp_tools + .iter() + .filter(|(name, _)| allowed.contains(name.as_str())) + .map(|(name, tool)| (name.clone(), tool.clone())) + .collect() } fn normalize_codex_apps_tool_title( @@ -1098,15 +1117,42 @@ async fn list_tools_for_client( client: &Arc, timeout: Option, ) -> Result> { + let total_start = Instant::now(); if server_name == CODEX_APPS_MCP_SERVER_NAME && let Some(cached_tools) = read_cached_codex_apps_tools() { + emit_duration( + MCP_TOOLS_LIST_DURATION_METRIC, + total_start.elapsed(), + &[("cache", "hit")], + ); return Ok(cached_tools); } + let fetch_start = Instant::now(); let tools = list_tools_for_client_uncached(server_name, client, timeout).await?; + emit_duration( + MCP_TOOLS_FETCH_UNCACHED_DURATION_METRIC, + fetch_start.elapsed(), + &[], + ); + if server_name == CODEX_APPS_MCP_SERVER_NAME { + let cache_write_start = Instant::now(); write_cached_codex_apps_tools(&tools); + emit_duration( + MCP_TOOLS_CACHE_WRITE_DURATION_METRIC, + cache_write_start.elapsed(), + &[], + ); + } + + if server_name == CODEX_APPS_MCP_SERVER_NAME { + emit_duration( + MCP_TOOLS_LIST_DURATION_METRIC, + total_start.elapsed(), + &[("cache", "miss")], + ); } Ok(tools) } @@ -1123,7 +1169,12 @@ fn read_cached_codex_apps_tools() -> Option> { return Some(cached.tools.clone()); } - *cache_guard = None; + if cache_guard + .as_ref() + .is_some_and(|cached| now >= cached.expires_at) + { + *cache_guard = None; + } None } @@ -1137,6 +1188,12 @@ fn write_cached_codex_apps_tools(tools: &[ToolInfo]) { }); } +fn emit_duration(metric: &str, duration: Duration, tags: &[(&str, &str)]) { + if let Some(metrics) = codex_otel::metrics::global() { + let _ = metrics.record_duration(metric, duration, tags); + } +} + async fn list_tools_for_client_uncached( server_name: &str, client: &Arc, @@ -1272,6 +1329,21 @@ mod tests { } } + fn with_clean_codex_apps_tools_cache(f: impl FnOnce() -> T) -> T { + let previous_cache = { + let mut cache_guard = CODEX_APPS_TOOLS_CACHE + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + cache_guard.take() + }; + let result = f(); + let mut cache_guard = CODEX_APPS_TOOLS_CACHE + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + *cache_guard = previous_cache; + result + } + #[test] fn test_qualify_tools_short_non_duplicated_names() { let tools = vec![ @@ -1424,6 +1496,47 @@ mod tests { assert_eq!(filtered[0].tool_name, "tool_a"); } + #[test] + fn codex_apps_tools_cache_is_overwritten_by_last_write() { + with_clean_codex_apps_tools_cache(|| { + let tools_gateway_1 = vec![create_test_tool(CODEX_APPS_MCP_SERVER_NAME, "one")]; + let tools_gateway_2 = vec![create_test_tool(CODEX_APPS_MCP_SERVER_NAME, "two")]; + + write_cached_codex_apps_tools(&tools_gateway_1); + let cached_gateway_1 = + read_cached_codex_apps_tools().expect("cache entry exists for first write"); + assert_eq!(cached_gateway_1[0].tool_name, "one"); + + write_cached_codex_apps_tools(&tools_gateway_2); + let cached_gateway_2 = + read_cached_codex_apps_tools().expect("cache entry exists for second write"); + assert_eq!(cached_gateway_2[0].tool_name, "two"); + }); + } + + #[test] + fn codex_apps_tools_cache_is_cleared_when_expired() { + with_clean_codex_apps_tools_cache(|| { + let tools = vec![create_test_tool(CODEX_APPS_MCP_SERVER_NAME, "stale_tool")]; + write_cached_codex_apps_tools(&tools); + + { + let mut cache_guard = CODEX_APPS_TOOLS_CACHE + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + cache_guard.as_mut().expect("cache exists").expires_at = + Instant::now() - Duration::from_secs(1); + } + + assert!(read_cached_codex_apps_tools().is_none()); + + let cache_guard = CODEX_APPS_TOOLS_CACHE + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + assert!(cache_guard.is_none()); + }); + } + #[test] fn mcp_init_error_display_prompts_for_github_pat() { let server_name = "github"; diff --git a/codex-rs/core/src/mcp_tool_call.rs b/codex-rs/core/src/mcp_tool_call.rs index 831cc6b3a1..745633b148 100644 --- a/codex-rs/core/src/mcp_tool_call.rs +++ b/codex-rs/core/src/mcp_tool_call.rs @@ -3,6 +3,8 @@ use std::time::Instant; use tracing::error; +use crate::analytics_client::AppInvocation; +use crate::analytics_client::build_track_events_context; use crate::codex::Session; use crate::codex::TurnContext; use crate::mcp::CODEX_APPS_MCP_SERVER_NAME; @@ -102,6 +104,7 @@ pub(crate) async fn handle_mcp_tool_call( tool_call_end_event.clone(), ) .await; + maybe_track_codex_app_used(sess.as_ref(), turn_context, &server, &tool_name).await; result } McpToolApprovalDecision::Decline => { @@ -166,6 +169,7 @@ pub(crate) async fn handle_mcp_tool_call( }); notify_mcp_tool_call_event(sess.as_ref(), turn_context, tool_call_end_event.clone()).await; + maybe_track_codex_app_used(sess.as_ref(), turn_context, &server, &tool_name).await; let status = if result.is_ok() { "ok" } else { "error" }; turn_context @@ -210,6 +214,50 @@ async fn notify_mcp_tool_call_event(sess: &Session, turn_context: &TurnContext, sess.send_event(turn_context, event).await; } +struct McpAppUsageMetadata { + connector_id: Option, + app_name: Option, +} + +async fn maybe_track_codex_app_used( + sess: &Session, + turn_context: &TurnContext, + server: &str, + tool_name: &str, +) { + if server != CODEX_APPS_MCP_SERVER_NAME { + return; + } + let metadata = lookup_mcp_app_usage_metadata(sess, server, tool_name).await; + let (connector_id, app_name) = metadata + .map(|metadata| (metadata.connector_id, metadata.app_name)) + .unwrap_or((None, None)); + let invoke_type = if let Some(connector_id) = connector_id.as_deref() { + let mentioned_connector_ids = sess.get_connector_selection().await; + if mentioned_connector_ids.contains(connector_id) { + "explicit" + } else { + "implicit" + } + } else { + "implicit" + }; + + let tracking = build_track_events_context( + turn_context.model_info.slug.clone(), + sess.conversation_id.to_string(), + turn_context.sub_id.clone(), + ); + sess.services.analytics_events_client.track_app_used( + tracking, + AppInvocation { + connector_id, + app_name, + invoke_type: Some(invoke_type.to_string()), + }, + ); +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum McpToolApprovalDecision { Accept, @@ -332,6 +380,31 @@ async fn lookup_mcp_tool_metadata( }) } +async fn lookup_mcp_app_usage_metadata( + sess: &Session, + server: &str, + tool_name: &str, +) -> Option { + let tools = sess + .services + .mcp_connection_manager + .read() + .await + .list_all_tools() + .await; + + tools.into_values().find_map(|tool_info| { + if tool_info.server_name == server && tool_info.tool_name == tool_name { + Some(McpAppUsageMetadata { + connector_id: tool_info.connector_id, + app_name: tool_info.connector_name, + }) + } else { + None + } + }) +} + fn build_mcp_tool_approval_question( question_id: String, tool_name: &str, diff --git a/codex-rs/core/src/memories/dispatch.rs b/codex-rs/core/src/memories/dispatch.rs deleted file mode 100644 index bfa6daa5a8..0000000000 --- a/codex-rs/core/src/memories/dispatch.rs +++ /dev/null @@ -1,688 +0,0 @@ -use crate::codex::Session; -use crate::config::Config; -use crate::config::Constrained; -use crate::memories::memory_root; -use crate::memories::metrics; -use crate::memories::phase_two; -use crate::memories::phase2::spawn_phase2_completion_task; -use crate::memories::prompts::build_consolidation_prompt; -use crate::memories::storage::rebuild_raw_memories_file_from_memories; -use crate::memories::storage::sync_rollout_summaries_from_memories; -use codex_protocol::protocol::AskForApproval; -use codex_protocol::protocol::SandboxPolicy; -use codex_protocol::protocol::SessionSource; -use codex_protocol::protocol::SubAgentSource; -use codex_protocol::user_input::UserInput; -use codex_utils_absolute_path::AbsolutePathBuf; -use std::sync::Arc; -use tracing::debug; -use tracing::info; -use tracing::warn; - -//TODO(jif) clean. - -fn completion_watermark( - claimed_watermark: i64, - latest_memories: &[codex_state::Stage1Output], -) -> i64 { - latest_memories - .iter() - .map(|memory| memory.source_updated_at.timestamp()) - .max() - .unwrap_or(claimed_watermark) - .max(claimed_watermark) -} - -pub(in crate::memories) async fn run_global_memory_consolidation( - session: &Arc, - config: Arc, -) -> bool { - let otel_manager = &session.services.otel_manager; - let Some(state_db) = session.services.state_db.as_deref() else { - warn!("state db unavailable; skipping global memory consolidation"); - otel_manager.counter( - metrics::MEMORY_PHASE_TWO_JOBS, - 1, - &[("status", "skipped_state_db_unavailable")], - ); - return false; - }; - - let claim = match state_db - .try_claim_global_phase2_job(session.conversation_id, phase_two::JOB_LEASE_SECONDS) - .await - { - Ok(claim) => claim, - Err(err) => { - warn!("state db try_claim_global_phase2_job failed during memories startup: {err}"); - otel_manager.counter( - metrics::MEMORY_PHASE_TWO_JOBS, - 1, - &[("status", "failed_claim")], - ); - return false; - } - }; - let (ownership_token, claimed_watermark) = match claim { - codex_state::Phase2JobClaimOutcome::Claimed { - ownership_token, - input_watermark, - } => { - otel_manager.counter(metrics::MEMORY_PHASE_TWO_JOBS, 1, &[("status", "claimed")]); - (ownership_token, input_watermark) - } - codex_state::Phase2JobClaimOutcome::SkippedNotDirty => { - debug!("memory phase-2 global lock is up-to-date; skipping consolidation"); - otel_manager.counter( - metrics::MEMORY_PHASE_TWO_JOBS, - 1, - &[("status", "skipped_not_dirty")], - ); - return false; - } - codex_state::Phase2JobClaimOutcome::SkippedRunning => { - debug!("memory phase-2 global consolidation already running; skipping"); - otel_manager.counter( - metrics::MEMORY_PHASE_TWO_JOBS, - 1, - &[("status", "skipped_running")], - ); - return false; - } - }; - - let root = memory_root(&config.codex_home); - let consolidation_config = { - let mut consolidation_config = config.as_ref().clone(); - consolidation_config.cwd = root.clone(); - consolidation_config.permissions.approval_policy = - Constrained::allow_only(AskForApproval::Never); - let mut writable_roots = Vec::new(); - match AbsolutePathBuf::from_absolute_path(consolidation_config.codex_home.clone()) { - Ok(codex_home) => writable_roots.push(codex_home), - Err(err) => warn!( - "memory phase-2 consolidation could not add codex_home writable root {}: {err}", - consolidation_config.codex_home.display() - ), - } - let consolidation_sandbox_policy = SandboxPolicy::WorkspaceWrite { - writable_roots, - read_only_access: Default::default(), - network_access: false, - exclude_tmpdir_env_var: false, - exclude_slash_tmp: false, - }; - if let Err(err) = consolidation_config - .permissions - .sandbox_policy - .set(consolidation_sandbox_policy) - { - warn!("memory phase-2 consolidation sandbox policy was rejected by constraints: {err}"); - otel_manager.counter( - metrics::MEMORY_PHASE_TWO_JOBS, - 1, - &[("status", "failed_sandbox_policy")], - ); - let _ = state_db - .mark_global_phase2_job_failed( - &ownership_token, - "consolidation sandbox policy was rejected by constraints", - phase_two::JOB_RETRY_DELAY_SECONDS, - ) - .await; - return false; - } - consolidation_config - }; - - let latest_memories = match state_db - .list_stage1_outputs_for_global(phase_two::MAX_RAW_MEMORIES_FOR_GLOBAL) - .await - { - Ok(memories) => memories, - Err(err) => { - warn!("state db list_stage1_outputs_for_global failed during consolidation: {err}"); - otel_manager.counter( - metrics::MEMORY_PHASE_TWO_JOBS, - 1, - &[("status", "failed_load_stage1_outputs")], - ); - let _ = state_db - .mark_global_phase2_job_failed( - &ownership_token, - "failed to read stage-1 outputs before global consolidation", - phase_two::JOB_RETRY_DELAY_SECONDS, - ) - .await; - return false; - } - }; - if !latest_memories.is_empty() { - otel_manager.counter( - metrics::MEMORY_PHASE_TWO_INPUT, - latest_memories.len() as i64, - &[], - ); - } - let completion_watermark = completion_watermark(claimed_watermark, &latest_memories); - if let Err(err) = sync_rollout_summaries_from_memories(&root, &latest_memories).await { - warn!("failed syncing local memory artifacts for global consolidation: {err}"); - otel_manager.counter( - metrics::MEMORY_PHASE_TWO_JOBS, - 1, - &[("status", "failed_sync_artifacts")], - ); - let _ = state_db - .mark_global_phase2_job_failed( - &ownership_token, - "failed syncing local memory artifacts", - phase_two::JOB_RETRY_DELAY_SECONDS, - ) - .await; - return false; - } - - if let Err(err) = rebuild_raw_memories_file_from_memories(&root, &latest_memories).await { - warn!("failed rebuilding raw memories aggregate for global consolidation: {err}"); - otel_manager.counter( - metrics::MEMORY_PHASE_TWO_JOBS, - 1, - &[("status", "failed_rebuild_raw_memories")], - ); - let _ = state_db - .mark_global_phase2_job_failed( - &ownership_token, - "failed rebuilding raw memories aggregate", - phase_two::JOB_RETRY_DELAY_SECONDS, - ) - .await; - return false; - } - if latest_memories.is_empty() { - debug!("memory phase-2 has no stage-1 outputs; finalized local memory artifacts"); - let _ = state_db - .mark_global_phase2_job_succeeded(&ownership_token, completion_watermark) - .await; - otel_manager.counter( - metrics::MEMORY_PHASE_TWO_JOBS, - 1, - &[("status", "succeeded_no_input")], - ); - return false; - } - - let prompt = build_consolidation_prompt(&root); - let input = vec![UserInput::Text { - text: prompt, - text_elements: vec![], - }]; - let source = SessionSource::SubAgent(SubAgentSource::Other( - phase_two::MEMORY_CONSOLIDATION_SUBAGENT_LABEL.to_string(), - )); - - match session - .services - .agent_control - .spawn_agent(consolidation_config, input, Some(source)) - .await - { - Ok(consolidation_agent_id) => { - info!( - "memory phase-2 global consolidation agent started: agent_id={consolidation_agent_id}" - ); - otel_manager.counter( - metrics::MEMORY_PHASE_TWO_JOBS, - 1, - &[("status", "agent_spawned")], - ); - spawn_phase2_completion_task( - session.as_ref(), - ownership_token, - completion_watermark, - consolidation_agent_id, - ); - true - } - Err(err) => { - warn!("failed to spawn global memory consolidation agent: {err}"); - otel_manager.counter( - metrics::MEMORY_PHASE_TWO_JOBS, - 1, - &[("status", "failed_spawn_agent")], - ); - let _ = state_db - .mark_global_phase2_job_failed( - &ownership_token, - "failed to spawn consolidation agent", - phase_two::JOB_RETRY_DELAY_SECONDS, - ) - .await; - false - } - } -} - -#[cfg(test)] -mod tests { - use super::completion_watermark; - use super::run_global_memory_consolidation; - use crate::CodexAuth; - use crate::ThreadManager; - use crate::agent::control::AgentControl; - use crate::codex::Session; - use crate::codex::make_session_and_context; - use crate::config::Config; - use crate::config::test_config; - use crate::memories::memory_root; - use crate::memories::raw_memories_file; - use crate::memories::rollout_summaries_dir; - use chrono::Utc; - use codex_protocol::ThreadId; - use codex_protocol::protocol::AskForApproval; - use codex_protocol::protocol::Op; - use codex_protocol::protocol::SandboxPolicy; - use codex_protocol::protocol::SessionSource; - use codex_state::Phase2JobClaimOutcome; - use codex_state::Stage1Output; - use codex_state::ThreadMetadataBuilder; - use pretty_assertions::assert_eq; - use std::path::PathBuf; - use std::sync::Arc; - use tempfile::TempDir; - - struct DispatchHarness { - _codex_home: TempDir, - config: Arc, - session: Arc, - manager: ThreadManager, - state_db: Arc, - } - - impl DispatchHarness { - async fn new() -> Self { - let codex_home = tempfile::tempdir().expect("create temp codex home"); - let mut config = test_config(); - config.codex_home = codex_home.path().to_path_buf(); - config.cwd = config.codex_home.clone(); - let config = Arc::new(config); - - let state_db = codex_state::StateRuntime::init( - config.codex_home.clone(), - config.model_provider_id.clone(), - None, - ) - .await - .expect("initialize state db"); - - let manager = ThreadManager::with_models_provider_and_home_for_tests( - CodexAuth::from_api_key("dummy"), - config.model_provider.clone(), - config.codex_home.clone(), - ); - let (mut session, _turn_context) = make_session_and_context().await; - session.services.state_db = Some(Arc::clone(&state_db)); - session.services.agent_control = manager.agent_control(); - - Self { - _codex_home: codex_home, - config, - session: Arc::new(session), - manager, - state_db, - } - } - - async fn seed_stage1_output(&self, source_updated_at: i64) { - let thread_id = ThreadId::new(); - let mut metadata_builder = ThreadMetadataBuilder::new( - thread_id, - self.config - .codex_home - .join(format!("rollout-{thread_id}.jsonl")), - Utc::now(), - SessionSource::Cli, - ); - metadata_builder.cwd = self.config.cwd.clone(); - metadata_builder.model_provider = Some(self.config.model_provider_id.clone()); - let metadata = metadata_builder.build(&self.config.model_provider_id); - - self.state_db - .upsert_thread(&metadata) - .await - .expect("upsert thread metadata"); - - let claim = self - .state_db - .try_claim_stage1_job( - thread_id, - self.session.conversation_id, - source_updated_at, - 3_600, - 64, - ) - .await - .expect("claim stage-1 job"); - let ownership_token = match claim { - codex_state::Stage1JobClaimOutcome::Claimed { ownership_token } => ownership_token, - other => panic!("unexpected stage-1 claim outcome: {other:?}"), - }; - assert!( - self.state_db - .mark_stage1_job_succeeded( - thread_id, - &ownership_token, - source_updated_at, - "raw memory", - "rollout summary", - ) - .await - .expect("mark stage-1 success"), - "stage-1 success should enqueue global consolidation" - ); - } - - async fn shutdown_threads(&self) { - self.manager - .remove_and_close_all_threads() - .await - .expect("shutdown spawned threads"); - } - - fn user_input_ops_count(&self) -> usize { - self.manager - .captured_ops() - .into_iter() - .filter(|(_, op)| matches!(op, Op::UserInput { .. })) - .count() - } - } - - #[test] - fn completion_watermark_never_regresses_below_claimed_input_watermark() { - let stage1_output = Stage1Output { - thread_id: ThreadId::new(), - source_updated_at: chrono::DateTime::::from_timestamp(123, 0) - .expect("valid source_updated_at timestamp"), - raw_memory: "raw memory".to_string(), - rollout_summary: "rollout summary".to_string(), - cwd: PathBuf::from("/tmp/workspace"), - generated_at: chrono::DateTime::::from_timestamp(124, 0) - .expect("valid generated_at timestamp"), - }; - - let completion = completion_watermark(1_000, &[stage1_output]); - assert_eq!(completion, 1_000); - } - - #[tokio::test] - async fn dispatch_reclaims_stale_global_lock_and_starts_consolidation() { - let harness = DispatchHarness::new().await; - harness.seed_stage1_output(100).await; - - let stale_claim = harness - .state_db - .try_claim_global_phase2_job(ThreadId::new(), 0) - .await - .expect("claim stale global lock"); - assert!( - matches!(stale_claim, Phase2JobClaimOutcome::Claimed { .. }), - "stale lock precondition should be claimed" - ); - - let scheduled = - run_global_memory_consolidation(&harness.session, Arc::clone(&harness.config)).await; - assert!( - scheduled, - "dispatch should reclaim stale lock and spawn one agent" - ); - - let running_claim = harness - .state_db - .try_claim_global_phase2_job(ThreadId::new(), 3_600) - .await - .expect("claim while running"); - assert_eq!(running_claim, Phase2JobClaimOutcome::SkippedRunning); - - let user_input_ops = harness.user_input_ops_count(); - assert_eq!(user_input_ops, 1); - let thread_ids = harness.manager.list_thread_ids().await; - assert_eq!(thread_ids.len(), 1); - let subagent = harness - .manager - .get_thread(thread_ids[0]) - .await - .expect("get consolidation thread"); - let config_snapshot = subagent.config_snapshot().await; - assert_eq!(config_snapshot.approval_policy, AskForApproval::Never); - assert_eq!(config_snapshot.cwd, memory_root(&harness.config.codex_home)); - match config_snapshot.sandbox_policy { - SandboxPolicy::WorkspaceWrite { writable_roots, .. } => { - assert!( - writable_roots - .iter() - .any(|root| root.as_path() == harness.config.codex_home.as_path()), - "consolidation subagent should have codex_home as writable root" - ); - } - other => panic!("unexpected sandbox policy: {other:?}"), - } - - harness.shutdown_threads().await; - } - - #[tokio::test] - async fn dispatch_schedules_only_one_agent_while_lock_is_running() { - let harness = DispatchHarness::new().await; - harness.seed_stage1_output(200).await; - - let first_run = - run_global_memory_consolidation(&harness.session, Arc::clone(&harness.config)).await; - let second_run = - run_global_memory_consolidation(&harness.session, Arc::clone(&harness.config)).await; - - assert!(first_run, "first dispatch should schedule consolidation"); - assert!( - !second_run, - "second dispatch should skip while the global lock is running" - ); - - let user_input_ops = harness.user_input_ops_count(); - assert_eq!(user_input_ops, 1); - - harness.shutdown_threads().await; - } - - #[tokio::test] - async fn dispatch_with_dirty_job_and_no_stage1_outputs_skips_spawn_and_clears_dirty_flag() { - let harness = DispatchHarness::new().await; - harness - .state_db - .enqueue_global_consolidation(999) - .await - .expect("enqueue global consolidation"); - - let scheduled = - run_global_memory_consolidation(&harness.session, Arc::clone(&harness.config)).await; - assert!( - !scheduled, - "dispatch should not spawn when no stage-1 outputs are available" - ); - assert_eq!(harness.user_input_ops_count(), 0); - - let claim = harness - .state_db - .try_claim_global_phase2_job(ThreadId::new(), 3_600) - .await - .expect("claim global job after empty dispatch"); - assert_eq!( - claim, - Phase2JobClaimOutcome::SkippedNotDirty, - "empty dispatch should finalize global job as up-to-date" - ); - - harness.shutdown_threads().await; - } - - #[tokio::test] - async fn dispatch_with_empty_stage1_outputs_rebuilds_local_artifacts() { - let harness = DispatchHarness::new().await; - let root = memory_root(&harness.config.codex_home); - let summaries_dir = rollout_summaries_dir(&root); - tokio::fs::create_dir_all(&summaries_dir) - .await - .expect("create rollout summaries dir"); - - let stale_summary_path = summaries_dir.join(format!("{}.md", ThreadId::new())); - tokio::fs::write(&stale_summary_path, "stale summary\n") - .await - .expect("write stale rollout summary"); - let raw_memories_path = raw_memories_file(&root); - tokio::fs::write(&raw_memories_path, "stale raw memories\n") - .await - .expect("write stale raw memories"); - let memory_index_path = root.join("MEMORY.md"); - tokio::fs::write(&memory_index_path, "stale memory index\n") - .await - .expect("write stale memory index"); - let memory_summary_path = root.join("memory_summary.md"); - tokio::fs::write(&memory_summary_path, "stale memory summary\n") - .await - .expect("write stale memory summary"); - let stale_skill_file = root.join("skills/demo/SKILL.md"); - tokio::fs::create_dir_all( - stale_skill_file - .parent() - .expect("skills subdirectory parent should exist"), - ) - .await - .expect("create stale skills dir"); - tokio::fs::write(&stale_skill_file, "stale skill\n") - .await - .expect("write stale skill"); - - harness - .state_db - .enqueue_global_consolidation(999) - .await - .expect("enqueue global consolidation"); - - let scheduled = - run_global_memory_consolidation(&harness.session, Arc::clone(&harness.config)).await; - assert!( - !scheduled, - "dispatch should skip subagent spawn when no stage-1 outputs are available" - ); - - assert!( - !tokio::fs::try_exists(&stale_summary_path) - .await - .expect("check stale summary existence"), - "empty consolidation should prune stale rollout summary files" - ); - let raw_memories = tokio::fs::read_to_string(&raw_memories_path) - .await - .expect("read rebuilt raw memories"); - assert_eq!(raw_memories, "# Raw Memories\n\nNo raw memories yet.\n"); - assert!( - !tokio::fs::try_exists(&memory_index_path) - .await - .expect("check memory index existence"), - "empty consolidation should remove stale MEMORY.md" - ); - assert!( - !tokio::fs::try_exists(&memory_summary_path) - .await - .expect("check memory summary existence"), - "empty consolidation should remove stale memory_summary.md" - ); - assert!( - !tokio::fs::try_exists(&stale_skill_file) - .await - .expect("check stale skill existence"), - "empty consolidation should remove stale skills artifacts" - ); - assert!( - !tokio::fs::try_exists(root.join("skills")) - .await - .expect("check skills dir existence"), - "empty consolidation should remove stale skills directory" - ); - - harness.shutdown_threads().await; - } - - #[tokio::test] - async fn dispatch_marks_job_for_retry_when_spawn_agent_fails() { - let codex_home = tempfile::tempdir().expect("create temp codex home"); - let mut config = test_config(); - config.codex_home = codex_home.path().to_path_buf(); - config.cwd = config.codex_home.clone(); - let config = Arc::new(config); - - let state_db = codex_state::StateRuntime::init( - config.codex_home.clone(), - config.model_provider_id.clone(), - None, - ) - .await - .expect("initialize state db"); - - let (mut session, _turn_context) = make_session_and_context().await; - session.services.state_db = Some(Arc::clone(&state_db)); - session.services.agent_control = AgentControl::default(); - let session = Arc::new(session); - - let thread_id = ThreadId::new(); - let mut metadata_builder = ThreadMetadataBuilder::new( - thread_id, - config.codex_home.join(format!("rollout-{thread_id}.jsonl")), - Utc::now(), - SessionSource::Cli, - ); - metadata_builder.cwd = config.cwd.clone(); - metadata_builder.model_provider = Some(config.model_provider_id.clone()); - let metadata = metadata_builder.build(&config.model_provider_id); - state_db - .upsert_thread(&metadata) - .await - .expect("upsert thread metadata"); - - let claim = state_db - .try_claim_stage1_job(thread_id, session.conversation_id, 100, 3_600, 64) - .await - .expect("claim stage-1 job"); - let ownership_token = match claim { - codex_state::Stage1JobClaimOutcome::Claimed { ownership_token } => ownership_token, - other => panic!("unexpected stage-1 claim outcome: {other:?}"), - }; - assert!( - state_db - .mark_stage1_job_succeeded( - thread_id, - &ownership_token, - 100, - "raw memory", - "rollout summary", - ) - .await - .expect("mark stage-1 success"), - "stage-1 success should enqueue global consolidation" - ); - - let scheduled = run_global_memory_consolidation(&session, Arc::clone(&config)).await; - assert!( - !scheduled, - "dispatch should return false when consolidation subagent cannot be spawned" - ); - - let retry_claim = state_db - .try_claim_global_phase2_job(ThreadId::new(), 3_600) - .await - .expect("claim global job after spawn failure"); - assert_eq!( - retry_claim, - Phase2JobClaimOutcome::SkippedNotDirty, - "spawn failures should leave the job in retry backoff instead of running" - ); - } -} diff --git a/codex-rs/core/src/memories/mod.rs b/codex-rs/core/src/memories/mod.rs index 30d6c42796..22fe0fb9c9 100644 --- a/codex-rs/core/src/memories/mod.rs +++ b/codex-rs/core/src/memories/mod.rs @@ -4,7 +4,6 @@ //! - Phase 1: select rollouts, extract stage-1 raw memories, persist stage-1 outputs, and enqueue consolidation. //! - Phase 2: claim a global consolidation lock, materialize consolidation inputs, and dispatch one consolidation agent. -mod dispatch; mod phase1; mod phase2; pub(crate) mod prompts; @@ -26,10 +25,10 @@ mod artifacts { /// Phase 1 (startup extraction). mod phase_one { + /// Default model used for phase 1. + pub(super) const MODEL: &str = "gpt-5.3-codex-spark"; /// Prompt used for phase 1. pub(super) const PROMPT: &str = include_str!("../../templates/memories/stage_one_system.md"); - /// Maximum number of rollout candidates processed per startup pass. - pub(super) const MAX_ROLLOUTS_PER_STARTUP: usize = 8; /// Concurrency cap for startup memory extraction and consolidation scheduling. pub(super) const CONCURRENCY_LIMIT: usize = 8; /// Fallback stage-1 rollout truncation limit (tokens) when model metadata @@ -44,10 +43,6 @@ mod phase_one { /// Keeping this below 100% leaves room for system instructions, prompt /// framing, and model output. pub(super) const CONTEXT_WINDOW_PERCENT: i64 = 70; - /// Maximum rollout age considered for phase-1 extraction. - pub(super) const MAX_ROLLOUT_AGE_DAYS: i64 = 30; - /// Minimum rollout idle time required before phase-1 extraction. - pub(super) const MIN_ROLLOUT_IDLE_HOURS: i64 = 12; /// Lease duration (seconds) for phase-1 job ownership. pub(super) const JOB_LEASE_SECONDS: i64 = 3_600; /// Backoff delay (seconds) before retrying a failed stage-1 extraction job. @@ -58,10 +53,8 @@ mod phase_one { /// Phase 2 (aka `Consolidation`). mod phase_two { - /// Subagent source label used to identify consolidation tasks. - pub(super) const MEMORY_CONSOLIDATION_SUBAGENT_LABEL: &str = "memory_consolidation"; - /// Maximum number of recent raw memories retained for global consolidation. - pub(super) const MAX_RAW_MEMORIES_FOR_GLOBAL: usize = 1_024; + /// Default model used for phase 2. + pub(super) const MODEL: &str = "gpt-5.3-codex"; /// Lease duration (seconds) for phase-2 consolidation job ownership. pub(super) const JOB_LEASE_SECONDS: i64 = 3_600; /// Backoff delay (seconds) before retrying a failed phase-2 consolidation @@ -76,6 +69,8 @@ mod metrics { pub(super) const MEMORY_PHASE_ONE_JOBS: &str = "codex.memory.phase1"; /// Number of raw memories produced by phase-1 startup extraction. pub(super) const MEMORY_PHASE_ONE_OUTPUT: &str = "codex.memory.phase1.output"; + /// Histogram for aggregate token usage across one phase-1 startup run. + pub(super) const MEMORY_PHASE_ONE_TOKEN_USAGE: &str = "codex.memory.phase1.token_usage"; /// Number of phase-2 startup jobs grouped by status. pub(super) const MEMORY_PHASE_TWO_JOBS: &str = "codex.memory.phase2"; /// Number of stage-1 memories included in each phase-2 consolidation step. diff --git a/codex-rs/core/src/memories/phase1.rs b/codex-rs/core/src/memories/phase1.rs index a7c171d3e0..146dc817ff 100644 --- a/codex-rs/core/src/memories/phase1.rs +++ b/codex-rs/core/src/memories/phase1.rs @@ -2,6 +2,8 @@ use crate::Prompt; use crate::RolloutRecorder; use crate::codex::Session; use crate::codex::TurnContext; +use crate::config::Config; +use crate::config::types::MemoriesConfig; use crate::error::CodexErr; use crate::memories::metrics; use crate::memories::phase_one; @@ -17,6 +19,7 @@ use codex_protocol::models::ResponseItem; use codex_protocol::openai_models::ModelInfo; use codex_protocol::openai_models::ReasoningEffort as ReasoningEffortConfig; use codex_protocol::protocol::RolloutItem; +use codex_protocol::protocol::TokenUsage; use codex_utils_sanitizer::redact_secrets; use futures::StreamExt; use serde::Deserialize; @@ -28,7 +31,7 @@ use tracing::info; use tracing::warn; #[derive(Clone, Debug)] -pub(in crate::memories) struct Phase1RequestContext { +pub(in crate::memories) struct RequestContext { pub(in crate::memories) model_info: ModelInfo, pub(in crate::memories) otel_manager: OtelManager, pub(in crate::memories) reasoning_effort: Option, @@ -36,18 +39,24 @@ pub(in crate::memories) struct Phase1RequestContext { pub(in crate::memories) turn_metadata_header: Option, } +struct JobResult { + outcome: JobOutcome, + token_usage: Option, +} + #[derive(Clone, Copy, Debug, Eq, PartialEq)] -enum PhaseOneJobOutcome { +enum JobOutcome { SucceededWithOutput, SucceededNoOutput, Failed, } -struct PhaseOneOutcomeCounts { +struct Stats { claimed: usize, succeeded_with_output: usize, succeeded_no_output: usize, failed: usize, + total_token_usage: Option, } /// Phase 1 model output payload. @@ -60,10 +69,9 @@ struct StageOneOutput { /// Compact summary line used for routing and indexing. #[serde(rename = "rollout_summary")] pub(crate) rollout_summary: String, - /// Optional slug accepted from stage-1 output for forward compatibility. - /// This is currently ignored by downstream storage and naming, which remain thread-id based. + /// Optional slug used to derive rollout summary artifact filenames. #[serde(default, rename = "rollout_slug")] - pub(crate) _rollout_slug: Option, + pub(crate) rollout_slug: Option, } /// Runs memory phase 1 in strict step order: @@ -71,9 +79,9 @@ struct StageOneOutput { /// 2) build one stage-1 request context /// 3) run stage-1 extraction jobs in parallel /// 4) emit metrics and logs -pub(in crate::memories) async fn run(session: &Arc) { +pub(in crate::memories) async fn run(session: &Arc, config: &Config) { // 1. Claim startup job. - let Some(claimed_candidates) = claim_startup_jobs(session).await else { + let Some(claimed_candidates) = claim_startup_jobs(session, &config.memories).await else { return; }; if claimed_candidates.is_empty() { @@ -86,13 +94,13 @@ pub(in crate::memories) async fn run(session: &Arc) { } // 2. Build request. - let stage_one_context = build_request_context(session).await; + let stage_one_context = build_request_context(session, config).await; // 3. Run the parallel sampling. let outcomes = run_jobs(session, claimed_candidates, stage_one_context).await; // 4. Metrics and logs. - let counts = count_outcomes(outcomes); + let counts = aggregate_stats(outcomes); emit_metrics(session, &counts); info!( "memory stage-1 extraction complete: {} job(s) claimed, {} succeeded ({} with output, {} no output), {} failed", @@ -110,7 +118,7 @@ pub fn output_schema() -> Value { "type": "object", "properties": { "rollout_summary": { "type": "string" }, - "rollout_slug": { "type": "string" }, + "rollout_slug": { "type": ["string", "null"] }, "raw_memory": { "type": "string" } }, "required": ["rollout_summary", "rollout_slug", "raw_memory"], @@ -118,22 +126,26 @@ pub fn output_schema() -> Value { }) } -impl Phase1RequestContext { +impl RequestContext { pub(in crate::memories) fn from_turn_context( turn_context: &TurnContext, turn_metadata_header: Option, + model_info: ModelInfo, ) -> Self { Self { - model_info: turn_context.model_info.clone(), + model_info, + turn_metadata_header, otel_manager: turn_context.otel_manager.clone(), reasoning_effort: turn_context.reasoning_effort, reasoning_summary: turn_context.reasoning_summary, - turn_metadata_header, } } } -async fn claim_startup_jobs(session: &Arc) -> Option> { +async fn claim_startup_jobs( + session: &Arc, + memories_config: &MemoriesConfig, +) -> Option> { let Some(state_db) = session.services.state_db.as_deref() else { // This should not happen. warn!("state db unavailable while claiming phase-1 startup jobs; skipping"); @@ -150,9 +162,9 @@ async fn claim_startup_jobs(session: &Arc) -> Option) -> Option) -> Phase1RequestContext { +async fn build_request_context(session: &Arc, config: &Config) -> RequestContext { + let model_name = config + .memories + .phase_1_model + .clone() + .unwrap_or(phase_one::MODEL.to_string()); + let model = session + .services + .models_manager + .get_model_info(&model_name, config) + .await; let turn_context = session.new_default_turn().await; - Phase1RequestContext::from_turn_context( + RequestContext::from_turn_context( turn_context.as_ref(), - turn_context.resolve_turn_metadata_header().await, + turn_context.turn_metadata_state.current_header_value(), + model, ) } async fn run_jobs( session: &Arc, claimed_candidates: Vec, - stage_one_context: Phase1RequestContext, -) -> Vec { + stage_one_context: RequestContext, +) -> Vec { futures::stream::iter(claimed_candidates.into_iter()) .map(|claim| { let session = Arc::clone(session); @@ -202,10 +225,10 @@ mod job { pub(in crate::memories) async fn run( session: &Session, claim: codex_state::Stage1JobClaim, - stage_one_context: &Phase1RequestContext, - ) -> PhaseOneJobOutcome { + stage_one_context: &RequestContext, + ) -> JobResult { let thread = claim.thread; - let stage_one_output = match sample( + let (stage_one_output, token_usage) = match sample( session, &thread.rollout_path, &thread.cwd, @@ -222,23 +245,33 @@ mod job { &reason.to_string(), ) .await; - return PhaseOneJobOutcome::Failed; + return JobResult { + outcome: JobOutcome::Failed, + token_usage: None, + }; } }; if stage_one_output.raw_memory.is_empty() || stage_one_output.rollout_summary.is_empty() { - return result::no_output(session, thread.id, &claim.ownership_token).await; + return JobResult { + outcome: result::no_output(session, thread.id, &claim.ownership_token).await, + token_usage, + }; } - result::success( - session, - thread.id, - &claim.ownership_token, - thread.updated_at.timestamp(), - &stage_one_output.raw_memory, - &stage_one_output.rollout_summary, - ) - .await + JobResult { + outcome: result::success( + session, + thread.id, + &claim.ownership_token, + thread.updated_at.timestamp(), + &stage_one_output.raw_memory, + &stage_one_output.rollout_summary, + stage_one_output.rollout_slug.as_deref(), + ) + .await, + token_usage, + } } /// Extract the rollout and perform the actual sampling. @@ -246,8 +279,8 @@ mod job { session: &Session, rollout_path: &Path, rollout_cwd: &Path, - stage_one_context: &Phase1RequestContext, - ) -> anyhow::Result { + stage_one_context: &RequestContext, + ) -> anyhow::Result<(StageOneOutput, Option)> { let (rollout_items, _, _) = RolloutRecorder::load_rollout_items(rollout_path).await?; let rollout_contents = serialize_filtered_rollout_response_items(&rollout_items)?; @@ -290,6 +323,7 @@ mod job { // TODO(jif) we should have a shared helper somewhere for this. // Unwrap the stream. let mut result = String::new(); + let mut token_usage = None; while let Some(message) = stream.next().await.transpose()? { match message { ResponseEvent::OutputTextDelta(delta) => result.push_str(&delta), @@ -301,7 +335,12 @@ mod job { result.push_str(&text); } } - ResponseEvent::Completed { .. } => break, + ResponseEvent::Completed { + token_usage: usage, .. + } => { + token_usage = usage; + break; + } _ => {} } } @@ -309,8 +348,9 @@ mod job { let mut output: StageOneOutput = serde_json::from_str(&result)?; output.raw_memory = redact_secrets(output.raw_memory); output.rollout_summary = redact_secrets(output.rollout_summary); + output.rollout_slug = output.rollout_slug.map(redact_secrets); - Ok(output) + Ok((output, token_usage)) } mod result { @@ -339,9 +379,9 @@ mod job { session: &Session, thread_id: codex_protocol::ThreadId, ownership_token: &str, - ) -> PhaseOneJobOutcome { + ) -> JobOutcome { let Some(state_db) = session.services.state_db.as_deref() else { - return PhaseOneJobOutcome::Failed; + return JobOutcome::Failed; }; if state_db @@ -349,9 +389,9 @@ mod job { .await .unwrap_or(false) { - PhaseOneJobOutcome::SucceededNoOutput + JobOutcome::SucceededNoOutput } else { - PhaseOneJobOutcome::Failed + JobOutcome::Failed } } @@ -362,9 +402,10 @@ mod job { source_updated_at: i64, raw_memory: &str, rollout_summary: &str, - ) -> PhaseOneJobOutcome { + rollout_slug: Option<&str>, + ) -> JobOutcome { let Some(state_db) = session.services.state_db.as_deref() else { - return PhaseOneJobOutcome::Failed; + return JobOutcome::Failed; }; if state_db @@ -374,13 +415,14 @@ mod job { source_updated_at, raw_memory, rollout_summary, + rollout_slug, ) .await .unwrap_or(false) { - PhaseOneJobOutcome::SucceededWithOutput + JobOutcome::SucceededWithOutput } else { - PhaseOneJobOutcome::Failed + JobOutcome::Failed } } } @@ -407,29 +449,37 @@ mod job { } } -fn count_outcomes(outcomes: Vec) -> PhaseOneOutcomeCounts { - let succeeded_with_output = outcomes - .iter() - .filter(|outcome| matches!(outcome, PhaseOneJobOutcome::SucceededWithOutput)) - .count(); - let succeeded_no_output = outcomes - .iter() - .filter(|outcome| matches!(outcome, PhaseOneJobOutcome::SucceededNoOutput)) - .count(); - let failed = outcomes - .iter() - .filter(|outcome| matches!(outcome, PhaseOneJobOutcome::Failed)) - .count(); +fn aggregate_stats(outcomes: Vec) -> Stats { + let claimed = outcomes.len(); + let mut succeeded_with_output = 0; + let mut succeeded_no_output = 0; + let mut failed = 0; + let mut total_token_usage = TokenUsage::default(); + let mut has_token_usage = false; - PhaseOneOutcomeCounts { - claimed: outcomes.len(), + for outcome in outcomes { + match outcome.outcome { + JobOutcome::SucceededWithOutput => succeeded_with_output += 1, + JobOutcome::SucceededNoOutput => succeeded_no_output += 1, + JobOutcome::Failed => failed += 1, + } + + if let Some(token_usage) = outcome.token_usage { + total_token_usage.add_assign(&token_usage); + has_token_usage = true; + } + } + + Stats { + claimed, succeeded_with_output, succeeded_no_output, failed, + total_token_usage: has_token_usage.then_some(total_token_usage), } } -fn emit_metrics(session: &Session, counts: &PhaseOneOutcomeCounts) { +fn emit_metrics(session: &Session, counts: &Stats) { if counts.claimed > 0 { session.services.otel_manager.counter( metrics::MEMORY_PHASE_ONE_JOBS, @@ -463,4 +513,102 @@ fn emit_metrics(session: &Session, counts: &PhaseOneOutcomeCounts) { &[("status", "failed")], ); } + if let Some(token_usage) = counts.total_token_usage.as_ref() { + session.services.otel_manager.histogram( + metrics::MEMORY_PHASE_ONE_TOKEN_USAGE, + token_usage.total_tokens.max(0), + &[("token_type", "total")], + ); + session.services.otel_manager.histogram( + metrics::MEMORY_PHASE_ONE_TOKEN_USAGE, + token_usage.input_tokens.max(0), + &[("token_type", "input")], + ); + session.services.otel_manager.histogram( + metrics::MEMORY_PHASE_ONE_TOKEN_USAGE, + token_usage.cached_input(), + &[("token_type", "cached_input")], + ); + session.services.otel_manager.histogram( + metrics::MEMORY_PHASE_ONE_TOKEN_USAGE, + token_usage.output_tokens.max(0), + &[("token_type", "output")], + ); + session.services.otel_manager.histogram( + metrics::MEMORY_PHASE_ONE_TOKEN_USAGE, + token_usage.reasoning_output_tokens.max(0), + &[("token_type", "reasoning_output")], + ); + } +} + +#[cfg(test)] +mod tests { + use super::JobOutcome; + use super::JobResult; + use super::aggregate_stats; + use codex_protocol::protocol::TokenUsage; + use pretty_assertions::assert_eq; + + #[test] + fn count_outcomes_sums_token_usage_across_all_jobs() { + let counts = aggregate_stats(vec![ + JobResult { + outcome: JobOutcome::SucceededWithOutput, + token_usage: Some(TokenUsage { + input_tokens: 10, + cached_input_tokens: 2, + output_tokens: 3, + reasoning_output_tokens: 1, + total_tokens: 13, + }), + }, + JobResult { + outcome: JobOutcome::SucceededNoOutput, + token_usage: Some(TokenUsage { + input_tokens: 7, + cached_input_tokens: 1, + output_tokens: 2, + reasoning_output_tokens: 0, + total_tokens: 9, + }), + }, + JobResult { + outcome: JobOutcome::Failed, + token_usage: None, + }, + ]); + + assert_eq!(counts.claimed, 3); + assert_eq!(counts.succeeded_with_output, 1); + assert_eq!(counts.succeeded_no_output, 1); + assert_eq!(counts.failed, 1); + assert_eq!( + counts.total_token_usage, + Some(TokenUsage { + input_tokens: 17, + cached_input_tokens: 3, + output_tokens: 5, + reasoning_output_tokens: 1, + total_tokens: 22, + }) + ); + } + + #[test] + fn count_outcomes_keeps_usage_empty_when_no_job_reports_it() { + let counts = aggregate_stats(vec![ + JobResult { + outcome: JobOutcome::SucceededWithOutput, + token_usage: None, + }, + JobResult { + outcome: JobOutcome::Failed, + token_usage: None, + }, + ]); + + assert_eq!(counts.claimed, 2); + assert_eq!(counts.total_token_usage, None); + } } diff --git a/codex-rs/core/src/memories/phase2.rs b/codex-rs/core/src/memories/phase2.rs index ef0f4642b6..28048498ce 100644 --- a/codex-rs/core/src/memories/phase2.rs +++ b/codex-rs/core/src/memories/phase2.rs @@ -1,136 +1,360 @@ use crate::agent::AgentStatus; use crate::agent::status::is_final as is_final_agent_status; use crate::codex::Session; +use crate::config::Config; +use crate::memories::memory_root; use crate::memories::metrics; use crate::memories::phase_two; +use crate::memories::prompts::build_consolidation_prompt; +use crate::memories::storage::rebuild_raw_memories_file_from_memories; +use crate::memories::storage::sync_rollout_summaries_from_memories; +use codex_config::Constrained; use codex_protocol::ThreadId; +use codex_protocol::protocol::AskForApproval; +use codex_protocol::protocol::SandboxPolicy; +use codex_protocol::protocol::SessionSource; +use codex_protocol::protocol::SubAgentSource; +use codex_protocol::user_input::UserInput; +use codex_state::StateRuntime; +use codex_utils_absolute_path::AbsolutePathBuf; use std::sync::Arc; use std::time::Duration; use tokio::sync::watch; -use tracing::debug; -use tracing::info; use tracing::warn; -pub(in crate::memories) fn spawn_phase2_completion_task( - session: &Session, - ownership_token: String, - completion_watermark: i64, - consolidation_agent_id: ThreadId, -) { - let state_db = session.services.state_db.clone(); - let agent_control = session.services.agent_control.clone(); - let otel_manager = session.services.otel_manager.clone(); - - tokio::spawn(async move { - let Some(state_db) = state_db else { - return; - }; - - let status_rx = match agent_control.subscribe_status(consolidation_agent_id).await { - Ok(status_rx) => status_rx, - Err(err) => { - warn!( - "failed to subscribe to global memory consolidation agent {consolidation_agent_id}: {err}" - ); - otel_manager.counter( - metrics::MEMORY_PHASE_TWO_JOBS, - 1, - &[("status", "failed_subscribe_status")], - ); - mark_phase2_failed_with_recovery( - state_db.as_ref(), - &ownership_token, - "failed to subscribe to consolidation agent status", - ) - .await; - return; - } - }; - - let final_status = run_phase2_completion_task( - Arc::clone(&state_db), - ownership_token, - completion_watermark, - consolidation_agent_id, - status_rx, - ) - .await; - if matches!(final_status, AgentStatus::Shutdown | AgentStatus::NotFound) { - otel_manager.counter( - metrics::MEMORY_PHASE_TWO_JOBS, - 1, - &[("status", "failed_agent_unavailable")], - ); - return; - } - if is_phase2_success(&final_status) { - otel_manager.counter( - metrics::MEMORY_PHASE_TWO_JOBS, - 1, - &[("status", "succeeded")], - ); - } else { - otel_manager.counter(metrics::MEMORY_PHASE_TWO_JOBS, 1, &[("status", "failed")]); - } - - tokio::spawn(async move { - if let Err(err) = agent_control.shutdown_agent(consolidation_agent_id).await { - warn!( - "failed to auto-close global memory consolidation agent {consolidation_agent_id}: {err}" - ); - } - }); - }); +#[derive(Debug, Clone, Default)] +struct Claim { + token: String, + watermark: i64, } -async fn run_phase2_completion_task( - state_db: Arc, - ownership_token: String, - completion_watermark: i64, - consolidation_agent_id: ThreadId, - mut status_rx: watch::Receiver, -) -> AgentStatus { - let final_status = { +#[derive(Debug, Clone, Default)] +struct Counters { + input: i64, +} + +/// Runs memory phase 2 (aka consolidation) in strict order. The method represents the linear +/// flow of the consolidation phase. +pub(super) async fn run(session: &Arc, config: Arc) { + let Some(db) = session.services.state_db.as_deref() else { + // This should not happen. + return; + }; + let root = memory_root(&config.codex_home); + let max_raw_memories = config.memories.max_raw_memories_for_global; + + // 1. Claim the job. + let claim = match job::claim(session, db).await { + Ok(claim) => claim, + Err(e) => { + session.services.otel_manager.counter( + metrics::MEMORY_PHASE_TWO_JOBS, + 1, + &[("status", e)], + ); + return; + } + }; + + // 2. Get the config for the agent + let Some(agent_config) = agent::get_config(config.clone()) else { + // If we can't get the config, we can't consolidate. + tracing::error!("failed to get agent config"); + job::failed(session, db, &claim, "failed_sandbox_policy").await; + return; + }; + + // 3. Query the memories + let raw_memories = match db.list_stage1_outputs_for_global(max_raw_memories).await { + Ok(memories) => memories, + Err(err) => { + tracing::error!("failed to list stage1 outputs from global: {}", err); + job::failed(session, db, &claim, "failed_load_stage1_outputs").await; + return; + } + }; + let new_watermark = get_watermark(claim.watermark, &raw_memories); + + // 4. Update the file system by syncing the raw memories with the one extracted from DB at + // step 3 + // [`rollout_summaries/`] + if let Err(err) = + sync_rollout_summaries_from_memories(&root, &raw_memories, max_raw_memories).await + { + tracing::error!("failed syncing local memory artifacts for global consolidation: {err}"); + job::failed(session, db, &claim, "failed_sync_artifacts").await; + return; + } + // [`raw_memories.md`] + if let Err(err) = + rebuild_raw_memories_file_from_memories(&root, &raw_memories, max_raw_memories).await + { + tracing::error!("failed syncing local memory artifacts for global consolidation: {err}"); + job::failed(session, db, &claim, "failed_rebuild_raw_memories").await; + return; + } + if raw_memories.is_empty() { + // We check only after sync of the file system. + job::succeed(session, db, &claim, new_watermark, "succeeded_no_input").await; + return; + } + + // 5. Spawn the agent + let prompt = agent::get_prompt(config); + let source = SessionSource::SubAgent(SubAgentSource::MemoryConsolidation); + let thread_id = match session + .services + .agent_control + .spawn_agent(agent_config, prompt, Some(source)) + .await + { + Ok(thread_id) => thread_id, + Err(err) => { + tracing::error!("failed to spawn global memory consolidation agent: {err}"); + job::failed(session, db, &claim, "failed_spawn_agent").await; + return; + } + }; + + // 6. Spawn the agent handler. + agent::handle(session, claim, new_watermark, thread_id); + + // 7. Metrics and logs. + let counters = Counters { + input: raw_memories.len() as i64, + }; + emit_metrics(session, counters); +} + +mod job { + use super::*; + + pub(super) async fn claim( + session: &Arc, + db: &StateRuntime, + ) -> Result { + let otel_manager = &session.services.otel_manager; + let claim = db + .try_claim_global_phase2_job(session.conversation_id, phase_two::JOB_LEASE_SECONDS) + .await + .map_err(|e| { + tracing::error!("failed to claim job: {}", e); + "failed_claim" + })?; + let (token, watermark) = match claim { + codex_state::Phase2JobClaimOutcome::Claimed { + ownership_token, + input_watermark, + } => { + otel_manager.counter(metrics::MEMORY_PHASE_TWO_JOBS, 1, &[("status", "claimed")]); + (ownership_token, input_watermark) + } + codex_state::Phase2JobClaimOutcome::SkippedNotDirty => return Err("skipped_not_dirty"), + codex_state::Phase2JobClaimOutcome::SkippedRunning => return Err("skipped_running"), + }; + + Ok(Claim { token, watermark }) + } + + pub(super) async fn failed( + session: &Arc, + db: &StateRuntime, + claim: &Claim, + reason: &'static str, + ) { + session.services.otel_manager.counter( + metrics::MEMORY_PHASE_TWO_JOBS, + 1, + &[("status", reason)], + ); + if matches!( + db.mark_global_phase2_job_failed( + &claim.token, + reason, + phase_two::JOB_RETRY_DELAY_SECONDS, + ) + .await, + Ok(false) + ) { + let _ = db + .mark_global_phase2_job_failed_if_unowned( + &claim.token, + reason, + phase_two::JOB_RETRY_DELAY_SECONDS, + ) + .await; + } + } + + pub(super) async fn succeed( + session: &Arc, + db: &StateRuntime, + claim: &Claim, + completion_watermark: i64, + reason: &'static str, + ) { + session.services.otel_manager.counter( + metrics::MEMORY_PHASE_TWO_JOBS, + 1, + &[("status", reason)], + ); + let _ = db + .mark_global_phase2_job_succeeded(&claim.token, completion_watermark) + .await; + } +} + +mod agent { + use super::*; + + pub(super) fn get_config(config: Arc) -> Option { + let root = memory_root(&config.codex_home); + let mut agent_config = config.as_ref().clone(); + + agent_config.cwd = root; + // Approval policy + agent_config.permissions.approval_policy = Constrained::allow_only(AskForApproval::Never); + + // Sandbox policy + let mut writable_roots = Vec::new(); + match AbsolutePathBuf::from_absolute_path(agent_config.codex_home.clone()) { + Ok(codex_home) => writable_roots.push(codex_home), + Err(err) => warn!( + "memory phase-2 consolidation could not add codex_home writable root {}: {err}", + agent_config.codex_home.display() + ), + } + // The consolidation agent only needs local codex_home write access and no network. + let consolidation_sandbox_policy = SandboxPolicy::WorkspaceWrite { + writable_roots, + read_only_access: Default::default(), + network_access: false, + exclude_tmpdir_env_var: false, + exclude_slash_tmp: false, + }; + agent_config + .permissions + .sandbox_policy + .set(consolidation_sandbox_policy) + .ok()?; + + agent_config.model = Some( + config + .memories + .phase_2_model + .clone() + .unwrap_or(phase_two::MODEL.to_string()), + ); + + Some(agent_config) + } + + pub(super) fn get_prompt(config: Arc) -> Vec { + let root = memory_root(&config.codex_home); + let prompt = build_consolidation_prompt(&root); + vec![UserInput::Text { + text: prompt, + text_elements: vec![], + }] + } + + /// Handle the agent while it is running. + pub(super) fn handle( + session: &Arc, + claim: Claim, + new_watermark: i64, + thread_id: ThreadId, + ) { + let Some(db) = session.services.state_db.clone() else { + return; + }; + let session = session.clone(); + + tokio::spawn(async move { + let agent_control = session.services.agent_control.clone(); + + // TODO(jif) we might have a very small race here. + let rx = match agent_control.subscribe_status(thread_id).await { + Ok(rx) => rx, + Err(err) => { + tracing::error!("agent_control.subscribe_status failed: {err:?}"); + job::failed(&session, &db, &claim, "failed_subscribe_status").await; + return; + } + }; + + // Loop the agent until we have the final status. + let final_status = loop_agent( + db.clone(), + claim.token.clone(), + new_watermark, + thread_id, + rx, + ) + .await; + + if matches!(final_status, AgentStatus::Completed(_)) { + job::succeed(&session, &db, &claim, new_watermark, "succeeded").await; + } else { + job::failed(&session, &db, &claim, "failed_agent").await; + } + + // Fire and forget close of the agent. + if !matches!(final_status, AgentStatus::Shutdown | AgentStatus::NotFound) { + tokio::spawn(async move { + if let Err(err) = agent_control.shutdown_agent(thread_id).await { + warn!( + "failed to auto-close global memory consolidation agent {thread_id}: {err}" + ); + } + }); + } else { + tracing::warn!("The agent was already gone"); + } + }); + } + + async fn loop_agent( + db: Arc, + token: String, + _new_watermark: i64, + thread_id: ThreadId, + mut rx: watch::Receiver, + ) -> AgentStatus { let mut heartbeat_interval = tokio::time::interval(Duration::from_secs(phase_two::JOB_HEARTBEAT_SECONDS)); heartbeat_interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); loop { - let status = status_rx.borrow().clone(); + let status = rx.borrow().clone(); if is_final_agent_status(&status) { break status; } tokio::select! { - changed = status_rx.changed() => { - if changed.is_err() { - warn!( - "lost status updates for global memory consolidation agent {consolidation_agent_id}" + update = rx.changed() => { + if update.is_err() { + tracing::warn!( + "lost status updates for global memory consolidation agent {thread_id}" ); break status; } } _ = heartbeat_interval.tick() => { - match state_db + match db .heartbeat_global_phase2_job( - &ownership_token, + &token, phase_two::JOB_LEASE_SECONDS, ) .await { Ok(true) => {} Ok(false) => { - warn!( - "memory phase-2 heartbeat lost global ownership; finalizing as failure" - ); break AgentStatus::Errored( "lost global phase-2 ownership during heartbeat".to_string(), ); } Err(err) => { - warn!( - "state db heartbeat_global_phase2_job failed during memories startup: {err}" - ); break AgentStatus::Errored(format!( "phase-2 heartbeat update failed: {err}" )); @@ -139,281 +363,30 @@ async fn run_phase2_completion_task( } } } - }; + } +} - let phase2_success = is_phase2_success(&final_status); - info!( - "memory phase-2 global consolidation complete: agent_id={consolidation_agent_id} success={phase2_success} final_status={final_status:?}" +pub(super) fn get_watermark( + claimed_watermark: i64, + latest_memories: &[codex_state::Stage1Output], +) -> i64 { + latest_memories + .iter() + .map(|memory| memory.source_updated_at.timestamp()) + .max() + .unwrap_or(claimed_watermark) + .max(claimed_watermark) // todo double check the claimed here. +} + +fn emit_metrics(session: &Arc, counters: Counters) { + let otel = session.services.otel_manager.clone(); + if counters.input > 0 { + otel.counter(metrics::MEMORY_PHASE_TWO_INPUT, counters.input, &[]); + } + + otel.counter( + metrics::MEMORY_PHASE_TWO_JOBS, + 1, + &[("status", "agent_spawned")], ); - - if phase2_success { - match state_db - .mark_global_phase2_job_succeeded(&ownership_token, completion_watermark) - .await - { - Ok(true) => {} - Ok(false) => { - debug!( - "memory phase-2 success finalization skipped after global ownership changed" - ); - } - Err(err) => { - warn!( - "state db mark_global_phase2_job_succeeded failed during memories startup: {err}" - ); - } - } - return final_status; - } - - let failure_reason = phase2_failure_reason(&final_status); - mark_phase2_failed_with_recovery(state_db.as_ref(), &ownership_token, &failure_reason).await; - warn!( - "memory phase-2 global consolidation agent finished with non-success status: agent_id={consolidation_agent_id} final_status={final_status:?}" - ); - final_status -} - -async fn mark_phase2_failed_with_recovery( - state_db: &codex_state::StateRuntime, - ownership_token: &str, - failure_reason: &str, -) { - match state_db - .mark_global_phase2_job_failed( - ownership_token, - failure_reason, - phase_two::JOB_RETRY_DELAY_SECONDS, - ) - .await - { - Ok(true) => {} - Ok(false) => match state_db - .mark_global_phase2_job_failed_if_unowned( - ownership_token, - failure_reason, - phase_two::JOB_RETRY_DELAY_SECONDS, - ) - .await - { - Ok(true) => { - debug!( - "memory phase-2 failure finalization applied fallback update for unowned running job" - ); - } - Ok(false) => { - debug!( - "memory phase-2 failure finalization skipped after global ownership changed" - ); - } - Err(err) => { - warn!( - "state db mark_global_phase2_job_failed_if_unowned failed during memories startup: {err}" - ); - } - }, - Err(err) => { - warn!("state db mark_global_phase2_job_failed failed during memories startup: {err}"); - } - } -} - -fn is_phase2_success(final_status: &AgentStatus) -> bool { - matches!(final_status, AgentStatus::Completed(_)) -} - -fn phase2_failure_reason(final_status: &AgentStatus) -> String { - format!("consolidation agent finished with status {final_status:?}") -} - -#[cfg(test)] -mod tests { - use super::is_phase2_success; - use super::phase2_failure_reason; - use super::run_phase2_completion_task; - use crate::agent::AgentStatus; - use codex_protocol::ThreadId; - use codex_state::Phase2JobClaimOutcome; - use pretty_assertions::assert_eq; - use std::sync::Arc; - - #[test] - fn phase2_success_only_for_completed_status() { - assert!(is_phase2_success(&AgentStatus::Completed(None))); - assert!(!is_phase2_success(&AgentStatus::Running)); - assert!(!is_phase2_success(&AgentStatus::Errored( - "oops".to_string() - ))); - } - - #[test] - fn phase2_failure_reason_includes_status() { - let status = AgentStatus::Errored("boom".to_string()); - let reason = phase2_failure_reason(&status); - assert!(reason.contains("consolidation agent finished with status")); - assert!(reason.contains("boom")); - } - - #[tokio::test] - async fn phase2_completion_marks_succeeded_for_completed_status() { - let codex_home = tempfile::tempdir().expect("create temp codex home"); - let state_db = Arc::new( - codex_state::StateRuntime::init( - codex_home.path().to_path_buf(), - "test-provider".to_string(), - None, - ) - .await - .expect("initialize state runtime"), - ); - let owner = ThreadId::new(); - state_db - .enqueue_global_consolidation(123) - .await - .expect("enqueue global consolidation"); - let claim = state_db - .try_claim_global_phase2_job(owner, 3_600) - .await - .expect("claim global phase-2 job"); - let ownership_token = match claim { - Phase2JobClaimOutcome::Claimed { - ownership_token, .. - } => ownership_token, - other => panic!("unexpected phase-2 claim outcome: {other:?}"), - }; - - let (_status_tx, status_rx) = tokio::sync::watch::channel(AgentStatus::Completed(None)); - run_phase2_completion_task( - Arc::clone(&state_db), - ownership_token.clone(), - 123, - ThreadId::new(), - status_rx, - ) - .await; - - let up_to_date_claim = state_db - .try_claim_global_phase2_job(ThreadId::new(), 3_600) - .await - .expect("claim up-to-date global job"); - assert_eq!(up_to_date_claim, Phase2JobClaimOutcome::SkippedNotDirty); - - state_db - .enqueue_global_consolidation(124) - .await - .expect("enqueue advanced consolidation watermark"); - let rerun_claim = state_db - .try_claim_global_phase2_job(ThreadId::new(), 3_600) - .await - .expect("claim rerun global job"); - assert!( - matches!(rerun_claim, Phase2JobClaimOutcome::Claimed { .. }), - "advanced watermark should be claimable after success finalization" - ); - } - - #[tokio::test] - async fn phase2_completion_marks_failed_when_status_updates_are_lost() { - let codex_home = tempfile::tempdir().expect("create temp codex home"); - let state_db = Arc::new( - codex_state::StateRuntime::init( - codex_home.path().to_path_buf(), - "test-provider".to_string(), - None, - ) - .await - .expect("initialize state runtime"), - ); - state_db - .enqueue_global_consolidation(456) - .await - .expect("enqueue global consolidation"); - let claim = state_db - .try_claim_global_phase2_job(ThreadId::new(), 3_600) - .await - .expect("claim global phase-2 job"); - let ownership_token = match claim { - Phase2JobClaimOutcome::Claimed { - ownership_token, .. - } => ownership_token, - other => panic!("unexpected phase-2 claim outcome: {other:?}"), - }; - - let (status_tx, status_rx) = tokio::sync::watch::channel(AgentStatus::Running); - drop(status_tx); - run_phase2_completion_task( - Arc::clone(&state_db), - ownership_token, - 456, - ThreadId::new(), - status_rx, - ) - .await; - - let claim = state_db - .try_claim_global_phase2_job(ThreadId::new(), 3_600) - .await - .expect("claim after failure finalization"); - assert_eq!( - claim, - Phase2JobClaimOutcome::SkippedNotDirty, - "failure finalization should leave global job in retry-backoff, not running ownership" - ); - } - - #[tokio::test] - async fn phase2_completion_heartbeat_loss_does_not_steal_active_other_owner() { - let codex_home = tempfile::tempdir().expect("create temp codex home"); - let state_db = Arc::new( - codex_state::StateRuntime::init( - codex_home.path().to_path_buf(), - "test-provider".to_string(), - None, - ) - .await - .expect("initialize state runtime"), - ); - state_db - .enqueue_global_consolidation(789) - .await - .expect("enqueue global consolidation"); - let claim = state_db - .try_claim_global_phase2_job(ThreadId::new(), 3_600) - .await - .expect("claim global phase-2 job"); - let claimed_token = match claim { - Phase2JobClaimOutcome::Claimed { - ownership_token, .. - } => ownership_token, - other => panic!("unexpected phase-2 claim outcome: {other:?}"), - }; - - let (_status_tx, status_rx) = tokio::sync::watch::channel(AgentStatus::Running); - run_phase2_completion_task( - Arc::clone(&state_db), - "non-owner-token".to_string(), - 789, - ThreadId::new(), - status_rx, - ) - .await; - - let claim = state_db - .try_claim_global_phase2_job(ThreadId::new(), 3_600) - .await - .expect("claim after heartbeat ownership loss"); - assert_eq!( - claim, - Phase2JobClaimOutcome::SkippedRunning, - "heartbeat ownership-loss handling should not steal a live owner lease" - ); - assert_eq!( - state_db - .mark_global_phase2_job_succeeded(claimed_token.as_str(), 789) - .await - .expect("mark original owner success"), - true, - "the original owner should still be able to finalize" - ); - } } diff --git a/codex-rs/core/src/memories/start.rs b/codex-rs/core/src/memories/start.rs index 5b47b6c2bd..b93846857f 100644 --- a/codex-rs/core/src/memories/start.rs +++ b/codex-rs/core/src/memories/start.rs @@ -2,6 +2,7 @@ use crate::codex::Session; use crate::config::Config; use crate::features::Feature; use crate::memories::phase1; +use crate::memories::phase2; use codex_protocol::protocol::SessionSource; use std::sync::Arc; use tracing::warn; @@ -34,8 +35,8 @@ pub(crate) fn start_memories_startup_task( }; // Run phase 1. - phase1::run(&session).await; + phase1::run(&session, &config).await; // Run phase 2. - crate::memories::dispatch::run_global_memory_consolidation(&session, config).await; + phase2::run(&session, config).await; }); } diff --git a/codex-rs/core/src/memories/storage.rs b/codex-rs/core/src/memories/storage.rs index dd39815b36..a596ea6f0c 100644 --- a/codex-rs/core/src/memories/storage.rs +++ b/codex-rs/core/src/memories/storage.rs @@ -5,7 +5,6 @@ use std::path::Path; use tracing::warn; use crate::memories::ensure_layout; -use crate::memories::phase_two; use crate::memories::raw_memories_file; use crate::memories::rollout_summaries_dir; @@ -15,25 +14,27 @@ use crate::memories::rollout_summaries_dir; pub(super) async fn rebuild_raw_memories_file_from_memories( root: &Path, memories: &[Stage1Output], + max_raw_memories_for_global: usize, ) -> std::io::Result<()> { ensure_layout(root).await?; - rebuild_raw_memories_file(root, memories).await + rebuild_raw_memories_file(root, memories, max_raw_memories_for_global).await } /// Syncs canonical rollout summary files from DB-backed stage-1 output rows. pub(super) async fn sync_rollout_summaries_from_memories( root: &Path, memories: &[Stage1Output], + max_raw_memories_for_global: usize, ) -> std::io::Result<()> { ensure_layout(root).await?; let retained = memories .iter() - .take(phase_two::MAX_RAW_MEMORIES_FOR_GLOBAL) + .take(max_raw_memories_for_global) .collect::>(); let keep = retained .iter() - .map(|memory| memory.thread_id.to_string()) + .map(|memory| rollout_summary_file_stem(memory)) .collect::>(); prune_rollout_summaries(root, &keep).await?; @@ -62,10 +63,14 @@ pub(super) async fn sync_rollout_summaries_from_memories( Ok(()) } -async fn rebuild_raw_memories_file(root: &Path, memories: &[Stage1Output]) -> std::io::Result<()> { +async fn rebuild_raw_memories_file( + root: &Path, + memories: &[Stage1Output], + max_raw_memories_for_global: usize, +) -> std::io::Result<()> { let retained = memories .iter() - .take(phase_two::MAX_RAW_MEMORIES_FOR_GLOBAL) + .take(max_raw_memories_for_global) .collect::>(); let mut body = String::from("# Raw Memories\n\n"); @@ -108,10 +113,10 @@ async fn prune_rollout_summaries(root: &Path, keep: &BTreeSet) -> std::i let Some(file_name) = path.file_name().and_then(|name| name.to_str()) else { continue; }; - let Some(thread_id) = extract_thread_id_from_rollout_summary_filename(file_name) else { + let Some(stem) = file_name.strip_suffix(".md") else { continue; }; - if !keep.contains(thread_id) + if !keep.contains(stem) && let Err(err) = tokio::fs::remove_file(&path).await && err.kind() != std::io::ErrorKind::NotFound { @@ -129,7 +134,8 @@ async fn write_rollout_summary_for_thread( root: &Path, memory: &Stage1Output, ) -> std::io::Result<()> { - let path = rollout_summaries_dir(root).join(format!("{}.md", memory.thread_id)); + let file_stem = rollout_summary_file_stem(memory); + let path = rollout_summaries_dir(root).join(format!("{file_stem}.md")); let mut body = String::new(); writeln!(body, "thread_id: {}", memory.thread_id) @@ -150,7 +156,85 @@ async fn write_rollout_summary_for_thread( tokio::fs::write(path, body).await } -fn extract_thread_id_from_rollout_summary_filename(file_name: &str) -> Option<&str> { - let stem = file_name.strip_suffix(".md")?; - if stem.is_empty() { None } else { Some(stem) } +fn rollout_summary_file_stem(memory: &Stage1Output) -> String { + const ROLLOUT_SLUG_MAX_LEN: usize = 20; + + let thread_id = memory.thread_id.to_string(); + let Some(raw_slug) = memory.rollout_slug.as_deref() else { + return thread_id; + }; + + let mut slug = String::with_capacity(ROLLOUT_SLUG_MAX_LEN); + for ch in raw_slug.chars() { + if slug.len() >= ROLLOUT_SLUG_MAX_LEN { + break; + } + + if ch.is_ascii_alphanumeric() { + slug.push(ch.to_ascii_lowercase()); + } else { + slug.push('_'); + } + } + + while slug.ends_with('_') { + slug.pop(); + } + + if slug.is_empty() { + thread_id + } else { + format!("{thread_id}-{slug}") + } +} + +#[cfg(test)] +mod tests { + use super::rollout_summary_file_stem; + use chrono::TimeZone; + use chrono::Utc; + use codex_protocol::ThreadId; + use codex_state::Stage1Output; + use pretty_assertions::assert_eq; + use std::path::PathBuf; + + fn stage1_output_with_slug(rollout_slug: Option<&str>) -> Stage1Output { + Stage1Output { + thread_id: ThreadId::new(), + source_updated_at: Utc.timestamp_opt(123, 0).single().expect("timestamp"), + raw_memory: "raw memory".to_string(), + rollout_summary: "summary".to_string(), + rollout_slug: rollout_slug.map(ToString::to_string), + cwd: PathBuf::from("/tmp/workspace"), + generated_at: Utc.timestamp_opt(124, 0).single().expect("timestamp"), + } + } + + #[test] + fn rollout_summary_file_stem_uses_thread_id_when_slug_missing() { + let memory = stage1_output_with_slug(None); + let thread_id = memory.thread_id.to_string(); + + assert_eq!(rollout_summary_file_stem(&memory), thread_id); + } + + #[test] + fn rollout_summary_file_stem_sanitizes_and_truncates_slug() { + let memory = + stage1_output_with_slug(Some("Unsafe Slug/With Spaces & Symbols + EXTRA_LONG_12345")); + let thread_id = memory.thread_id.to_string(); + + assert_eq!( + rollout_summary_file_stem(&memory), + format!("{thread_id}-unsafe_slug_with_spa") + ); + } + + #[test] + fn rollout_summary_file_stem_uses_thread_id_when_slug_is_empty() { + let memory = stage1_output_with_slug(Some("")); + let thread_id = memory.thread_id.to_string(); + + assert_eq!(rollout_summary_file_stem(&memory), thread_id); + } } diff --git a/codex-rs/core/src/memories/tests.rs b/codex-rs/core/src/memories/tests.rs index 1baea576c2..92fd69e4df 100644 --- a/codex-rs/core/src/memories/tests.rs +++ b/codex-rs/core/src/memories/tests.rs @@ -1,5 +1,6 @@ use super::storage::rebuild_raw_memories_file_from_memories; use super::storage::sync_rollout_summaries_from_memories; +use crate::config::types::DEFAULT_MEMORIES_MAX_RAW_MEMORIES_FOR_GLOBAL; use crate::memories::ensure_layout; use crate::memories::memory_root; use crate::memories::raw_memories_file; @@ -21,7 +22,7 @@ fn memory_root_uses_shared_global_path() { } #[test] -fn stage_one_output_schema_requires_all_declared_properties() { +fn stage_one_output_schema_requires_rollout_slug_and_keeps_it_nullable() { let schema = crate::memories::phase1::output_schema(); let properties = schema .get("properties") @@ -32,16 +33,34 @@ fn stage_one_output_schema_requires_all_declared_properties() { .and_then(Value::as_array) .expect("required array"); - let mut property_keys = properties.keys().map(String::as_str).collect::>(); - property_keys.sort_unstable(); - let mut required_keys = required .iter() .map(|key| key.as_str().expect("required key string")) .collect::>(); required_keys.sort_unstable(); - assert_eq!(required_keys, property_keys); + assert!( + properties.contains_key("rollout_slug"), + "schema should declare rollout_slug" + ); + + let rollout_slug_type = properties + .get("rollout_slug") + .and_then(Value::as_object) + .and_then(|schema| schema.get("type")) + .and_then(Value::as_array) + .expect("rollout_slug type array"); + let mut rollout_slug_types = rollout_slug_type + .iter() + .map(|entry| entry.as_str().expect("type entry string")) + .collect::>(); + rollout_slug_types.sort_unstable(); + + assert_eq!( + required_keys, + vec!["raw_memory", "rollout_slug", "rollout_summary"] + ); + assert_eq!(rollout_slug_types, vec!["null", "string"]); } #[tokio::test] @@ -66,16 +85,25 @@ async fn sync_rollout_summaries_and_raw_memories_file_keeps_latest_memories_only source_updated_at: Utc.timestamp_opt(100, 0).single().expect("timestamp"), raw_memory: "raw memory".to_string(), rollout_summary: "short summary".to_string(), + rollout_slug: None, cwd: PathBuf::from("/tmp/workspace"), generated_at: Utc.timestamp_opt(101, 0).single().expect("timestamp"), }]; - sync_rollout_summaries_from_memories(&root, &memories) - .await - .expect("sync rollout summaries"); - rebuild_raw_memories_file_from_memories(&root, &memories) - .await - .expect("rebuild raw memories"); + sync_rollout_summaries_from_memories( + &root, + &memories, + DEFAULT_MEMORIES_MAX_RAW_MEMORIES_FOR_GLOBAL, + ) + .await + .expect("sync rollout summaries"); + rebuild_raw_memories_file_from_memories( + &root, + &memories, + DEFAULT_MEMORIES_MAX_RAW_MEMORIES_FOR_GLOBAL, + ) + .await + .expect("rebuild raw memories"); assert!(keep_path.is_file()); assert!(!drop_path.exists()); @@ -87,3 +115,578 @@ async fn sync_rollout_summaries_and_raw_memories_file_keeps_latest_memories_only assert!(raw_memories.contains(&keep_id)); assert!(raw_memories.contains("cwd: /tmp/workspace")); } + +#[tokio::test] +async fn sync_rollout_summaries_uses_thread_id_and_sanitized_slug_filename() { + let dir = tempdir().expect("tempdir"); + let root = dir.path().join("memory"); + ensure_layout(&root).await.expect("ensure layout"); + + let thread_id = ThreadId::new(); + let stale_unslugged_path = rollout_summaries_dir(&root).join(format!("{thread_id}.md")); + let stale_old_slug_path = + rollout_summaries_dir(&root).join(format!("{thread_id}--old-slug.md")); + tokio::fs::write(&stale_unslugged_path, "stale") + .await + .expect("write stale unslugged file"); + tokio::fs::write(&stale_old_slug_path, "stale") + .await + .expect("write stale old-slug file"); + + let memories = vec![Stage1Output { + thread_id, + source_updated_at: Utc.timestamp_opt(200, 0).single().expect("timestamp"), + raw_memory: "raw memory".to_string(), + rollout_summary: "short summary".to_string(), + rollout_slug: Some("Unsafe Slug/With Spaces & Symbols + EXTRA_LONG_12345".to_string()), + cwd: PathBuf::from("/tmp/workspace"), + generated_at: Utc.timestamp_opt(201, 0).single().expect("timestamp"), + }]; + + sync_rollout_summaries_from_memories( + &root, + &memories, + DEFAULT_MEMORIES_MAX_RAW_MEMORIES_FOR_GLOBAL, + ) + .await + .expect("sync rollout summaries"); + + let mut dir = tokio::fs::read_dir(rollout_summaries_dir(&root)) + .await + .expect("open rollout summaries dir"); + let mut files = Vec::new(); + while let Some(entry) = dir.next_entry().await.expect("read dir entry") { + files.push(entry.file_name().to_string_lossy().to_string()); + } + files.sort_unstable(); + + assert_eq!(files.len(), 1); + let file_name = &files[0]; + let stem = file_name + .strip_suffix(".md") + .expect("rollout summary file should end with .md"); + let slug = stem + .strip_prefix(&format!("{thread_id}-")) + .expect("rollout summary filename should include thread id and slug"); + assert!(slug.len() <= 20, "slug should be capped at 20 chars"); + assert!( + slug.chars() + .all(|ch| ch.is_ascii_lowercase() || ch.is_ascii_digit() || ch == '_'), + "slug should be file-safe lowercase ascii with underscores" + ); + + let summary = tokio::fs::read_to_string(rollout_summaries_dir(&root).join(file_name)) + .await + .expect("read rollout summary"); + assert!(summary.contains(&format!("thread_id: {thread_id}"))); + assert!( + !tokio::fs::try_exists(&stale_unslugged_path) + .await + .expect("check stale unslugged path"), + "slugged sync should prune stale unslugged filename for same thread" + ); + assert!( + !tokio::fs::try_exists(&stale_old_slug_path) + .await + .expect("check stale old slug path"), + "slugged sync should prune stale slugged filename for same thread" + ); +} + +mod phase2 { + use crate::CodexAuth; + use crate::ThreadManager; + use crate::agent::AgentControl; + use crate::codex::Session; + use crate::codex::make_session_and_context; + use crate::config::Config; + use crate::config::test_config; + use crate::memories::memory_root; + use crate::memories::phase2; + use crate::memories::raw_memories_file; + use crate::memories::rollout_summaries_dir; + use chrono::Utc; + use codex_config::Constrained; + use codex_protocol::ThreadId; + use codex_protocol::protocol::AskForApproval; + use codex_protocol::protocol::Op; + use codex_protocol::protocol::SandboxPolicy; + use codex_protocol::protocol::SessionSource; + use codex_state::Phase2JobClaimOutcome; + use codex_state::Stage1Output; + use codex_state::ThreadMetadataBuilder; + use std::path::PathBuf; + use std::sync::Arc; + use tempfile::TempDir; + + fn stage1_output_with_source_updated_at(source_updated_at: i64) -> Stage1Output { + Stage1Output { + thread_id: ThreadId::new(), + source_updated_at: chrono::DateTime::::from_timestamp(source_updated_at, 0) + .expect("valid source_updated_at timestamp"), + raw_memory: "raw memory".to_string(), + rollout_summary: "rollout summary".to_string(), + rollout_slug: None, + cwd: PathBuf::from("/tmp/workspace"), + generated_at: chrono::DateTime::::from_timestamp(source_updated_at + 1, 0) + .expect("valid generated_at timestamp"), + } + } + + struct DispatchHarness { + _codex_home: TempDir, + config: Arc, + session: Arc, + manager: ThreadManager, + state_db: Arc, + } + + impl DispatchHarness { + async fn new() -> Self { + let codex_home = tempfile::tempdir().expect("create temp codex home"); + let mut config = test_config(); + config.codex_home = codex_home.path().to_path_buf(); + config.cwd = config.codex_home.clone(); + let config = Arc::new(config); + + let state_db = codex_state::StateRuntime::init( + config.codex_home.clone(), + config.model_provider_id.clone(), + None, + ) + .await + .expect("initialize state db"); + + let manager = ThreadManager::with_models_provider_and_home_for_tests( + CodexAuth::from_api_key("dummy"), + config.model_provider.clone(), + config.codex_home.clone(), + ); + let (mut session, _turn_context) = make_session_and_context().await; + session.services.state_db = Some(Arc::clone(&state_db)); + session.services.agent_control = manager.agent_control(); + + Self { + _codex_home: codex_home, + config, + session: Arc::new(session), + manager, + state_db, + } + } + + async fn seed_stage1_output(&self, source_updated_at: i64) { + let thread_id = ThreadId::new(); + let mut metadata_builder = ThreadMetadataBuilder::new( + thread_id, + self.config + .codex_home + .join(format!("rollout-{thread_id}.jsonl")), + Utc::now(), + SessionSource::Cli, + ); + metadata_builder.cwd = self.config.cwd.clone(); + metadata_builder.model_provider = Some(self.config.model_provider_id.clone()); + let metadata = metadata_builder.build(&self.config.model_provider_id); + + self.state_db + .upsert_thread(&metadata) + .await + .expect("upsert thread metadata"); + + let claim = self + .state_db + .try_claim_stage1_job( + thread_id, + self.session.conversation_id, + source_updated_at, + 3_600, + 64, + ) + .await + .expect("claim stage-1 job"); + let ownership_token = match claim { + codex_state::Stage1JobClaimOutcome::Claimed { ownership_token } => ownership_token, + other => panic!("unexpected stage-1 claim outcome: {other:?}"), + }; + assert!( + self.state_db + .mark_stage1_job_succeeded( + thread_id, + &ownership_token, + source_updated_at, + "raw memory", + "rollout summary", + None, + ) + .await + .expect("mark stage-1 success"), + "stage-1 success should enqueue global consolidation" + ); + } + + async fn shutdown_threads(&self) { + self.manager + .remove_and_close_all_threads() + .await + .expect("shutdown spawned threads"); + } + + fn user_input_ops_count(&self) -> usize { + self.manager + .captured_ops() + .into_iter() + .filter(|(_, op)| matches!(op, Op::UserInput { .. })) + .count() + } + } + + #[test] + fn completion_watermark_never_regresses_below_claimed_input_watermark() { + let stage1_output = stage1_output_with_source_updated_at(123); + + let completion = phase2::get_watermark(1_000, &[stage1_output]); + pretty_assertions::assert_eq!(completion, 1_000); + } + + #[test] + fn completion_watermark_uses_claimed_watermark_when_there_are_no_memories() { + let completion = phase2::get_watermark(777, &[]); + pretty_assertions::assert_eq!(completion, 777); + } + + #[test] + fn completion_watermark_uses_latest_memory_timestamp_when_it_is_newer() { + let older = stage1_output_with_source_updated_at(123); + let newer = stage1_output_with_source_updated_at(456); + + let completion = phase2::get_watermark(200, &[older, newer]); + pretty_assertions::assert_eq!(completion, 456); + } + + #[tokio::test] + async fn dispatch_skips_when_global_job_is_not_dirty() { + let harness = DispatchHarness::new().await; + + phase2::run(&harness.session, Arc::clone(&harness.config)).await; + + pretty_assertions::assert_eq!(harness.user_input_ops_count(), 0); + let thread_ids = harness.manager.list_thread_ids().await; + pretty_assertions::assert_eq!(thread_ids.len(), 0); + } + + #[tokio::test] + async fn dispatch_skips_when_global_job_is_already_running() { + let harness = DispatchHarness::new().await; + harness + .state_db + .enqueue_global_consolidation(123) + .await + .expect("enqueue global consolidation"); + let claimed = harness + .state_db + .try_claim_global_phase2_job(ThreadId::new(), 3_600) + .await + .expect("claim running global lock"); + assert!( + matches!(claimed, Phase2JobClaimOutcome::Claimed { .. }), + "precondition should claim the running lock" + ); + + phase2::run(&harness.session, Arc::clone(&harness.config)).await; + + let running_claim = harness + .state_db + .try_claim_global_phase2_job(ThreadId::new(), 3_600) + .await + .expect("claim while lock is still running"); + pretty_assertions::assert_eq!(running_claim, Phase2JobClaimOutcome::SkippedRunning); + pretty_assertions::assert_eq!(harness.user_input_ops_count(), 0); + let thread_ids = harness.manager.list_thread_ids().await; + pretty_assertions::assert_eq!(thread_ids.len(), 0); + } + + #[tokio::test] + async fn dispatch_reclaims_stale_global_lock_and_starts_consolidation() { + let harness = DispatchHarness::new().await; + harness.seed_stage1_output(100).await; + + let stale_claim = harness + .state_db + .try_claim_global_phase2_job(ThreadId::new(), 0) + .await + .expect("claim stale global lock"); + assert!( + matches!(stale_claim, Phase2JobClaimOutcome::Claimed { .. }), + "stale lock precondition should be claimed" + ); + + phase2::run(&harness.session, Arc::clone(&harness.config)).await; + + let running_claim = harness + .state_db + .try_claim_global_phase2_job(ThreadId::new(), 3_600) + .await + .expect("claim while running"); + pretty_assertions::assert_eq!(running_claim, Phase2JobClaimOutcome::SkippedRunning); + + let user_input_ops = harness.user_input_ops_count(); + pretty_assertions::assert_eq!(user_input_ops, 1); + let thread_ids = harness.manager.list_thread_ids().await; + pretty_assertions::assert_eq!(thread_ids.len(), 1); + let subagent = harness + .manager + .get_thread(thread_ids[0]) + .await + .expect("get consolidation thread"); + let config_snapshot = subagent.config_snapshot().await; + pretty_assertions::assert_eq!(config_snapshot.approval_policy, AskForApproval::Never); + pretty_assertions::assert_eq!(config_snapshot.cwd, memory_root(&harness.config.codex_home)); + match config_snapshot.sandbox_policy { + SandboxPolicy::WorkspaceWrite { writable_roots, .. } => { + assert!( + writable_roots + .iter() + .any(|root| root.as_path() == harness.config.codex_home.as_path()), + "consolidation subagent should have codex_home as writable root" + ); + } + other => panic!("unexpected sandbox policy: {other:?}"), + } + + harness.shutdown_threads().await; + } + + #[tokio::test] + async fn dispatch_with_empty_stage1_outputs_rebuilds_local_artifacts() { + let harness = DispatchHarness::new().await; + let root = memory_root(&harness.config.codex_home); + let summaries_dir = rollout_summaries_dir(&root); + tokio::fs::create_dir_all(&summaries_dir) + .await + .expect("create rollout summaries dir"); + + let stale_summary_path = summaries_dir.join(format!("{}.md", ThreadId::new())); + tokio::fs::write(&stale_summary_path, "stale summary\n") + .await + .expect("write stale rollout summary"); + let raw_memories_path = raw_memories_file(&root); + tokio::fs::write(&raw_memories_path, "stale raw memories\n") + .await + .expect("write stale raw memories"); + let memory_index_path = root.join("MEMORY.md"); + tokio::fs::write(&memory_index_path, "stale memory index\n") + .await + .expect("write stale memory index"); + let memory_summary_path = root.join("memory_summary.md"); + tokio::fs::write(&memory_summary_path, "stale memory summary\n") + .await + .expect("write stale memory summary"); + let stale_skill_file = root.join("skills/demo/SKILL.md"); + tokio::fs::create_dir_all( + stale_skill_file + .parent() + .expect("skills subdirectory parent should exist"), + ) + .await + .expect("create stale skills dir"); + tokio::fs::write(&stale_skill_file, "stale skill\n") + .await + .expect("write stale skill"); + + harness + .state_db + .enqueue_global_consolidation(999) + .await + .expect("enqueue global consolidation"); + + phase2::run(&harness.session, Arc::clone(&harness.config)).await; + + assert!( + !tokio::fs::try_exists(&stale_summary_path) + .await + .expect("check stale summary existence"), + "empty consolidation should prune stale rollout summary files" + ); + let raw_memories = tokio::fs::read_to_string(&raw_memories_path) + .await + .expect("read rebuilt raw memories"); + pretty_assertions::assert_eq!(raw_memories, "# Raw Memories\n\nNo raw memories yet.\n"); + assert!( + !tokio::fs::try_exists(&memory_index_path) + .await + .expect("check memory index existence"), + "empty consolidation should remove stale MEMORY.md" + ); + assert!( + !tokio::fs::try_exists(&memory_summary_path) + .await + .expect("check memory summary existence"), + "empty consolidation should remove stale memory_summary.md" + ); + assert!( + !tokio::fs::try_exists(&stale_skill_file) + .await + .expect("check stale skill existence"), + "empty consolidation should remove stale skills artifacts" + ); + assert!( + !tokio::fs::try_exists(root.join("skills")) + .await + .expect("check skills dir existence"), + "empty consolidation should remove stale skills directory" + ); + let next_claim = harness + .state_db + .try_claim_global_phase2_job(ThreadId::new(), 3_600) + .await + .expect("claim global job after empty consolidation success"); + pretty_assertions::assert_eq!(next_claim, Phase2JobClaimOutcome::SkippedNotDirty); + pretty_assertions::assert_eq!(harness.user_input_ops_count(), 0); + let thread_ids = harness.manager.list_thread_ids().await; + pretty_assertions::assert_eq!(thread_ids.len(), 0); + + harness.shutdown_threads().await; + } + + #[tokio::test] + async fn dispatch_marks_job_for_retry_when_sandbox_policy_cannot_be_overridden() { + let harness = DispatchHarness::new().await; + harness + .state_db + .enqueue_global_consolidation(99) + .await + .expect("enqueue global consolidation"); + let mut constrained_config = harness.config.as_ref().clone(); + constrained_config.permissions.sandbox_policy = + Constrained::allow_only(SandboxPolicy::DangerFullAccess); + + phase2::run(&harness.session, Arc::new(constrained_config)).await; + + let retry_claim = harness + .state_db + .try_claim_global_phase2_job(ThreadId::new(), 3_600) + .await + .expect("claim global job after sandbox policy failure"); + pretty_assertions::assert_eq!(retry_claim, Phase2JobClaimOutcome::SkippedNotDirty); + pretty_assertions::assert_eq!(harness.user_input_ops_count(), 0); + let thread_ids = harness.manager.list_thread_ids().await; + pretty_assertions::assert_eq!(thread_ids.len(), 0); + } + + #[tokio::test] + async fn dispatch_marks_job_for_retry_when_syncing_artifacts_fails() { + let harness = DispatchHarness::new().await; + harness.seed_stage1_output(100).await; + let root = memory_root(&harness.config.codex_home); + tokio::fs::write(&root, "not a directory") + .await + .expect("create file at memory root"); + + phase2::run(&harness.session, Arc::clone(&harness.config)).await; + + let retry_claim = harness + .state_db + .try_claim_global_phase2_job(ThreadId::new(), 3_600) + .await + .expect("claim global job after sync failure"); + pretty_assertions::assert_eq!(retry_claim, Phase2JobClaimOutcome::SkippedNotDirty); + pretty_assertions::assert_eq!(harness.user_input_ops_count(), 0); + let thread_ids = harness.manager.list_thread_ids().await; + pretty_assertions::assert_eq!(thread_ids.len(), 0); + } + + #[tokio::test] + async fn dispatch_marks_job_for_retry_when_rebuilding_raw_memories_fails() { + let harness = DispatchHarness::new().await; + harness.seed_stage1_output(100).await; + let root = memory_root(&harness.config.codex_home); + tokio::fs::create_dir_all(raw_memories_file(&root)) + .await + .expect("create raw_memories.md as a directory"); + + phase2::run(&harness.session, Arc::clone(&harness.config)).await; + + let retry_claim = harness + .state_db + .try_claim_global_phase2_job(ThreadId::new(), 3_600) + .await + .expect("claim global job after rebuild failure"); + pretty_assertions::assert_eq!(retry_claim, Phase2JobClaimOutcome::SkippedNotDirty); + pretty_assertions::assert_eq!(harness.user_input_ops_count(), 0); + let thread_ids = harness.manager.list_thread_ids().await; + pretty_assertions::assert_eq!(thread_ids.len(), 0); + } + + #[tokio::test] + async fn dispatch_marks_job_for_retry_when_spawn_agent_fails() { + let codex_home = tempfile::tempdir().expect("create temp codex home"); + let mut config = test_config(); + config.codex_home = codex_home.path().to_path_buf(); + config.cwd = config.codex_home.clone(); + let config = Arc::new(config); + + let state_db = codex_state::StateRuntime::init( + config.codex_home.clone(), + config.model_provider_id.clone(), + None, + ) + .await + .expect("initialize state db"); + + let (mut session, _turn_context) = make_session_and_context().await; + session.services.state_db = Some(Arc::clone(&state_db)); + session.services.agent_control = AgentControl::default(); + let session = Arc::new(session); + + let thread_id = ThreadId::new(); + let mut metadata_builder = ThreadMetadataBuilder::new( + thread_id, + config.codex_home.join(format!("rollout-{thread_id}.jsonl")), + Utc::now(), + SessionSource::Cli, + ); + metadata_builder.cwd = config.cwd.clone(); + metadata_builder.model_provider = Some(config.model_provider_id.clone()); + let metadata = metadata_builder.build(&config.model_provider_id); + state_db + .upsert_thread(&metadata) + .await + .expect("upsert thread metadata"); + + let claim = state_db + .try_claim_stage1_job(thread_id, session.conversation_id, 100, 3_600, 64) + .await + .expect("claim stage-1 job"); + let ownership_token = match claim { + codex_state::Stage1JobClaimOutcome::Claimed { ownership_token } => ownership_token, + other => panic!("unexpected stage-1 claim outcome: {other:?}"), + }; + assert!( + state_db + .mark_stage1_job_succeeded( + thread_id, + &ownership_token, + 100, + "raw memory", + "rollout summary", + None, + ) + .await + .expect("mark stage-1 success"), + "stage-1 success should enqueue global consolidation" + ); + + phase2::run(&session, Arc::clone(&config)).await; + + let retry_claim = state_db + .try_claim_global_phase2_job(ThreadId::new(), 3_600) + .await + .expect("claim global job after spawn failure"); + pretty_assertions::assert_eq!( + retry_claim, + Phase2JobClaimOutcome::SkippedNotDirty, + "spawn failures should leave the job in retry backoff instead of running" + ); + } +} diff --git a/codex-rs/core/src/models_manager/manager.rs b/codex-rs/core/src/models_manager/manager.rs index 15b3dfe0d0..dd769da2e8 100644 --- a/codex-rs/core/src/models_manager/manager.rs +++ b/codex-rs/core/src/models_manager/manager.rs @@ -140,15 +140,16 @@ impl ModelsManager { let remote = self .find_remote_model_by_longest_prefix(model, config) .await; - let model = if let Some(remote) = remote { + let model_info = if let Some(remote) = remote { ModelInfo { slug: model.to_string(), + used_fallback_model_metadata: false, ..remote } } else { model_info::model_info_from_slug(model) }; - model_info::with_config_overrides(model, config) + model_info::with_config_overrides(model_info, config) } async fn find_remote_model_by_longest_prefix( @@ -472,6 +473,37 @@ mod tests { } } + #[tokio::test] + async fn get_model_info_tracks_fallback_usage() { + let codex_home = tempdir().expect("temp dir"); + let mut config = ConfigBuilder::default() + .codex_home(codex_home.path().to_path_buf()) + .build() + .await + .expect("load default test config"); + config.features.enable(Feature::RemoteModels); + let auth_manager = + AuthManager::from_auth_for_testing(CodexAuth::from_api_key("Test API Key")); + let manager = ModelsManager::new(codex_home.path().to_path_buf(), auth_manager); + let known_slug = manager + .get_remote_models(&config) + .await + .first() + .expect("bundled models should include at least one model") + .slug + .clone(); + + let known = manager.get_model_info(known_slug.as_str(), &config).await; + assert!(!known.used_fallback_model_metadata); + assert_eq!(known.slug, known_slug); + + let unknown = manager + .get_model_info("model-that-does-not-exist", &config) + .await; + assert!(unknown.used_fallback_model_metadata); + assert_eq!(unknown.slug, "model-that-does-not-exist"); + } + #[tokio::test] async fn refresh_available_models_sorts_by_priority() { let server = MockServer::start().await; diff --git a/codex-rs/core/src/models_manager/model_info.rs b/codex-rs/core/src/models_manager/model_info.rs index 6a29cd96f9..c5ad8899a1 100644 --- a/codex-rs/core/src/models_manager/model_info.rs +++ b/codex-rs/core/src/models_manager/model_info.rs @@ -81,6 +81,7 @@ pub(crate) fn model_info_from_slug(slug: &str) -> ModelInfo { experimental_supported_tools: Vec::new(), input_modalities: default_input_modalities(), prefer_websockets: false, + used_fallback_model_metadata: true, // this is the fallback model metadata } } diff --git a/codex-rs/core/src/network_policy_decision.rs b/codex-rs/core/src/network_policy_decision.rs new file mode 100644 index 0000000000..3da6d34d6c --- /dev/null +++ b/codex-rs/core/src/network_policy_decision.rs @@ -0,0 +1,254 @@ +use codex_network_proxy::BlockedRequest; +use codex_network_proxy::NetworkDecisionSource; +use codex_network_proxy::NetworkPolicyDecision; +use codex_protocol::approvals::NetworkApprovalContext; +use codex_protocol::approvals::NetworkApprovalProtocol; +use serde::Deserialize; + +#[derive(Debug, Clone, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct NetworkPolicyDecisionPayload { + pub decision: NetworkPolicyDecision, + pub source: NetworkDecisionSource, + #[serde(default)] + pub protocol: Option, + pub host: Option, + pub reason: Option, + pub port: Option, +} + +impl NetworkPolicyDecisionPayload { + pub(crate) fn is_ask_from_decider(&self) -> bool { + self.decision == NetworkPolicyDecision::Ask && self.source == NetworkDecisionSource::Decider + } +} + +fn parse_network_policy_decision(value: &str) -> Option { + match value { + "deny" => Some(NetworkPolicyDecision::Deny), + "ask" => Some(NetworkPolicyDecision::Ask), + _ => None, + } +} + +pub(crate) fn network_approval_context_from_payload( + payload: &NetworkPolicyDecisionPayload, +) -> Option { + if !payload.is_ask_from_decider() { + return None; + } + + let protocol = payload.protocol?; + + let host = payload.host.as_deref()?.trim(); + if host.is_empty() { + return None; + } + + Some(NetworkApprovalContext { + host: host.to_string(), + protocol, + }) +} + +pub(crate) fn denied_network_policy_message(blocked: &BlockedRequest) -> Option { + let decision = blocked + .decision + .as_deref() + .and_then(parse_network_policy_decision); + if decision != Some(NetworkPolicyDecision::Deny) { + return None; + } + + let host = blocked.host.trim(); + if host.is_empty() { + return Some("Network access was blocked by policy.".to_string()); + } + + let detail = match blocked.reason.as_str() { + "denied" => "domain is explicitly denied by policy and cannot be approved from this prompt", + "not_allowed" => "domain is not on the allowlist for the current sandbox mode", + "not_allowed_local" => "local/private network addresses are blocked by policy", + "method_not_allowed" => "request method is blocked by the current network mode", + "proxy_disabled" => "managed network proxy is disabled", + _ => "request is blocked by network policy", + }; + + Some(format!( + "Network access to \"{host}\" was blocked: {detail}." + )) +} + +#[cfg(test)] +mod tests { + use super::*; + use codex_network_proxy::BlockedRequest; + use pretty_assertions::assert_eq; + + #[test] + fn network_approval_context_requires_ask_from_decider() { + let payload = NetworkPolicyDecisionPayload { + decision: NetworkPolicyDecision::Deny, + source: NetworkDecisionSource::Decider, + protocol: Some(NetworkApprovalProtocol::Https), + host: Some("example.com".to_string()), + reason: Some("not_allowed".to_string()), + port: Some(443), + }; + + assert_eq!(network_approval_context_from_payload(&payload), None); + } + + #[test] + fn network_approval_context_maps_http_https_and_socks_protocols() { + let http_payload = NetworkPolicyDecisionPayload { + decision: NetworkPolicyDecision::Ask, + source: NetworkDecisionSource::Decider, + protocol: Some(NetworkApprovalProtocol::Http), + host: Some("example.com".to_string()), + reason: Some("not_allowed".to_string()), + port: Some(80), + }; + assert_eq!( + network_approval_context_from_payload(&http_payload), + Some(NetworkApprovalContext { + host: "example.com".to_string(), + protocol: NetworkApprovalProtocol::Http, + }) + ); + + let https_payload = NetworkPolicyDecisionPayload { + decision: NetworkPolicyDecision::Ask, + source: NetworkDecisionSource::Decider, + protocol: Some(NetworkApprovalProtocol::Https), + host: Some("example.com".to_string()), + reason: Some("not_allowed".to_string()), + port: Some(443), + }; + assert_eq!( + network_approval_context_from_payload(&https_payload), + Some(NetworkApprovalContext { + host: "example.com".to_string(), + protocol: NetworkApprovalProtocol::Https, + }) + ); + + let http_connect_payload = NetworkPolicyDecisionPayload { + decision: NetworkPolicyDecision::Ask, + source: NetworkDecisionSource::Decider, + protocol: Some(NetworkApprovalProtocol::Https), + host: Some("example.com".to_string()), + reason: Some("not_allowed".to_string()), + port: Some(443), + }; + assert_eq!( + network_approval_context_from_payload(&http_connect_payload), + Some(NetworkApprovalContext { + host: "example.com".to_string(), + protocol: NetworkApprovalProtocol::Https, + }) + ); + + let socks5_tcp_payload = NetworkPolicyDecisionPayload { + decision: NetworkPolicyDecision::Ask, + source: NetworkDecisionSource::Decider, + protocol: Some(NetworkApprovalProtocol::Socks5Tcp), + host: Some("example.com".to_string()), + reason: Some("not_allowed".to_string()), + port: Some(443), + }; + assert_eq!( + network_approval_context_from_payload(&socks5_tcp_payload), + Some(NetworkApprovalContext { + host: "example.com".to_string(), + protocol: NetworkApprovalProtocol::Socks5Tcp, + }) + ); + + let socks5_udp_payload = NetworkPolicyDecisionPayload { + decision: NetworkPolicyDecision::Ask, + source: NetworkDecisionSource::Decider, + protocol: Some(NetworkApprovalProtocol::Socks5Udp), + host: Some("example.com".to_string()), + reason: Some("not_allowed".to_string()), + port: Some(443), + }; + assert_eq!( + network_approval_context_from_payload(&socks5_udp_payload), + Some(NetworkApprovalContext { + host: "example.com".to_string(), + protocol: NetworkApprovalProtocol::Socks5Udp, + }) + ); + } + + #[test] + fn network_policy_decision_payload_deserializes_proxy_protocol_aliases() { + let payload: NetworkPolicyDecisionPayload = serde_json::from_str( + r#"{ + "decision":"ask", + "source":"decider", + "protocol":"https_connect", + "host":"example.com", + "reason":"not_allowed", + "port":443 + }"#, + ) + .expect("payload should deserialize"); + assert_eq!(payload.protocol, Some(NetworkApprovalProtocol::Https)); + + let payload: NetworkPolicyDecisionPayload = serde_json::from_str( + r#"{ + "decision":"ask", + "source":"decider", + "protocol":"http-connect", + "host":"example.com", + "reason":"not_allowed", + "port":443 + }"#, + ) + .expect("payload should deserialize"); + assert_eq!(payload.protocol, Some(NetworkApprovalProtocol::Https)); + } + + #[test] + fn denied_network_policy_message_requires_deny_decision() { + let blocked = BlockedRequest { + host: "example.com".to_string(), + reason: "not_allowed".to_string(), + client: None, + method: Some("GET".to_string()), + mode: None, + protocol: "http".to_string(), + attempt_id: Some("attempt-1".to_string()), + decision: Some("ask".to_string()), + source: Some("decider".to_string()), + port: Some(80), + timestamp: 0, + }; + assert_eq!(denied_network_policy_message(&blocked), None); + } + + #[test] + fn denied_network_policy_message_for_denylist_block_is_explicit() { + let blocked = BlockedRequest { + host: "example.com".to_string(), + reason: "denied".to_string(), + client: None, + method: Some("GET".to_string()), + mode: None, + protocol: "http".to_string(), + attempt_id: Some("attempt-1".to_string()), + decision: Some("deny".to_string()), + source: Some("baseline_policy".to_string()), + port: Some(80), + timestamp: 0, + }; + assert_eq!( + denied_network_policy_message(&blocked), + Some( + "Network access to \"example.com\" was blocked: domain is explicitly denied by policy and cannot be approved from this prompt.".to_string() + ) + ); + } +} diff --git a/codex-rs/core/src/rollout/list.rs b/codex-rs/core/src/rollout/list.rs index af9223119c..9b2b00c78b 100644 --- a/codex-rs/core/src/rollout/list.rs +++ b/codex-rs/core/src/rollout/list.rs @@ -1214,7 +1214,7 @@ async fn find_thread_path_by_id_str_in_subdir( let found = results.matches.into_iter().next().map(|m| m.full_path()); if let Some(found_path) = found.as_ref() { - tracing::error!("state db missing rollout path for thread {id_str}"); + tracing::debug!("state db missing rollout path for thread {id_str}"); state_db::record_discrepancy("find_thread_path_by_id_str_in_subdir", "falling_back"); state_db::read_repair_rollout_path( state_db_ctx.as_deref(), diff --git a/codex-rs/core/src/sandbox_tags.rs b/codex-rs/core/src/sandbox_tags.rs index d81647e711..56c9e0f184 100644 --- a/codex-rs/core/src/sandbox_tags.rs +++ b/codex-rs/core/src/sandbox_tags.rs @@ -6,6 +6,7 @@ use codex_protocol::config_types::WindowsSandboxLevel; pub(crate) fn sandbox_tag( policy: &SandboxPolicy, windows_sandbox_level: WindowsSandboxLevel, + use_linux_sandbox_bwrap: bool, ) -> &'static str { if matches!(policy, SandboxPolicy::DangerFullAccess) { return "none"; @@ -17,8 +18,61 @@ pub(crate) fn sandbox_tag( { return "windows_elevated"; } + if cfg!(target_os = "linux") && use_linux_sandbox_bwrap { + return "linux_bubblewrap"; + } get_platform_sandbox(windows_sandbox_level != WindowsSandboxLevel::Disabled) .map(SandboxType::as_metric_tag) .unwrap_or("none") } + +#[cfg(test)] +mod tests { + use super::sandbox_tag; + use crate::exec::SandboxType; + use crate::protocol::SandboxPolicy; + use crate::safety::get_platform_sandbox; + use codex_protocol::config_types::WindowsSandboxLevel; + use codex_protocol::protocol::NetworkAccess; + use pretty_assertions::assert_eq; + + #[test] + fn danger_full_access_is_untagged_even_when_bubblewrap_is_enabled() { + let actual = sandbox_tag( + &SandboxPolicy::DangerFullAccess, + WindowsSandboxLevel::Disabled, + true, + ); + assert_eq!(actual, "none"); + } + + #[test] + fn external_sandbox_keeps_external_tag_when_bubblewrap_is_enabled() { + let actual = sandbox_tag( + &SandboxPolicy::ExternalSandbox { + network_access: NetworkAccess::Enabled, + }, + WindowsSandboxLevel::Disabled, + true, + ); + assert_eq!(actual, "external"); + } + + #[test] + fn bubblewrap_feature_sets_distinct_linux_tag() { + let actual = sandbox_tag( + &SandboxPolicy::new_read_only_policy(), + WindowsSandboxLevel::Disabled, + true, + ); + let expected = if cfg!(target_os = "linux") { + "linux_bubblewrap" + } else { + get_platform_sandbox(false) + .map(SandboxType::as_metric_tag) + .unwrap_or("none") + }; + assert_eq!(actual, expected); + } +} diff --git a/codex-rs/core/src/sandboxing/mod.rs b/codex-rs/core/src/sandboxing/mod.rs index 9a72b6e249..487278b619 100644 --- a/codex-rs/core/src/sandboxing/mod.rs +++ b/codex-rs/core/src/sandboxing/mod.rs @@ -46,6 +46,7 @@ pub struct ExecRequest { pub cwd: PathBuf, pub env: HashMap, pub network: Option, + pub network_attempt_id: Option, pub expiration: ExecExpiration, pub sandbox: SandboxType, pub windows_sandbox_level: WindowsSandboxLevel, @@ -214,6 +215,7 @@ impl SandboxManager { cwd: spec.cwd, env, network: network.cloned(), + network_attempt_id: None, expiration: spec.expiration, sandbox, windows_sandbox_level, diff --git a/codex-rs/core/src/seatbelt.rs b/codex-rs/core/src/seatbelt.rs index 29c05e0183..7f738797b8 100644 --- a/codex-rs/core/src/seatbelt.rs +++ b/codex-rs/core/src/seatbelt.rs @@ -14,7 +14,6 @@ use tokio::process::Child; use url::Url; use crate::protocol::SandboxPolicy; -use crate::seatbelt_permissions::MacOsPreferencesPermission; use crate::seatbelt_permissions::MacOsSeatbeltProfileExtensions; use crate::seatbelt_permissions::build_seatbelt_extensions; use crate::spawn::CODEX_SANDBOX_ENV_VAR; @@ -299,10 +298,7 @@ pub(crate) fn create_seatbelt_command_args_with_extensions( let seatbelt_extensions = extensions.map_or_else( || { // Backward-compatibility default when no extension profile is provided. - build_seatbelt_extensions(&MacOsSeatbeltProfileExtensions { - macos_preferences: MacOsPreferencesPermission::ReadOnly, - ..Default::default() - }) + build_seatbelt_extensions(&MacOsSeatbeltProfileExtensions::default()) }, build_seatbelt_extensions, ); @@ -480,7 +476,7 @@ mod tests { } #[test] - fn seatbelt_args_omit_macos_extensions_when_profile_is_empty() { + fn seatbelt_args_default_extension_profile_keeps_preferences_read_access() { let cwd = std::env::temp_dir(); let args = create_seatbelt_command_args_with_extensions( vec!["echo".to_string(), "ok".to_string()], @@ -494,7 +490,7 @@ mod tests { assert!(!policy.contains("appleevent-send")); assert!(!policy.contains("com.apple.axserver")); assert!(!policy.contains("com.apple.CalendarAgent")); - assert!(!policy.contains("user-preference-read")); + assert!(policy.contains("(allow user-preference-read)")); assert!(!policy.contains("user-preference-write")); } diff --git a/codex-rs/core/src/seatbelt_permissions.rs b/codex-rs/core/src/seatbelt_permissions.rs index 29313d7a3c..aacc3a7784 100644 --- a/codex-rs/core/src/seatbelt_permissions.rs +++ b/codex-rs/core/src/seatbelt_permissions.rs @@ -6,10 +6,12 @@ use std::path::PathBuf; #[allow(dead_code)] #[derive(Debug, Clone, PartialEq, Eq, Default)] pub enum MacOsPreferencesPermission { + // IMPORTANT: ReadOnly needs to be the default because it's the security-sensitive default. + // it's important for allowing cf prefs to work. #[default] - None, ReadOnly, ReadWrite, + None, } #[allow(dead_code)] @@ -164,7 +166,6 @@ mod tests { use super::MacOsPreferencesPermission; use super::MacOsSeatbeltProfileExtensions; use super::build_seatbelt_extensions; - use pretty_assertions::assert_eq; #[test] fn preferences_read_only_emits_read_clauses_only() { @@ -239,8 +240,9 @@ mod tests { } #[test] - fn empty_extensions_emit_empty_policy() { + fn default_extensions_emit_preferences_read_only_policy() { let policy = build_seatbelt_extensions(&MacOsSeatbeltProfileExtensions::default()); - assert_eq!(policy.policy, ""); + assert!(policy.policy.contains("(allow user-preference-read)")); + assert!(!policy.policy.contains("(allow user-preference-write)")); } } diff --git a/codex-rs/core/src/skills/injection.rs b/codex-rs/core/src/skills/injection.rs index 660785d712..4dec3fe399 100644 --- a/codex-rs/core/src/skills/injection.rs +++ b/codex-rs/core/src/skills/injection.rs @@ -476,6 +476,7 @@ mod tests { interface: None, dependencies: None, policy: None, + permissions: None, path: PathBuf::from(path), scope: codex_protocol::protocol::SkillScope::User, } diff --git a/codex-rs/core/src/skills/loader.rs b/codex-rs/core/src/skills/loader.rs index 6821d1b8fe..e00bd42976 100644 --- a/codex-rs/core/src/skills/loader.rs +++ b/codex-rs/core/src/skills/loader.rs @@ -1,4 +1,5 @@ use crate::config::Config; +use crate::config::Permissions; use crate::config_loader::ConfigLayerStack; use crate::config_loader::ConfigLayerStackOrdering; use crate::config_loader::default_project_root_markers; @@ -11,6 +12,8 @@ use crate::skills::model::SkillLoadOutcome; use crate::skills::model::SkillMetadata; use crate::skills::model::SkillPolicy; use crate::skills::model::SkillToolDependency; +use crate::skills::permissions::SkillManifestPermissions; +use crate::skills::permissions::compile_permission_profile; use crate::skills::system::system_cache_root_dir; use codex_app_server_protocol::ConfigLayerSource; use codex_protocol::protocol::SkillScope; @@ -50,6 +53,8 @@ struct SkillMetadataFile { dependencies: Option, #[serde(default)] policy: Option, + #[serde(default)] + permissions: Option, } #[derive(Debug, Default, Deserialize)] @@ -490,7 +495,7 @@ fn parse_skill_file(path: &Path, scope: SkillScope) -> Result Result, Option, Option, + Option, ) { // Fail open: optional metadata should not block loading SKILL.md. let Some(skill_dir) = skill_path.parent() else { - return (None, None, None); + return (None, None, None, None); }; let metadata_path = skill_dir .join(SKILLS_METADATA_DIR) .join(SKILLS_METADATA_FILENAME); if !metadata_path.exists() { - return (None, None, None); + return (None, None, None, None); } let contents = match fs::read_to_string(&metadata_path) { @@ -542,7 +549,7 @@ fn load_skill_metadata( path = metadata_path.display(), label = SKILLS_METADATA_FILENAME ); - return (None, None, None); + return (None, None, None, None); } }; @@ -554,7 +561,7 @@ fn load_skill_metadata( path = metadata_path.display(), label = SKILLS_METADATA_FILENAME ); - return (None, None, None); + return (None, None, None, None); } }; @@ -562,12 +569,14 @@ fn load_skill_metadata( interface, dependencies, policy, + permissions, } = parsed; ( resolve_interface(interface, skill_dir), resolve_dependencies(dependencies), resolve_policy(policy), + compile_permission_profile(skill_dir, permissions), ) } @@ -801,7 +810,9 @@ mod tests { use crate::config::ConfigBuilder; use crate::config::ConfigOverrides; use crate::config::ConfigToml; + use crate::config::Constrained; use crate::config::ProjectConfig; + use crate::config::types::ShellEnvironmentPolicy; use crate::config_loader::ConfigLayerEntry; use crate::config_loader::ConfigLayerStack; use crate::config_loader::ConfigRequirements; @@ -1021,6 +1032,7 @@ mod tests { interface: None, dependencies: None, policy: None, + permissions: None, path: normalized(&skill_path), scope: SkillScope::User, }] @@ -1168,6 +1180,7 @@ mod tests { ], }), policy: None, + permissions: None, path: normalized(&skill_path), scope: SkillScope::User, }] @@ -1223,6 +1236,7 @@ interface: }), dependencies: None, policy: None, + permissions: None, path: normalized(skill_path.as_path()), scope: SkillScope::User, }] @@ -1295,6 +1309,219 @@ policy: {} ); } + #[tokio::test] + async fn loads_skill_permissions_from_yaml() { + let codex_home = tempfile::tempdir().expect("tempdir"); + let skill_path = write_skill(&codex_home, "demo", "permissions-skill", "from yaml"); + let skill_dir = skill_path.parent().expect("skill dir"); + fs::create_dir_all(skill_dir.join("data")).expect("create read path"); + fs::create_dir_all(skill_dir.join("output")).expect("create write path"); + + write_skill_metadata_at( + skill_dir, + r#" +permissions: + network: true + file_system: + read: + - "./data" + - "./data" + write: + - "./output" +"#, + ); + + let cfg = make_config(&codex_home).await; + let outcome = load_skills(&cfg); + + assert!( + outcome.errors.is_empty(), + "unexpected errors: {:?}", + outcome.errors + ); + assert_eq!(outcome.skills.len(), 1); + #[cfg(target_os = "macos")] + let macos_seatbelt_profile_extensions = + Some(crate::seatbelt_permissions::MacOsSeatbeltProfileExtensions::default()); + #[cfg(not(target_os = "macos"))] + let macos_seatbelt_profile_extensions = None; + assert_eq!( + outcome.skills[0].permissions, + Some(Permissions { + approval_policy: Constrained::allow_any(crate::protocol::AskForApproval::Never), + sandbox_policy: Constrained::allow_any( + crate::protocol::SandboxPolicy::WorkspaceWrite { + writable_roots: vec![ + AbsolutePathBuf::try_from(normalized( + skill_dir.join("output").as_path(), + )) + .expect("absolute output path") + ], + read_only_access: crate::protocol::ReadOnlyAccess::Restricted { + include_platform_defaults: true, + readable_roots: vec![ + AbsolutePathBuf::try_from(normalized( + skill_dir.join("data").as_path(), + )) + .expect("absolute data path") + ], + }, + network_access: true, + exclude_tmpdir_env_var: false, + exclude_slash_tmp: false, + } + ), + network: None, + shell_environment_policy: ShellEnvironmentPolicy::default(), + windows_sandbox_mode: None, + macos_seatbelt_profile_extensions, + }) + ); + } + + #[tokio::test] + async fn empty_skill_permissions_do_not_create_profile() { + let codex_home = tempfile::tempdir().expect("tempdir"); + let skill_path = write_skill(&codex_home, "demo", "permissions-empty", "from yaml"); + let skill_dir = skill_path.parent().expect("skill dir"); + + write_skill_metadata_at( + skill_dir, + r#" +permissions: {} +"#, + ); + + let cfg = make_config(&codex_home).await; + let outcome = load_skills(&cfg); + + assert!( + outcome.errors.is_empty(), + "unexpected errors: {:?}", + outcome.errors + ); + assert_eq!(outcome.skills.len(), 1); + #[cfg(target_os = "macos")] + let expected = Some(Permissions { + approval_policy: Constrained::allow_any(crate::protocol::AskForApproval::Never), + sandbox_policy: Constrained::allow_any( + crate::protocol::SandboxPolicy::new_read_only_policy(), + ), + network: None, + shell_environment_policy: ShellEnvironmentPolicy::default(), + windows_sandbox_mode: None, + macos_seatbelt_profile_extensions: Some( + crate::seatbelt_permissions::MacOsSeatbeltProfileExtensions::default(), + ), + }); + #[cfg(not(target_os = "macos"))] + let expected = Some(Permissions { + approval_policy: Constrained::allow_any(crate::protocol::AskForApproval::Never), + sandbox_policy: Constrained::allow_any( + crate::protocol::SandboxPolicy::new_read_only_policy(), + ), + network: None, + shell_environment_policy: ShellEnvironmentPolicy::default(), + windows_sandbox_mode: None, + macos_seatbelt_profile_extensions: None, + }); + assert_eq!(outcome.skills[0].permissions, expected); + } + + #[cfg(target_os = "macos")] + #[tokio::test] + async fn loads_skill_macos_permissions_from_yaml() { + let codex_home = tempfile::tempdir().expect("tempdir"); + let skill_path = write_skill(&codex_home, "demo", "permissions-macos", "from yaml"); + let skill_dir = skill_path.parent().expect("skill dir"); + + write_skill_metadata_at( + skill_dir, + r#" +permissions: + macos: + preferences: "readwrite" + automations: + - "com.apple.Notes" + accessibility: true + calendar: true +"#, + ); + + let cfg = make_config(&codex_home).await; + let outcome = load_skills(&cfg); + + assert!( + outcome.errors.is_empty(), + "unexpected errors: {:?}", + outcome.errors + ); + assert_eq!(outcome.skills.len(), 1); + let profile = outcome.skills[0] + .permissions + .as_ref() + .expect("permission profile"); + assert_eq!( + profile.macos_seatbelt_profile_extensions, + Some( + crate::seatbelt_permissions::MacOsSeatbeltProfileExtensions { + macos_preferences: + crate::seatbelt_permissions::MacOsPreferencesPermission::ReadWrite, + macos_automation: + crate::seatbelt_permissions::MacOsAutomationPermission::BundleIds(vec![ + "com.apple.Notes".to_string() + ],), + macos_accessibility: true, + macos_calendar: true, + } + ) + ); + } + + #[cfg(not(target_os = "macos"))] + #[tokio::test] + async fn loads_skill_macos_permissions_from_yaml_non_macos_does_not_create_profile() { + let codex_home = tempfile::tempdir().expect("tempdir"); + let skill_path = write_skill(&codex_home, "demo", "permissions-macos", "from yaml"); + let skill_dir = skill_path.parent().expect("skill dir"); + + write_skill_metadata_at( + skill_dir, + r#" +permissions: + macos: + preferences: "readwrite" + automations: + - "com.apple.Notes" + accessibility: true + calendar: true +"#, + ); + + let cfg = make_config(&codex_home).await; + let outcome = load_skills(&cfg); + + assert!( + outcome.errors.is_empty(), + "unexpected errors: {:?}", + outcome.errors + ); + assert_eq!(outcome.skills.len(), 1); + assert_eq!( + outcome.skills[0].permissions, + Some(Permissions { + approval_policy: Constrained::allow_any(crate::protocol::AskForApproval::Never), + sandbox_policy: Constrained::allow_any( + crate::protocol::SandboxPolicy::new_read_only_policy(), + ), + network: None, + shell_environment_policy: ShellEnvironmentPolicy::default(), + windows_sandbox_mode: None, + macos_seatbelt_profile_extensions: None, + }) + ); + } + #[tokio::test] async fn accepts_icon_paths_under_assets_dir() { let codex_home = tempfile::tempdir().expect("tempdir"); @@ -1339,6 +1566,7 @@ policy: {} }), dependencies: None, policy: None, + permissions: None, path: normalized(&skill_path), scope: SkillScope::User, }] @@ -1379,6 +1607,7 @@ policy: {} interface: None, dependencies: None, policy: None, + permissions: None, path: normalized(&skill_path), scope: SkillScope::User, }] @@ -1432,6 +1661,7 @@ policy: {} }), dependencies: None, policy: None, + permissions: None, path: normalized(&skill_path), scope: SkillScope::User, }] @@ -1473,6 +1703,7 @@ policy: {} interface: None, dependencies: None, policy: None, + permissions: None, path: normalized(&skill_path), scope: SkillScope::User, }] @@ -1517,6 +1748,7 @@ policy: {} interface: None, dependencies: None, policy: None, + permissions: None, path: normalized(&shared_skill_path), scope: SkillScope::User, }] @@ -1577,6 +1809,7 @@ policy: {} interface: None, dependencies: None, policy: None, + permissions: None, path: normalized(&skill_path), scope: SkillScope::User, }] @@ -1613,6 +1846,7 @@ policy: {} interface: None, dependencies: None, policy: None, + permissions: None, path: normalized(&shared_skill_path), scope: SkillScope::Admin, }] @@ -1653,6 +1887,7 @@ policy: {} interface: None, dependencies: None, policy: None, + permissions: None, path: normalized(&linked_skill_path), scope: SkillScope::Repo, }] @@ -1720,6 +1955,7 @@ policy: {} interface: None, dependencies: None, policy: None, + permissions: None, path: normalized(&within_depth_path), scope: SkillScope::User, }] @@ -1747,6 +1983,7 @@ policy: {} interface: None, dependencies: None, policy: None, + permissions: None, path: normalized(&skill_path), scope: SkillScope::User, }] @@ -1778,6 +2015,7 @@ policy: {} interface: None, dependencies: None, policy: None, + permissions: None, path: normalized(&skill_path), scope: SkillScope::User, }] @@ -1890,6 +2128,7 @@ policy: {} interface: None, dependencies: None, policy: None, + permissions: None, path: normalized(&skill_path), scope: SkillScope::Repo, }] @@ -1925,6 +2164,7 @@ policy: {} interface: None, dependencies: None, policy: None, + permissions: None, path: normalized(&skill_path), scope: SkillScope::Repo, }] @@ -1978,6 +2218,7 @@ policy: {} interface: None, dependencies: None, policy: None, + permissions: None, path: normalized(&nested_skill_path), scope: SkillScope::Repo, }, @@ -1988,6 +2229,7 @@ policy: {} interface: None, dependencies: None, policy: None, + permissions: None, path: normalized(&root_skill_path), scope: SkillScope::Repo, }, @@ -2027,6 +2269,7 @@ policy: {} interface: None, dependencies: None, policy: None, + permissions: None, path: normalized(&skill_path), scope: SkillScope::Repo, }] @@ -2064,6 +2307,7 @@ policy: {} interface: None, dependencies: None, policy: None, + permissions: None, path: normalized(&skill_path), scope: SkillScope::Repo, }] @@ -2105,6 +2349,7 @@ policy: {} interface: None, dependencies: None, policy: None, + permissions: None, path: normalized(&repo_skill_path), scope: SkillScope::Repo, }, @@ -2115,6 +2360,7 @@ policy: {} interface: None, dependencies: None, policy: None, + permissions: None, path: normalized(&user_skill_path), scope: SkillScope::User, }, @@ -2179,6 +2425,7 @@ policy: {} interface: None, dependencies: None, policy: None, + permissions: None, path: first_path, scope: SkillScope::Repo, }, @@ -2189,6 +2436,7 @@ policy: {} interface: None, dependencies: None, policy: None, + permissions: None, path: second_path, scope: SkillScope::Repo, }, @@ -2260,6 +2508,7 @@ policy: {} interface: None, dependencies: None, policy: None, + permissions: None, path: normalized(&skill_path), scope: SkillScope::Repo, }] @@ -2318,6 +2567,7 @@ policy: {} interface: None, dependencies: None, policy: None, + permissions: None, path: normalized(&skill_path), scope: SkillScope::System, }] diff --git a/codex-rs/core/src/skills/mod.rs b/codex-rs/core/src/skills/mod.rs index 17405129ec..18f7180f99 100644 --- a/codex-rs/core/src/skills/mod.rs +++ b/codex-rs/core/src/skills/mod.rs @@ -3,6 +3,7 @@ pub mod injection; pub mod loader; pub mod manager; pub mod model; +pub mod permissions; pub mod remote; pub mod render; pub mod system; diff --git a/codex-rs/core/src/skills/model.rs b/codex-rs/core/src/skills/model.rs index bc75b7e365..ac642db43d 100644 --- a/codex-rs/core/src/skills/model.rs +++ b/codex-rs/core/src/skills/model.rs @@ -1,9 +1,10 @@ use std::collections::HashSet; use std::path::PathBuf; +use crate::config::Permissions; use codex_protocol::protocol::SkillScope; -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq)] pub struct SkillMetadata { pub name: String, pub description: String, @@ -11,6 +12,8 @@ pub struct SkillMetadata { pub interface: Option, pub dependencies: Option, pub policy: Option, + // This is an experimental field. + pub permissions: Option, pub path: PathBuf, pub scope: SkillScope, } diff --git a/codex-rs/core/src/skills/permissions.rs b/codex-rs/core/src/skills/permissions.rs new file mode 100644 index 0000000000..1cf18e1052 --- /dev/null +++ b/codex-rs/core/src/skills/permissions.rs @@ -0,0 +1,510 @@ +use std::collections::HashSet; +use std::path::Component; +use std::path::Path; +use std::path::PathBuf; + +use codex_utils_absolute_path::AbsolutePathBuf; +use dirs::home_dir; +use dunce::canonicalize as canonicalize_path; +use serde::Deserialize; +use tracing::warn; + +use crate::config::Constrained; +use crate::config::Permissions; +use crate::config::types::ShellEnvironmentPolicy; +use crate::protocol::AskForApproval; +use crate::protocol::ReadOnlyAccess; +use crate::protocol::SandboxPolicy; +#[cfg(target_os = "macos")] +use crate::seatbelt_permissions::MacOsSeatbeltProfileExtensions; +#[cfg(not(target_os = "macos"))] +type MacOsSeatbeltProfileExtensions = (); + +#[derive(Debug, Clone, PartialEq, Eq, Default, Deserialize)] +pub(crate) struct SkillManifestPermissions { + #[serde(default)] + pub(crate) network: bool, + #[serde(default)] + pub(crate) file_system: SkillManifestFileSystemPermissions, + #[serde(default)] + pub(crate) macos: SkillManifestMacOsPermissions, +} + +#[derive(Debug, Clone, PartialEq, Eq, Default, Deserialize)] +pub(crate) struct SkillManifestFileSystemPermissions { + #[serde(default)] + pub(crate) read: Vec, + #[serde(default)] + pub(crate) write: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Default, Deserialize)] +pub(crate) struct SkillManifestMacOsPermissions { + #[serde(default)] + pub(crate) preferences: Option, + #[serde(default)] + pub(crate) automations: Option, + #[serde(default)] + pub(crate) accessibility: bool, + #[serde(default)] + pub(crate) calendar: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] +#[serde(untagged)] +pub(crate) enum MacOsPreferencesValue { + Bool(bool), + Mode(String), +} + +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] +#[serde(untagged)] +pub(crate) enum MacOsAutomationValue { + Bool(bool), + BundleIds(Vec), +} + +pub(crate) fn compile_permission_profile( + skill_dir: &Path, + permissions: Option, +) -> Option { + let permissions = permissions?; + let fs_read = normalize_permission_paths( + skill_dir, + &permissions.file_system.read, + "permissions.file_system.read", + ); + let fs_write = normalize_permission_paths( + skill_dir, + &permissions.file_system.write, + "permissions.file_system.write", + ); + let sandbox_policy = if !fs_write.is_empty() { + SandboxPolicy::WorkspaceWrite { + writable_roots: fs_write, + read_only_access: if fs_read.is_empty() { + ReadOnlyAccess::FullAccess + } else { + ReadOnlyAccess::Restricted { + include_platform_defaults: true, + readable_roots: fs_read, + } + }, + network_access: permissions.network, + exclude_tmpdir_env_var: false, + exclude_slash_tmp: false, + } + } else if !fs_read.is_empty() { + SandboxPolicy::ReadOnly { + access: ReadOnlyAccess::Restricted { + include_platform_defaults: true, + readable_roots: fs_read, + }, + } + } else { + // Default sandbox policy + SandboxPolicy::new_read_only_policy() + }; + let macos_seatbelt_profile_extensions = + build_macos_seatbelt_profile_extensions(&permissions.macos); + + Some(Permissions { + approval_policy: Constrained::allow_any(AskForApproval::Never), + sandbox_policy: Constrained::allow_any(sandbox_policy), + network: None, + shell_environment_policy: ShellEnvironmentPolicy::default(), + windows_sandbox_mode: None, + macos_seatbelt_profile_extensions, + }) +} + +fn normalize_permission_paths( + skill_dir: &Path, + values: &[String], + field: &str, +) -> Vec { + let mut paths = Vec::new(); + let mut seen = HashSet::new(); + + for value in values { + let Some(path) = normalize_permission_path(skill_dir, value, field) else { + continue; + }; + if seen.insert(path.clone()) { + paths.push(path); + } + } + + paths +} + +fn normalize_permission_path( + skill_dir: &Path, + value: &str, + field: &str, +) -> Option { + let trimmed = value.trim(); + if trimmed.is_empty() { + warn!("ignoring {field}: value is empty"); + return None; + } + + let expanded = expand_home(trimmed); + let path = PathBuf::from(expanded); + let absolute = if path.is_absolute() { + path + } else { + skill_dir.join(path) + }; + let normalized = normalize_lexically(&absolute); + let canonicalized = canonicalize_path(&normalized).unwrap_or(normalized); + match AbsolutePathBuf::from_absolute_path(&canonicalized) { + Ok(path) => Some(path), + Err(error) => { + warn!("ignoring {field}: expected absolute path, got {canonicalized:?}: {error}"); + None + } + } +} + +fn expand_home(path: &str) -> String { + if path == "~" { + if let Some(home) = home_dir() { + return home.to_string_lossy().to_string(); + } + return path.to_string(); + } + if let Some(rest) = path.strip_prefix("~/") + && let Some(home) = home_dir() + { + return home.join(rest).to_string_lossy().to_string(); + } + path.to_string() +} + +#[cfg(target_os = "macos")] +fn build_macos_seatbelt_profile_extensions( + permissions: &SkillManifestMacOsPermissions, +) -> Option { + let defaults = MacOsSeatbeltProfileExtensions::default(); + + let extensions = MacOsSeatbeltProfileExtensions { + macos_preferences: resolve_macos_preferences_permission( + permissions.preferences.as_ref(), + defaults.macos_preferences, + ), + macos_automation: resolve_macos_automation_permission( + permissions.automations.as_ref(), + defaults.macos_automation, + ), + macos_accessibility: permissions.accessibility, + macos_calendar: permissions.calendar, + }; + Some(extensions) +} + +#[cfg(target_os = "macos")] +fn resolve_macos_preferences_permission( + value: Option<&MacOsPreferencesValue>, + default: crate::seatbelt_permissions::MacOsPreferencesPermission, +) -> crate::seatbelt_permissions::MacOsPreferencesPermission { + use crate::seatbelt_permissions::MacOsPreferencesPermission; + + match value { + Some(MacOsPreferencesValue::Bool(true)) => MacOsPreferencesPermission::ReadOnly, + Some(MacOsPreferencesValue::Bool(false)) => MacOsPreferencesPermission::None, + Some(MacOsPreferencesValue::Mode(mode)) => { + let mode = mode.trim(); + if mode.eq_ignore_ascii_case("readonly") || mode.eq_ignore_ascii_case("read-only") { + MacOsPreferencesPermission::ReadOnly + } else if mode.eq_ignore_ascii_case("readwrite") + || mode.eq_ignore_ascii_case("read-write") + { + MacOsPreferencesPermission::ReadWrite + } else { + warn!( + "ignoring permissions.macos.preferences: expected true/false, readonly, or readwrite" + ); + default + } + } + None => default, + } +} + +#[cfg(target_os = "macos")] +fn resolve_macos_automation_permission( + value: Option<&MacOsAutomationValue>, + default: crate::seatbelt_permissions::MacOsAutomationPermission, +) -> crate::seatbelt_permissions::MacOsAutomationPermission { + use crate::seatbelt_permissions::MacOsAutomationPermission; + + match value { + Some(MacOsAutomationValue::Bool(true)) => MacOsAutomationPermission::All, + Some(MacOsAutomationValue::Bool(false)) => MacOsAutomationPermission::None, + Some(MacOsAutomationValue::BundleIds(bundle_ids)) => { + let bundle_ids = bundle_ids + .iter() + .map(|bundle_id| bundle_id.trim()) + .filter(|bundle_id| !bundle_id.is_empty()) + .map(ToOwned::to_owned) + .collect::>(); + if bundle_ids.is_empty() { + MacOsAutomationPermission::None + } else { + MacOsAutomationPermission::BundleIds(bundle_ids) + } + } + None => default, + } +} + +#[cfg(not(target_os = "macos"))] +fn build_macos_seatbelt_profile_extensions( + _: &SkillManifestMacOsPermissions, +) -> Option { + None +} + +fn normalize_lexically(path: &Path) -> PathBuf { + let mut normalized = PathBuf::new(); + for component in path.components() { + match component { + Component::CurDir => {} + Component::ParentDir => { + normalized.pop(); + } + Component::RootDir | Component::Prefix(_) | Component::Normal(_) => { + normalized.push(component.as_os_str()); + } + } + } + normalized +} + +#[cfg(test)] +mod tests { + use super::SkillManifestFileSystemPermissions; + #[cfg(target_os = "macos")] + use super::SkillManifestMacOsPermissions; + use super::SkillManifestPermissions; + use super::compile_permission_profile; + use crate::config::Constrained; + use crate::config::Permissions; + use crate::config::types::ShellEnvironmentPolicy; + use crate::protocol::AskForApproval; + use crate::protocol::ReadOnlyAccess; + use crate::protocol::SandboxPolicy; + use codex_utils_absolute_path::AbsolutePathBuf; + use pretty_assertions::assert_eq; + use std::fs; + + #[test] + fn compile_permission_profile_normalizes_paths() { + let tempdir = tempfile::tempdir().expect("tempdir"); + let skill_dir = tempdir.path().join("skill"); + fs::create_dir_all(skill_dir.join("scripts")).expect("skill dir"); + let read_dir = skill_dir.join("data"); + fs::create_dir_all(&read_dir).expect("read dir"); + + let profile = compile_permission_profile( + &skill_dir, + Some(SkillManifestPermissions { + network: true, + file_system: SkillManifestFileSystemPermissions { + read: vec![ + "./data".to_string(), + "./data".to_string(), + "scripts/../data".to_string(), + ], + write: vec!["./output".to_string()], + }, + ..Default::default() + }), + ) + .expect("profile"); + + assert_eq!( + profile, + Permissions { + approval_policy: Constrained::allow_any(AskForApproval::Never), + sandbox_policy: Constrained::allow_any(SandboxPolicy::WorkspaceWrite { + writable_roots: vec![ + AbsolutePathBuf::try_from(skill_dir.join("output")) + .expect("absolute output path") + ], + read_only_access: ReadOnlyAccess::Restricted { + include_platform_defaults: true, + readable_roots: vec![ + AbsolutePathBuf::try_from( + dunce::canonicalize(&read_dir).unwrap_or(read_dir) + ) + .expect("absolute read path") + ], + }, + network_access: true, + exclude_tmpdir_env_var: false, + exclude_slash_tmp: false, + }), + network: None, + shell_environment_policy: ShellEnvironmentPolicy::default(), + windows_sandbox_mode: None, + #[cfg(target_os = "macos")] + macos_seatbelt_profile_extensions: Some( + crate::seatbelt_permissions::MacOsSeatbeltProfileExtensions::default(), + ), + #[cfg(not(target_os = "macos"))] + macos_seatbelt_profile_extensions: None, + } + ); + } + + #[test] + fn compile_permission_profile_without_permissions_has_empty_profile() { + let tempdir = tempfile::tempdir().expect("tempdir"); + let skill_dir = tempdir.path().join("skill"); + fs::create_dir_all(&skill_dir).expect("skill dir"); + + let profile = compile_permission_profile(&skill_dir, None); + + assert_eq!(profile, None); + } + + #[test] + fn compile_permission_profile_with_network_only_uses_read_only_policy() { + let tempdir = tempfile::tempdir().expect("tempdir"); + let skill_dir = tempdir.path().join("skill"); + fs::create_dir_all(&skill_dir).expect("skill dir"); + + let profile = compile_permission_profile( + &skill_dir, + Some(SkillManifestPermissions { + network: true, + ..Default::default() + }), + ) + .expect("profile"); + + assert_eq!( + profile, + Permissions { + approval_policy: Constrained::allow_any(AskForApproval::Never), + sandbox_policy: Constrained::allow_any(SandboxPolicy::new_read_only_policy()), + network: None, + shell_environment_policy: ShellEnvironmentPolicy::default(), + windows_sandbox_mode: None, + #[cfg(target_os = "macos")] + macos_seatbelt_profile_extensions: Some( + crate::seatbelt_permissions::MacOsSeatbeltProfileExtensions::default(), + ), + #[cfg(not(target_os = "macos"))] + macos_seatbelt_profile_extensions: None, + } + ); + } + + #[test] + fn compile_permission_profile_with_network_and_read_only_paths_uses_read_only_policy() { + let tempdir = tempfile::tempdir().expect("tempdir"); + let skill_dir = tempdir.path().join("skill"); + let read_dir = skill_dir.join("data"); + fs::create_dir_all(&read_dir).expect("read dir"); + + let profile = compile_permission_profile( + &skill_dir, + Some(SkillManifestPermissions { + network: true, + file_system: SkillManifestFileSystemPermissions { + read: vec!["./data".to_string()], + write: Vec::new(), + }, + ..Default::default() + }), + ) + .expect("profile"); + + assert_eq!( + profile, + Permissions { + approval_policy: Constrained::allow_any(AskForApproval::Never), + sandbox_policy: Constrained::allow_any(SandboxPolicy::ReadOnly { + access: ReadOnlyAccess::Restricted { + include_platform_defaults: true, + readable_roots: vec![ + AbsolutePathBuf::try_from( + dunce::canonicalize(&read_dir).unwrap_or(read_dir) + ) + .expect("absolute read path") + ], + }, + }), + network: None, + shell_environment_policy: ShellEnvironmentPolicy::default(), + windows_sandbox_mode: None, + #[cfg(target_os = "macos")] + macos_seatbelt_profile_extensions: Some( + crate::seatbelt_permissions::MacOsSeatbeltProfileExtensions::default(), + ), + #[cfg(not(target_os = "macos"))] + macos_seatbelt_profile_extensions: None, + } + ); + } + + #[cfg(target_os = "macos")] + #[test] + fn compile_permission_profile_builds_macos_permission_file() { + let tempdir = tempfile::tempdir().expect("tempdir"); + let skill_dir = tempdir.path().join("skill"); + fs::create_dir_all(&skill_dir).expect("skill dir"); + + let profile = compile_permission_profile( + &skill_dir, + Some(SkillManifestPermissions { + macos: SkillManifestMacOsPermissions { + preferences: Some(super::MacOsPreferencesValue::Mode("readwrite".to_string())), + automations: Some(super::MacOsAutomationValue::BundleIds(vec![ + "com.apple.Notes".to_string(), + ])), + accessibility: true, + calendar: true, + }, + ..Default::default() + }), + ) + .expect("profile"); + + assert_eq!( + profile.macos_seatbelt_profile_extensions, + Some( + crate::seatbelt_permissions::MacOsSeatbeltProfileExtensions { + macos_preferences: + crate::seatbelt_permissions::MacOsPreferencesPermission::ReadWrite, + macos_automation: + crate::seatbelt_permissions::MacOsAutomationPermission::BundleIds(vec![ + "com.apple.Notes".to_string() + ],), + macos_accessibility: true, + macos_calendar: true, + } + ) + ); + } + + #[cfg(target_os = "macos")] + #[test] + fn compile_permission_profile_uses_macos_defaults_when_values_missing() { + let tempdir = tempfile::tempdir().expect("tempdir"); + let skill_dir = tempdir.path().join("skill"); + fs::create_dir_all(&skill_dir).expect("skill dir"); + + let profile = + compile_permission_profile(&skill_dir, Some(SkillManifestPermissions::default())) + .expect("profile"); + + assert_eq!( + profile.macos_seatbelt_profile_extensions, + Some(crate::seatbelt_permissions::MacOsSeatbeltProfileExtensions::default()) + ); + } +} diff --git a/codex-rs/core/src/state/service.rs b/codex-rs/core/src/state/service.rs index 3aec0e5350..6f640131f6 100644 --- a/codex-rs/core/src/state/service.rs +++ b/codex-rs/core/src/state/service.rs @@ -12,6 +12,7 @@ use crate::mcp_connection_manager::McpConnectionManager; use crate::models_manager::manager::ModelsManager; use crate::skills::SkillsManager; use crate::state_db::StateDbHandle; +use crate::tools::network_approval::NetworkApprovalService; use crate::tools::sandboxing::ApprovalStore; use crate::unified_exec::UnifiedExecProcessManager; use codex_hooks::Hooks; @@ -40,6 +41,7 @@ pub(crate) struct SessionServices { pub(crate) file_watcher: Arc, pub(crate) agent_control: AgentControl, pub(crate) network_proxy: Option, + pub(crate) network_approval: Arc, pub(crate) state_db: Option, /// Session-scoped model client shared across turns. pub(crate) model_client: ModelClient, diff --git a/codex-rs/core/src/state/session.rs b/codex-rs/core/src/state/session.rs index 70fa78a280..c86f0cdfd2 100644 --- a/codex-rs/core/src/state/session.rs +++ b/codex-rs/core/src/state/session.rs @@ -170,6 +170,27 @@ impl SessionState { merged } + pub(crate) fn set_mcp_tool_selection(&mut self, tool_names: Vec) { + if tool_names.is_empty() { + self.active_mcp_tool_selection = None; + return; + } + + let mut selected = Vec::new(); + let mut seen = HashSet::new(); + for tool_name in tool_names { + if seen.insert(tool_name.clone()) { + selected.push(tool_name); + } + } + + self.active_mcp_tool_selection = if selected.is_empty() { + None + } else { + Some(selected) + }; + } + pub(crate) fn get_mcp_tool_selection(&self) -> Option> { self.active_mcp_tool_selection.clone() } @@ -293,6 +314,40 @@ mod tests { assert_eq!(state.get_mcp_tool_selection(), None); } + #[tokio::test] + async fn set_mcp_tool_selection_deduplicates_and_preserves_order() { + let session_configuration = make_session_configuration_for_tests().await; + let mut state = SessionState::new(session_configuration); + state.merge_mcp_tool_selection(vec!["mcp__rmcp__old".to_string()]); + + state.set_mcp_tool_selection(vec![ + "mcp__rmcp__echo".to_string(), + "mcp__rmcp__image".to_string(), + "mcp__rmcp__echo".to_string(), + "mcp__rmcp__search".to_string(), + ]); + + assert_eq!( + state.get_mcp_tool_selection(), + Some(vec![ + "mcp__rmcp__echo".to_string(), + "mcp__rmcp__image".to_string(), + "mcp__rmcp__search".to_string(), + ]) + ); + } + + #[tokio::test] + async fn set_mcp_tool_selection_empty_input_clears_selection() { + let session_configuration = make_session_configuration_for_tests().await; + let mut state = SessionState::new(session_configuration); + state.merge_mcp_tool_selection(vec!["mcp__rmcp__echo".to_string()]); + + state.set_mcp_tool_selection(Vec::new()); + + assert_eq!(state.get_mcp_tool_selection(), None); + } + #[tokio::test] // Verifies connector merging deduplicates repeated IDs. async fn merge_connector_selection_deduplicates_entries() { diff --git a/codex-rs/core/src/tasks/mod.rs b/codex-rs/core/src/tasks/mod.rs index e05912f5fd..eb04619ee9 100644 --- a/codex-rs/core/src/tasks/mod.rs +++ b/codex-rs/core/src/tasks/mod.rs @@ -120,7 +120,6 @@ impl Session { task: T, ) { self.abort_all_tasks(TurnAbortReason::Replaced).await; - self.clear_mcp_tool_selection().await; self.clear_connector_selection().await; self.seed_initial_context_if_needed(turn_context.as_ref()) .await; @@ -197,6 +196,10 @@ impl Session { turn_context: Arc, last_agent_message: Option, ) { + turn_context + .turn_metadata_state + .cancel_git_enrichment_task(); + let mut active = self.active_turn.lock().await; let mut pending_input = Vec::::new(); let mut should_clear_active_turn = false; @@ -260,6 +263,9 @@ impl Session { trace!(task_kind = ?task.kind, sub_id, "aborting running task"); task.cancellation_token.cancel(); + task.turn_context + .turn_metadata_state + .cancel_git_enrichment_task(); let session_task = task.task; select! { diff --git a/codex-rs/core/src/tasks/regular.rs b/codex-rs/core/src/tasks/regular.rs index 8782d7cedb..1428f7775b 100644 --- a/codex-rs/core/src/tasks/regular.rs +++ b/codex-rs/core/src/tasks/regular.rs @@ -10,7 +10,6 @@ use async_trait::async_trait; use codex_otel::OtelManager; use codex_protocol::openai_models::ModelInfo; use codex_protocol::user_input::UserInput; -use futures::future::BoxFuture; use tokio::task::JoinHandle; use tokio_util::sync::CancellationToken; use tracing::Instrument; @@ -39,13 +38,11 @@ impl RegularTask { model_client: ModelClient, otel_manager: OtelManager, model_info: ModelInfo, - turn_metadata_header: BoxFuture<'static, Option>, ) -> Self { let prewarmed_session_task = tokio::spawn(async move { let mut client_session = model_client.new_session(); - let turn_metadata_header = turn_metadata_header.await; match client_session - .prewarm_websocket(&otel_manager, &model_info, turn_metadata_header.as_deref()) + .prewarm_websocket(&otel_manager, &model_info) .await { Ok(()) => Some(client_session), diff --git a/codex-rs/core/src/tasks/user_shell.rs b/codex-rs/core/src/tasks/user_shell.rs index 43950a8da0..35088efce8 100644 --- a/codex-rs/core/src/tasks/user_shell.rs +++ b/codex-rs/core/src/tasks/user_shell.rs @@ -150,6 +150,7 @@ pub(crate) async fn execute_user_shell_command( Some(session.conversation_id), ), network: turn_context.network.clone(), + network_attempt_id: None, // TODO(zhao-oai): Now that we have ExecExpiration::Cancellation, we // should use that instead of an "arbitrarily large" timeout here. expiration: USER_SHELL_TIMEOUT_MS.into(), diff --git a/codex-rs/core/src/tools/events.rs b/codex-rs/core/src/tools/events.rs index 65972f5a5a..2bacd188d5 100644 --- a/codex-rs/core/src/tools/events.rs +++ b/codex-rs/core/src/tools/events.rs @@ -314,7 +314,7 @@ impl ToolEmitter { (event, result) } Err(ToolError::Codex(CodexErr::Sandbox(SandboxErr::Timeout { output }))) - | Err(ToolError::Codex(CodexErr::Sandbox(SandboxErr::Denied { output }))) => { + | Err(ToolError::Codex(CodexErr::Sandbox(SandboxErr::Denied { output, .. }))) => { let response = self.format_exec_output_for_model(&output, ctx); let event = ToolEventStage::Failure(ToolEventFailure::Output(*output)); let result = Err(FunctionCallError::RespondToModel(response)); diff --git a/codex-rs/core/src/tools/handlers/apply_patch.rs b/codex-rs/core/src/tools/handlers/apply_patch.rs index aced569ce4..49c679e520 100644 --- a/codex-rs/core/src/tools/handlers/apply_patch.rs +++ b/codex-rs/core/src/tools/handlers/apply_patch.rs @@ -143,10 +143,12 @@ impl ToolHandler for ApplyPatchHandler { turn: turn.as_ref(), call_id: call_id.clone(), tool_name: tool_name.to_string(), + network_attempt_id: None, }; let out = orchestrator .run(&mut runtime, &req, &tool_ctx, &turn, turn.approval_policy) - .await; + .await + .map(|result| result.output); let event_ctx = ToolEventCtx::new( session.as_ref(), turn.as_ref(), @@ -232,10 +234,12 @@ pub(crate) async fn intercept_apply_patch( turn, call_id: call_id.to_string(), tool_name: tool_name.to_string(), + network_attempt_id: None, }; let out = orchestrator .run(&mut runtime, &req, &tool_ctx, turn, turn.approval_policy) - .await; + .await + .map(|result| result.output); let event_ctx = ToolEventCtx::new(session, turn, call_id, tracker.as_ref().copied()); let content = emitter.finish(event_ctx, out).await?; diff --git a/codex-rs/core/src/tools/handlers/mod.rs b/codex-rs/core/src/tools/handlers/mod.rs index d07d2e68bf..9f4a1a16ce 100644 --- a/codex-rs/core/src/tools/handlers/mod.rs +++ b/codex-rs/core/src/tools/handlers/mod.rs @@ -33,6 +33,7 @@ pub use read_file::ReadFileHandler; pub use request_user_input::RequestUserInputHandler; pub(crate) use request_user_input::request_user_input_tool_description; pub(crate) use search_tool_bm25::DEFAULT_LIMIT as SEARCH_TOOL_BM25_DEFAULT_LIMIT; +pub(crate) use search_tool_bm25::SEARCH_TOOL_BM25_TOOL_NAME; pub use search_tool_bm25::SearchToolBm25Handler; pub use shell::ShellCommandHandler; pub use shell::ShellHandler; diff --git a/codex-rs/core/src/tools/handlers/search_tool_bm25.rs b/codex-rs/core/src/tools/handlers/search_tool_bm25.rs index 1da8a3d03f..e535abac91 100644 --- a/codex-rs/core/src/tools/handlers/search_tool_bm25.rs +++ b/codex-rs/core/src/tools/handlers/search_tool_bm25.rs @@ -2,11 +2,16 @@ use async_trait::async_trait; use bm25::Document; use bm25::Language; use bm25::SearchEngineBuilder; +use codex_app_server_protocol::AppInfo; use codex_protocol::models::FunctionCallOutputBody; use serde::Deserialize; use serde_json::json; +use std::collections::HashMap; +use std::collections::HashSet; +use crate::connectors; use crate::function_tool::FunctionCallError; +use crate::mcp::CODEX_APPS_MCP_SERVER_NAME; use crate::mcp_connection_manager::ToolInfo; use crate::tools::context::ToolInvocation; use crate::tools::context::ToolOutput; @@ -17,6 +22,7 @@ use crate::tools::registry::ToolKind; pub struct SearchToolBm25Handler; +pub(crate) const SEARCH_TOOL_BM25_TOOL_NAME: &str = "search_tool_bm25"; pub(crate) const DEFAULT_LIMIT: usize = 8; fn default_limit() -> usize { @@ -36,7 +42,6 @@ struct ToolEntry { server_name: String, title: Option, description: Option, - connector_id: Option, connector_name: Option, input_keys: Vec, search_text: String, @@ -60,7 +65,6 @@ impl ToolEntry { .tool .description .map(|description| description.to_string()), - connector_id: info.connector_id, connector_name: info.connector_name, input_keys, search_text, @@ -76,15 +80,18 @@ impl ToolHandler for SearchToolBm25Handler { async fn handle(&self, invocation: ToolInvocation) -> Result { let ToolInvocation { - payload, session, .. + payload, + session, + turn, + .. } = invocation; let arguments = match payload { ToolPayload::Function { arguments } => arguments, _ => { - return Err(FunctionCallError::Fatal( - "search_tool_bm25 handler received unsupported payload".to_string(), - )); + return Err(FunctionCallError::Fatal(format!( + "{SEARCH_TOOL_BM25_TOOL_NAME} handler received unsupported payload" + ))); } }; @@ -112,6 +119,12 @@ impl ToolHandler for SearchToolBm25Handler { .list_all_tools() .await; + let connectors = connectors::with_app_enabled_state( + connectors::accessible_connectors_from_mcp_tools(&mcp_tools), + &turn.config, + ); + let mcp_tools = filter_codex_apps_mcp_tools(mcp_tools, &connectors); + let mut entries: Vec = mcp_tools .into_iter() .map(|(name, info)| ToolEntry::new(name, info)) @@ -154,7 +167,6 @@ impl ToolHandler for SearchToolBm25Handler { "server": entry.server_name.clone(), "title": entry.title.clone(), "description": entry.description.clone(), - "connector_id": entry.connector_id.clone(), "connector_name": entry.connector_name.clone(), "input_keys": entry.input_keys.clone(), "score": result.score, @@ -178,6 +190,28 @@ impl ToolHandler for SearchToolBm25Handler { } } +fn filter_codex_apps_mcp_tools( + mut mcp_tools: HashMap, + connectors: &[AppInfo], +) -> HashMap { + let enabled_connectors: HashSet<&str> = connectors + .iter() + .filter(|connector| connector.is_enabled) + .map(|connector| connector.id.as_str()) + .collect(); + + mcp_tools.retain(|_, tool| { + if tool.server_name != CODEX_APPS_MCP_SERVER_NAME { + return false; + } + + tool.connector_id + .as_deref() + .is_some_and(|connector_id| enabled_connectors.contains(connector_id)) + }); + mcp_tools +} + fn build_search_text(name: &str, info: &ToolInfo, input_keys: &[String]) -> String { let mut parts = vec![ name.to_string(), @@ -203,15 +237,112 @@ fn build_search_text(name: &str, info: &ToolInfo, input_keys: &[String]) -> Stri parts.push(connector_name.to_string()); } - if let Some(connector_id) = info.connector_id.as_deref() - && !connector_id.trim().is_empty() - { - parts.push(connector_id.to_string()); - } - if !input_keys.is_empty() { parts.extend(input_keys.iter().cloned()); } parts.join(" ") } + +#[cfg(test)] +mod tests { + use super::*; + use codex_app_server_protocol::AppInfo; + use pretty_assertions::assert_eq; + use rmcp::model::JsonObject; + use rmcp::model::Tool; + use std::sync::Arc; + + fn make_connector(id: &str, enabled: bool) -> AppInfo { + AppInfo { + id: id.to_string(), + name: id.to_string(), + description: None, + logo_url: None, + logo_url_dark: None, + distribution_channel: None, + install_url: None, + is_accessible: true, + is_enabled: enabled, + } + } + + fn make_tool( + qualified_name: &str, + server_name: &str, + tool_name: &str, + connector_id: Option<&str>, + ) -> (String, ToolInfo) { + ( + qualified_name.to_string(), + ToolInfo { + server_name: server_name.to_string(), + tool_name: tool_name.to_string(), + tool: Tool { + name: tool_name.to_string().into(), + title: None, + description: Some(format!("Test tool: {tool_name}").into()), + input_schema: Arc::new(JsonObject::default()), + output_schema: None, + annotations: None, + execution: None, + icons: None, + meta: None, + }, + connector_id: connector_id.map(str::to_string), + connector_name: connector_id.map(str::to_string), + }, + ) + } + + #[test] + fn filter_codex_apps_mcp_tools_keeps_enabled_apps_only() { + let mcp_tools = HashMap::from([ + make_tool( + "mcp__codex_apps__calendar_create_event", + CODEX_APPS_MCP_SERVER_NAME, + "calendar_create_event", + Some("calendar"), + ), + make_tool( + "mcp__codex_apps__drive_search", + CODEX_APPS_MCP_SERVER_NAME, + "drive_search", + Some("drive"), + ), + make_tool("mcp__rmcp__echo", "rmcp", "echo", None), + ]); + let connectors = vec![ + make_connector("calendar", false), + make_connector("drive", true), + ]; + + let mut filtered: Vec = filter_codex_apps_mcp_tools(mcp_tools, &connectors) + .into_keys() + .collect(); + filtered.sort(); + + assert_eq!(filtered, vec!["mcp__codex_apps__drive_search".to_string()]); + } + + #[test] + fn filter_codex_apps_mcp_tools_drops_apps_without_connector_id() { + let mcp_tools = HashMap::from([ + make_tool( + "mcp__codex_apps__unknown", + CODEX_APPS_MCP_SERVER_NAME, + "unknown", + None, + ), + make_tool("mcp__rmcp__echo", "rmcp", "echo", None), + ]); + + let mut filtered: Vec = + filter_codex_apps_mcp_tools(mcp_tools, &[make_connector("calendar", true)]) + .into_keys() + .collect(); + filtered.sort(); + + assert_eq!(filtered, Vec::::new()); + } +} diff --git a/codex-rs/core/src/tools/handlers/shell.rs b/codex-rs/core/src/tools/handlers/shell.rs index b9e2a97d6d..67f4c04166 100644 --- a/codex-rs/core/src/tools/handlers/shell.rs +++ b/codex-rs/core/src/tools/handlers/shell.rs @@ -54,6 +54,7 @@ impl ShellHandler { expiration: params.timeout_ms.into(), env: create_env(&turn_context.shell_environment_policy, Some(thread_id)), network: turn_context.network.clone(), + network_attempt_id: None, sandbox_permissions: params.sandbox_permissions.unwrap_or_default(), windows_sandbox_level: turn_context.windows_sandbox_level, justification: params.justification.clone(), @@ -83,6 +84,7 @@ impl ShellCommandHandler { expiration: params.timeout_ms.into(), env: create_env(&turn_context.shell_environment_policy, Some(thread_id)), network: turn_context.network.clone(), + network_attempt_id: None, sandbox_permissions: params.sandbox_permissions.unwrap_or_default(), windows_sandbox_level: turn_context.windows_sandbox_level, justification: params.justification.clone(), @@ -325,10 +327,12 @@ impl ShellHandler { turn: turn.as_ref(), call_id: call_id.clone(), tool_name, + network_attempt_id: None, }; let out = orchestrator .run(&mut runtime, &req, &tool_ctx, &turn, turn.approval_policy) - .await; + .await + .map(|result| result.output); let event_ctx = ToolEventCtx::new(session.as_ref(), turn.as_ref(), &call_id, None); let content = emitter.finish(event_ctx, out).await?; Ok(ToolOutput::Function { diff --git a/codex-rs/core/src/tools/js_repl/mod.rs b/codex-rs/core/src/tools/js_repl/mod.rs index 8b362ab8f3..23efb0c67e 100644 --- a/codex-rs/core/src/tools/js_repl/mod.rs +++ b/codex-rs/core/src/tools/js_repl/mod.rs @@ -1,5 +1,8 @@ use std::collections::HashMap; +use std::collections::VecDeque; use std::fmt; +#[cfg(unix)] +use std::os::unix::process::ExitStatusExt; use std::path::Path; use std::path::PathBuf; use std::sync::Arc; @@ -38,6 +41,13 @@ pub(crate) const JS_REPL_PRAGMA_PREFIX: &str = "// codex-js-repl:"; const KERNEL_SOURCE: &str = include_str!("kernel.js"); const MERIYAH_UMD: &str = include_str!("meriyah.umd.min.js"); const JS_REPL_MIN_NODE_VERSION: &str = include_str!("../../../../node-version.txt"); +const JS_REPL_STDERR_TAIL_LINE_LIMIT: usize = 20; +const JS_REPL_STDERR_TAIL_LINE_MAX_BYTES: usize = 512; +const JS_REPL_STDERR_TAIL_MAX_BYTES: usize = 4_096; +const JS_REPL_STDERR_TAIL_SEPARATOR: &str = " | "; +const JS_REPL_EXEC_ID_LOG_LIMIT: usize = 8; +const JS_REPL_MODEL_DIAG_STDERR_MAX_BYTES: usize = 1_024; +const JS_REPL_MODEL_DIAG_ERROR_MAX_BYTES: usize = 256; /// Per-task js_repl handle stored on the turn context. pub(crate) struct JsReplHandle { @@ -85,7 +95,8 @@ pub struct JsExecResult { } struct KernelState { - _child: Child, + child: Arc>, + recent_stderr: Arc>>, stdin: Arc>, pending_execs: Arc>>>, exec_contexts: Arc>>, @@ -105,6 +116,151 @@ struct ExecToolCalls { notify: Arc, } +enum KernelStreamEnd { + Shutdown, + StdoutEof, + StdoutReadError(String), +} + +impl KernelStreamEnd { + fn reason(&self) -> &'static str { + match self { + Self::Shutdown => "shutdown", + Self::StdoutEof => "stdout_eof", + Self::StdoutReadError(_) => "stdout_read_error", + } + } + + fn error(&self) -> Option<&str> { + match self { + Self::StdoutReadError(err) => Some(err), + _ => None, + } + } +} + +struct KernelDebugSnapshot { + pid: Option, + status: String, + stderr_tail: String, +} + +fn format_exit_status(status: std::process::ExitStatus) -> String { + if let Some(code) = status.code() { + return format!("code={code}"); + } + #[cfg(unix)] + if let Some(signal) = status.signal() { + return format!("signal={signal}"); + } + "unknown".to_string() +} + +fn format_stderr_tail(lines: &VecDeque) -> String { + if lines.is_empty() { + return "".to_string(); + } + lines + .iter() + .cloned() + .collect::>() + .join(JS_REPL_STDERR_TAIL_SEPARATOR) +} + +fn truncate_utf8_prefix_by_bytes(input: &str, max_bytes: usize) -> String { + if input.len() <= max_bytes { + return input.to_string(); + } + if max_bytes == 0 { + return String::new(); + } + let mut end = max_bytes; + while end > 0 && !input.is_char_boundary(end) { + end -= 1; + } + input[..end].to_string() +} + +fn stderr_tail_formatted_bytes(lines: &VecDeque) -> usize { + if lines.is_empty() { + return 0; + } + let payload_bytes: usize = lines.iter().map(String::len).sum(); + let separator_bytes = JS_REPL_STDERR_TAIL_SEPARATOR.len() * (lines.len() - 1); + payload_bytes + separator_bytes +} + +fn stderr_tail_bytes_with_candidate(lines: &VecDeque, line: &str) -> usize { + if lines.is_empty() { + return line.len(); + } + stderr_tail_formatted_bytes(lines) + JS_REPL_STDERR_TAIL_SEPARATOR.len() + line.len() +} + +fn push_stderr_tail_line(lines: &mut VecDeque, line: &str) -> String { + let max_line_bytes = JS_REPL_STDERR_TAIL_LINE_MAX_BYTES.min(JS_REPL_STDERR_TAIL_MAX_BYTES); + let bounded_line = truncate_utf8_prefix_by_bytes(line, max_line_bytes); + if bounded_line.is_empty() { + return bounded_line; + } + + while !lines.is_empty() + && (lines.len() >= JS_REPL_STDERR_TAIL_LINE_LIMIT + || stderr_tail_bytes_with_candidate(lines, &bounded_line) + > JS_REPL_STDERR_TAIL_MAX_BYTES) + { + lines.pop_front(); + } + + lines.push_back(bounded_line.clone()); + bounded_line +} + +fn is_kernel_status_exited(status: &str) -> bool { + status.starts_with("exited(") +} + +fn should_include_model_diagnostics_for_write_error( + err_message: &str, + snapshot: &KernelDebugSnapshot, +) -> bool { + is_kernel_status_exited(&snapshot.status) + || err_message.to_ascii_lowercase().contains("broken pipe") +} + +fn format_model_kernel_failure_details( + reason: &str, + stream_error: Option<&str>, + snapshot: &KernelDebugSnapshot, +) -> String { + let payload = serde_json::json!({ + "reason": reason, + "stream_error": stream_error + .map(|err| truncate_utf8_prefix_by_bytes(err, JS_REPL_MODEL_DIAG_ERROR_MAX_BYTES)), + "kernel_pid": snapshot.pid, + "kernel_status": snapshot.status, + "kernel_stderr_tail": truncate_utf8_prefix_by_bytes( + &snapshot.stderr_tail, + JS_REPL_MODEL_DIAG_STDERR_MAX_BYTES, + ), + }); + let encoded = serde_json::to_string(&payload) + .unwrap_or_else(|err| format!(r#"{{"reason":"serialization_error","error":"{err}"}}"#)); + format!("js_repl diagnostics: {encoded}") +} + +fn with_model_kernel_failure_message( + base_message: &str, + reason: &str, + stream_error: Option<&str>, + snapshot: &KernelDebugSnapshot, +) -> String { + format!( + "{base_message}\n\n{}", + format_model_kernel_failure_details(reason, stream_error, snapshot) + ) +} + pub struct JsReplManager { node_path: Option, codex_home: PathBuf, @@ -150,15 +306,15 @@ impl JsReplManager { async fn wait_for_exec_tool_calls(&self, exec_id: &str) { loop { - let notify = { + let notified = { let calls = self.exec_tool_calls.lock().await; calls .get(exec_id) .filter(|state| state.in_flight > 0) - .map(|state| Arc::clone(&state.notify)) + .map(|state| Arc::clone(&state.notify).notified_owned()) }; - match notify { - Some(notify) => notify.notified().await, + match notified { + Some(notified) => notified.await, None => return, } } @@ -166,15 +322,15 @@ impl JsReplManager { async fn wait_for_all_exec_tool_calls(&self) { loop { - let notify = { + let notified = { let calls = self.exec_tool_calls.lock().await; calls .values() .find(|state| state.in_flight > 0) - .map(|state| Arc::clone(&state.notify)) + .map(|state| Arc::clone(&state.notify).notified_owned()) }; - match notify { - Some(notify) => notify.notified().await, + match notified { + Some(notified) => notified.await, None => return, } } @@ -221,15 +377,15 @@ impl JsReplManager { exec_id: &str, ) { loop { - let notify = { + let notified = { let calls = exec_tool_calls.lock().await; calls .get(exec_id) .filter(|state| state.in_flight > 0) - .map(|state| Arc::clone(&state.notify)) + .map(|state| Arc::clone(&state.notify).notified_owned()) }; - match notify { - Some(notify) => notify.notified().await, + match notified { + Some(notified) => notified.await, None => return, } } @@ -258,6 +414,7 @@ impl JsReplManager { }; if let Some(state) = state { state.shutdown.cancel(); + Self::kill_kernel_child(&state.child, "reset").await; } } @@ -272,7 +429,7 @@ impl JsReplManager { FunctionCallError::RespondToModel("js_repl execution unavailable".to_string()) })?; - let (stdin, pending_execs, exec_contexts) = { + let (stdin, pending_execs, exec_contexts, child, recent_stderr) = { let mut kernel = self.kernel.lock().await; if kernel.is_none() { let state = self @@ -294,6 +451,8 @@ impl JsReplManager { Arc::clone(&state.stdin), Arc::clone(&state.pending_execs), Arc::clone(&state.exec_contexts), + Arc::clone(&state.child), + Arc::clone(&state.recent_stderr), ) }; @@ -324,7 +483,28 @@ impl JsReplManager { pending_execs.lock().await.remove(&req_id); exec_contexts.lock().await.remove(&req_id); self.clear_exec_tool_calls(&req_id).await; - return Err(err); + let snapshot = Self::kernel_debug_snapshot(&child, &recent_stderr).await; + let err_message = err.to_string(); + warn!( + exec_id = %req_id, + error = %err_message, + kernel_pid = ?snapshot.pid, + kernel_status = %snapshot.status, + kernel_stderr_tail = %snapshot.stderr_tail, + "failed to submit js_repl exec request to kernel" + ); + let message = + if should_include_model_diagnostics_for_write_error(&err_message, &snapshot) { + with_model_kernel_failure_message( + &err_message, + "write_failed", + Some(&err_message), + &snapshot, + ) + } else { + err_message + }; + return Err(FunctionCallError::RespondToModel(message)); } let timeout_ms = args.timeout_ms.unwrap_or(30_000); @@ -336,9 +516,18 @@ impl JsReplManager { exec_contexts.lock().await.remove(&req_id); self.wait_for_exec_tool_calls(&req_id).await; self.clear_exec_tool_calls(&req_id).await; - return Err(FunctionCallError::RespondToModel( - "js_repl kernel closed unexpectedly".to_string(), - )); + let snapshot = Self::kernel_debug_snapshot(&child, &recent_stderr).await; + let message = if is_kernel_status_exited(&snapshot.status) { + with_model_kernel_failure_message( + "js_repl kernel closed unexpectedly", + "response_channel_closed", + None, + &snapshot, + ) + } else { + "js_repl kernel closed unexpectedly".to_string() + }; + return Err(FunctionCallError::RespondToModel(message)); } Err(_) => { self.reset().await?; @@ -464,9 +653,15 @@ impl JsReplManager { let exec_contexts: Arc>> = Arc::new(Mutex::new(HashMap::new())); let stdin_arc = Arc::new(Mutex::new(stdin)); + let child = Arc::new(Mutex::new(child)); + let recent_stderr = Arc::new(Mutex::new(VecDeque::with_capacity( + JS_REPL_STDERR_TAIL_LINE_LIMIT, + ))); tokio::spawn(Self::read_stdout( stdout, + Arc::clone(&child), + Arc::clone(&recent_stderr), Arc::clone(&pending_execs), Arc::clone(&exec_contexts), Arc::clone(&self.exec_tool_calls), @@ -474,13 +669,18 @@ impl JsReplManager { shutdown.clone(), )); if let Some(stderr) = stderr { - tokio::spawn(Self::read_stderr(stderr, shutdown.clone())); + tokio::spawn(Self::read_stderr( + stderr, + Arc::clone(&recent_stderr), + shutdown.clone(), + )); } else { warn!("js_repl kernel missing stderr"); } Ok(KernelState { - _child: child, + child, + recent_stderr, stdin: stdin_arc, pending_execs, exec_contexts, @@ -514,8 +714,96 @@ impl JsReplManager { Ok(()) } + async fn kernel_stderr_tail_snapshot(recent_stderr: &Arc>>) -> String { + let tail = recent_stderr.lock().await; + format_stderr_tail(&tail) + } + + async fn kernel_debug_snapshot( + child: &Arc>, + recent_stderr: &Arc>>, + ) -> KernelDebugSnapshot { + let (pid, status) = { + let mut guard = child.lock().await; + let pid = guard.id(); + let status = match guard.try_wait() { + Ok(Some(status)) => format!("exited({})", format_exit_status(status)), + Ok(None) => "running".to_string(), + Err(err) => format!("unknown ({err})"), + }; + (pid, status) + }; + let stderr_tail = { + let tail = recent_stderr.lock().await; + format_stderr_tail(&tail) + }; + KernelDebugSnapshot { + pid, + status, + stderr_tail, + } + } + + async fn kill_kernel_child(child: &Arc>, reason: &'static str) { + let mut guard = child.lock().await; + let pid = guard.id(); + match guard.try_wait() { + Ok(Some(_)) => return, + Ok(None) => {} + Err(err) => { + warn!( + kernel_pid = ?pid, + kill_reason = reason, + error = %err, + "failed to inspect js_repl kernel before kill" + ); + } + } + + if let Err(err) = guard.start_kill() { + warn!( + kernel_pid = ?pid, + kill_reason = reason, + error = %err, + "failed to send kill signal to js_repl kernel" + ); + return; + } + + match tokio::time::timeout(Duration::from_secs(2), guard.wait()).await { + Ok(Ok(_status)) => {} + Ok(Err(err)) => { + warn!( + kernel_pid = ?pid, + kill_reason = reason, + error = %err, + "failed while waiting for js_repl kernel exit" + ); + } + Err(_) => { + warn!( + kernel_pid = ?pid, + kill_reason = reason, + "timed out waiting for js_repl kernel to exit after kill" + ); + } + } + } + + fn truncate_id_list(ids: &[String]) -> Vec { + if ids.len() <= JS_REPL_EXEC_ID_LOG_LIMIT { + return ids.to_vec(); + } + let mut output = ids[..JS_REPL_EXEC_ID_LOG_LIMIT].to_vec(); + output.push(format!("...+{}", ids.len() - JS_REPL_EXEC_ID_LOG_LIMIT)); + output + } + + #[allow(clippy::too_many_arguments)] async fn read_stdout( stdout: tokio::process::ChildStdout, + child: Arc>, + recent_stderr: Arc>>, pending_execs: Arc>>>, exec_contexts: Arc>>, exec_tool_calls: Arc>>, @@ -523,17 +811,13 @@ impl JsReplManager { shutdown: CancellationToken, ) { let mut reader = BufReader::new(stdout).lines(); - - loop { + let end_reason = loop { let line = tokio::select! { - _ = shutdown.cancelled() => break, + _ = shutdown.cancelled() => break KernelStreamEnd::Shutdown, res = reader.next_line() => match res { Ok(Some(line)) => line, - Ok(None) => break, - Err(err) => { - warn!("js_repl kernel stream ended: {err}"); - break; - } + Ok(None) => break KernelStreamEnd::StdoutEof, + Err(err) => break KernelStreamEnd::StdoutReadError(err.to_string()), }, }; @@ -571,6 +855,8 @@ impl JsReplManager { } KernelToHost::RunTool(req) => { if !JsReplManager::begin_exec_tool_call(&exec_tool_calls, &req.exec_id).await { + let exec_id = req.exec_id.clone(); + let tool_call_id = req.id.clone(); let payload = HostToKernel::RunToolResult(RunToolResult { id: req.id, ok: false, @@ -578,15 +864,28 @@ impl JsReplManager { error: Some("js_repl exec context not found".to_string()), }); if let Err(err) = JsReplManager::write_message(&stdin, &payload).await { - warn!("failed to reply to kernel run_tool request: {err}"); + let snapshot = + JsReplManager::kernel_debug_snapshot(&child, &recent_stderr).await; + warn!( + exec_id = %exec_id, + tool_call_id = %tool_call_id, + error = %err, + kernel_pid = ?snapshot.pid, + kernel_status = %snapshot.status, + kernel_stderr_tail = %snapshot.stderr_tail, + "failed to reply to kernel run_tool request" + ); } continue; } let stdin_clone = Arc::clone(&stdin); let exec_contexts = Arc::clone(&exec_contexts); let exec_tool_calls = Arc::clone(&exec_tool_calls); + let recent_stderr = Arc::clone(&recent_stderr); tokio::spawn(async move { let exec_id = req.exec_id.clone(); + let tool_call_id = req.id.clone(); + let tool_name = req.tool_name.clone(); let context = { exec_contexts.lock().await.get(&exec_id).cloned() }; let result = match context { Some(ctx) => JsReplManager::run_tool_request(ctx, req).await, @@ -601,12 +900,21 @@ impl JsReplManager { let payload = HostToKernel::RunToolResult(result); if let Err(err) = JsReplManager::write_message(&stdin_clone, &payload).await { - warn!("failed to reply to kernel run_tool request: {err}"); + let stderr_tail = + JsReplManager::kernel_stderr_tail_snapshot(&recent_stderr).await; + warn!( + exec_id = %exec_id, + tool_call_id = %tool_call_id, + tool_name = %tool_name, + error = %err, + kernel_stderr_tail = %stderr_tail, + "failed to reply to kernel run_tool request" + ); } }); } } - } + }; let exec_ids = { let mut contexts = exec_contexts.lock().await; @@ -618,12 +926,47 @@ impl JsReplManager { JsReplManager::wait_for_exec_tool_calls_map(&exec_tool_calls, &exec_id).await; JsReplManager::clear_exec_tool_calls_map(&exec_tool_calls, &exec_id).await; } + let unexpected_snapshot = if matches!(end_reason, KernelStreamEnd::Shutdown) { + None + } else { + Some(Self::kernel_debug_snapshot(&child, &recent_stderr).await) + }; + let kernel_failure_message = unexpected_snapshot.as_ref().map(|snapshot| { + with_model_kernel_failure_message( + "js_repl kernel exited unexpectedly", + end_reason.reason(), + end_reason.error(), + snapshot, + ) + }); + let kernel_exit_message = kernel_failure_message + .clone() + .unwrap_or_else(|| "js_repl kernel exited unexpectedly".to_string()); + let mut pending = pending_execs.lock().await; + let pending_exec_ids = pending.keys().cloned().collect::>(); for (_id, tx) in pending.drain() { let _ = tx.send(ExecResultMessage::Err { - message: "js_repl kernel exited unexpectedly".to_string(), + message: kernel_exit_message.clone(), }); } + drop(pending); + + if !matches!(end_reason, KernelStreamEnd::Shutdown) { + let mut pending_exec_ids = pending_exec_ids; + pending_exec_ids.sort_unstable(); + let snapshot = Self::kernel_debug_snapshot(&child, &recent_stderr).await; + warn!( + reason = %end_reason.reason(), + stream_error = %end_reason.error().unwrap_or(""), + kernel_pid = ?snapshot.pid, + kernel_status = %snapshot.status, + pending_exec_count = pending_exec_ids.len(), + pending_exec_ids = ?Self::truncate_id_list(&pending_exec_ids), + kernel_stderr_tail = %snapshot.stderr_tail, + "js_repl kernel terminated unexpectedly" + ); + } } async fn run_tool_request(exec: ExecContext, req: RunToolRequest) -> RunToolResult { @@ -653,6 +996,7 @@ impl JsReplManager { .map(|(name, tool)| (name, tool.tool)) .collect(), ), + None, exec.turn.dynamic_tools.as_slice(), ); @@ -712,7 +1056,11 @@ impl JsReplManager { } } - async fn read_stderr(stderr: tokio::process::ChildStderr, shutdown: CancellationToken) { + async fn read_stderr( + stderr: tokio::process::ChildStderr, + recent_stderr: Arc>>, + shutdown: CancellationToken, + ) { let mut reader = BufReader::new(stderr).lines(); loop { @@ -729,7 +1077,14 @@ impl JsReplManager { }; let trimmed = line.trim(); if !trimmed.is_empty() { - warn!("js_repl stderr: {trimmed}"); + let bounded_line = { + let mut tail = recent_stderr.lock().await; + push_stderr_tail_line(&mut tail, trimmed) + }; + if bounded_line.is_empty() { + continue; + } + warn!("js_repl stderr: {bounded_line}"); } } } @@ -934,6 +1289,116 @@ mod tests { ); } + #[test] + fn truncate_utf8_prefix_by_bytes_preserves_character_boundaries() { + let input = "aé🙂z"; + assert_eq!(truncate_utf8_prefix_by_bytes(input, 0), ""); + assert_eq!(truncate_utf8_prefix_by_bytes(input, 1), "a"); + assert_eq!(truncate_utf8_prefix_by_bytes(input, 2), "a"); + assert_eq!(truncate_utf8_prefix_by_bytes(input, 3), "aé"); + assert_eq!(truncate_utf8_prefix_by_bytes(input, 6), "aé"); + assert_eq!(truncate_utf8_prefix_by_bytes(input, 7), "aé🙂"); + assert_eq!(truncate_utf8_prefix_by_bytes(input, 8), "aé🙂z"); + } + + #[test] + fn stderr_tail_applies_line_and_byte_limits() { + let mut lines = VecDeque::new(); + let per_line_cap = JS_REPL_STDERR_TAIL_LINE_MAX_BYTES.min(JS_REPL_STDERR_TAIL_MAX_BYTES); + let long = "x".repeat(per_line_cap + 128); + let bounded = push_stderr_tail_line(&mut lines, &long); + assert_eq!(bounded.len(), per_line_cap); + + for i in 0..50 { + let line = format!("line-{i}-{}", "y".repeat(200)); + push_stderr_tail_line(&mut lines, &line); + } + + assert!(lines.len() <= JS_REPL_STDERR_TAIL_LINE_LIMIT); + assert!(lines.iter().all(|line| line.len() <= per_line_cap)); + assert!(stderr_tail_formatted_bytes(&lines) <= JS_REPL_STDERR_TAIL_MAX_BYTES); + assert_eq!( + format_stderr_tail(&lines).len(), + stderr_tail_formatted_bytes(&lines) + ); + } + + #[test] + fn model_kernel_failure_details_are_structured_and_truncated() { + let snapshot = KernelDebugSnapshot { + pid: Some(42), + status: "exited(code=1)".to_string(), + stderr_tail: "s".repeat(JS_REPL_MODEL_DIAG_STDERR_MAX_BYTES + 400), + }; + let stream_error = "e".repeat(JS_REPL_MODEL_DIAG_ERROR_MAX_BYTES + 200); + let message = with_model_kernel_failure_message( + "js_repl kernel exited unexpectedly", + "stdout_eof", + Some(&stream_error), + &snapshot, + ); + assert!(message.starts_with("js_repl kernel exited unexpectedly\n\njs_repl diagnostics: ")); + let (_prefix, encoded) = message + .split_once("js_repl diagnostics: ") + .expect("diagnostics suffix should be present"); + let parsed: serde_json::Value = + serde_json::from_str(encoded).expect("diagnostics should be valid json"); + assert_eq!( + parsed.get("reason").and_then(|v| v.as_str()), + Some("stdout_eof") + ); + assert_eq!( + parsed.get("kernel_pid").and_then(serde_json::Value::as_u64), + Some(42) + ); + assert_eq!( + parsed.get("kernel_status").and_then(|v| v.as_str()), + Some("exited(code=1)") + ); + assert!( + parsed + .get("kernel_stderr_tail") + .and_then(|v| v.as_str()) + .expect("kernel_stderr_tail should be present") + .len() + <= JS_REPL_MODEL_DIAG_STDERR_MAX_BYTES + ); + assert!( + parsed + .get("stream_error") + .and_then(|v| v.as_str()) + .expect("stream_error should be present") + .len() + <= JS_REPL_MODEL_DIAG_ERROR_MAX_BYTES + ); + } + + #[test] + fn write_error_diagnostics_only_attach_for_likely_kernel_failures() { + let running = KernelDebugSnapshot { + pid: Some(7), + status: "running".to_string(), + stderr_tail: "".to_string(), + }; + let exited = KernelDebugSnapshot { + pid: Some(7), + status: "exited(code=1)".to_string(), + stderr_tail: "".to_string(), + }; + assert!(!should_include_model_diagnostics_for_write_error( + "failed to flush kernel message: other io error", + &running + )); + assert!(should_include_model_diagnostics_for_write_error( + "failed to write to kernel: Broken pipe (os error 32)", + &running + )); + assert!(should_include_model_diagnostics_for_write_error( + "failed to write to kernel: some other io error", + &exited + )); + } + #[test] fn js_repl_internal_tool_guard_matches_expected_names() { assert!(is_js_repl_internal_tool("js_repl")); @@ -942,6 +1407,40 @@ mod tests { assert!(!is_js_repl_internal_tool("list_mcp_resources")); } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn wait_for_exec_tool_calls_map_drains_inflight_calls_without_hanging() { + let exec_tool_calls = Arc::new(Mutex::new(HashMap::new())); + + for _ in 0..128 { + let exec_id = Uuid::new_v4().to_string(); + exec_tool_calls + .lock() + .await + .insert(exec_id.clone(), ExecToolCalls::default()); + assert!(JsReplManager::begin_exec_tool_call(&exec_tool_calls, &exec_id).await); + + let wait_map = Arc::clone(&exec_tool_calls); + let wait_exec_id = exec_id.clone(); + let waiter = tokio::spawn(async move { + JsReplManager::wait_for_exec_tool_calls_map(&wait_map, &wait_exec_id).await; + }); + + let finish_map = Arc::clone(&exec_tool_calls); + let finish_exec_id = exec_id.clone(); + let finisher = tokio::spawn(async move { + tokio::task::yield_now().await; + JsReplManager::finish_exec_tool_call(&finish_map, &finish_exec_id).await; + }); + + tokio::time::timeout(Duration::from_secs(1), waiter) + .await + .expect("wait_for_exec_tool_calls_map should not hang") + .expect("wait task should not panic"); + finisher.await.expect("finish task should not panic"); + + JsReplManager::clear_exec_tool_calls_map(&exec_tool_calls, &exec_id).await; + } + } async fn can_run_js_repl_runtime_tests() -> bool { if std::env::var_os("CODEX_SANDBOX").is_some() { return false; @@ -1036,6 +1535,117 @@ mod tests { Ok(()) } + #[tokio::test] + async fn js_repl_timeout_kills_kernel_process() -> anyhow::Result<()> { + if !can_run_js_repl_runtime_tests().await { + return Ok(()); + } + + let (session, turn) = make_session_and_context().await; + let session = Arc::new(session); + let turn = Arc::new(turn); + let tracker = Arc::new(tokio::sync::Mutex::new(TurnDiffTracker::default())); + let manager = turn.js_repl.manager().await?; + + manager + .execute( + Arc::clone(&session), + Arc::clone(&turn), + Arc::clone(&tracker), + JsReplArgs { + code: "console.log('warmup');".to_string(), + timeout_ms: Some(10_000), + }, + ) + .await?; + + let child = { + let guard = manager.kernel.lock().await; + let state = guard.as_ref().expect("kernel should exist after warmup"); + Arc::clone(&state.child) + }; + + let result = manager + .execute( + session, + turn, + tracker, + JsReplArgs { + code: "while (true) {}".to_string(), + timeout_ms: Some(50), + }, + ) + .await + .expect_err("expected timeout error"); + + assert_eq!( + result.to_string(), + "js_repl execution timed out; kernel reset, rerun your request" + ); + + let exit_state = { + let mut child = child.lock().await; + child.try_wait()? + }; + assert!( + exit_state.is_some(), + "timed out js_repl execution should kill previous kernel process" + ); + Ok(()) + } + + #[tokio::test] + async fn js_repl_kernel_failure_includes_model_diagnostics() -> anyhow::Result<()> { + if !can_run_js_repl_runtime_tests().await { + return Ok(()); + } + + let (session, turn) = make_session_and_context().await; + let session = Arc::new(session); + let turn = Arc::new(turn); + let tracker = Arc::new(tokio::sync::Mutex::new(TurnDiffTracker::default())); + let manager = turn.js_repl.manager().await?; + + manager + .execute( + Arc::clone(&session), + Arc::clone(&turn), + Arc::clone(&tracker), + JsReplArgs { + code: "console.log('warmup');".to_string(), + timeout_ms: Some(10_000), + }, + ) + .await?; + + let child = { + let guard = manager.kernel.lock().await; + let state = guard.as_ref().expect("kernel should exist after warmup"); + Arc::clone(&state.child) + }; + JsReplManager::kill_kernel_child(&child, "test_crash").await; + tokio::time::sleep(Duration::from_millis(50)).await; + + let err = manager + .execute( + session, + turn, + tracker, + JsReplArgs { + code: "console.log('after-kill');".to_string(), + timeout_ms: Some(10_000), + }, + ) + .await + .expect_err("expected kernel failure after forced kill"); + + let message = err.to_string(); + assert!(message.contains("js_repl diagnostics:")); + assert!(message.contains("\"reason\":\"write_failed\"")); + assert!(message.contains("\"kernel_status\":\"exited(")); + Ok(()) + } + #[tokio::test] async fn js_repl_can_call_tools() -> anyhow::Result<()> { if !can_run_js_repl_runtime_tests().await { @@ -1170,7 +1780,7 @@ console.log("cell-complete"); Ok(()) } - #[tokio::test] + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn js_repl_can_attach_image_via_view_image_tool() -> anyhow::Result<()> { if !can_run_js_repl_runtime_tests().await { return Ok(()); @@ -1219,7 +1829,6 @@ console.log(out.output?.body?.text ?? ""); ) .await?; assert!(result.output.contains("function_call_output")); - assert!(result.output.contains("attached local image path")); let pending_input = session.get_pending_input().await; let image_url = pending_input diff --git a/codex-rs/core/src/tools/mod.rs b/codex-rs/core/src/tools/mod.rs index 3307d65313..27fe515126 100644 --- a/codex-rs/core/src/tools/mod.rs +++ b/codex-rs/core/src/tools/mod.rs @@ -2,6 +2,7 @@ pub mod context; pub mod events; pub(crate) mod handlers; pub mod js_repl; +pub(crate) mod network_approval; pub mod orchestrator; pub mod parallel; pub mod registry; diff --git a/codex-rs/core/src/tools/network_approval.rs b/codex-rs/core/src/tools/network_approval.rs new file mode 100644 index 0000000000..51e6e3c98e --- /dev/null +++ b/codex-rs/core/src/tools/network_approval.rs @@ -0,0 +1,675 @@ +use crate::codex::Session; +use crate::network_policy_decision::denied_network_policy_message; +use crate::tools::sandboxing::ToolError; +use codex_network_proxy::BlockedRequest; +use codex_network_proxy::BlockedRequestObserver; +use codex_network_proxy::NetworkDecision; +use codex_network_proxy::NetworkPolicyDecider; +use codex_network_proxy::NetworkPolicyRequest; +use codex_network_proxy::NetworkProtocol; +use codex_network_proxy::NetworkProxy; +use codex_protocol::approvals::NetworkApprovalContext; +use codex_protocol::approvals::NetworkApprovalProtocol; +use codex_protocol::protocol::ReviewDecision; +use std::collections::HashMap; +use std::collections::HashSet; +use std::path::PathBuf; +use std::sync::Arc; +use tokio::sync::Mutex; +use tokio::sync::RwLock; +use uuid::Uuid; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum NetworkApprovalMode { + Immediate, + Deferred, +} + +#[derive(Clone, Debug)] +pub(crate) struct NetworkApprovalSpec { + pub command: Vec, + pub cwd: PathBuf, + pub network: Option, + pub mode: NetworkApprovalMode, +} + +#[derive(Clone, Debug)] +pub(crate) struct DeferredNetworkApproval { + attempt_id: String, +} + +impl DeferredNetworkApproval { + pub(crate) fn attempt_id(&self) -> &str { + &self.attempt_id + } +} + +#[derive(Debug)] +pub(crate) struct ActiveNetworkApproval { + attempt_id: Option, + mode: NetworkApprovalMode, +} + +impl ActiveNetworkApproval { + pub(crate) fn attempt_id(&self) -> Option<&str> { + self.attempt_id.as_deref() + } + + pub(crate) fn mode(&self) -> NetworkApprovalMode { + self.mode + } + + pub(crate) fn into_deferred(self) -> Option { + match (self.mode, self.attempt_id) { + (NetworkApprovalMode::Deferred, Some(attempt_id)) => { + Some(DeferredNetworkApproval { attempt_id }) + } + _ => None, + } + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) enum NetworkApprovalOutcome { + DeniedByUser, + DeniedByPolicy(String), +} + +struct NetworkApprovalAttempt { + turn_id: String, + call_id: String, + command: Vec, + cwd: PathBuf, + approved_hosts: Mutex>, + outcome: Mutex>, +} + +pub(crate) struct NetworkApprovalService { + attempts: Mutex>>, + session_approved_hosts: Mutex>, +} + +impl Default for NetworkApprovalService { + fn default() -> Self { + Self { + attempts: Mutex::new(HashMap::new()), + session_approved_hosts: Mutex::new(HashSet::new()), + } + } +} + +impl NetworkApprovalService { + pub(crate) async fn register_attempt( + &self, + attempt_id: String, + turn_id: String, + call_id: String, + command: Vec, + cwd: PathBuf, + ) { + let mut attempts = self.attempts.lock().await; + attempts.insert( + attempt_id, + Arc::new(NetworkApprovalAttempt { + turn_id, + call_id, + command, + cwd, + approved_hosts: Mutex::new(HashSet::new()), + outcome: Mutex::new(None), + }), + ); + } + + pub(crate) async fn unregister_attempt(&self, attempt_id: &str) { + let mut attempts = self.attempts.lock().await; + attempts.remove(attempt_id); + } + + pub(crate) async fn take_outcome(&self, attempt_id: &str) -> Option { + let attempt = { + let attempts = self.attempts.lock().await; + attempts.get(attempt_id).cloned() + }?; + let mut outcome = attempt.outcome.lock().await; + outcome.take() + } + + pub(crate) async fn take_user_denial_outcome(&self, attempt_id: &str) -> bool { + let attempt = { + let attempts = self.attempts.lock().await; + attempts.get(attempt_id).cloned() + }; + let Some(attempt) = attempt else { + return false; + }; + let mut outcome = attempt.outcome.lock().await; + if matches!(outcome.as_ref(), Some(NetworkApprovalOutcome::DeniedByUser)) { + outcome.take(); + return true; + } + false + } + + async fn resolve_attempt_for_request( + &self, + request: &NetworkPolicyRequest, + ) -> Option> { + let attempts = self.attempts.lock().await; + + if let Some(attempt_id) = request.attempt_id.as_deref() { + if let Some(attempt) = attempts.get(attempt_id).cloned() { + return Some(attempt); + } + return None; + } + + if attempts.len() == 1 { + return attempts.values().next().cloned(); + } + + None + } + + async fn resolve_attempt_for_blocked_request( + &self, + blocked: &BlockedRequest, + ) -> Option> { + let attempts = self.attempts.lock().await; + + if let Some(attempt_id) = blocked.attempt_id.as_deref() { + if let Some(attempt) = attempts.get(attempt_id).cloned() { + return Some(attempt); + } + return None; + } + + if attempts.len() == 1 { + return attempts.values().next().cloned(); + } + + None + } + + pub(crate) async fn record_blocked_request(&self, blocked: BlockedRequest) { + let Some(message) = denied_network_policy_message(&blocked) else { + return; + }; + + let Some(attempt) = self.resolve_attempt_for_blocked_request(&blocked).await else { + return; + }; + + let mut outcome = attempt.outcome.lock().await; + if matches!(outcome.as_ref(), Some(NetworkApprovalOutcome::DeniedByUser)) { + return; + } + *outcome = Some(NetworkApprovalOutcome::DeniedByPolicy(message)); + } + + pub(crate) async fn handle_inline_policy_request( + &self, + session: &Session, + request: NetworkPolicyRequest, + ) -> NetworkDecision { + const REASON_NOT_ALLOWED: &str = "not_allowed"; + + { + let approved_hosts = self.session_approved_hosts.lock().await; + if approved_hosts.contains(request.host.as_str()) { + return NetworkDecision::Allow; + } + } + + let Some(attempt) = self.resolve_attempt_for_request(&request).await else { + return NetworkDecision::deny(REASON_NOT_ALLOWED); + }; + + { + let approved_hosts = attempt.approved_hosts.lock().await; + if approved_hosts.contains(request.host.as_str()) { + return NetworkDecision::Allow; + } + } + + let protocol = match request.protocol { + NetworkProtocol::Http => NetworkApprovalProtocol::Http, + NetworkProtocol::HttpsConnect => NetworkApprovalProtocol::Https, + NetworkProtocol::Socks5Tcp => NetworkApprovalProtocol::Socks5Tcp, + NetworkProtocol::Socks5Udp => NetworkApprovalProtocol::Socks5Udp, + }; + + let Some(turn_context) = session.turn_context_for_sub_id(&attempt.turn_id).await else { + return NetworkDecision::deny(REASON_NOT_ALLOWED); + }; + + let approval_decision = session + .request_command_approval( + turn_context.as_ref(), + attempt.call_id.clone(), + attempt.command.clone(), + attempt.cwd.clone(), + Some(format!( + "Network access to \"{}\" is blocked by policy.", + request.host + )), + Some(NetworkApprovalContext { + host: request.host.clone(), + protocol, + }), + None, + ) + .await; + + match approval_decision { + ReviewDecision::Approved | ReviewDecision::ApprovedExecpolicyAmendment { .. } => { + let mut approved_hosts = attempt.approved_hosts.lock().await; + approved_hosts.insert(request.host); + NetworkDecision::Allow + } + ReviewDecision::ApprovedForSession => { + let mut approved_hosts = self.session_approved_hosts.lock().await; + approved_hosts.insert(request.host); + NetworkDecision::Allow + } + ReviewDecision::Denied | ReviewDecision::Abort => { + let mut outcome = attempt.outcome.lock().await; + *outcome = Some(NetworkApprovalOutcome::DeniedByUser); + NetworkDecision::deny(REASON_NOT_ALLOWED) + } + } + } +} + +pub(crate) fn build_blocked_request_observer( + network_approval: Arc, +) -> Arc { + Arc::new(move |blocked: BlockedRequest| { + let network_approval = Arc::clone(&network_approval); + async move { + network_approval.record_blocked_request(blocked).await; + } + }) +} + +pub(crate) fn build_network_policy_decider( + network_approval: Arc, + network_policy_decider_session: Arc>>, +) -> Arc { + Arc::new(move |request: NetworkPolicyRequest| { + let network_approval = Arc::clone(&network_approval); + let network_policy_decider_session = Arc::clone(&network_policy_decider_session); + async move { + let Some(session) = network_policy_decider_session.read().await.upgrade() else { + return NetworkDecision::ask("not_allowed"); + }; + network_approval + .handle_inline_policy_request(session.as_ref(), request) + .await + } + }) +} + +pub(crate) async fn begin_network_approval( + session: &Session, + turn_id: &str, + call_id: &str, + has_managed_network_requirements: bool, + spec: Option, +) -> Option { + let spec = spec?; + if !has_managed_network_requirements || spec.network.is_none() { + return None; + } + + let attempt_id = Uuid::new_v4().to_string(); + session + .services + .network_approval + .register_attempt( + attempt_id.clone(), + turn_id.to_string(), + call_id.to_string(), + spec.command, + spec.cwd, + ) + .await; + + Some(ActiveNetworkApproval { + attempt_id: Some(attempt_id), + mode: spec.mode, + }) +} + +pub(crate) async fn finish_immediate_network_approval( + session: &Session, + active: ActiveNetworkApproval, +) -> Result<(), ToolError> { + let Some(attempt_id) = active.attempt_id.as_deref() else { + return Ok(()); + }; + + let approval_outcome = session + .services + .network_approval + .take_outcome(attempt_id) + .await; + + session + .services + .network_approval + .unregister_attempt(attempt_id) + .await; + + match approval_outcome { + Some(NetworkApprovalOutcome::DeniedByUser) => { + Err(ToolError::Rejected("rejected by user".to_string())) + } + Some(NetworkApprovalOutcome::DeniedByPolicy(message)) => Err(ToolError::Rejected(message)), + None => Ok(()), + } +} + +pub(crate) async fn deferred_rejection_message( + session: &Session, + deferred: &DeferredNetworkApproval, +) -> Option { + match session + .services + .network_approval + .take_outcome(deferred.attempt_id()) + .await + { + Some(NetworkApprovalOutcome::DeniedByUser) => Some("rejected by user".to_string()), + Some(NetworkApprovalOutcome::DeniedByPolicy(message)) => Some(message), + None => None, + } +} + +pub(crate) async fn finish_deferred_network_approval( + session: &Session, + deferred: Option, +) { + let Some(deferred) = deferred else { + return; + }; + session + .services + .network_approval + .unregister_attempt(deferred.attempt_id()) + .await; +} + +#[cfg(test)] +mod tests { + use super::*; + use codex_network_proxy::BlockedRequestArgs; + use codex_network_proxy::NetworkPolicyRequestArgs; + use pretty_assertions::assert_eq; + + fn http_request(host: &str, attempt_id: Option<&str>) -> NetworkPolicyRequest { + NetworkPolicyRequest::new(NetworkPolicyRequestArgs { + protocol: NetworkProtocol::Http, + host: host.to_string(), + port: 80, + client_addr: None, + method: Some("GET".to_string()), + command: None, + exec_policy_hint: None, + attempt_id: attempt_id.map(ToString::to_string), + }) + } + + #[tokio::test] + async fn resolve_attempt_for_request_falls_back_to_single_active_attempt() { + let service = NetworkApprovalService::default(); + service + .register_attempt( + "attempt-1".to_string(), + "turn-1".to_string(), + "call-1".to_string(), + vec!["curl".to_string(), "example.com".to_string()], + std::env::temp_dir(), + ) + .await; + + let resolved = service + .resolve_attempt_for_request(&http_request("example.com", None)) + .await + .expect("single active attempt should be used as fallback"); + assert_eq!(resolved.call_id, "call-1"); + } + + #[tokio::test] + async fn resolve_attempt_for_request_returns_exact_attempt_match() { + let service = NetworkApprovalService::default(); + service + .register_attempt( + "attempt-1".to_string(), + "turn-1".to_string(), + "call-1".to_string(), + vec!["curl".to_string(), "example.com".to_string()], + std::env::temp_dir(), + ) + .await; + service + .register_attempt( + "attempt-2".to_string(), + "turn-2".to_string(), + "call-2".to_string(), + vec!["curl".to_string(), "openai.com".to_string()], + std::env::temp_dir(), + ) + .await; + + let resolved = service + .resolve_attempt_for_request(&http_request("openai.com", Some("attempt-2"))) + .await + .expect("attempt-2 should resolve"); + assert_eq!(resolved.call_id, "call-2"); + } + + #[tokio::test] + async fn resolve_attempt_for_request_returns_none_for_unknown_attempt_id() { + let service = NetworkApprovalService::default(); + service + .register_attempt( + "attempt-1".to_string(), + "turn-1".to_string(), + "call-1".to_string(), + vec!["curl".to_string(), "example.com".to_string()], + std::env::temp_dir(), + ) + .await; + + let resolved = service + .resolve_attempt_for_request(&http_request("example.com", Some("attempt-unknown"))) + .await; + assert!(resolved.is_none()); + } + + #[tokio::test] + async fn resolve_attempt_for_request_returns_none_when_ambiguous() { + let service = NetworkApprovalService::default(); + service + .register_attempt( + "attempt-1".to_string(), + "turn-1".to_string(), + "call-1".to_string(), + vec!["curl".to_string(), "example.com".to_string()], + std::env::temp_dir(), + ) + .await; + service + .register_attempt( + "attempt-2".to_string(), + "turn-2".to_string(), + "call-2".to_string(), + vec!["curl".to_string(), "robinhood.com".to_string()], + std::env::temp_dir(), + ) + .await; + + let resolved = service + .resolve_attempt_for_request(&http_request("example.com", None)) + .await; + assert!(resolved.is_none()); + } + + #[tokio::test] + async fn take_outcome_clears_stored_value() { + let service = NetworkApprovalService::default(); + service + .register_attempt( + "attempt-1".to_string(), + "turn-1".to_string(), + "call-1".to_string(), + vec!["curl".to_string(), "example.com".to_string()], + std::env::temp_dir(), + ) + .await; + + let attempt = { + let attempts = service.attempts.lock().await; + attempts + .get("attempt-1") + .cloned() + .expect("attempt should exist") + }; + { + let mut outcome = attempt.outcome.lock().await; + *outcome = Some(NetworkApprovalOutcome::DeniedByUser); + } + + assert_eq!( + service.take_outcome("attempt-1").await, + Some(NetworkApprovalOutcome::DeniedByUser) + ); + assert_eq!(service.take_outcome("attempt-1").await, None); + } + + #[tokio::test] + async fn take_user_denial_outcome_preserves_policy_denial() { + let service = NetworkApprovalService::default(); + service + .register_attempt( + "attempt-1".to_string(), + "turn-1".to_string(), + "call-1".to_string(), + vec!["curl".to_string(), "example.com".to_string()], + std::env::temp_dir(), + ) + .await; + + let attempt = { + let attempts = service.attempts.lock().await; + attempts + .get("attempt-1") + .cloned() + .expect("attempt should exist") + }; + { + let mut outcome = attempt.outcome.lock().await; + *outcome = Some(NetworkApprovalOutcome::DeniedByPolicy( + "policy denied".to_string(), + )); + } + + assert!(!service.take_user_denial_outcome("attempt-1").await); + assert_eq!( + service.take_outcome("attempt-1").await, + Some(NetworkApprovalOutcome::DeniedByPolicy( + "policy denied".to_string(), + )) + ); + } + + #[tokio::test] + async fn record_blocked_request_stores_policy_denial_outcome() { + let service = NetworkApprovalService::default(); + service + .register_attempt( + "attempt-1".to_string(), + "turn-1".to_string(), + "call-1".to_string(), + vec!["curl".to_string(), "example.com".to_string()], + std::env::temp_dir(), + ) + .await; + + service + .record_blocked_request(BlockedRequest::new(BlockedRequestArgs { + host: "example.com".to_string(), + reason: "denied".to_string(), + client: None, + method: Some("GET".to_string()), + mode: None, + protocol: "http".to_string(), + attempt_id: Some("attempt-1".to_string()), + decision: Some("deny".to_string()), + source: Some("baseline_policy".to_string()), + port: Some(80), + })) + .await; + + let outcome = service + .take_outcome("attempt-1") + .await + .expect("outcome should be recorded"); + match outcome { + NetworkApprovalOutcome::DeniedByPolicy(message) => { + assert_eq!( + message, + "Network access to \"example.com\" was blocked: domain is explicitly denied by policy and cannot be approved from this prompt.".to_string() + ); + } + NetworkApprovalOutcome::DeniedByUser => panic!("expected policy denial"), + } + } + + #[tokio::test] + async fn record_blocked_request_does_not_override_user_denial() { + let service = NetworkApprovalService::default(); + service + .register_attempt( + "attempt-1".to_string(), + "turn-1".to_string(), + "call-1".to_string(), + vec!["curl".to_string(), "example.com".to_string()], + std::env::temp_dir(), + ) + .await; + + let attempt = { + let attempts = service.attempts.lock().await; + attempts + .get("attempt-1") + .cloned() + .expect("attempt should exist") + }; + { + let mut outcome = attempt.outcome.lock().await; + *outcome = Some(NetworkApprovalOutcome::DeniedByUser); + } + + service + .record_blocked_request(BlockedRequest::new(BlockedRequestArgs { + host: "example.com".to_string(), + reason: "denied".to_string(), + client: None, + method: Some("GET".to_string()), + mode: None, + protocol: "http".to_string(), + attempt_id: Some("attempt-1".to_string()), + decision: Some("deny".to_string()), + source: Some("baseline_policy".to_string()), + port: Some(80), + })) + .await; + + assert_eq!( + service.take_outcome("attempt-1").await, + Some(NetworkApprovalOutcome::DeniedByUser) + ); + } +} diff --git a/codex-rs/core/src/tools/orchestrator.rs b/codex-rs/core/src/tools/orchestrator.rs index 3f410e1b26..cdbaac0254 100644 --- a/codex-rs/core/src/tools/orchestrator.rs +++ b/codex-rs/core/src/tools/orchestrator.rs @@ -10,7 +10,13 @@ use crate::error::CodexErr; use crate::error::SandboxErr; use crate::exec::ExecToolCallOutput; use crate::features::Feature; +use crate::network_policy_decision::network_approval_context_from_payload; use crate::sandboxing::SandboxManager; +use crate::tools::network_approval::DeferredNetworkApproval; +use crate::tools::network_approval::NetworkApprovalMode; +use crate::tools::network_approval::begin_network_approval; +use crate::tools::network_approval::finish_deferred_network_approval; +use crate::tools::network_approval::finish_immediate_network_approval; use crate::tools::sandboxing::ApprovalCtx; use crate::tools::sandboxing::ExecApprovalRequirement; use crate::tools::sandboxing::SandboxAttempt; @@ -27,6 +33,11 @@ pub(crate) struct ToolOrchestrator { sandbox: SandboxManager, } +pub(crate) struct OrchestratorRunResult { + pub output: Out, + pub deferred_network_approval: Option, +} + impl ToolOrchestrator { pub fn new() -> Self { Self { @@ -34,6 +45,60 @@ impl ToolOrchestrator { } } + async fn run_attempt( + tool: &mut T, + req: &Rq, + tool_ctx: &ToolCtx<'_>, + attempt: &SandboxAttempt<'_>, + has_managed_network_requirements: bool, + ) -> (Result, Option) + where + T: ToolRuntime, + { + let network_approval = begin_network_approval( + tool_ctx.session, + &tool_ctx.turn.sub_id, + &tool_ctx.call_id, + has_managed_network_requirements, + tool.network_approval_spec(req, tool_ctx), + ) + .await; + + let attempt_tool_ctx = ToolCtx { + session: tool_ctx.session, + turn: tool_ctx.turn, + call_id: tool_ctx.call_id.clone(), + tool_name: tool_ctx.tool_name.clone(), + network_attempt_id: network_approval.as_ref().and_then(|network_approval| { + network_approval.attempt_id().map(ToString::to_string) + }), + }; + let run_result = tool.run(req, attempt, &attempt_tool_ctx).await; + + let Some(network_approval) = network_approval else { + return (run_result, None); + }; + + match network_approval.mode() { + NetworkApprovalMode::Immediate => { + let finalize_result = + finish_immediate_network_approval(tool_ctx.session, network_approval).await; + if let Err(err) = finalize_result { + return (Err(err), None); + } + (run_result, None) + } + NetworkApprovalMode::Deferred => { + let deferred = network_approval.into_deferred(); + if run_result.is_err() { + finish_deferred_network_approval(tool_ctx.session, deferred).await; + return (run_result, None); + } + (run_result, deferred) + } + } + } + pub async fn run( &mut self, tool: &mut T, @@ -41,7 +106,7 @@ impl ToolOrchestrator { tool_ctx: &ToolCtx<'_>, turn_ctx: &crate::codex::TurnContext, approval_policy: AskForApproval, - ) -> Result + ) -> Result, ToolError> where T: ToolRuntime, { @@ -70,6 +135,7 @@ impl ToolOrchestrator { turn: turn_ctx, call_id: &tool_ctx.call_id, retry_reason: reason, + network_approval_context: None, }; let decision = tool.start_approval_async(req, approval_ctx).await; @@ -118,33 +184,86 @@ impl ToolOrchestrator { windows_sandbox_level: turn_ctx.windows_sandbox_level, }; - match tool.run(req, &initial_attempt, tool_ctx).await { + let (first_result, first_deferred_network_approval) = Self::run_attempt( + tool, + req, + tool_ctx, + &initial_attempt, + has_managed_network_requirements, + ) + .await; + match first_result { Ok(out) => { // We have a successful initial result - Ok(out) + Ok(OrchestratorRunResult { + output: out, + deferred_network_approval: first_deferred_network_approval, + }) } - Err(ToolError::Codex(CodexErr::Sandbox(SandboxErr::Denied { output }))) => { + Err(ToolError::Codex(CodexErr::Sandbox(SandboxErr::Denied { + output, + network_policy_decision, + }))) => { + let network_approval_context = if has_managed_network_requirements { + network_policy_decision + .as_ref() + .and_then(network_approval_context_from_payload) + } else { + None + }; + if network_policy_decision.is_some() && network_approval_context.is_none() { + return Err(ToolError::Codex(CodexErr::Sandbox(SandboxErr::Denied { + output, + network_policy_decision, + }))); + } if !tool.escalate_on_failure() { return Err(ToolError::Codex(CodexErr::Sandbox(SandboxErr::Denied { output, + network_policy_decision, }))); } // Under `Never` or `OnRequest`, do not retry without sandbox; surface a concise // sandbox denial that preserves the original output. if !tool.wants_no_sandbox_approval(approval_policy) { - return Err(ToolError::Codex(CodexErr::Sandbox(SandboxErr::Denied { - output, - }))); + let allow_on_request_network_prompt = + matches!(approval_policy, AskForApproval::OnRequest) + && network_approval_context.is_some() + && matches!( + default_exec_approval_requirement( + approval_policy, + &turn_ctx.sandbox_policy + ), + ExecApprovalRequirement::NeedsApproval { .. } + ); + if !allow_on_request_network_prompt { + return Err(ToolError::Codex(CodexErr::Sandbox(SandboxErr::Denied { + output, + network_policy_decision, + }))); + } } + let retry_reason = + if let Some(network_approval_context) = network_approval_context.as_ref() { + format!( + "Network access to \"{}\" is blocked by policy.", + network_approval_context.host + ) + } else { + build_denial_reason_from_output(output.as_ref()) + }; // Ask for approval before retrying with the escalated sandbox. - if !tool.should_bypass_approval(approval_policy, already_approved) { - let reason_msg = build_denial_reason_from_output(output.as_ref()); + let bypass_retry_approval = tool + .should_bypass_approval(approval_policy, already_approved) + && network_approval_context.is_none(); + if !bypass_retry_approval { let approval_ctx = ApprovalCtx { session: tool_ctx.session, turn: turn_ctx, call_id: &tool_ctx.call_id, - retry_reason: Some(reason_msg), + retry_reason: Some(retry_reason), + network_approval_context: network_approval_context.clone(), }; let decision = tool.start_approval_async(req, approval_ctx).await; @@ -172,9 +291,20 @@ impl ToolOrchestrator { }; // Second attempt. - (*tool).run(req, &escalated_attempt, tool_ctx).await + let (retry_result, retry_deferred_network_approval) = Self::run_attempt( + tool, + req, + tool_ctx, + &escalated_attempt, + has_managed_network_requirements, + ) + .await; + retry_result.map(|output| OrchestratorRunResult { + output, + deferred_network_approval: retry_deferred_network_approval, + }) } - other => other, + Err(err) => Err(err), } } } diff --git a/codex-rs/core/src/tools/registry.rs b/codex-rs/core/src/tools/registry.rs index 8ab702c0c2..ffde584b19 100644 --- a/codex-rs/core/src/tools/registry.rs +++ b/codex-rs/core/src/tools/registry.rs @@ -4,6 +4,7 @@ use std::time::Duration; use std::time::Instant; use crate::client_common::tools::ToolSpec; +use crate::features::Feature; use crate::function_tool::FunctionCallError; use crate::protocol::SandboxPolicy; use crate::sandbox_tags::sandbox_tag; @@ -88,6 +89,10 @@ impl ToolRegistry { sandbox_tag( &invocation.turn.sandbox_policy, invocation.turn.windows_sandbox_level, + invocation + .turn + .features + .enabled(Feature::UseLinuxSandboxBwrap), ), ), ( @@ -356,8 +361,12 @@ async fn dispatch_after_tool_use_hook(dispatch: AfterToolUseHookDispatch<'_>) { success: dispatch.success, duration_ms: u64::try_from(dispatch.duration.as_millis()).unwrap_or(u64::MAX), mutating: dispatch.mutating, - sandbox: sandbox_tag(&turn.sandbox_policy, turn.windows_sandbox_level) - .to_string(), + sandbox: sandbox_tag( + &turn.sandbox_policy, + turn.windows_sandbox_level, + turn.features.enabled(Feature::UseLinuxSandboxBwrap), + ) + .to_string(), sandbox_policy: sandbox_policy_tag(&turn.sandbox_policy).to_string(), output_preview: dispatch.output_preview.clone(), }, diff --git a/codex-rs/core/src/tools/router.rs b/codex-rs/core/src/tools/router.rs index 1cc8b91778..1f1b7f3e00 100644 --- a/codex-rs/core/src/tools/router.rs +++ b/codex-rs/core/src/tools/router.rs @@ -2,6 +2,7 @@ use crate::client_common::tools::ToolSpec; use crate::codex::Session; use crate::codex::TurnContext; use crate::function_tool::FunctionCallError; +use crate::mcp_connection_manager::ToolInfo; use crate::sandboxing::SandboxPermissions; use crate::tools::context::SharedTurnDiffTracker; use crate::tools::context::ToolInvocation; @@ -43,9 +44,10 @@ impl ToolRouter { pub fn from_config( config: &ToolsConfig, mcp_tools: Option>, + app_tools: Option>, dynamic_tools: &[DynamicToolSpec], ) -> Self { - let builder = build_specs(config, mcp_tools, dynamic_tools); + let builder = build_specs(config, mcp_tools, app_tools, dynamic_tools); let (specs, registry) = builder.build(); Self { registry, specs } @@ -238,6 +240,7 @@ mod tests { .await .list_all_tools() .await; + let app_tools = Some(mcp_tools.clone()); let router = ToolRouter::from_config( &turn.tools_config, Some( @@ -246,6 +249,7 @@ mod tests { .map(|(name, tool)| (name, tool.tool)) .collect(), ), + app_tools, turn.dynamic_tools.as_slice(), ); @@ -289,6 +293,7 @@ mod tests { .await .list_all_tools() .await; + let app_tools = Some(mcp_tools.clone()); let router = ToolRouter::from_config( &turn.tools_config, Some( @@ -297,6 +302,7 @@ mod tests { .map(|(name, tool)| (name, tool.tool)) .collect(), ), + app_tools, turn.dynamic_tools.as_slice(), ); diff --git a/codex-rs/core/src/tools/runtimes/shell.rs b/codex-rs/core/src/tools/runtimes/shell.rs index fd076f005e..3b7f970d77 100644 --- a/codex-rs/core/src/tools/runtimes/shell.rs +++ b/codex-rs/core/src/tools/runtimes/shell.rs @@ -11,6 +11,8 @@ use crate::powershell::prefix_powershell_script_with_utf8; use crate::sandboxing::SandboxPermissions; use crate::sandboxing::execute_env; use crate::shell::ShellType; +use crate::tools::network_approval::NetworkApprovalMode; +use crate::tools::network_approval::NetworkApprovalSpec; use crate::tools::runtimes::build_command_spec; use crate::tools::runtimes::maybe_wrap_shell_lc_with_snapshot; use crate::tools::sandboxing::Approvable; @@ -109,6 +111,7 @@ impl Approvable for ShellRuntime { command, cwd, reason, + ctx.network_approval_context.clone(), req.exec_approval_requirement .proposed_execpolicy_amendment() .cloned(), @@ -141,6 +144,20 @@ impl Approvable for ShellRuntime { } impl ToolRuntime for ShellRuntime { + fn network_approval_spec( + &self, + req: &ShellRequest, + _ctx: &ToolCtx<'_>, + ) -> Option { + req.network.as_ref()?; + Some(NetworkApprovalSpec { + command: req.command.clone(), + cwd: req.cwd.clone(), + network: req.network.clone(), + mode: NetworkApprovalMode::Immediate, + }) + } + async fn run( &mut self, req: &ShellRequest, @@ -167,9 +184,10 @@ impl ToolRuntime for ShellRuntime { req.sandbox_permissions, req.justification.clone(), )?; - let env = attempt + let mut env = attempt .env_for(spec, req.network.as_ref()) .map_err(|err| ToolError::Codex(err.into()))?; + env.network_attempt_id = ctx.network_attempt_id.clone(); let out = execute_env(env, attempt.policy, Self::stdout_stream(ctx)) .await .map_err(ToolError::Codex)?; diff --git a/codex-rs/core/src/tools/runtimes/unified_exec.rs b/codex-rs/core/src/tools/runtimes/unified_exec.rs index f3bd2e64b1..4c6c611075 100644 --- a/codex-rs/core/src/tools/runtimes/unified_exec.rs +++ b/codex-rs/core/src/tools/runtimes/unified_exec.rs @@ -12,6 +12,8 @@ use crate::features::Feature; use crate::powershell::prefix_powershell_script_with_utf8; use crate::sandboxing::SandboxPermissions; use crate::shell::ShellType; +use crate::tools::network_approval::NetworkApprovalMode; +use crate::tools::network_approval::NetworkApprovalSpec; use crate::tools::runtimes::build_command_spec; use crate::tools::runtimes::maybe_wrap_shell_lc_with_snapshot; use crate::tools::sandboxing::Approvable; @@ -110,6 +112,7 @@ impl Approvable for UnifiedExecRuntime<'_> { command, cwd, reason, + ctx.network_approval_context.clone(), req.exec_approval_requirement .proposed_execpolicy_amendment() .cloned(), @@ -145,6 +148,20 @@ impl Approvable for UnifiedExecRuntime<'_> { } impl<'a> ToolRuntime for UnifiedExecRuntime<'a> { + fn network_approval_spec( + &self, + req: &UnifiedExecRequest, + _ctx: &ToolCtx<'_>, + ) -> Option { + req.network.as_ref()?; + Some(NetworkApprovalSpec { + command: req.command.clone(), + cwd: req.cwd.clone(), + network: req.network.clone(), + mode: NetworkApprovalMode::Deferred, + }) + } + async fn run( &mut self, req: &UnifiedExecRequest, @@ -165,7 +182,7 @@ impl<'a> ToolRuntime for UnifiedExecRunt let mut env = req.env.clone(); if let Some(network) = req.network.as_ref() { - network.apply_to_env(&mut env); + network.apply_to_env_for_attempt(&mut env, ctx.network_attempt_id.as_deref()); } let spec = build_command_spec( &command, @@ -186,6 +203,7 @@ impl<'a> ToolRuntime for UnifiedExecRunt UnifiedExecError::SandboxDenied { output, .. } => { ToolError::Codex(CodexErr::Sandbox(SandboxErr::Denied { output: Box::new(output), + network_policy_decision: None, })) } other => ToolError::Rejected(other.to_string()), diff --git a/codex-rs/core/src/tools/sandboxing.rs b/codex-rs/core/src/tools/sandboxing.rs index 3671760844..04dc10c3e1 100644 --- a/codex-rs/core/src/tools/sandboxing.rs +++ b/codex-rs/core/src/tools/sandboxing.rs @@ -12,8 +12,10 @@ use crate::sandboxing::CommandSpec; use crate::sandboxing::SandboxManager; use crate::sandboxing::SandboxTransformError; use crate::state::SessionServices; +use crate::tools::network_approval::NetworkApprovalSpec; use codex_network_proxy::NetworkProxy; use codex_protocol::approvals::ExecPolicyAmendment; +use codex_protocol::approvals::NetworkApprovalContext; use codex_protocol::protocol::AskForApproval; use codex_protocol::protocol::ReviewDecision; use std::collections::HashMap; @@ -110,6 +112,7 @@ pub(crate) struct ApprovalCtx<'a> { pub turn: &'a TurnContext, pub call_id: &'a str, pub retry_reason: Option, + pub network_approval_context: Option, } // Specifies what tool orchestrator should do with a given tool call. @@ -252,6 +255,7 @@ pub(crate) struct ToolCtx<'a> { pub turn: &'a TurnContext, pub call_id: String, pub tool_name: String, + pub network_attempt_id: Option, } #[derive(Debug)] @@ -261,6 +265,10 @@ pub(crate) enum ToolError { } pub(crate) trait ToolRuntime: Approvable + Sandboxable { + fn network_approval_spec(&self, _req: &Req, _ctx: &ToolCtx<'_>) -> Option { + None + } + async fn run( &mut self, req: &Req, diff --git a/codex-rs/core/src/tools/spec.rs b/codex-rs/core/src/tools/spec.rs index 82e34298c1..20b27ccce7 100644 --- a/codex-rs/core/src/tools/spec.rs +++ b/codex-rs/core/src/tools/spec.rs @@ -5,8 +5,10 @@ use crate::client_common::tools::ResponsesApiTool; use crate::client_common::tools::ToolSpec; use crate::features::Feature; use crate::features::Features; +use crate::mcp_connection_manager::ToolInfo; use crate::tools::handlers::PLAN_TOOL; use crate::tools::handlers::SEARCH_TOOL_BM25_DEFAULT_LIMIT; +use crate::tools::handlers::SEARCH_TOOL_BM25_TOOL_NAME; use crate::tools::handlers::apply_patch::create_apply_patch_freeform_tool; use crate::tools::handlers::apply_patch::create_apply_patch_json_tool; use crate::tools::handlers::collab::DEFAULT_WAIT_TIMEOUT_MS; @@ -27,6 +29,9 @@ use serde_json::json; use std::collections::BTreeMap; use std::collections::HashMap; +const SEARCH_TOOL_BM25_DESCRIPTION_TEMPLATE: &str = + include_str!("../../templates/search_tool/tool_description.md"); + #[derive(Debug, Clone)] pub(crate) struct ToolsConfig { pub shell_type: ConfigShellToolType, @@ -880,12 +885,12 @@ fn create_grep_files_tool() -> ToolSpec { }) } -fn create_search_tool_bm25_tool() -> ToolSpec { +fn create_search_tool_bm25_tool(app_tools: &HashMap) -> ToolSpec { let properties = BTreeMap::from([ ( "query".to_string(), JsonSchema::String { - description: Some("Search query for MCP tools.".to_string()), + description: Some("Search query for apps tools.".to_string()), }, ), ( @@ -897,10 +902,20 @@ fn create_search_tool_bm25_tool() -> ToolSpec { }, ), ]); + let mut app_names = app_tools + .values() + .filter_map(|tool| tool.connector_name.clone()) + .collect::>(); + app_names.sort(); + app_names.dedup(); + let app_names = app_names.join(", "); + + let description = + SEARCH_TOOL_BM25_DESCRIPTION_TEMPLATE.replace("{{app_names}}", app_names.as_str()); ToolSpec::Function(ResponsesApiTool { - name: "search_tool_bm25".to_string(), - description: "Searches MCP tool metadata with BM25 and exposes matching tools for the next model call.".to_string(), + name: SEARCH_TOOL_BM25_TOOL_NAME.to_string(), + description, strict: false, parameters: JsonSchema::Object { properties, @@ -1390,6 +1405,7 @@ fn sanitize_json_schema(value: &mut JsonValue) { pub(crate) fn build_specs( config: &ToolsConfig, mcp_tools: Option>, + app_tools: Option>, dynamic_tools: &[DynamicToolSpec], ) -> ToolRegistryBuilder { use crate::tools::handlers::ApplyPatchHandler; @@ -1488,9 +1504,11 @@ pub(crate) fn build_specs( builder.register_handler("request_user_input", request_user_input_handler); } - if config.search_tool { - builder.push_spec_with_parallel_support(create_search_tool_bm25_tool(), true); - builder.register_handler("search_tool_bm25", search_tool_handler); + if config.search_tool + && let Some(app_tools) = app_tools + { + builder.push_spec_with_parallel_support(create_search_tool_bm25_tool(&app_tools), true); + builder.register_handler(SEARCH_TOOL_BM25_TOOL_NAME, search_tool_handler); } if let Some(apply_patch_tool_type) = &config.apply_patch_tool_type { @@ -1795,7 +1813,7 @@ mod tests { features: &features, web_search_mode: Some(WebSearchMode::Live), }); - let (tools, _) = build_specs(&config, None, &[]).build(); + let (tools, _) = build_specs(&config, None, None, &[]).build(); // Build actual map name -> spec use std::collections::BTreeMap; @@ -1860,7 +1878,7 @@ mod tests { features: &features, web_search_mode: Some(WebSearchMode::Cached), }); - let (tools, _) = build_specs(&tools_config, None, &[]).build(); + let (tools, _) = build_specs(&tools_config, None, None, &[]).build(); assert_contains_tool_names( &tools, &[ @@ -1885,7 +1903,7 @@ mod tests { features: &features, web_search_mode: Some(WebSearchMode::Cached), }); - let (tools, _) = build_specs(&tools_config, None, &[]).build(); + let (tools, _) = build_specs(&tools_config, None, None, &[]).build(); assert!( !tools.iter().any(|t| t.spec.name() == "request_user_input"), "request_user_input should be disabled when collaboration_modes feature is off" @@ -1897,7 +1915,7 @@ mod tests { features: &features, web_search_mode: Some(WebSearchMode::Cached), }); - let (tools, _) = build_specs(&tools_config, None, &[]).build(); + let (tools, _) = build_specs(&tools_config, None, None, &[]).build(); assert_contains_tool_names(&tools, &["request_user_input"]); } @@ -1913,7 +1931,7 @@ mod tests { features: &features, web_search_mode: Some(WebSearchMode::Cached), }); - let (tools, _) = build_specs(&tools_config, None, &[]).build(); + let (tools, _) = build_specs(&tools_config, None, None, &[]).build(); assert!( !tools.iter().any(|tool| tool.spec.name() == "js_repl"), @@ -1938,7 +1956,7 @@ mod tests { features: &features, web_search_mode: Some(WebSearchMode::Cached), }); - let (tools, _) = build_specs(&tools_config, None, &[]).build(); + let (tools, _) = build_specs(&tools_config, None, None, &[]).build(); assert_contains_tool_names(&tools, &["js_repl", "js_repl_reset"]); } @@ -1956,7 +1974,7 @@ mod tests { features: &features, web_search_mode: Some(WebSearchMode::Cached), }); - let (tools, _) = build_specs(&tools_config, None, &[]).build(); + let (tools, _) = build_specs(&tools_config, None, None, &[]).build(); let filtered = filter_tools_for_model( tools.iter().map(|tool| tool.spec.clone()).collect(), &tools_config, @@ -1994,7 +2012,7 @@ mod tests { "additionalProperties": false }), }]; - let (tools, _) = build_specs(&tools_config, None, &dynamic_tools).build(); + let (tools, _) = build_specs(&tools_config, None, None, &dynamic_tools).build(); assert!( tools.iter().any(|tool| tool.spec.name() == "dynamic_echo"), "expected dynamic tool in full router specs" @@ -2025,7 +2043,7 @@ mod tests { features, web_search_mode, }); - let (tools, _) = build_specs(&tools_config, Some(HashMap::new()), &[]).build(); + let (tools, _) = build_specs(&tools_config, Some(HashMap::new()), None, &[]).build(); let tool_names = tools.iter().map(|t| t.spec.name()).collect::>(); assert_eq!(&tool_names, &expected_tools,); } @@ -2058,7 +2076,7 @@ mod tests { features: &features, web_search_mode: Some(WebSearchMode::Cached), }); - let (tools, _) = build_specs(&tools_config, None, &[]).build(); + let (tools, _) = build_specs(&tools_config, None, None, &[]).build(); let tool = find_tool(&tools, "web_search"); assert_eq!( @@ -2081,7 +2099,7 @@ mod tests { features: &features, web_search_mode: Some(WebSearchMode::Live), }); - let (tools, _) = build_specs(&tools_config, None, &[]).build(); + let (tools, _) = build_specs(&tools_config, None, None, &[]).build(); let tool = find_tool(&tools, "web_search"); assert_eq!( @@ -2306,7 +2324,7 @@ mod tests { features: &features, web_search_mode: Some(WebSearchMode::Live), }); - let (tools, _) = build_specs(&tools_config, Some(HashMap::new()), &[]).build(); + let (tools, _) = build_specs(&tools_config, Some(HashMap::new()), None, &[]).build(); // Only check the shell variant and a couple of core tools. let mut subset = vec!["exec_command", "write_stdin", "update_plan"]; @@ -2329,7 +2347,7 @@ mod tests { features: &features, web_search_mode: Some(WebSearchMode::Cached), }); - let (tools, _) = build_specs(&tools_config, None, &[]).build(); + let (tools, _) = build_specs(&tools_config, None, None, &[]).build(); assert!(find_tool(&tools, "exec_command").supports_parallel_tool_calls); assert!(!find_tool(&tools, "write_stdin").supports_parallel_tool_calls); @@ -2353,7 +2371,7 @@ mod tests { features: &features, web_search_mode: Some(WebSearchMode::Cached), }); - let (tools, _) = build_specs(&tools_config, None, &[]).build(); + let (tools, _) = build_specs(&tools_config, None, None, &[]).build(); assert!( tools @@ -2409,6 +2427,7 @@ mod tests { }), ), )])), + None, &[], ) .build(); @@ -2486,7 +2505,7 @@ mod tests { ), ]); - let (tools, _) = build_specs(&tools_config, Some(tools_map), &[]).build(); + let (tools, _) = build_specs(&tools_config, Some(tools_map), None, &[]).build(); // Only assert that the MCP tools themselves are sorted by fully-qualified name. let mcp_names: Vec<_> = tools @@ -2502,6 +2521,73 @@ mod tests { assert_eq!(mcp_names, expected); } + #[test] + fn search_tool_description_includes_only_codex_apps_connector_names() { + let config = test_config(); + let model_info = + ModelsManager::construct_model_info_offline_for_tests("gpt-5-codex", &config); + let mut features = Features::with_defaults(); + features.enable(Feature::Apps); + let tools_config = ToolsConfig::new(&ToolsConfigParams { + model_info: &model_info, + features: &features, + web_search_mode: Some(WebSearchMode::Cached), + }); + + let (tools, _) = build_specs( + &tools_config, + Some(HashMap::from([ + ( + "mcp__codex_apps__calendar_create_event".to_string(), + mcp_tool( + "calendar_create_event", + "Create calendar event", + serde_json::json!({"type": "object"}), + ), + ), + ( + "mcp__rmcp__echo".to_string(), + mcp_tool("echo", "Echo", serde_json::json!({"type": "object"})), + ), + ])), + Some(HashMap::from([ + ( + "mcp__codex_apps__calendar_create_event".to_string(), + ToolInfo { + server_name: crate::mcp::CODEX_APPS_MCP_SERVER_NAME.to_string(), + tool_name: "calendar_create_event".to_string(), + tool: mcp_tool( + "calendar_create_event", + "Create calendar event", + serde_json::json!({"type": "object"}), + ), + connector_id: Some("calendar".to_string()), + connector_name: Some("Calendar".to_string()), + }, + ), + ( + "mcp__rmcp__echo".to_string(), + ToolInfo { + server_name: "rmcp".to_string(), + tool_name: "echo".to_string(), + tool: mcp_tool("echo", "Echo", serde_json::json!({"type": "object"})), + connector_id: None, + connector_name: None, + }, + ), + ])), + &[], + ) + .build(); + + let search_tool = find_tool(&tools, SEARCH_TOOL_BM25_TOOL_NAME); + let ToolSpec::Function(ResponsesApiTool { description, .. }) = &search_tool.spec else { + panic!("expected function tool"); + }; + assert!(description.contains("Calendar")); + assert!(!description.contains("mcp__rmcp__echo")); + } + #[test] fn test_mcp_tool_property_missing_type_defaults_to_string() { let config = test_config(); @@ -2530,6 +2616,7 @@ mod tests { }), ), )])), + None, &[], ) .build(); @@ -2581,6 +2668,7 @@ mod tests { }), ), )])), + None, &[], ) .build(); @@ -2631,6 +2719,7 @@ mod tests { }), ), )])), + None, &[], ) .build(); @@ -2685,6 +2774,7 @@ mod tests { }), ), )])), + None, &[], ) .build(); @@ -2811,6 +2901,7 @@ Examples of valid command strings: }), ), )])), + None, &[], ) .build(); diff --git a/codex-rs/core/src/turn_metadata.rs b/codex-rs/core/src/turn_metadata.rs index e17a78804a..0814897d91 100644 --- a/codex-rs/core/src/turn_metadata.rs +++ b/codex-rs/core/src/turn_metadata.rs @@ -1,88 +1,349 @@ -//! Helpers for computing and resolving optional per-turn metadata headers. -//! -//! This module owns both metadata construction and the shared timeout policy used by -//! turn execution and startup websocket prewarm. Keeping timeout behavior centralized -//! ensures both call sites treat timeout as the same best-effort fallback condition. - use std::collections::BTreeMap; -use std::future::Future; use std::path::Path; -use std::time::Duration; +use std::path::PathBuf; +use std::sync::Arc; +use std::sync::Mutex; +use std::sync::RwLock; use serde::Serialize; -use tracing::warn; +use tokio::task::JoinHandle; use crate::git_info::get_git_remote_urls_assume_git_repo; use crate::git_info::get_git_repo_root; +use crate::git_info::get_has_changes; use crate::git_info::get_head_commit_hash; +use crate::sandbox_tags::sandbox_tag; +use codex_protocol::config_types::WindowsSandboxLevel; +use codex_protocol::protocol::SandboxPolicy; -pub(crate) const TURN_METADATA_HEADER_TIMEOUT: Duration = Duration::from_millis(250); +#[derive(Clone, Debug, Default)] +struct WorkspaceGitMetadata { + associated_remote_urls: Option>, + latest_git_commit_hash: Option, + has_changes: Option, +} -/// Resolves turn metadata with a shared timeout policy. -/// -/// On timeout, this logs a warning and returns the provided fallback header. -/// -/// Keeping this helper centralized avoids drift between turn-time metadata resolution and startup -/// websocket prewarm, both of which need identical timeout semantics. -pub(crate) async fn resolve_turn_metadata_header_with_timeout( - build_header: F, - fallback_on_timeout: Option, -) -> Option -where - F: Future>, -{ - match tokio::time::timeout(TURN_METADATA_HEADER_TIMEOUT, build_header).await { - Ok(header) => header, - Err(_) => { - warn!( - "timed out after {}ms while building turn metadata header", - TURN_METADATA_HEADER_TIMEOUT.as_millis() - ); - fallback_on_timeout - } +impl WorkspaceGitMetadata { + fn is_empty(&self) -> bool { + self.associated_remote_urls.is_none() + && self.latest_git_commit_hash.is_none() + && self.has_changes.is_none() } } -#[derive(Serialize)] +#[derive(Clone, Debug, Serialize, Default)] struct TurnMetadataWorkspace { #[serde(default, skip_serializing_if = "Option::is_none")] associated_remote_urls: Option>, #[serde(default, skip_serializing_if = "Option::is_none")] latest_git_commit_hash: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + has_changes: Option, } -#[derive(Serialize)] -struct TurnMetadata { +impl From for TurnMetadataWorkspace { + fn from(value: WorkspaceGitMetadata) -> Self { + Self { + associated_remote_urls: value.associated_remote_urls, + latest_git_commit_hash: value.latest_git_commit_hash, + has_changes: value.has_changes, + } + } +} + +#[derive(Clone, Debug, Serialize, Default)] +pub(crate) struct TurnMetadataBag { + #[serde(default, skip_serializing_if = "Option::is_none")] + turn_id: Option, #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] workspaces: BTreeMap, #[serde(default, skip_serializing_if = "Option::is_none")] sandbox: Option, } -pub async fn build_turn_metadata_header(cwd: &Path, sandbox: Option<&str>) -> Option { - let repo_root = get_git_repo_root(cwd); +impl TurnMetadataBag { + fn to_header_value(&self) -> Option { + serde_json::to_string(self).ok() + } +} - let (latest_git_commit_hash, associated_remote_urls) = tokio::join!( +fn build_turn_metadata_bag( + turn_id: Option, + sandbox: Option, + repo_root: Option, + workspace_git_metadata: Option, +) -> TurnMetadataBag { + let mut workspaces = BTreeMap::new(); + if let (Some(repo_root), Some(workspace_git_metadata)) = (repo_root, workspace_git_metadata) + && !workspace_git_metadata.is_empty() + { + workspaces.insert(repo_root, workspace_git_metadata.into()); + } + + TurnMetadataBag { + turn_id, + workspaces, + sandbox, + } +} + +pub async fn build_turn_metadata_header(cwd: &Path, sandbox: Option<&str>) -> Option { + let repo_root = get_git_repo_root(cwd).map(|root| root.to_string_lossy().into_owned()); + + let (latest_git_commit_hash, associated_remote_urls, has_changes) = tokio::join!( get_head_commit_hash(cwd), - get_git_remote_urls_assume_git_repo(cwd) + get_git_remote_urls_assume_git_repo(cwd), + get_has_changes(cwd), ); - if latest_git_commit_hash.is_none() && associated_remote_urls.is_none() && sandbox.is_none() { + if latest_git_commit_hash.is_none() + && associated_remote_urls.is_none() + && has_changes.is_none() + && sandbox.is_none() + { return None; } - let mut workspaces = BTreeMap::new(); - if let Some(repo_root) = repo_root { - workspaces.insert( - repo_root.to_string_lossy().into_owned(), - TurnMetadataWorkspace { - associated_remote_urls, - latest_git_commit_hash, - }, + build_turn_metadata_bag( + None, + sandbox.map(ToString::to_string), + repo_root, + Some(WorkspaceGitMetadata { + associated_remote_urls, + latest_git_commit_hash, + has_changes, + }), + ) + .to_header_value() +} + +#[derive(Clone, Debug)] +pub(crate) struct TurnMetadataState { + cwd: PathBuf, + repo_root: Option, + base_metadata: TurnMetadataBag, + base_header: String, + enriched_header: Arc>>, + enrichment_task: Arc>>>, +} + +impl TurnMetadataState { + pub(crate) fn new( + turn_id: String, + cwd: PathBuf, + sandbox_policy: &SandboxPolicy, + windows_sandbox_level: WindowsSandboxLevel, + use_linux_sandbox_bwrap: bool, + ) -> Self { + let repo_root = get_git_repo_root(&cwd).map(|root| root.to_string_lossy().into_owned()); + let sandbox = Some( + sandbox_tag( + sandbox_policy, + windows_sandbox_level, + use_linux_sandbox_bwrap, + ) + .to_string(), + ); + let base_metadata = build_turn_metadata_bag(Some(turn_id), sandbox, None, None); + let base_header = base_metadata + .to_header_value() + .unwrap_or_else(|| "{}".to_string()); + + Self { + cwd, + repo_root, + base_metadata, + base_header, + enriched_header: Arc::new(RwLock::new(None)), + enrichment_task: Arc::new(Mutex::new(None)), + } + } + + pub(crate) fn current_header_value(&self) -> Option { + if let Some(header) = self + .enriched_header + .read() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .as_ref() + .cloned() + { + return Some(header); + } + Some(self.base_header.clone()) + } + + pub(crate) fn spawn_git_enrichment_task(&self) { + if self.repo_root.is_none() { + return; + } + + let mut task_guard = self + .enrichment_task + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if task_guard.is_some() { + return; + } + + let state = self.clone(); + *task_guard = Some(tokio::spawn(async move { + let workspace_git_metadata = state.fetch_workspace_git_metadata().await; + let Some(repo_root) = state.repo_root.clone() else { + return; + }; + + let enriched_metadata = build_turn_metadata_bag( + state.base_metadata.turn_id.clone(), + state.base_metadata.sandbox.clone(), + Some(repo_root), + Some(workspace_git_metadata), + ); + if enriched_metadata.workspaces.is_empty() { + return; + } + + if let Some(header_value) = enriched_metadata.to_header_value() { + *state + .enriched_header + .write() + .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(header_value); + } + })); + } + + pub(crate) fn cancel_git_enrichment_task(&self) { + let mut task_guard = self + .enrichment_task + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if let Some(task) = task_guard.take() { + task.abort(); + } + } + + async fn fetch_workspace_git_metadata(&self) -> WorkspaceGitMetadata { + let (latest_git_commit_hash, associated_remote_urls, has_changes) = tokio::join!( + get_head_commit_hash(&self.cwd), + get_git_remote_urls_assume_git_repo(&self.cwd), + get_has_changes(&self.cwd), + ); + + WorkspaceGitMetadata { + associated_remote_urls, + latest_git_commit_hash, + has_changes, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + use serde_json::Value; + use tempfile::TempDir; + use tokio::process::Command; + + #[tokio::test] + async fn build_turn_metadata_header_includes_has_changes_for_clean_repo() { + let temp_dir = TempDir::new().expect("temp dir"); + let repo_path = temp_dir.path().join("repo"); + std::fs::create_dir_all(&repo_path).expect("create repo"); + + Command::new("git") + .args(["init"]) + .current_dir(&repo_path) + .output() + .await + .expect("git init"); + Command::new("git") + .args(["config", "user.name", "Test User"]) + .current_dir(&repo_path) + .output() + .await + .expect("git config user.name"); + Command::new("git") + .args(["config", "user.email", "test@example.com"]) + .current_dir(&repo_path) + .output() + .await + .expect("git config user.email"); + + std::fs::write(repo_path.join("README.md"), "hello").expect("write file"); + Command::new("git") + .args(["add", "."]) + .current_dir(&repo_path) + .output() + .await + .expect("git add"); + Command::new("git") + .args(["commit", "-m", "initial"]) + .current_dir(&repo_path) + .output() + .await + .expect("git commit"); + + let header = build_turn_metadata_header(&repo_path, Some("none")) + .await + .expect("header"); + let parsed: Value = serde_json::from_str(&header).expect("valid json"); + let workspace = parsed + .get("workspaces") + .and_then(Value::as_object) + .and_then(|workspaces| workspaces.values().next()) + .cloned() + .expect("workspace"); + + assert_eq!( + workspace.get("has_changes").and_then(Value::as_bool), + Some(false) ); } - serde_json::to_string(&TurnMetadata { - workspaces, - sandbox: sandbox.map(ToString::to_string), - }) - .ok() + + #[test] + fn turn_metadata_state_respects_linux_bubblewrap_toggle() { + let temp_dir = TempDir::new().expect("temp dir"); + let cwd = temp_dir.path().to_path_buf(); + let sandbox_policy = SandboxPolicy::new_read_only_policy(); + + let without_bubblewrap = TurnMetadataState::new( + "turn-a".to_string(), + cwd.clone(), + &sandbox_policy, + WindowsSandboxLevel::Disabled, + false, + ); + let with_bubblewrap = TurnMetadataState::new( + "turn-b".to_string(), + cwd, + &sandbox_policy, + WindowsSandboxLevel::Disabled, + true, + ); + + let without_bubblewrap_header = without_bubblewrap + .current_header_value() + .expect("without_bubblewrap_header"); + let with_bubblewrap_header = with_bubblewrap + .current_header_value() + .expect("with_bubblewrap_header"); + + let without_bubblewrap_json: Value = + serde_json::from_str(&without_bubblewrap_header).expect("without_bubblewrap_json"); + let with_bubblewrap_json: Value = + serde_json::from_str(&with_bubblewrap_header).expect("with_bubblewrap_json"); + + let without_bubblewrap_sandbox = without_bubblewrap_json + .get("sandbox") + .and_then(Value::as_str); + let with_bubblewrap_sandbox = with_bubblewrap_json.get("sandbox").and_then(Value::as_str); + + let expected_with_bubblewrap = + sandbox_tag(&sandbox_policy, WindowsSandboxLevel::Disabled, true); + assert_eq!(with_bubblewrap_sandbox, Some(expected_with_bubblewrap)); + + if cfg!(target_os = "linux") { + assert_eq!(with_bubblewrap_sandbox, Some("linux_bubblewrap")); + assert_ne!(with_bubblewrap_sandbox, without_bubblewrap_sandbox); + } + } } diff --git a/codex-rs/core/src/unified_exec/async_watcher.rs b/codex-rs/core/src/unified_exec/async_watcher.rs index 1fbb1e8f6d..772baab00b 100644 --- a/codex-rs/core/src/unified_exec/async_watcher.rs +++ b/codex-rs/core/src/unified_exec/async_watcher.rs @@ -139,6 +139,43 @@ pub(crate) fn spawn_exit_watcher( }); } +pub(crate) fn spawn_network_denial_watcher( + process: Arc, + session: Arc, + process_id: String, + network_attempt_id: String, +) { + let exit_token = process.cancellation_token(); + tokio::spawn(async move { + let mut poll = tokio::time::interval(Duration::from_millis(100)); + poll.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + + loop { + tokio::select! { + _ = exit_token.cancelled() => { + break; + } + _ = poll.tick() => { + if session + .services + .network_approval + .take_user_denial_outcome(&network_attempt_id) + .await + { + process.terminate(); + session + .services + .unified_exec_manager + .release_process_id(&process_id) + .await; + break; + } + } + } + } + }); +} + async fn process_chunk( pending: &mut Vec, transcript: &Arc>, diff --git a/codex-rs/core/src/unified_exec/mod.rs b/codex-rs/core/src/unified_exec/mod.rs index 84785270f8..a907f348f2 100644 --- a/codex-rs/core/src/unified_exec/mod.rs +++ b/codex-rs/core/src/unified_exec/mod.rs @@ -25,6 +25,7 @@ use std::collections::HashMap; use std::collections::HashSet; use std::path::PathBuf; use std::sync::Arc; +use std::sync::Weak; use std::time::Duration; use codex_network_proxy::NetworkProxy; @@ -144,6 +145,8 @@ struct ProcessEntry { process_id: String, command: Vec, tty: bool, + network_attempt_id: Option, + session: Weak, last_used: tokio::time::Instant, } diff --git a/codex-rs/core/src/unified_exec/process_manager.rs b/codex-rs/core/src/unified_exec/process_manager.rs index 251a4910ba..197cda479f 100644 --- a/codex-rs/core/src/unified_exec/process_manager.rs +++ b/codex-rs/core/src/unified_exec/process_manager.rs @@ -19,6 +19,9 @@ use crate::sandboxing::ExecRequest; use crate::tools::events::ToolEmitter; use crate::tools::events::ToolEventCtx; use crate::tools::events::ToolEventStage; +use crate::tools::network_approval::DeferredNetworkApproval; +use crate::tools::network_approval::deferred_rejection_message; +use crate::tools::network_approval::finish_deferred_network_approval; use crate::tools::orchestrator::ToolOrchestrator; use crate::tools::runtimes::unified_exec::UnifiedExecRequest as UnifiedExecToolRequest; use crate::tools::runtimes::unified_exec::UnifiedExecRuntime; @@ -40,6 +43,7 @@ use crate::unified_exec::WARNING_UNIFIED_EXEC_PROCESSES; use crate::unified_exec::WriteStdinRequest; use crate::unified_exec::async_watcher::emit_exec_end_for_unified_exec; use crate::unified_exec::async_watcher::spawn_exit_watcher; +use crate::unified_exec::async_watcher::spawn_network_denial_watcher; use crate::unified_exec::async_watcher::start_streaming_output; use crate::unified_exec::clamp_yield_time; use crate::unified_exec::generate_chunk_id; @@ -130,8 +134,25 @@ impl UnifiedExecProcessManager { } pub(crate) async fn release_process_id(&self, process_id: &str) { - let mut store = self.process_store.lock().await; - store.remove(process_id); + let removed = { + let mut store = self.process_store.lock().await; + store.remove(process_id) + }; + if let Some(entry) = removed { + Self::unregister_network_attempt_for_entry(&entry).await; + } + } + + async fn unregister_network_attempt_for_entry(entry: &ProcessEntry) { + if let Some(attempt_id) = entry.network_attempt_id.as_deref() + && let Some(session) = entry.session.upgrade() + { + session + .services + .network_approval + .unregister_attempt(attempt_id) + .await; + } } pub(crate) async fn exec_command( @@ -143,13 +164,14 @@ impl UnifiedExecProcessManager { .workdir .clone() .unwrap_or_else(|| context.turn.cwd.clone()); - let process = self .open_session_with_sandbox(&request, cwd.clone(), context) .await; - let process = match process { - Ok(process) => Arc::new(process), + let (process, mut deferred_network_approval) = match process { + Ok((process, deferred_network_approval)) => { + (Arc::new(process), deferred_network_approval) + } Err(err) => { self.release_process_id(&request.process_id).await; return Err(err); @@ -172,7 +194,6 @@ impl UnifiedExecProcessManager { emitter.emit(event_ctx, ToolEventStage::Begin).await; start_streaming_output(&process, context, Arc::clone(&transcript)); - let max_tokens = resolve_max_tokens(request.max_output_tokens); let yield_time_ms = clamp_yield_time(request.yield_time_ms); @@ -205,6 +226,7 @@ impl UnifiedExecProcessManager { let has_exited = process.has_exited() || exit_code.is_some(); let chunk_id = generate_chunk_id(); let process_id = request.process_id.clone(); + if has_exited { // Short‑lived command: emit ExecCommandEnd immediately using the // same helper as the background watcher, so all end events share @@ -215,7 +237,7 @@ impl UnifiedExecProcessManager { Arc::clone(&context.turn), context.call_id.clone(), request.command.clone(), - cwd, + cwd.clone(), Some(process_id), Arc::clone(&transcript), output.clone(), @@ -225,12 +247,45 @@ impl UnifiedExecProcessManager { .await; self.release_process_id(&request.process_id).await; + if let Some(deferred) = deferred_network_approval.as_ref() + && let Some(message) = + deferred_rejection_message(context.session.as_ref(), deferred).await + { + finish_deferred_network_approval( + context.session.as_ref(), + deferred_network_approval.take(), + ) + .await; + return Err(UnifiedExecError::create_process(message)); + } + finish_deferred_network_approval( + context.session.as_ref(), + deferred_network_approval.take(), + ) + .await; process.check_for_sandbox_denial_with_text(&text).await?; } else { + if let Some(deferred) = deferred_network_approval.as_ref() + && let Some(message) = + deferred_rejection_message(context.session.as_ref(), deferred).await + { + process.terminate(); + finish_deferred_network_approval( + context.session.as_ref(), + deferred_network_approval.take(), + ) + .await; + self.release_process_id(&request.process_id).await; + return Err(UnifiedExecError::create_process(message)); + } + // Long‑lived command: persist the process so write_stdin can reuse // it, and register a background watcher that will emit // ExecCommandEnd when the PTY eventually exits (even if no further // tool calls are made). + let network_attempt_id = deferred_network_approval + .as_ref() + .map(|deferred| deferred.attempt_id().to_string()); self.store_process( Arc::clone(&process), context, @@ -239,6 +294,7 @@ impl UnifiedExecProcessManager { start, process_id, request.tty, + network_attempt_id, Arc::clone(&transcript), ) .await; @@ -358,29 +414,35 @@ impl UnifiedExecProcessManager { } async fn refresh_process_state(&self, process_id: &str) -> ProcessStatus { - let mut store = self.process_store.lock().await; - let Some(entry) = store.processes.get(process_id) else { - return ProcessStatus::Unknown; - }; - - let exit_code = entry.process.exit_code(); - let process_id = entry.process_id.clone(); - - if entry.process.has_exited() { - let Some(entry) = store.remove(&process_id) else { + let status = { + let mut store = self.process_store.lock().await; + let Some(entry) = store.processes.get(process_id) else { return ProcessStatus::Unknown; }; - ProcessStatus::Exited { - exit_code, - entry: Box::new(entry), - } - } else { - ProcessStatus::Alive { - exit_code, - call_id: entry.call_id.clone(), - process_id, + + let exit_code = entry.process.exit_code(); + let process_id = entry.process_id.clone(); + + if entry.process.has_exited() { + let Some(entry) = store.remove(&process_id) else { + return ProcessStatus::Unknown; + }; + ProcessStatus::Exited { + exit_code, + entry: Box::new(entry), + } + } else { + ProcessStatus::Alive { + exit_code, + call_id: entry.call_id.clone(), + process_id, + } } + }; + if let ProcessStatus::Exited { entry, .. } = &status { + Self::unregister_network_attempt_for_entry(entry).await; } + status } async fn prepare_process_handles( @@ -437,22 +499,32 @@ impl UnifiedExecProcessManager { started_at: Instant, process_id: String, tty: bool, + network_attempt_id: Option, transcript: Arc>, ) { + let network_attempt_id_for_watcher = network_attempt_id.clone(); let entry = ProcessEntry { process: Arc::clone(&process), call_id: context.call_id.clone(), process_id: process_id.clone(), command: command.to_vec(), tty, + network_attempt_id, + session: Arc::downgrade(&context.session), last_used: started_at, }; - let number_processes = { + let (number_processes, pruned_entry) = { let mut store = self.process_store.lock().await; - Self::prune_processes_if_needed(&mut store); + let pruned_entry = Self::prune_processes_if_needed(&mut store); store.processes.insert(process_id.clone(), entry); - store.processes.len() + (store.processes.len(), pruned_entry) }; + // prune_processes_if_needed runs while holding process_store; do async + // network-approval cleanup only after dropping that lock. + if let Some(pruned_entry) = pruned_entry { + Self::unregister_network_attempt_for_entry(&pruned_entry).await; + pruned_entry.process.terminate(); + } if number_processes >= WARNING_UNIFIED_EXEC_PROCESSES { context @@ -471,10 +543,21 @@ impl UnifiedExecProcessManager { context.call_id.clone(), command.to_vec(), cwd, - process_id, + process_id.clone(), transcript, started_at, ); + + if context.turn.config.managed_network_requirements_enabled() + && let Some(network_attempt_id) = network_attempt_id_for_watcher + { + spawn_network_denial_watcher( + Arc::clone(&process), + Arc::clone(&context.session), + process_id, + network_attempt_id, + ); + } } pub(crate) async fn open_session_with_exec_env( @@ -516,7 +599,7 @@ impl UnifiedExecProcessManager { request: &ExecCommandRequest, cwd: PathBuf, context: &UnifiedExecContext, - ) -> Result { + ) -> Result<(UnifiedExecProcess, Option), UnifiedExecError> { let env = apply_unified_exec_env(create_env( &context.turn.shell_environment_policy, Some(context.session.conversation_id), @@ -550,6 +633,7 @@ impl UnifiedExecProcessManager { turn: context.turn.as_ref(), call_id: context.call_id.clone(), tool_name: "exec_command".to_string(), + network_attempt_id: None, }; orchestrator .run( @@ -560,6 +644,7 @@ impl UnifiedExecProcessManager { context.turn.approval_policy, ) .await + .map(|result| (result.output, result.deferred_network_approval)) .map_err(|e| UnifiedExecError::create_process(format!("{e:?}"))) } @@ -643,9 +728,9 @@ impl UnifiedExecProcessManager { collected } - fn prune_processes_if_needed(store: &mut ProcessStore) -> bool { + fn prune_processes_if_needed(store: &mut ProcessStore) -> Option { if store.processes.len() < MAX_UNIFIED_EXEC_PROCESSES { - return false; + return None; } let meta: Vec<(String, Instant, bool)> = store @@ -655,13 +740,10 @@ impl UnifiedExecProcessManager { .collect(); if let Some(process_id) = Self::process_id_to_prune_from_meta(&meta) { - if let Some(entry) = store.remove(&process_id) { - entry.process.terminate(); - } - return true; + return store.remove(&process_id); } - false + None } // Centralized pruning policy so we can easily swap strategies later. @@ -706,6 +788,7 @@ impl UnifiedExecProcessManager { }; for entry in entries { + Self::unregister_network_attempt_for_entry(&entry).await; entry.process.terminate(); } } diff --git a/codex-rs/core/templates/memories/read_path.md b/codex-rs/core/templates/memories/read_path.md index 5f06131fa9..98728ef8b8 100644 --- a/codex-rs/core/templates/memories/read_path.md +++ b/codex-rs/core/templates/memories/read_path.md @@ -1,15 +1,21 @@ ## Memory -You have access to a memory folder with guidance from prior runs. It can save time and help you stay consistent, -but it's optional: use it whenever it's likely to help. +You have access to a memory folder with guidance from prior runs. It can save +time and help you stay consistent. Use it whenever it is likely to help. -Decision boundary: should you use memory for the new user query? -- You can SKIP memory when the new user query is trivial (e.g. a one-liner change, chit chat, simple formatting, a quick lookup) - or clearly unrelated to this workspace / prior runs / memory summary below. -- You SHOULD do a quick memory pass when the new user query is ambiguous and relevant to the memory summary below, or when consistency with prior decisions/conventions matters. +Decision boundary: should you use memory for a new user query? +- You may skip memory when the new query is trivial (for example, +a one-line change, chit-chat, or simple formatting) or clearly +unrelated to this workspace or the memory summary below. +- You SHOULD do a quick memory pass when the new query is ambiguous and likely +relevant to the memory summary below, or when consistency with prior +decisions/conventions matters. +Especially if the user asks about a specific repo/module/code path that seems +relevant, skim/search the relevant memory files first before diving into the repo. Memory layout (general -> specific): -- {{ base_path }}/memory_summary.md (already provided below; do NOT open again) +- {{ base_path }}/memory_summary.md (already provided below; do NOT open +again) - {{ base_path }}/MEMORY.md (searchable registry; primary file to query) - {{ base_path }}/skills// (skill folder) - SKILL.md (entrypoint instructions) @@ -19,18 +25,38 @@ Memory layout (general -> specific): - {{ base_path }}/rollout_summaries/ (per-rollout recaps + evidence snippets) Quick memory pass (when applicable): -1) Skim the MEMORY_SUMMARY included below and extract a few task-relevant keywords (e.g. repo / module names, error strings, etc.). -2) Search {{ base_path }}/MEMORY.md for those keywords, and for any referenced rollout summary files and skills. -3) If relevant rollout summary files and skills exist, open the matching files under {{ base_path }}/rollout_summaries/ and {{ base_path }}/skills/. +1) Skim the MEMORY_SUMMARY included below and extract a few task-relevant +keywords (for example repo/module names, error strings, etc.). +2) Search {{ base_path }}/MEMORY.md for those keywords, and for any referenced +rollout summary files and skills. +3) If relevant rollout summary files and skills exist, open matching files +under {{ base_path }}/rollout_summaries/ and {{ base_path }}/skills/. 4) If nothing relevant turns up, proceed normally without memory. -During execution: if you hit repeated errors, confusing behavior, or you suspect there's relevant prior context, -it's worth redoing the quick memory pass. Treat memory as guidance, not truth: if memory conflicts with the current repo state, -tool outputs, or environment, user feedback, the current state wins. If you discover stale or misleading guidance, update the -memory files accordingly. +During execution: if you hit repeated errors, confusing behavior, or you suspect +there is relevant prior context, it is worth redoing the quick memory pass. + +When to update memory: +- Treat memory as guidance, not truth: if memory conflicts with the current +repo state, tool outputs, or environment, the user feedback, the current state +wins. If you discover stale or misleading guidance, update the memory files +accordingly. +- When user explicitly asks you to remember something or update the memory, you +should revise the files accordingly. Usually you should directly update +memory_summary.md (such as general tips and user profile section) and MEMORY.md. + +Memory citation requirements: +- If ANY relevant memory files were used: you must output exactly one final +line: + Memory used: `:-`, `:-`, ... + - Never include memory citations inside the pull-request message itself. + - Never cite blank lines; double-check ranges. + - Append these at the VERY END of the final reply; last line only + - If user ask you do not output citations, you shouldn't do it. ========= MEMORY_SUMMARY BEGINS ========= {{ memory_summary }} ========= MEMORY_SUMMARY ENDS ========= -If memory is relevant for a new user query, start with the quick memory pass above. +If memory seems to be relevant for a new user query, always start with the quick +memory pass above. diff --git a/codex-rs/core/templates/search_tool/developer_instructions.md b/codex-rs/core/templates/search_tool/developer_instructions.md deleted file mode 100644 index 5ff6a83e50..0000000000 --- a/codex-rs/core/templates/search_tool/developer_instructions.md +++ /dev/null @@ -1,29 +0,0 @@ -# MCP tool discovery - -When `search_tool_bm25` is available, MCP tools (`mcp__...`) are hidden until you search for them. - -Follow this workflow: - -1. Call `search_tool_bm25` with: - - `query` (required): focused terms that describe the capability you need. - - `limit` (optional): maximum number of tools to return (default `8`). -2. Use the returned `tools` list to decide which MCP tools are relevant. -3. Matching tools are added to `active_selected_tools`. Only tools in `active_selected_tools` are available for the remainder of the current turn. -4. Repeated searches in the same turn are additive: new matches are unioned into `active_selected_tools`. -5. `active_selected_tools` resets at the start of the next turn. - -Notes: -- Core tools remain available without searching. -- If you are unsure, start with `limit` between 5 and 10 to see a broader set of tools. -- `query` is matched against MCP tool metadata fields: - - `name` - - `tool_name` - - `server_name` - - `title` - - `description` - - `connector_name` - - `connector_id` - - input schema property keys (`input_keys`) -- When the user asks to search/lookup/query any external system (logs, tickets, metrics, Slack, etc.), you must call `search_tool_bm25` first before running any shell command or repo search. -- Only use shell commands if (a) MCP tools for that system are not available or not sufficient, and (b) the user explicitly wants a local file/CLI search. -- If unsure which system/tool applies, ask a clarifying question after checking MCP tools. diff --git a/codex-rs/core/templates/search_tool/tool_description.md b/codex-rs/core/templates/search_tool/tool_description.md new file mode 100644 index 0000000000..0c84a6f52e --- /dev/null +++ b/codex-rs/core/templates/search_tool/tool_description.md @@ -0,0 +1,28 @@ +# Apps tool discovery + +Searches over apps tool metadata with BM25 and exposes matching tools for the next model call. + +MCP tools of the apps ({{app_names}}) are hidden until you search for them with this tool (`search_tool_bm25`). + +Follow this workflow: + +1. Call `search_tool_bm25` with: + - `query` (required): focused terms that describe the capability you need. + - `limit` (optional): maximum number of tools to return (default `8`). +2. Use the returned `tools` list to decide which Apps tools are relevant. +3. Matching tools are added to available `tools` and available for the remainder of the current session/thread. +4. Repeated searches in the same session/thread are additive: new matches are unioned into `tools`. + +Notes: +- Core tools remain available without searching. +- If you are unsure, start with `limit` between 5 and 10 to see a broader set of tools. +- `query` is matched against Apps tool metadata fields: + - `name` + - `tool_name` + - `server_name` + - `title` + - `description` + - `connector_name` + - input schema property keys (`input_keys`) +- If the needed app is already explicit in the prompt (for example an `apps://...` mention) or already present in the current `tools` list, you can call that tool directly. +- Do not use `search_tool_bm25` for non-apps/local tasks (filesystem, repo search, or shell-only workflows) or anything not related to {{app_names}}. diff --git a/codex-rs/core/tests/common/apps_test_server.rs b/codex-rs/core/tests/common/apps_test_server.rs new file mode 100644 index 0000000000..c562e390e1 --- /dev/null +++ b/codex-rs/core/tests/common/apps_test_server.rs @@ -0,0 +1,158 @@ +use anyhow::Result; +use serde_json::Value; +use serde_json::json; +use wiremock::Mock; +use wiremock::MockServer; +use wiremock::Request; +use wiremock::Respond; +use wiremock::ResponseTemplate; +use wiremock::matchers::method; +use wiremock::matchers::path; +use wiremock::matchers::path_regex; + +const CONNECTOR_ID: &str = "calendar"; +const CONNECTOR_NAME: &str = "Calendar"; +const PROTOCOL_VERSION: &str = "2025-11-25"; +const SERVER_NAME: &str = "codex-apps-test"; +const SERVER_VERSION: &str = "1.0.0"; + +#[derive(Clone)] +pub struct AppsTestServer { + pub chatgpt_base_url: String, +} + +impl AppsTestServer { + pub async fn mount(server: &MockServer) -> Result { + mount_oauth_metadata(server).await; + mount_streamable_http_json_rpc(server).await; + Ok(Self { + chatgpt_base_url: server.uri(), + }) + } +} + +async fn mount_oauth_metadata(server: &MockServer) { + Mock::given(method("GET")) + .and(path("/.well-known/oauth-authorization-server/mcp")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "authorization_endpoint": format!("{}/oauth/authorize", server.uri()), + "token_endpoint": format!("{}/oauth/token", server.uri()), + "scopes_supported": [""], + }))) + .mount(server) + .await; +} + +async fn mount_streamable_http_json_rpc(server: &MockServer) { + Mock::given(method("POST")) + .and(path_regex("^/api/codex/apps/?$")) + .respond_with(CodexAppsJsonRpcResponder) + .mount(server) + .await; +} + +struct CodexAppsJsonRpcResponder; + +impl Respond for CodexAppsJsonRpcResponder { + fn respond(&self, request: &Request) -> ResponseTemplate { + let body: Value = match serde_json::from_slice(&request.body) { + Ok(body) => body, + Err(error) => { + return ResponseTemplate::new(400).set_body_json(json!({ + "error": format!("invalid JSON-RPC body: {error}"), + })); + } + }; + + let Some(method) = body.get("method").and_then(Value::as_str) else { + return ResponseTemplate::new(400).set_body_json(json!({ + "error": "missing method in JSON-RPC request", + })); + }; + + match method { + "initialize" => { + let id = body.get("id").cloned().unwrap_or(Value::Null); + let protocol_version = body + .pointer("/params/protocolVersion") + .and_then(Value::as_str) + .unwrap_or(PROTOCOL_VERSION); + ResponseTemplate::new(200).set_body_json(json!({ + "jsonrpc": "2.0", + "id": id, + "result": { + "protocolVersion": protocol_version, + "capabilities": { + "tools": { + "listChanged": true + } + }, + "serverInfo": { + "name": SERVER_NAME, + "version": SERVER_VERSION + } + } + })) + } + "notifications/initialized" => ResponseTemplate::new(202), + "tools/list" => { + let id = body.get("id").cloned().unwrap_or(Value::Null); + ResponseTemplate::new(200).set_body_json(json!({ + "jsonrpc": "2.0", + "id": id, + "result": { + "tools": [ + { + "name": "calendar_create_event", + "description": "Create a calendar event.", + "inputSchema": { + "type": "object", + "properties": { + "title": { "type": "string" }, + "starts_at": { "type": "string" }, + "timezone": { "type": "string" } + }, + "required": ["title", "starts_at"], + "additionalProperties": false + }, + "_meta": { + "connector_id": CONNECTOR_ID, + "connector_name": CONNECTOR_NAME + } + }, + { + "name": "calendar_list_events", + "description": "List calendar events.", + "inputSchema": { + "type": "object", + "properties": { + "query": { "type": "string" }, + "limit": { "type": "integer" } + }, + "additionalProperties": false + }, + "_meta": { + "connector_id": CONNECTOR_ID, + "connector_name": CONNECTOR_NAME + } + } + ], + "nextCursor": null + } + })) + } + method if method.starts_with("notifications/") => ResponseTemplate::new(202), + _ => { + let id = body.get("id").cloned().unwrap_or(Value::Null); + ResponseTemplate::new(200).set_body_json(json!({ + "jsonrpc": "2.0", + "id": id, + "error": { + "code": -32601, + "message": format!("method not found: {method}") + } + })) + } + } + } +} diff --git a/codex-rs/core/tests/common/context_snapshot.rs b/codex-rs/core/tests/common/context_snapshot.rs new file mode 100644 index 0000000000..f9362ee30c --- /dev/null +++ b/codex-rs/core/tests/common/context_snapshot.rs @@ -0,0 +1,339 @@ +use serde_json::Value; + +use crate::responses::ResponsesRequest; + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub enum ContextSnapshotRenderMode { + #[default] + RedactedText, + FullText, + KindOnly, + KindWithTextPrefix { + max_chars: usize, + }, +} + +#[derive(Debug, Clone)] +pub struct ContextSnapshotOptions { + render_mode: ContextSnapshotRenderMode, +} + +impl Default for ContextSnapshotOptions { + fn default() -> Self { + Self { + render_mode: ContextSnapshotRenderMode::RedactedText, + } + } +} + +impl ContextSnapshotOptions { + pub fn render_mode(mut self, render_mode: ContextSnapshotRenderMode) -> Self { + self.render_mode = render_mode; + self + } +} + +pub fn format_request_input_snapshot( + request: &ResponsesRequest, + options: &ContextSnapshotOptions, +) -> String { + let items = request.input(); + format_response_items_snapshot(items.as_slice(), options) +} + +pub fn format_response_items_snapshot(items: &[Value], options: &ContextSnapshotOptions) -> String { + items + .iter() + .enumerate() + .map(|(idx, item)| { + let Some(item_type) = item.get("type").and_then(Value::as_str) else { + return format!("{idx:02}:"); + }; + + if options.render_mode == ContextSnapshotRenderMode::KindOnly { + return if item_type == "message" { + let role = item.get("role").and_then(Value::as_str).unwrap_or("unknown"); + format!("{idx:02}:message/{role}") + } else { + format!("{idx:02}:{item_type}") + }; + } + + match item_type { + "message" => { + let role = item.get("role").and_then(Value::as_str).unwrap_or("unknown"); + let text = item + .get("content") + .and_then(Value::as_array) + .map(|content| { + content + .iter() + .map(|entry| { + if let Some(text) = entry.get("text").and_then(Value::as_str) { + return format_snapshot_text(text, options); + } + let Some(content_type) = + entry.get("type").and_then(Value::as_str) + else { + return "".to_string(); + }; + let Some(content_object) = entry.as_object() else { + return format!("<{content_type}>"); + }; + let mut extra_keys = content_object + .keys() + .filter(|key| *key != "type" && *key != "text") + .cloned() + .collect::>(); + extra_keys.sort(); + if extra_keys.is_empty() { + format!("<{content_type}>") + } else { + format!("<{content_type}:{}>", extra_keys.join(",")) + } + }) + .collect::>() + .join(" | ") + }) + .filter(|text| !text.is_empty()) + .unwrap_or_else(|| "".to_string()); + format!("{idx:02}:message/{role}:{text}") + } + "function_call" => { + let name = item.get("name").and_then(Value::as_str).unwrap_or("unknown"); + format!("{idx:02}:function_call/{name}") + } + "function_call_output" => { + let output = item + .get("output") + .and_then(Value::as_str) + .map(|output| format_snapshot_text(output, options)) + .unwrap_or_else(|| "".to_string()); + format!("{idx:02}:function_call_output:{output}") + } + "local_shell_call" => { + let command = item + .get("action") + .and_then(|action| action.get("command")) + .and_then(Value::as_array) + .map(|parts| { + parts + .iter() + .filter_map(Value::as_str) + .collect::>() + .join(" ") + }) + .map(|command| format_snapshot_text(&command, options)) + .filter(|cmd| !cmd.is_empty()) + .unwrap_or_else(|| "".to_string()); + format!("{idx:02}:local_shell_call:{command}") + } + "reasoning" => { + let summary_text = item + .get("summary") + .and_then(Value::as_array) + .and_then(|summary| summary.first()) + .and_then(|entry| entry.get("text")) + .and_then(Value::as_str) + .map(|text| format_snapshot_text(text, options)) + .unwrap_or_else(|| "".to_string()); + let has_encrypted_content = item + .get("encrypted_content") + .and_then(Value::as_str) + .is_some_and(|value| !value.is_empty()); + format!( + "{idx:02}:reasoning:summary={summary_text}:encrypted={has_encrypted_content}" + ) + } + "compaction" => { + let has_encrypted_content = item + .get("encrypted_content") + .and_then(Value::as_str) + .is_some_and(|value| !value.is_empty()); + format!("{idx:02}:compaction:encrypted={has_encrypted_content}") + } + other => format!("{idx:02}:{other}"), + } + }) + .collect::>() + .join("\n") +} + +pub fn format_labeled_requests_snapshot( + scenario: &str, + sections: &[(&str, &ResponsesRequest)], + options: &ContextSnapshotOptions, +) -> String { + let sections = sections + .iter() + .map(|(title, request)| { + format!( + "## {title}\n{}", + format_request_input_snapshot(request, options) + ) + }) + .collect::>() + .join("\n\n"); + format!("Scenario: {scenario}\n\n{sections}") +} + +pub fn format_labeled_items_snapshot( + scenario: &str, + sections: &[(&str, &[Value])], + options: &ContextSnapshotOptions, +) -> String { + let sections = sections + .iter() + .map(|(title, items)| { + format!( + "## {title}\n{}", + format_response_items_snapshot(items, options) + ) + }) + .collect::>() + .join("\n\n"); + format!("Scenario: {scenario}\n\n{sections}") +} + +fn format_snapshot_text(text: &str, options: &ContextSnapshotOptions) -> String { + match options.render_mode { + ContextSnapshotRenderMode::RedactedText => { + canonicalize_snapshot_text(text).replace('\n', "\\n") + } + ContextSnapshotRenderMode::FullText => text.replace('\n', "\\n"), + ContextSnapshotRenderMode::KindWithTextPrefix { max_chars } => { + let normalized = canonicalize_snapshot_text(text).replace('\n', "\\n"); + if normalized.chars().count() <= max_chars { + normalized + } else { + let prefix = normalized.chars().take(max_chars).collect::(); + format!("{prefix}...") + } + } + ContextSnapshotRenderMode::KindOnly => unreachable!(), + } +} + +fn canonicalize_snapshot_text(text: &str) -> String { + if text.starts_with("") { + return "".to_string(); + } + if text.starts_with("# AGENTS.md instructions for ") { + return "".to_string(); + } + if text.starts_with("") { + if let (Some(cwd_start), Some(cwd_end)) = (text.find(""), text.find("")) { + let cwd = &text[cwd_start + "".len()..cwd_end]; + return if cwd.ends_with("PRETURN_CONTEXT_DIFF_CWD") { + "".to_string() + } else { + ">".to_string() + }; + } + return "".to_string(); + } + if text.starts_with("You are performing a CONTEXT CHECKPOINT COMPACTION.") { + return "".to_string(); + } + if text.starts_with("Another language model started to solve this problem") + && let Some((_, summary)) = text.split_once('\n') + { + return format!("\n{summary}"); + } + text.to_string() +} + +#[cfg(test)] +mod tests { + use super::ContextSnapshotOptions; + use super::ContextSnapshotRenderMode; + use super::format_response_items_snapshot; + use pretty_assertions::assert_eq; + use serde_json::json; + + #[test] + fn full_text_mode_preserves_unredacted_text() { + let items = vec![json!({ + "type": "message", + "role": "user", + "content": [{ + "type": "input_text", + "text": "# AGENTS.md instructions for /tmp/example" + }] + })]; + + let rendered = format_response_items_snapshot( + &items, + &ContextSnapshotOptions::default().render_mode(ContextSnapshotRenderMode::FullText), + ); + + assert_eq!( + rendered, + "00:message/user:# AGENTS.md instructions for /tmp/example" + ); + } + + #[test] + fn redacted_text_mode_keeps_canonical_placeholders() { + let items = vec![json!({ + "type": "message", + "role": "user", + "content": [{ + "type": "input_text", + "text": "# AGENTS.md instructions for /tmp/example" + }] + })]; + + let rendered = format_response_items_snapshot( + &items, + &ContextSnapshotOptions::default().render_mode(ContextSnapshotRenderMode::RedactedText), + ); + + assert_eq!(rendered, "00:message/user:"); + } + + #[test] + fn image_only_message_is_rendered_as_non_text_span() { + let items = vec![json!({ + "type": "message", + "role": "user", + "content": [{ + "type": "input_image", + "image_url": "data:image/png;base64,AAAA" + }] + })]; + + let rendered = format_response_items_snapshot(&items, &ContextSnapshotOptions::default()); + + assert_eq!(rendered, "00:message/user:"); + } + + #[test] + fn mixed_text_and_image_message_keeps_image_span() { + let items = vec![json!({ + "type": "message", + "role": "user", + "content": [ + { + "type": "input_text", + "text": "" + }, + { + "type": "input_image", + "image_url": "data:image/png;base64,AAAA" + }, + { + "type": "input_text", + "text": "" + } + ] + })]; + + let rendered = format_response_items_snapshot(&items, &ContextSnapshotOptions::default()); + + assert_eq!( + rendered, + "00:message/user: | | " + ); + } +} diff --git a/codex-rs/core/tests/common/lib.rs b/codex-rs/core/tests/common/lib.rs index 87a8e1adb2..d2f9c9ac59 100644 --- a/codex-rs/core/tests/common/lib.rs +++ b/codex-rs/core/tests/common/lib.rs @@ -12,6 +12,8 @@ use codex_utils_absolute_path::AbsolutePathBuf; use regex_lite::Regex; use std::path::PathBuf; +pub mod apps_test_server; +pub mod context_snapshot; pub mod process; pub mod responses; pub mod streaming_sse; diff --git a/codex-rs/core/tests/common/responses.rs b/codex-rs/core/tests/common/responses.rs index 9faca23888..c434c89717 100644 --- a/codex-rs/core/tests/common/responses.rs +++ b/codex-rs/core/tests/common/responses.rs @@ -5,6 +5,8 @@ use std::time::Duration; use anyhow::Result; use base64::Engine; +use codex_protocol::models::ContentItem; +use codex_protocol::models::ResponseItem; use codex_protocol::openai_models::ModelsResponse; use futures::SinkExt; use futures::StreamExt; @@ -112,6 +114,14 @@ impl ResponsesRequest { self.0.body.clone() } + pub fn body_contains_text(&self, text: &str) -> bool { + let json_fragment = serde_json::to_string(text) + .expect("serialize text to JSON") + .trim_matches('"') + .to_string(); + self.body_json().to_string().contains(&json_fragment) + } + pub fn instructions_text(&self) -> String { self.body_json()["instructions"] .as_str() @@ -131,6 +141,22 @@ impl ResponsesRequest { .collect() } + /// Returns all `input_image` `image_url` spans from `message` inputs for the provided role. + pub fn message_input_image_urls(&self, role: &str) -> Vec { + self.inputs_of_type("message") + .into_iter() + .filter(|item| item.get("role").and_then(Value::as_str) == Some(role)) + .filter_map(|item| item.get("content").and_then(Value::as_array).cloned()) + .flatten() + .filter(|span| span.get("type").and_then(Value::as_str) == Some("input_image")) + .filter_map(|span| { + span.get("image_url") + .and_then(Value::as_str) + .map(str::to_owned) + }) + .collect() + } + pub fn input(&self) -> Vec { self.body_json()["input"] .as_array() @@ -480,6 +506,18 @@ pub fn ev_assistant_message(id: &str, text: &str) -> Value { }) } +pub fn user_message_item(text: &str) -> ResponseItem { + ResponseItem::Message { + id: None, + role: "user".to_string(), + content: vec![ContentItem::InputText { + text: text.to_string(), + }], + end_turn: None, + phase: None, + } +} + pub fn ev_message_item_added(id: &str, text: &str) -> Value { serde_json::json!({ "type": "response.output_item.added", @@ -818,15 +856,24 @@ where } pub async fn mount_compact_json_once(server: &MockServer, body: serde_json::Value) -> ResponseMock { - let (mock, response_mock) = compact_mock(); - mock.respond_with( + mount_compact_response_once( + server, ResponseTemplate::new(200) .insert_header("content-type", "application/json") - .set_body_json(body.clone()), + .set_body_json(body), ) - .up_to_n_times(1) - .mount(server) - .await; + .await +} + +pub async fn mount_compact_response_once( + server: &MockServer, + response: ResponseTemplate, +) -> ResponseMock { + let (mock, response_mock) = compact_mock(); + mock.respond_with(response) + .up_to_n_times(1) + .mount(server) + .await; response_mock } diff --git a/codex-rs/core/tests/responses_headers.rs b/codex-rs/core/tests/responses_headers.rs index d22e5c66e4..71c8b4717a 100644 --- a/codex-rs/core/tests/responses_headers.rs +++ b/codex-rs/core/tests/responses_headers.rs @@ -375,6 +375,15 @@ async fn responses_stream_includes_turn_metadata_header_for_git_workspace_e2e() .expect("x-codex-turn-metadata header should be present"); let initial_parsed: serde_json::Value = serde_json::from_str(&initial_header).expect("x-codex-turn-metadata should be valid JSON"); + let initial_turn_id = initial_parsed + .get("turn_id") + .and_then(serde_json::Value::as_str) + .expect("turn_id should be present") + .to_string(); + assert!( + !initial_turn_id.is_empty(), + "turn_id should not be empty in x-codex-turn-metadata" + ); assert_eq!( initial_parsed .get("sandbox") @@ -424,56 +433,95 @@ async fn responses_stream_includes_turn_metadata_header_for_git_workspace_e2e() .trim() .to_string(); - let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(5); - loop { - let request_recorder = responses::mount_sse_once(&server, response_body.clone()).await; - tokio::time::sleep(std::time::Duration::from_millis(50)).await; - test.submit_turn("hello") - .await - .expect("submit post-git turn prompt"); + let first_response = responses::sse(vec![ + responses::ev_response_created("resp-2"), + responses::ev_reasoning_item("rsn-1", &["thinking"], &[]), + responses::ev_shell_command_call("call-1", "echo turn-metadata"), + responses::ev_completed("resp-2"), + ]); + let follow_up_response = responses::sse(vec![ + responses::ev_response_created("resp-3"), + responses::ev_assistant_message("msg-1", "done"), + responses::ev_completed("resp-3"), + ]); + let request_log = responses::mount_response_sequence( + &server, + vec![ + responses::sse_response(first_response), + responses::sse_response(follow_up_response), + ], + ) + .await; - let maybe_metadata = request_recorder - .single_request() + test.submit_turn("hello") + .await + .expect("submit post-git turn prompt"); + + let requests = request_log.requests(); + assert_eq!(requests.len(), 2, "expected two requests in one turn"); + + let first_parsed: serde_json::Value = serde_json::from_str( + &requests[0] .header("x-codex-turn-metadata") - .and_then(|header_value| { - let parsed: serde_json::Value = serde_json::from_str(&header_value).ok()?; - let workspace = parsed - .get("workspaces") - .and_then(serde_json::Value::as_object) - .and_then(|workspaces| workspaces.values().next()) - .cloned()?; - Some((parsed, workspace)) - }); - let Some((parsed, workspace)) = maybe_metadata else { - if tokio::time::Instant::now() >= deadline { - break; - } - tokio::time::sleep(std::time::Duration::from_millis(25)).await; - continue; - }; + .expect("first request should include turn metadata"), + ) + .expect("first metadata should be valid json"); + let second_parsed: serde_json::Value = serde_json::from_str( + &requests[1] + .header("x-codex-turn-metadata") + .expect("second request should include turn metadata"), + ) + .expect("second metadata should be valid json"); - assert_eq!( - parsed.get("sandbox").and_then(serde_json::Value::as_str), - Some("none") - ); - assert_eq!( - workspace - .get("latest_git_commit_hash") - .and_then(serde_json::Value::as_str), - Some(expected_head.as_str()) - ); - assert_eq!( - workspace - .get("associated_remote_urls") - .and_then(serde_json::Value::as_object) - .and_then(|remotes| remotes.get("origin")) - .and_then(serde_json::Value::as_str), - Some(expected_origin.as_str()) - ); - return; - } + let first_turn_id = first_parsed + .get("turn_id") + .and_then(serde_json::Value::as_str) + .expect("first turn_id should be present"); + let second_turn_id = second_parsed + .get("turn_id") + .and_then(serde_json::Value::as_str) + .expect("second turn_id should be present"); + assert_eq!( + first_turn_id, second_turn_id, + "requests should share turn_id" + ); + assert_ne!( + second_turn_id, + initial_turn_id.as_str(), + "post-git turn should have a new turn_id" + ); - panic!( - "x-codex-turn-metadata with git workspace info was never observed within 5s after git setup" + assert_eq!( + second_parsed + .get("sandbox") + .and_then(serde_json::Value::as_str), + Some("none") + ); + + let workspace = second_parsed + .get("workspaces") + .and_then(serde_json::Value::as_object) + .and_then(|workspaces| workspaces.values().next()) + .cloned() + .expect("second request should include git workspace metadata"); + assert_eq!( + workspace + .get("latest_git_commit_hash") + .and_then(serde_json::Value::as_str), + Some(expected_head.as_str()) + ); + assert_eq!( + workspace + .get("associated_remote_urls") + .and_then(serde_json::Value::as_object) + .and_then(|remotes| remotes.get("origin")) + .and_then(serde_json::Value::as_str), + Some(expected_origin.as_str()) + ); + assert_eq!( + workspace + .get("has_changes") + .and_then(serde_json::Value::as_bool), + Some(false) ); } diff --git a/codex-rs/core/tests/suite/approvals.rs b/codex-rs/core/tests/suite/approvals.rs index 7683339ceb..72de76bab8 100644 --- a/codex-rs/core/tests/suite/approvals.rs +++ b/codex-rs/core/tests/suite/approvals.rs @@ -731,6 +731,42 @@ fn scenarios() -> Vec { stdout_contains: "trusted-unless", }, }, + ScenarioSpec { + name: "cat_redirect_unless_trusted_requires_approval", + approval_policy: UnlessTrusted, + sandbox_policy: workspace_write(false), + action: ActionKind::RunCommand { + command: r#"cat < "hello" > /var/test.txt"#, + }, + sandbox_permissions: SandboxPermissions::UseDefault, + features: vec![], + model_override: Some("gpt-5"), + outcome: Outcome::ExecApproval { + decision: ReviewDecision::Denied, + expected_reason: None, + }, + expectation: Expectation::CommandFailure { + output_contains: "rejected by user", + }, + }, + ScenarioSpec { + name: "cat_redirect_on_request_requires_approval", + approval_policy: OnRequest, + sandbox_policy: workspace_write(false), + action: ActionKind::RunCommand { + command: r#"cat < "hello" > /var/test.txt"#, + }, + sandbox_permissions: SandboxPermissions::RequireEscalated, + features: vec![], + model_override: Some("gpt-5"), + outcome: Outcome::ExecApproval { + decision: ReviewDecision::Denied, + expected_reason: None, + }, + expectation: Expectation::CommandFailure { + output_contains: "rejected by user", + }, + }, ScenarioSpec { name: "danger_full_access_on_failure_allows_outside_write", approval_policy: OnFailure, @@ -1853,3 +1889,72 @@ async fn approving_execpolicy_amendment_persists_policy_and_skips_future_prompts Ok(()) } + +#[tokio::test(flavor = "current_thread")] +#[cfg(unix)] +async fn heredoc_with_chained_allowed_prefix_still_requires_approval() -> Result<()> { + skip_if_no_network!(Ok(())); + + let server = start_mock_server().await; + let approval_policy = AskForApproval::UnlessTrusted; + let sandbox_policy = SandboxPolicy::new_read_only_policy(); + let sandbox_policy_for_config = sandbox_policy.clone(); + let mut builder = test_codex().with_config(move |config| { + config.permissions.approval_policy = Constrained::allow_any(approval_policy); + config.permissions.sandbox_policy = Constrained::allow_any(sandbox_policy_for_config); + }); + let test = builder.build(&server).await?; + + let rules_dir = test.home.path().join("rules"); + fs::create_dir_all(&rules_dir)?; + fs::write( + rules_dir.join("default.rules"), + r#"prefix_rule(pattern=["touch", "allow-prefix.txt"], decision="allow")"#, + )?; + + let call_id = "heredoc-with-chained-prefix"; + let command = "cat <<'EOF' > /tmp/test.txt && touch allow-prefix.txt\nhello\nEOF"; + let (event, expected_command) = ActionKind::RunCommand { command } + .prepare(&test, &server, call_id, SandboxPermissions::UseDefault) + .await?; + let expected_command = + expected_command.expect("heredoc chained command scenario should produce a shell command"); + + let _ = mount_sse_once( + &server, + sse(vec![ + ev_response_created("resp-heredoc-prefix-1"), + event, + ev_completed("resp-heredoc-prefix-1"), + ]), + ) + .await; + let _ = mount_sse_once( + &server, + sse(vec![ + ev_assistant_message("msg-heredoc-prefix-1", "done"), + ev_completed("resp-heredoc-prefix-2"), + ]), + ) + .await; + + submit_turn( + &test, + "heredoc chained prefix", + approval_policy, + sandbox_policy.clone(), + ) + .await?; + + let approval = expect_exec_approval(&test, expected_command.as_str()).await; + test.codex + .submit(Op::ExecApproval { + id: approval.call_id, + turn_id: None, + decision: ReviewDecision::Denied, + }) + .await?; + wait_for_completion(&test).await; + + Ok(()) +} diff --git a/codex-rs/core/tests/suite/client.rs b/codex-rs/core/tests/suite/client.rs index 7c6799df92..bbdbc73233 100644 --- a/codex-rs/core/tests/suite/client.rs +++ b/codex-rs/core/tests/suite/client.rs @@ -34,6 +34,7 @@ use codex_protocol::models::ReasoningItemReasoningSummary; use codex_protocol::models::WebSearchAction; use codex_protocol::openai_models::ReasoningEffort; use codex_protocol::user_input::UserInput; +use core_test_support::apps_test_server::AppsTestServer; use core_test_support::load_default_config_for_test; use core_test_support::responses::ev_completed; use core_test_support::responses::ev_completed_with_tokens; @@ -671,6 +672,10 @@ async fn includes_user_instructions_message_in_request() { async fn includes_apps_guidance_as_developer_message_when_enabled() { skip_if_no_network!(); let server = MockServer::start().await; + let apps_server = AppsTestServer::mount(&server) + .await + .expect("mount apps MCP mock"); + let apps_base_url = apps_server.chatgpt_base_url.clone(); let resp_mock = mount_sse_once( &server, @@ -680,8 +685,10 @@ async fn includes_apps_guidance_as_developer_message_when_enabled() { let mut builder = test_codex() .with_auth(CodexAuth::from_api_key("Test API Key")) - .with_config(|config| { + .with_config(move |config| { config.features.enable(Feature::Apps); + config.features.disable(Feature::AppsMcpGateway); + config.chatgpt_base_url = apps_base_url; }); let codex = builder .build(&server) diff --git a/codex-rs/core/tests/suite/client_websockets.rs b/codex-rs/core/tests/suite/client_websockets.rs index d436cac599..1db81ecf86 100755 --- a/codex-rs/core/tests/suite/client_websockets.rs +++ b/codex-rs/core/tests/suite/client_websockets.rs @@ -106,7 +106,7 @@ async fn responses_websocket_preconnect_reuses_connection() { let harness = websocket_harness(&server).await; let mut client_session = harness.client.new_session(); client_session - .prewarm_websocket(&harness.otel_manager, &harness.model_info, None) + .prewarm_websocket(&harness.otel_manager, &harness.model_info) .await .expect("websocket prewarm failed"); let prompt = prompt_with_input(vec![message_item("hello")]); @@ -131,7 +131,7 @@ async fn responses_websocket_preconnect_is_reused_even_with_header_changes() { let harness = websocket_harness(&server).await; let mut client_session = harness.client.new_session(); client_session - .prewarm_websocket(&harness.otel_manager, &harness.model_info, None) + .prewarm_websocket(&harness.otel_manager, &harness.model_info) .await .expect("websocket prewarm failed"); let prompt = prompt_with_input(vec![message_item("hello")]); @@ -172,7 +172,7 @@ async fn responses_websocket_prewarm_uses_model_preference_when_feature_disabled let harness = websocket_harness_with_options(&server, false, false, false, true).await; let mut client_session = harness.client.new_session(); client_session - .prewarm_websocket(&harness.otel_manager, &harness.model_info, None) + .prewarm_websocket(&harness.otel_manager, &harness.model_info) .await .expect("websocket prewarm failed"); diff --git a/codex-rs/core/tests/suite/compact.rs b/codex-rs/core/tests/suite/compact.rs index 914673bdb3..25278073ec 100644 --- a/codex-rs/core/tests/suite/compact.rs +++ b/codex-rs/core/tests/suite/compact.rs @@ -20,6 +20,9 @@ use codex_protocol::items::TurnItem; use codex_protocol::openai_models::ModelInfo; use codex_protocol::openai_models::ModelsResponse; use codex_protocol::user_input::UserInput; +use core_test_support::context_snapshot; +use core_test_support::context_snapshot::ContextSnapshotOptions; +use core_test_support::context_snapshot::ContextSnapshotRenderMode; use core_test_support::responses::ev_local_shell_call; use core_test_support::responses::ev_reasoning_item; use core_test_support::responses::mount_models_once; @@ -27,7 +30,7 @@ use core_test_support::skip_if_no_network; use core_test_support::test_codex::test_codex; use core_test_support::wait_for_event; use core_test_support::wait_for_event_match; -use std::collections::VecDeque; +use std::path::PathBuf; use core_test_support::responses::ev_assistant_message; use core_test_support::responses::ev_completed; @@ -60,10 +63,11 @@ const SECOND_AUTO_SUMMARY: &str = "SECOND_AUTO_SUMMARY"; const FINAL_REPLY: &str = "FINAL_REPLY"; const CONTEXT_LIMIT_MESSAGE: &str = "Your input exceeds the context window of this model. Please adjust your input and try again."; -const DUMMY_FUNCTION_NAME: &str = "unsupported_tool"; +const DUMMY_FUNCTION_NAME: &str = "test_tool"; const DUMMY_CALL_ID: &str = "call-multi-auto"; const FUNCTION_CALL_LIMIT_MSG: &str = "function call limit push"; const POST_AUTO_USER_MSG: &str = "post auto follow-up"; +const PRETURN_CONTEXT_DIFF_CWD: &str = "/tmp/PRETURN_CONTEXT_DIFF_CWD"; pub(super) const COMPACT_WARNING_MESSAGE: &str = "Heads up: Long threads and multiple compactions can cause the model to be less accurate. Start a new thread when possible to keep threads small and targeted."; @@ -75,23 +79,6 @@ fn summary_with_prefix(summary: &str) -> String { format!("{SUMMARY_PREFIX}\n{summary}") } -fn drop_call_id(value: &mut serde_json::Value) { - match value { - serde_json::Value::Object(obj) => { - obj.retain(|k, _| k != "call_id"); - for v in obj.values_mut() { - drop_call_id(v); - } - } - serde_json::Value::Array(arr) => { - for v in arr { - drop_call_id(v); - } - } - _ => {} - } -} - fn set_test_compact_prompt(config: &mut Config) { config.compact_prompt = Some(SUMMARIZATION_PROMPT.to_string()); } @@ -142,6 +129,15 @@ fn assert_pre_sampling_switch_compaction_requests( body_contains_text(&compact_body, SUMMARIZATION_PROMPT), "pre-sampling compact request should include summarization prompt" ); + assert!( + !compact_body.contains(""), + "pre-sampling compact request should strip trailing model-switch update item" + ); + let follow_up_body = follow_up.to_string(); + assert!( + follow_up_body.contains(""), + "follow-up request after successful model-switch compaction should include model-switch update item" + ); } async fn assert_compaction_uses_turn_lifecycle_id(codex: &std::sync::Arc) { @@ -185,6 +181,21 @@ async fn assert_compaction_uses_turn_lifecycle_id(codex: &std::sync::Arc ContextSnapshotOptions { + ContextSnapshotOptions::default() + .render_mode(ContextSnapshotRenderMode::KindWithTextPrefix { max_chars: 64 }) +} + +fn format_labeled_requests_snapshot( + scenario: &str, + sections: &[(&str, &core_test_support::responses::ResponsesRequest)], +) -> String { + context_snapshot::format_labeled_requests_snapshot( + scenario, + sections, + &context_snapshot_options(), + ) +} #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn summarize_context_three_requests_and_instructions() { @@ -399,8 +410,15 @@ async fn manual_compact_uses_custom_prompt() { skip_if_no_network!(); let server = start_mock_server().await; - let sse_stream = sse(vec![ev_completed("r1")]); - let response_mock = mount_sse_once(&server, sse_stream).await; + let first_turn = sse(vec![ + ev_assistant_message("m0", FIRST_REPLY), + ev_completed_with_tokens("r0", 80), + ]); + let compact_turn = sse(vec![ + ev_assistant_message("m1", SUMMARY_TEXT), + ev_completed_with_tokens("r1", 100), + ]); + let request_log = mount_sse_sequence(&server, vec![first_turn, compact_turn]).await; let custom_prompt = "Use this compact prompt instead"; @@ -415,6 +433,18 @@ async fn manual_compact_uses_custom_prompt() { .expect("create conversation") .codex; + codex + .submit(Op::UserInput { + items: vec![UserInput::Text { + text: "USER_ONE".to_string(), + text_elements: Vec::new(), + }], + final_output_json_schema: None, + }) + .await + .expect("submit first user turn"); + wait_for_event(&codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await; + codex.submit(Op::Compact).await.expect("trigger compact"); let warning_event = wait_for_event(&codex, |ev| matches!(ev, EventMsg::Warning(_))).await; let EventMsg::Warning(WarningEvent { message }) = warning_event else { @@ -423,7 +453,13 @@ async fn manual_compact_uses_custom_prompt() { assert_eq!(message, COMPACT_WARNING_MESSAGE); wait_for_event(&codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await; - let body = response_mock.single_request().body_json(); + let requests = request_log.requests(); + assert_eq!( + requests.len(), + 2, + "expected first turn and compact requests" + ); + let body = requests[1].body_json(); let input = body .get("input") @@ -1733,6 +1769,21 @@ async fn pre_sampling_compact_runs_on_switch_to_smaller_context_model() { previous_model, next_model, ); + + insta::assert_snapshot!( + "pre_sampling_model_switch_compaction_shapes", + format_labeled_requests_snapshot( + "Pre-sampling compaction on model switch to a smaller context window: current behavior compacts using prior-turn history only (incoming user message excluded), and the follow-up request carries compacted history plus the new user message.", + &[ + ("Initial Request (Previous Model)", &requests[0]), + ("Pre-sampling Compaction Request", &requests[1]), + ( + "Post-Compaction Follow-up Request (Next Model)", + &requests[2] + ), + ] + ) + ); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] @@ -2124,6 +2175,82 @@ async fn manual_compact_retries_after_context_window_error() { } } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +// TODO(ccunningham): Re-enable after the follow-up compaction behavior PR lands. +// Current main behavior around non-context manual /compact failures is known-incorrect. +#[ignore = "behavior change covered in follow-up compaction PR"] +async fn manual_compact_non_context_failure_retries_then_emits_task_error() { + skip_if_no_network!(); + + let server = start_mock_server().await; + + let user_turn = sse(vec![ + ev_assistant_message("m1", FIRST_REPLY), + ev_completed("r1"), + ]); + let compact_failed_1 = sse_failed( + "resp-fail-1", + "server_error", + "temporary compact failure one", + ); + let compact_failed_2 = sse_failed( + "resp-fail-2", + "server_error", + "temporary compact failure two", + ); + + mount_sse_sequence(&server, vec![user_turn, compact_failed_1, compact_failed_2]).await; + + let mut model_provider = non_openai_model_provider(&server); + model_provider.stream_max_retries = Some(1); + + let codex = test_codex() + .with_config(move |config| { + config.model_provider = model_provider; + set_test_compact_prompt(config); + config.model_auto_compact_token_limit = Some(200_000); + }) + .build(&server) + .await + .expect("build codex") + .codex; + + codex + .submit(Op::UserInput { + items: vec![UserInput::Text { + text: "first turn".into(), + text_elements: Vec::new(), + }], + final_output_json_schema: None, + }) + .await + .expect("submit user input"); + wait_for_event(&codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await; + + codex.submit(Op::Compact).await.expect("trigger compact"); + + let reconnect_message = wait_for_event_match(&codex, |event| match event { + EventMsg::StreamError(stream_error) => Some(stream_error.message.clone()), + _ => None, + }) + .await; + assert!( + reconnect_message.contains("Reconnecting... 1/1"), + "expected reconnect stream error message, got {reconnect_message}" + ); + + let task_error_message = wait_for_event_match(&codex, |event| match event { + EventMsg::Error(err) => Some(err.message.clone()), + _ => None, + }) + .await; + assert!( + task_error_message.contains("Error running local compact task"), + "expected local compact task error prefix, got {task_error_message}" + ); + wait_for_event(&codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await; +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn manual_compact_twice_preserves_latest_user_messages() { skip_if_no_network!(); @@ -2228,114 +2355,89 @@ async fn manual_compact_twice_preserves_latest_user_messages() { 5, "expected exactly 5 requests (user turn, compact, user turn, compact, final turn)" ); - let contains_user_text = |input: &[serde_json::Value], expected: &str| -> bool { - input.iter().any(|item| { - item.get("type").and_then(|v| v.as_str()) == Some("message") - && item.get("role").and_then(|v| v.as_str()) == Some("user") - && item - .get("content") - .and_then(|v| v.as_array()) - .is_some_and(|arr| { - arr.iter().any(|entry| { - entry.get("text").and_then(|v| v.as_str()) == Some(expected) - }) - }) - }) + let contains_user_text = |request: &core_test_support::responses::ResponsesRequest, + expected: &str| { + request + .message_input_texts("user") + .iter() + .any(|text| text == expected) }; - let first_turn_input = requests[0].input(); assert!( - contains_user_text(&first_turn_input, first_user_message), + contains_user_text(&requests[0], first_user_message), "first turn request missing first user message" ); assert!( - !contains_user_text(&first_turn_input, SUMMARIZATION_PROMPT), + !contains_user_text(&requests[0], SUMMARIZATION_PROMPT), "first turn request should not include summarization prompt" ); - let first_compact_input = requests[1].input(); assert!( - contains_user_text(&first_compact_input, first_user_message), + contains_user_text(&requests[1], first_user_message), "first compact request should include history before compaction" ); - let second_turn_input = requests[2].input(); assert!( - contains_user_text(&second_turn_input, second_user_message), + contains_user_text(&requests[2], second_user_message), "second turn request missing second user message" ); assert!( - contains_user_text(&second_turn_input, first_user_message), + contains_user_text(&requests[2], first_user_message), "second turn request should include the compacted user history" ); - let second_compact_input = requests[3].input(); assert!( - contains_user_text(&second_compact_input, second_user_message), + contains_user_text(&requests[3], second_user_message), "second compact request should include latest history" ); - let first_compact_has_prompt = contains_user_text(&first_compact_input, SUMMARIZATION_PROMPT); - let second_compact_has_prompt = contains_user_text(&second_compact_input, SUMMARIZATION_PROMPT); + insta::assert_snapshot!( + "manual_compact_with_history_shapes", + format_labeled_requests_snapshot( + "Manual /compact with prior user history compacts existing history and the follow-up turn includes the compact summary plus new user message.", + &[ + ("Local Compaction Request", &requests[1]), + ("Local Post-Compaction History Layout", &requests[2]), + ] + ) + ); + + let first_compact_has_prompt = contains_user_text(&requests[1], SUMMARIZATION_PROMPT); + let second_compact_has_prompt = contains_user_text(&requests[3], SUMMARIZATION_PROMPT); assert_eq!( first_compact_has_prompt, second_compact_has_prompt, "compact requests should consistently include or omit the summarization prompt" ); - let mut final_output = requests + let first_request_user_texts = requests[0].message_input_texts("user"); + let first_turn_user_index = first_request_user_texts + .len() + .checked_sub(1) + .unwrap_or_else(|| panic!("first turn request missing user messages")); + assert_eq!( + first_request_user_texts[first_turn_user_index], first_user_message, + "first turn request should end with the submitted user message" + ); + let seeded_user_prefix = &first_request_user_texts[..first_turn_user_index]; + + let final_request_user_texts = requests .last() .unwrap_or_else(|| panic!("final turn request missing for {final_user_message}")) - .input() - .into_iter() - .collect::>(); - - // Permissions developer message - final_output.pop_front(); - // User instructions (project docs/skills) - final_output.pop_front(); - // Environment context - final_output.pop_front(); - - let _ = final_output - .iter_mut() - .map(drop_call_id) - .collect::>(); - + .message_input_texts("user"); + assert!( + final_request_user_texts + .as_slice() + .starts_with(seeded_user_prefix), + "final request should start with seeded user prefix from first request: {seeded_user_prefix:?}" + ); + let final_output = &final_request_user_texts[seeded_user_prefix.len()..]; let expected = vec![ - json!({ - "content": vec![json!({ - "text": first_user_message, - "type": "input_text", - })], - "role": "user", - "type": "message", - }), - json!({ - "content": vec![json!({ - "text": second_user_message, - "type": "input_text", - })], - "role": "user", - "type": "message", - }), - json!({ - "content": vec![json!({ - "text": expected_second_summary, - "type": "input_text", - })], - "role": "user", - "type": "message", - }), - json!({ - "content": vec![json!({ - "text": final_user_message, - "type": "input_text", - })], - "role": "user", - "type": "message", - }), + first_user_message.to_string(), + second_user_message.to_string(), + expected_second_summary, + final_user_message.to_string(), ]; - assert_eq!(final_output, expected); + assert_eq!(final_output, expected.as_slice()); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] @@ -2450,7 +2552,7 @@ async fn auto_compact_allows_multiple_attempts_when_interleaved_with_other_turn_ } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn auto_compact_triggers_after_function_call_over_95_percent_usage() { +async fn snapshot_request_shape_mid_turn_continuation_compaction() { skip_if_no_network!(); let server = start_mock_server().await; @@ -2458,29 +2560,25 @@ async fn auto_compact_triggers_after_function_call_over_95_percent_usage() { let context_window = 100; let limit = context_window * 90 / 100; let over_limit_tokens = context_window * 95 / 100 + 1; - let follow_up_user = "FOLLOW_UP_AFTER_LIMIT"; let first_turn = sse(vec![ ev_function_call(DUMMY_CALL_ID, DUMMY_FUNCTION_NAME, "{}"), - ev_completed_with_tokens("r1", 50), - ]); - let function_call_follow_up = sse(vec![ - ev_assistant_message("m2", FINAL_REPLY), - ev_completed_with_tokens("r2", over_limit_tokens), + ev_completed_with_tokens("r1", over_limit_tokens), ]); let auto_summary_payload = auto_summary(AUTO_SUMMARY_TEXT); let auto_compact_turn = sse(vec![ - ev_assistant_message("m3", &auto_summary_payload), + ev_assistant_message("m2", &auto_summary_payload), ev_completed_with_tokens("r3", 10), ]); - let post_auto_compact_turn = sse(vec![ev_completed_with_tokens("r4", 10)]); + let post_auto_compact_turn = sse(vec![ + ev_assistant_message("m3", FINAL_REPLY), + ev_completed_with_tokens("r4", 10), + ]); // Mount responses in order and keep mocks only for the ones we assert on. let first_turn_mock = mount_sse_once(&server, first_turn).await; - let follow_up_mock = mount_sse_once(&server, function_call_follow_up).await; let auto_compact_mock = mount_sse_once(&server, auto_compact_turn).await; - // We don't assert on the post-compact request, so no need to keep its mock. - mount_sse_once(&server, post_auto_compact_turn).await; + let post_auto_compact_mock = mount_sse_once(&server, post_auto_compact_turn).await; let model_provider = non_openai_model_provider(&server); @@ -2505,19 +2603,6 @@ async fn auto_compact_triggers_after_function_call_over_95_percent_usage() { wait_for_event(&codex, |msg| matches!(msg, EventMsg::TurnComplete(_))).await; - codex - .submit(Op::UserInput { - items: vec![UserInput::Text { - text: follow_up_user.into(), - text_elements: Vec::new(), - }], - final_output_json_schema: None, - }) - .await - .unwrap(); - - wait_for_event(&codex, |msg| matches!(msg, EventMsg::TurnComplete(_))).await; - // Assert first request captured expected user message that triggers function call. let first_request = first_turn_mock.single_request().input(); assert!( @@ -2534,7 +2619,7 @@ async fn auto_compact_triggers_after_function_call_over_95_percent_usage() { "first request should include the user message that triggers the function call" ); - let function_call_output = follow_up_mock + let function_call_output = auto_compact_mock .single_request() .function_call_output(DUMMY_CALL_ID); let output_text = function_call_output @@ -2549,7 +2634,24 @@ async fn auto_compact_triggers_after_function_call_over_95_percent_usage() { let auto_compact_body = auto_compact_mock.single_request().body_json().to_string(); assert!( body_contains_text(&auto_compact_body, SUMMARIZATION_PROMPT), - "auto compact request should include the summarization prompt after exceeding 95% (limit {limit})" + "mid-turn auto compact request should include the summarization prompt after exceeding 95% (limit {limit})" + ); + + insta::assert_snapshot!( + "mid_turn_compaction_shapes", + format_labeled_requests_snapshot( + "True mid-turn continuation compaction after tool output: compact request includes tool artifacts, and the continuation request includes the summary in the same turn.", + &[ + ( + "Local Compaction Request", + &auto_compact_mock.single_request() + ), + ( + "Local Post-Compaction History Layout", + &post_auto_compact_mock.single_request() + ), + ] + ) ); } @@ -2819,3 +2921,270 @@ async fn auto_compact_runs_when_reasoning_header_clears_between_turns() { "remote compaction should run once after the reasoning header clears" ); } + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +// TODO(ccunningham): Update once pre-turn compaction includes incoming user input. +async fn snapshot_request_shape_pre_turn_compaction_including_incoming_user_message() { + skip_if_no_network!(); + + let server = start_mock_server().await; + + let sse1 = sse(vec![ + ev_assistant_message("m1", FIRST_REPLY), + ev_completed_with_tokens("r1", 60), + ]); + let sse2 = sse(vec![ + ev_assistant_message("m2", "SECOND_REPLY"), + ev_completed_with_tokens("r2", 500), + ]); + let sse3 = sse(vec![ + ev_assistant_message("m3", "PRE_TURN_SUMMARY"), + ev_completed_with_tokens("r3", 100), + ]); + let sse4 = sse(vec![ + ev_assistant_message("m4", FINAL_REPLY), + ev_completed_with_tokens("r4", 80), + ]); + let request_log = mount_sse_sequence(&server, vec![sse1, sse2, sse3, sse4]).await; + + let model_provider = non_openai_model_provider(&server); + let codex = test_codex() + .with_config(move |config| { + config.model_provider = model_provider; + set_test_compact_prompt(config); + config.model_auto_compact_token_limit = Some(200); + }) + .build(&server) + .await + .expect("build codex") + .codex; + + for user in ["USER_ONE", "USER_TWO"] { + codex + .submit(Op::UserInput { + items: vec![UserInput::Text { + text: user.to_string(), + text_elements: Vec::new(), + }], + final_output_json_schema: None, + }) + .await + .expect("submit user input"); + wait_for_event(&codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await; + } + codex + .submit(Op::OverrideTurnContext { + cwd: Some(PathBuf::from(PRETURN_CONTEXT_DIFF_CWD)), + approval_policy: None, + sandbox_policy: None, + windows_sandbox_level: None, + model: None, + effort: None, + summary: None, + collaboration_mode: None, + personality: None, + }) + .await + .expect("override turn context"); + let image_url = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR4nGNgYAAAAAMAASsJTYQAAAAASUVORK5CYII=" + .to_string(); + codex + .submit(Op::UserInput { + items: vec![ + UserInput::Image { + image_url: image_url.clone(), + }, + UserInput::Text { + text: "USER_THREE".to_string(), + text_elements: Vec::new(), + }, + ], + final_output_json_schema: None, + }) + .await + .expect("submit user input"); + wait_for_event(&codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await; + + let requests = request_log.requests(); + assert_eq!(requests.len(), 4, "expected user, user, compact, follow-up"); + + insta::assert_snapshot!( + "pre_turn_compaction_including_incoming_shapes", + format_labeled_requests_snapshot( + "Pre-turn auto-compaction with a context override emits the context diff in the compact request while the incoming user message is still excluded.", + &[ + ("Local Compaction Request", &requests[2]), + ("Local Post-Compaction History Layout", &requests[3]), + ] + ) + ); + let compact_request_user_texts = requests[2].message_input_texts("user"); + assert!( + !compact_request_user_texts + .iter() + .any(|text| text == "USER_THREE"), + "current behavior excludes incoming user message from pre-turn compaction input" + ); + let follow_up_user_texts = requests[3].message_input_texts("user"); + assert!( + follow_up_user_texts.iter().any(|text| text == "USER_THREE"), + "expected post-compaction follow-up request to keep incoming user text" + ); + let follow_up_user_images = requests[3].message_input_image_urls("user"); + assert!( + follow_up_user_images + .iter() + .any(|url| url == image_url.as_str()), + "expected post-compaction follow-up request to keep incoming user image content" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +// TODO(ccunningham): Update once pre-turn compaction context-overflow handling includes incoming +// user input and emits richer oversized-input messaging. +async fn snapshot_request_shape_pre_turn_compaction_context_window_exceeded() { + skip_if_no_network!(); + + let server = start_mock_server().await; + + let first_turn = sse(vec![ + ev_assistant_message("m1", FIRST_REPLY), + ev_completed_with_tokens("r1", 500), + ]); + let mut responses = vec![first_turn]; + responses.extend( + (0..6).map(|_| { + sse_failed( + "compact-failed", + "context_length_exceeded", + "Your input exceeds the context window of this model. Please adjust your input and try again.", + ) + }), + ); + let request_log = mount_sse_sequence(&server, responses).await; + + let mut model_provider = non_openai_model_provider(&server); + model_provider.stream_max_retries = Some(0); + let codex = test_codex() + .with_config(move |config| { + config.model_provider = model_provider; + set_test_compact_prompt(config); + config.model_auto_compact_token_limit = Some(200); + }) + .build(&server) + .await + .expect("build codex") + .codex; + + codex + .submit(Op::UserInput { + items: vec![UserInput::Text { + text: "USER_ONE".to_string(), + text_elements: Vec::new(), + }], + final_output_json_schema: None, + }) + .await + .expect("submit first user"); + wait_for_event(&codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await; + + codex + .submit(Op::UserInput { + items: vec![UserInput::Text { + text: "USER_TWO".to_string(), + text_elements: Vec::new(), + }], + final_output_json_schema: None, + }) + .await + .expect("submit second user"); + let error_message = wait_for_event_match(&codex, |event| match event { + EventMsg::Error(err) => Some(err.message.clone()), + _ => None, + }) + .await; + wait_for_event(&codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await; + + let requests = request_log.requests(); + assert!( + requests.len() >= 2, + "expected first turn and at least one compaction request" + ); + + insta::assert_snapshot!( + "pre_turn_compaction_context_window_exceeded_shapes", + format_labeled_requests_snapshot( + "Pre-turn auto-compaction context-window failure: compaction request excludes the incoming user message and the turn errors.", + &[( + "Local Compaction Request (Incoming User Excluded)", + &requests[1] + ),] + ) + ); + + assert!( + error_message.contains("ran out of room in the model's context window"), + "expected context window exceeded message, got {error_message}" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn snapshot_request_shape_manual_compact_without_previous_user_messages() { + skip_if_no_network!(); + + let server = start_mock_server().await; + + let compact_turn = sse(vec![ + ev_assistant_message("m1", "MANUAL_EMPTY_SUMMARY"), + ev_completed_with_tokens("r1", 90), + ]); + let follow_up_turn = sse(vec![ + ev_assistant_message("m2", FINAL_REPLY), + ev_completed_with_tokens("r2", 80), + ]); + let request_log = mount_sse_sequence(&server, vec![compact_turn, follow_up_turn]).await; + + let model_provider = non_openai_model_provider(&server); + let codex = test_codex() + .with_config(move |config| { + config.model_provider = model_provider; + set_test_compact_prompt(config); + }) + .build(&server) + .await + .expect("build codex") + .codex; + + codex.submit(Op::Compact).await.expect("run /compact"); + wait_for_event(&codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await; + + codex + .submit(Op::UserInput { + items: vec![UserInput::Text { + text: "AFTER_MANUAL_EMPTY_COMPACT".to_string(), + text_elements: Vec::new(), + }], + final_output_json_schema: None, + }) + .await + .expect("submit follow-up user input"); + wait_for_event(&codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await; + + let requests = request_log.requests(); + assert_eq!( + requests.len(), + 2, + "expected manual /compact request and follow-up turn request" + ); + + insta::assert_snapshot!( + "manual_compact_without_prev_user_shapes", + format_labeled_requests_snapshot( + "Manual /compact with no prior user turn currently still issues a compaction request; follow-up turn carries canonical context and the new user message.", + &[ + ("Local Compaction Request", &requests[0]), + ("Local Post-Compaction History Layout", &requests[1]), + ] + ) + ); +} diff --git a/codex-rs/core/tests/suite/compact_remote.rs b/codex-rs/core/tests/suite/compact_remote.rs index 3311427d4b..6c31b8dbeb 100644 --- a/codex-rs/core/tests/suite/compact_remote.rs +++ b/codex-rs/core/tests/suite/compact_remote.rs @@ -1,9 +1,11 @@ #![allow(clippy::expect_used)] use std::fs; +use std::path::PathBuf; use anyhow::Result; use codex_core::CodexAuth; +use codex_core::compact::SUMMARY_PREFIX; use codex_core::protocol::EventMsg; use codex_core::protocol::ItemCompletedEvent; use codex_core::protocol::ItemStartedEvent; @@ -14,6 +16,9 @@ use codex_protocol::items::TurnItem; use codex_protocol::models::ContentItem; use codex_protocol::models::ResponseItem; use codex_protocol::user_input::UserInput; +use core_test_support::context_snapshot; +use core_test_support::context_snapshot::ContextSnapshotOptions; +use core_test_support::context_snapshot::ContextSnapshotRenderMode; use core_test_support::responses; use core_test_support::responses::mount_sse_once; use core_test_support::responses::sse; @@ -23,6 +28,7 @@ use core_test_support::test_codex::test_codex; use core_test_support::wait_for_event; use core_test_support::wait_for_event_match; use pretty_assertions::assert_eq; +use wiremock::ResponseTemplate; fn approx_token_count(text: &str) -> i64 { i64::try_from(text.len().saturating_add(3) / 4).unwrap_or(i64::MAX) @@ -39,6 +45,29 @@ fn estimate_compact_payload_tokens(request: &responses::ResponsesRequest) -> i64 .saturating_add(approx_token_count(&request.instructions_text())) } +const PRETURN_CONTEXT_DIFF_CWD: &str = "/tmp/PRETURN_CONTEXT_DIFF_CWD"; +const DUMMY_FUNCTION_NAME: &str = "test_tool"; + +fn summary_with_prefix(summary: &str) -> String { + format!("{SUMMARY_PREFIX}\n{summary}") +} + +fn context_snapshot_options() -> ContextSnapshotOptions { + ContextSnapshotOptions::default() + .render_mode(ContextSnapshotRenderMode::KindWithTextPrefix { max_chars: 64 }) +} + +fn format_labeled_requests_snapshot( + scenario: &str, + sections: &[(&str, &responses::ResponsesRequest)], +) -> String { + context_snapshot::format_labeled_requests_snapshot( + scenario, + sections, + &context_snapshot_options(), + ) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn remote_compact_replaces_history_for_followups() -> Result<()> { skip_if_no_network!(Ok(())); @@ -65,15 +94,7 @@ async fn remote_compact_replaces_history_for_followups() -> Result<()> { .await; let compacted_history = vec![ - ResponseItem::Message { - id: None, - role: "user".to_string(), - content: vec![ContentItem::InputText { - text: "REMOTE_COMPACTED_SUMMARY".to_string(), - }], - end_turn: None, - phase: None, - }, + responses::user_message_item("REMOTE_COMPACTED_SUMMARY"), ResponseItem::Compaction { encrypted_content: "ENCRYPTED_COMPACTION_SUMMARY".to_string(), }, @@ -134,12 +155,9 @@ async fn remote_compact_replaces_history_for_followups() -> Result<()> { "expected compact request to include assistant history" ); - let follow_up_body = responses_mock - .requests() - .last() - .expect("follow-up request missing") - .body_json() - .to_string(); + let response_requests = responses_mock.requests(); + let follow_up_request = response_requests.last().expect("follow-up request missing"); + let follow_up_body = follow_up_request.body_json().to_string(); assert!( follow_up_body.contains("REMOTE_COMPACTED_SUMMARY"), "expected follow-up request to use compacted history" @@ -152,6 +170,21 @@ async fn remote_compact_replaces_history_for_followups() -> Result<()> { !follow_up_body.contains("FIRST_REMOTE_REPLY"), "expected follow-up request to drop pre-compaction assistant messages" ); + assert!( + !follow_up_body.contains("hello remote compact"), + "expected follow-up request to drop compacted-away user turns when remote output omits them" + ); + + insta::assert_snapshot!( + "remote_manual_compact_with_history_shapes", + format_labeled_requests_snapshot( + "Remote manual /compact where remote compact output is summary-only: follow-up layout uses returned summary plus new user message.", + &[ + ("Remote Compaction Request", &compact_request), + ("Remote Post-Compaction History Layout", follow_up_request), + ] + ) + ); Ok(()) } @@ -184,15 +217,7 @@ async fn remote_compact_runs_automatically() -> Result<()> { .await; let compacted_history = vec![ - ResponseItem::Message { - id: None, - role: "user".to_string(), - content: vec![ContentItem::InputText { - text: "REMOTE_COMPACTED_SUMMARY".to_string(), - }], - end_turn: None, - phase: None, - }, + responses::user_message_item("REMOTE_COMPACTED_SUMMARY"), ResponseItem::Compaction { encrypted_content: "ENCRYPTED_COMPACTION_SUMMARY".to_string(), }, @@ -221,7 +246,8 @@ async fn remote_compact_runs_automatically() -> Result<()> { wait_for_event(&codex, |event| matches!(event, EventMsg::TurnComplete(_))).await; assert!(message); assert_eq!(compact_mock.requests().len(), 1); - let follow_up_body = responses_mock.single_request().body_json().to_string(); + let follow_up_request = responses_mock.single_request(); + let follow_up_body = follow_up_request.body_json().to_string(); assert!(follow_up_body.contains("REMOTE_COMPACTED_SUMMARY")); assert!(follow_up_body.contains("ENCRYPTED_COMPACTION_SUMMARY")); @@ -497,7 +523,7 @@ async fn auto_remote_compact_failure_stops_agent_loop() -> Result<()> { ) .await; - let compact_mock = responses::mount_compact_json_once( + let first_compact_mock = responses::mount_compact_json_once( harness.server(), serde_json::json!({ "output": "invalid compact payload shape" }), ) @@ -541,18 +567,29 @@ async fn auto_remote_compact_failure_stops_agent_loop() -> Result<()> { assert!( error_message.contains("Error running remote compact task"), - "expected compact failure error, got {error_message}" + "expected remote compact task error prefix, got {error_message}" ); assert_eq!( - compact_mock.requests().len(), + first_compact_mock.requests().len(), 1, - "expected exactly one remote compact attempt" + "expected first remote compact attempt with incoming items" ); assert!( post_compact_turn_mock.requests().is_empty(), "expected agent loop to stop after compaction failure" ); + insta::assert_snapshot!( + "remote_pre_turn_compaction_failure_shapes", + format_labeled_requests_snapshot( + "Remote pre-turn auto-compaction parse failure: compaction request excludes the incoming user message and the turn stops.", + &[( + "Remote Compaction Request (Incoming User Excluded)", + &first_compact_mock.single_request() + ),] + ) + ); + Ok(()) } @@ -778,15 +815,7 @@ async fn remote_manual_compact_emits_context_compaction_items() -> Result<()> { .await; let compacted_history = vec![ - ResponseItem::Message { - id: None, - role: "user".to_string(), - content: vec![ContentItem::InputText { - text: "REMOTE_COMPACTED_SUMMARY".to_string(), - }], - end_turn: None, - phase: None, - }, + responses::user_message_item("REMOTE_COMPACTED_SUMMARY"), ResponseItem::Compaction { encrypted_content: "ENCRYPTED_COMPACTION_SUMMARY".to_string(), }, @@ -851,6 +880,68 @@ async fn remote_manual_compact_emits_context_compaction_items() -> Result<()> { } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn remote_manual_compact_failure_emits_task_error_event() -> Result<()> { + skip_if_no_network!(Ok(())); + + let harness = TestCodexHarness::with_builder( + test_codex().with_auth(CodexAuth::create_dummy_chatgpt_auth_for_testing()), + ) + .await?; + let codex = harness.test().codex.clone(); + + mount_sse_once( + harness.server(), + sse(vec![ + responses::ev_assistant_message("m1", "REMOTE_REPLY"), + responses::ev_completed("resp-1"), + ]), + ) + .await; + + let compact_mock = responses::mount_compact_json_once( + harness.server(), + serde_json::json!({ "output": "invalid compact payload shape" }), + ) + .await; + + codex + .submit(Op::UserInput { + items: vec![UserInput::Text { + text: "manual remote compact".into(), + text_elements: Vec::new(), + }], + final_output_json_schema: None, + }) + .await?; + wait_for_event(&codex, |event| matches!(event, EventMsg::TurnComplete(_))).await; + + codex.submit(Op::Compact).await?; + + let error_message = wait_for_event_match(&codex, |event| match event { + EventMsg::Error(err) => Some(err.message.clone()), + _ => None, + }) + .await; + assert!( + error_message.contains("Error running remote compact task"), + "expected remote compact task error prefix, got {error_message}" + ); + assert!( + error_message.contains("invalid compact payload shape") + || error_message.contains("invalid type: string"), + "expected invalid compact payload details, got {error_message}" + ); + wait_for_event(&codex, |event| matches!(event, EventMsg::TurnComplete(_))).await; + + assert_eq!(compact_mock.requests().len(), 1); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +// TODO(ccunningham): Re-enable after the follow-up compaction behavior PR lands. +// Current main behavior for rollout replacement-history persistence is known-incorrect. +#[ignore = "behavior change covered in follow-up compaction PR"] async fn remote_compact_persists_replacement_history_in_rollout() -> Result<()> { skip_if_no_network!(Ok(())); @@ -876,15 +967,7 @@ async fn remote_compact_persists_replacement_history_in_rollout() -> Result<()> .await; let compacted_history = vec![ - ResponseItem::Message { - id: None, - role: "user".to_string(), - content: vec![ContentItem::InputText { - text: "COMPACTED_USER_SUMMARY".to_string(), - }], - end_turn: None, - phase: None, - }, + responses::user_message_item("COMPACTED_USER_SUMMARY"), ResponseItem::Compaction { encrypted_content: "ENCRYPTED_COMPACTION_SUMMARY".to_string(), }, @@ -980,11 +1063,11 @@ async fn remote_compact_persists_replacement_history_in_rollout() -> Result<()> ) }); - if has_compacted_user_summary - && has_compaction_item - && has_compacted_assistant_note - && has_permissions_developer_message - { + if has_compacted_user_summary && has_compaction_item && has_compacted_assistant_note { + assert!( + !has_permissions_developer_message, + "manual remote compact rollout replacement history should not inject permissions context" + ); saw_compacted_history = true; break; } @@ -1036,15 +1119,7 @@ async fn remote_compact_and_resume_refresh_stale_developer_instructions() -> Res .await; let compacted_history = vec![ - ResponseItem::Message { - id: None, - role: "user".to_string(), - content: vec![ContentItem::InputText { - text: "REMOTE_COMPACTED_SUMMARY".to_string(), - }], - end_turn: None, - phase: None, - }, + responses::user_message_item("REMOTE_COMPACTED_SUMMARY"), ResponseItem::Message { id: None, role: "developer".to_string(), @@ -1177,15 +1252,7 @@ async fn remote_compact_refreshes_stale_developer_instructions_without_resume() .await; let compacted_history = vec![ - ResponseItem::Message { - id: None, - role: "user".to_string(), - content: vec![ContentItem::InputText { - text: "REMOTE_COMPACTED_SUMMARY".to_string(), - }], - end_turn: None, - phase: None, - }, + responses::user_message_item("REMOTE_COMPACTED_SUMMARY"), ResponseItem::Message { id: None, role: "developer".to_string(), @@ -1250,3 +1317,344 @@ async fn remote_compact_refreshes_stale_developer_instructions_without_resume() Ok(()) } + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +// TODO(ccunningham): Update once remote pre-turn compaction includes incoming user input. +async fn snapshot_request_shape_remote_pre_turn_compaction_including_incoming_user_message() +-> Result<()> { + skip_if_no_network!(Ok(())); + + let harness = TestCodexHarness::with_builder( + test_codex() + .with_auth(CodexAuth::create_dummy_chatgpt_auth_for_testing()) + .with_config(|config| { + config.model_auto_compact_token_limit = Some(200); + }), + ) + .await?; + let codex = harness.test().codex.clone(); + + let responses_mock = responses::mount_sse_sequence( + harness.server(), + vec![ + responses::sse(vec![ + responses::ev_assistant_message("m1", "REMOTE_FIRST_REPLY"), + responses::ev_completed_with_tokens("r1", 60), + ]), + responses::sse(vec![ + responses::ev_assistant_message("m2", "REMOTE_SECOND_REPLY"), + responses::ev_completed_with_tokens("r2", 500), + ]), + responses::sse(vec![ + responses::ev_assistant_message("m3", "REMOTE_FINAL_REPLY"), + responses::ev_completed_with_tokens("r3", 80), + ]), + ], + ) + .await; + + let compacted_history = vec![ + responses::user_message_item("USER_ONE"), + responses::user_message_item("USER_TWO"), + responses::user_message_item(&summary_with_prefix("REMOTE_PRE_TURN_SUMMARY")), + ]; + let compact_mock = responses::mount_compact_json_once( + harness.server(), + serde_json::json!({ "output": compacted_history }), + ) + .await; + + for user in ["USER_ONE", "USER_TWO", "USER_THREE"] { + if user == "USER_THREE" { + codex + .submit(Op::OverrideTurnContext { + cwd: Some(PathBuf::from(PRETURN_CONTEXT_DIFF_CWD)), + approval_policy: None, + sandbox_policy: None, + windows_sandbox_level: None, + model: None, + effort: None, + summary: None, + collaboration_mode: None, + personality: None, + }) + .await?; + } + codex + .submit(Op::UserInput { + items: vec![UserInput::Text { + text: user.to_string(), + text_elements: Vec::new(), + }], + final_output_json_schema: None, + }) + .await?; + wait_for_event(&codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await; + } + + assert_eq!(compact_mock.requests().len(), 1); + let requests = responses_mock.requests(); + assert_eq!( + requests.len(), + 3, + "expected user, user, and post-compact turn" + ); + + let compact_request = compact_mock.single_request(); + insta::assert_snapshot!( + "remote_pre_turn_compaction_including_incoming_shapes", + format_labeled_requests_snapshot( + "Remote pre-turn auto-compaction with a context override emits the context diff in the compact request while excluding the incoming user message.", + &[ + ("Remote Compaction Request", &compact_request), + ("Remote Post-Compaction History Layout", &requests[2]), + ] + ) + ); + assert_eq!( + requests[2] + .message_input_texts("user") + .iter() + .filter(|text| text.as_str() == "USER_THREE") + .count(), + 1, + "post-compaction request should contain incoming user exactly once from runtime append" + ); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +// TODO(ccunningham): Update once remote pre-turn compaction context-overflow handling includes +// incoming user input and emits richer oversized-input messaging. +async fn snapshot_request_shape_remote_pre_turn_compaction_context_window_exceeded() -> Result<()> { + skip_if_no_network!(Ok(())); + + let harness = TestCodexHarness::with_builder( + test_codex() + .with_auth(CodexAuth::create_dummy_chatgpt_auth_for_testing()) + .with_config(|config| { + config.model_auto_compact_token_limit = Some(200); + }), + ) + .await?; + let codex = harness.test().codex.clone(); + + let responses_mock = responses::mount_sse_sequence( + harness.server(), + vec![responses::sse(vec![ + responses::ev_assistant_message("m1", "REMOTE_FIRST_REPLY"), + responses::ev_completed_with_tokens("r1", 500), + ])], + ) + .await; + + let compact_mock = responses::mount_compact_response_once( + harness.server(), + ResponseTemplate::new(400).set_body_json(serde_json::json!({ + "error": { + "code": "context_length_exceeded", + "message": "Your input exceeds the context window of this model. Please adjust your input and try again." + } + })), + ) + .await; + let post_compact_turn_mock = responses::mount_sse_once( + harness.server(), + responses::sse(vec![ + responses::ev_assistant_message("m2", "REMOTE_POST_COMPACT_SHOULD_NOT_RUN"), + responses::ev_completed_with_tokens("r2", 80), + ]), + ) + .await; + + codex + .submit(Op::UserInput { + items: vec![UserInput::Text { + text: "USER_ONE".to_string(), + text_elements: Vec::new(), + }], + final_output_json_schema: None, + }) + .await?; + wait_for_event(&codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await; + + codex + .submit(Op::UserInput { + items: vec![UserInput::Text { + text: "USER_TWO".to_string(), + text_elements: Vec::new(), + }], + final_output_json_schema: None, + }) + .await?; + let error_message = wait_for_event_match(&codex, |event| match event { + EventMsg::Error(err) => Some(err.message.clone()), + _ => None, + }) + .await; + wait_for_event(&codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await; + + assert_eq!(compact_mock.requests().len(), 1); + let requests = responses_mock.requests(); + assert_eq!( + requests.len(), + 1, + "expected no post-compaction follow-up turn request after compact failure" + ); + assert!( + post_compact_turn_mock.requests().is_empty(), + "expected turn to stop after compaction failure" + ); + + let include_attempt_request = compact_mock.single_request(); + insta::assert_snapshot!( + "remote_pre_turn_compaction_context_window_exceeded_shapes", + format_labeled_requests_snapshot( + "Remote pre-turn auto-compaction context-window failure: compaction request excludes the incoming user message and the turn errors.", + &[( + "Remote Compaction Request (Incoming User Excluded)", + &include_attempt_request + ),] + ) + ); + assert!( + error_message.to_lowercase().contains("context window"), + "expected context window failure to surface, got {error_message}" + ); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn snapshot_request_shape_remote_mid_turn_continuation_compaction() -> Result<()> { + skip_if_no_network!(Ok(())); + + let harness = TestCodexHarness::with_builder( + test_codex() + .with_auth(CodexAuth::create_dummy_chatgpt_auth_for_testing()) + .with_config(|config| { + config.model_auto_compact_token_limit = Some(200); + }), + ) + .await?; + let codex = harness.test().codex.clone(); + + let responses_mock = responses::mount_sse_sequence( + harness.server(), + vec![ + responses::sse(vec![ + responses::ev_function_call("call-remote-mid-turn", DUMMY_FUNCTION_NAME, "{}"), + responses::ev_completed_with_tokens("r1", 500), + ]), + responses::sse(vec![ + responses::ev_assistant_message("m2", "REMOTE_MID_TURN_FINAL_REPLY"), + responses::ev_completed_with_tokens("r2", 80), + ]), + ], + ) + .await; + + let compacted_history = vec![ + responses::user_message_item("USER_ONE"), + responses::user_message_item(&summary_with_prefix("REMOTE_MID_TURN_SUMMARY")), + ]; + let compact_mock = responses::mount_compact_json_once( + harness.server(), + serde_json::json!({ "output": compacted_history }), + ) + .await; + + codex + .submit(Op::UserInput { + items: vec![UserInput::Text { + text: "USER_ONE".to_string(), + text_elements: Vec::new(), + }], + final_output_json_schema: None, + }) + .await?; + wait_for_event(&codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await; + + assert_eq!(compact_mock.requests().len(), 1); + let requests = responses_mock.requests(); + assert_eq!( + requests.len(), + 2, + "expected initial and post-compact requests" + ); + + let compact_request = compact_mock.single_request(); + insta::assert_snapshot!( + "remote_mid_turn_compaction_shapes", + format_labeled_requests_snapshot( + "Remote mid-turn continuation compaction after tool output: compact request includes tool artifacts and follow-up request includes the summary.", + &[ + ("Remote Compaction Request", &compact_request), + ("Remote Post-Compaction History Layout", &requests[1]), + ] + ) + ); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +// TODO(ccunningham): Update once manual remote /compact with no prior user turn becomes a no-op. +async fn snapshot_request_shape_remote_manual_compact_without_previous_user_messages() -> Result<()> +{ + skip_if_no_network!(Ok(())); + + let harness = TestCodexHarness::with_builder( + test_codex().with_auth(CodexAuth::create_dummy_chatgpt_auth_for_testing()), + ) + .await?; + let codex = harness.test().codex.clone(); + + let responses_mock = responses::mount_sse_once( + harness.server(), + responses::sse(vec![ + responses::ev_assistant_message("m1", "REMOTE_MANUAL_EMPTY_FOLLOW_UP_REPLY"), + responses::ev_completed_with_tokens("r1", 80), + ]), + ) + .await; + + let compact_mock = + responses::mount_compact_json_once(harness.server(), serde_json::json!({ "output": [] })) + .await; + + codex.submit(Op::Compact).await?; + wait_for_event(&codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await; + + codex + .submit(Op::UserInput { + items: vec![UserInput::Text { + text: "USER_ONE".to_string(), + text_elements: Vec::new(), + }], + final_output_json_schema: None, + }) + .await?; + wait_for_event(&codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await; + + assert_eq!( + compact_mock.requests().len(), + 1, + "current behavior still issues remote compaction for manual /compact without prior user" + ); + let compact_request = compact_mock.single_request(); + let follow_up_request = responses_mock.single_request(); + insta::assert_snapshot!( + "remote_manual_compact_without_prev_user_shapes", + format_labeled_requests_snapshot( + "Remote manual /compact with no prior user turn still issues a compact request; follow-up turn carries canonical context and new user message.", + &[ + ("Remote Compaction Request", &compact_request), + ("Remote Post-Compaction History Layout", &follow_up_request), + ] + ) + ); + + Ok(()) +} diff --git a/codex-rs/core/tests/suite/exec.rs b/codex-rs/core/tests/suite/exec.rs index c3cb15875b..c857affd2b 100644 --- a/codex-rs/core/tests/suite/exec.rs +++ b/codex-rs/core/tests/suite/exec.rs @@ -37,6 +37,7 @@ async fn run_test_cmd(tmp: TempDir, cmd: Vec<&str>) -> Result Result< supported_in_api: true, input_modalities: default_input_modalities(), prefer_websockets: false, + used_fallback_model_metadata: false, priority: 1, upgrade: None, base_instructions: "base instructions".to_string(), @@ -324,32 +324,14 @@ async fn model_change_from_image_to_text_strips_prior_image_content() -> Result< assert_eq!(requests.len(), 2, "expected two model requests"); let first_request = requests.first().expect("expected first request"); - let first_has_input_image = first_request.inputs_of_type("message").iter().any(|item| { - item.get("content") - .and_then(Value::as_array) - .is_some_and(|content| { - content - .iter() - .any(|span| span.get("type").and_then(Value::as_str) == Some("input_image")) - }) - }); assert!( - first_has_input_image, + !first_request.message_input_image_urls("user").is_empty(), "first request should include the uploaded image" ); let second_request = requests.last().expect("expected second request"); - let second_has_input_image = second_request.inputs_of_type("message").iter().any(|item| { - item.get("content") - .and_then(Value::as_array) - .is_some_and(|content| { - content - .iter() - .any(|span| span.get("type").and_then(Value::as_str) == Some("input_image")) - }) - }); assert!( - !second_has_input_image, + second_request.message_input_image_urls("user").is_empty(), "second request should strip unsupported image content" ); let second_user_texts = second_request.message_input_texts("user"); @@ -404,6 +386,7 @@ async fn model_switch_to_smaller_model_updates_token_context_window() -> Result< supported_in_api: true, input_modalities: default_input_modalities(), prefer_websockets: false, + used_fallback_model_metadata: false, priority: 1, upgrade: None, base_instructions: "base instructions".to_string(), diff --git a/codex-rs/core/tests/suite/models_cache_ttl.rs b/codex-rs/core/tests/suite/models_cache_ttl.rs index 59c7dde7d7..780937c111 100644 --- a/codex-rs/core/tests/suite/models_cache_ttl.rs +++ b/codex-rs/core/tests/suite/models_cache_ttl.rs @@ -352,5 +352,6 @@ fn test_remote_model(slug: &str, priority: i32) -> ModelInfo { experimental_supported_tools: Vec::new(), input_modalities: default_input_modalities(), prefer_websockets: false, + used_fallback_model_metadata: false, } } diff --git a/codex-rs/core/tests/suite/personality.rs b/codex-rs/core/tests/suite/personality.rs index 9397765c90..51f99720aa 100644 --- a/codex-rs/core/tests/suite/personality.rs +++ b/codex-rs/core/tests/suite/personality.rs @@ -616,6 +616,7 @@ async fn ignores_remote_personality_if_remote_models_disabled() -> anyhow::Resul experimental_supported_tools: Vec::new(), input_modalities: default_input_modalities(), prefer_websockets: false, + used_fallback_model_metadata: false, }; let _models_mock = mount_models_once( @@ -733,6 +734,7 @@ async fn remote_model_friendly_personality_instructions_with_feature() -> anyhow experimental_supported_tools: Vec::new(), input_modalities: default_input_modalities(), prefer_websockets: false, + used_fallback_model_metadata: false, }; let _models_mock = mount_models_once( @@ -845,6 +847,7 @@ async fn user_turn_personality_remote_model_template_includes_update_message() - experimental_supported_tools: Vec::new(), input_modalities: default_input_modalities(), prefer_websockets: false, + used_fallback_model_metadata: false, }; let _models_mock = mount_models_once( diff --git a/codex-rs/core/tests/suite/remote_models.rs b/codex-rs/core/tests/suite/remote_models.rs index 3af3a5952b..29ba047d01 100644 --- a/codex-rs/core/tests/suite/remote_models.rs +++ b/codex-rs/core/tests/suite/remote_models.rs @@ -225,6 +225,7 @@ async fn remote_models_remote_model_uses_unified_exec() -> Result<()> { supported_in_api: true, input_modalities: default_input_modalities(), prefer_websockets: false, + used_fallback_model_metadata: false, priority: 1, upgrade: None, base_instructions: "base instructions".to_string(), @@ -464,6 +465,7 @@ async fn remote_models_apply_remote_base_instructions() -> Result<()> { supported_in_api: true, input_modalities: default_input_modalities(), prefer_websockets: false, + used_fallback_model_metadata: false, priority: 1, upgrade: None, base_instructions: remote_base.to_string(), @@ -948,6 +950,7 @@ fn test_remote_model_with_policy( supported_in_api: true, input_modalities: default_input_modalities(), prefer_websockets: false, + used_fallback_model_metadata: false, priority, upgrade: None, base_instructions: "base instructions".to_string(), diff --git a/codex-rs/core/tests/suite/resume_warning.rs b/codex-rs/core/tests/suite/resume_warning.rs index eb4e9742a4..61726f913c 100644 --- a/codex-rs/core/tests/suite/resume_warning.rs +++ b/codex-rs/core/tests/suite/resume_warning.rs @@ -24,6 +24,7 @@ fn resume_history( cwd: config.cwd.clone(), approval_policy: config.permissions.approval_policy.value(), sandbox_policy: config.permissions.sandbox_policy.get().clone(), + network: None, model: previous_model.to_string(), personality: None, collaboration_mode: None, @@ -73,7 +74,14 @@ async fn emits_warning_when_resumed_model_differs() { .expect("resume conversation"); // Assert: a Warning event is emitted describing the model mismatch. - let warning = wait_for_event(&conversation, |ev| matches!(ev, EventMsg::Warning(_))).await; + let warning = wait_for_event(&conversation, |ev| { + matches!( + ev, + EventMsg::Warning(WarningEvent { message }) + if message.contains("previous-model") && message.contains("current-model") + ) + }) + .await; let EventMsg::Warning(WarningEvent { message }) = warning else { panic!("expected warning event"); }; diff --git a/codex-rs/core/tests/suite/rmcp_client.rs b/codex-rs/core/tests/suite/rmcp_client.rs index 46df9ca17e..467f4e354d 100644 --- a/codex-rs/core/tests/suite/rmcp_client.rs +++ b/codex-rs/core/tests/suite/rmcp_client.rs @@ -410,6 +410,7 @@ async fn stdio_image_responses_are_sanitized_for_text_only_model() -> anyhow::Re experimental_supported_tools: Vec::new(), input_modalities: vec![InputModality::Text], prefer_websockets: false, + used_fallback_model_metadata: false, }], }, ) diff --git a/codex-rs/core/tests/suite/search_tool.rs b/codex-rs/core/tests/suite/search_tool.rs index c819c03b25..7d2de4eb9a 100644 --- a/codex-rs/core/tests/suite/search_tool.rs +++ b/codex-rs/core/tests/suite/search_tool.rs @@ -1,14 +1,22 @@ #![cfg(not(target_os = "windows"))] #![allow(clippy::unwrap_used, clippy::expect_used)] +use std::sync::Arc; use std::time::Duration; use anyhow::Result; +use codex_core::CodexThread; +use codex_core::NewThread; +use codex_core::config::Config; use codex_core::config::types::McpServerConfig; use codex_core::config::types::McpServerTransportConfig; use codex_core::features::Feature; use codex_core::protocol::AskForApproval; +use codex_core::protocol::EventMsg; +use codex_core::protocol::Op; use codex_core::protocol::SandboxPolicy; +use codex_protocol::user_input::UserInput; +use core_test_support::apps_test_server::AppsTestServer; use core_test_support::responses::ResponsesRequest; use core_test_support::responses::ev_assistant_message; use core_test_support::responses::ev_completed; @@ -19,15 +27,24 @@ use core_test_support::responses::sse; use core_test_support::responses::start_mock_server; use core_test_support::skip_if_no_network; use core_test_support::stdio_server_bin; +use core_test_support::test_codex::TestCodexBuilder; use core_test_support::test_codex::test_codex; +use core_test_support::wait_for_event; use pretty_assertions::assert_eq; use serde_json::Value; use serde_json::json; const SEARCH_TOOL_INSTRUCTION_SNIPPETS: [&str; 2] = [ - "MCP tools (`mcp__...`) are hidden until you search for them.", - "Matching tools are added to `active_selected_tools`.", + "MCP tools of the apps (Calendar) are hidden until you search for them with this tool", + "Matching tools are added to available `tools` and available for the remainder of the current session/thread.", ]; +const SEARCH_TOOL_BM25_TOOL_NAME: &str = "search_tool_bm25"; +const CALENDAR_CREATE_TOOL: &str = "mcp__codex_apps__calendar_create_event"; +const CALENDAR_LIST_TOOL: &str = "mcp__codex_apps__calendar_list_events"; +const RMCP_ECHO_TOOL: &str = "mcp__rmcp__echo"; +const RMCP_IMAGE_TOOL: &str = "mcp__rmcp__image"; +const CALENDAR_CREATE_QUERY: &str = "create calendar event"; +const CALENDAR_LIST_QUERY: &str = "list calendar events"; fn tool_names(body: &Value) -> Vec { body.get("tools") @@ -46,38 +63,32 @@ fn tool_names(body: &Value) -> Vec { .unwrap_or_default() } -fn developer_messages(body: &Value) -> Vec { - body.get("input") +fn search_tool_description(body: &Value) -> Option { + body.get("tools") .and_then(Value::as_array) - .map(|items| { - items - .iter() - .filter_map(|item| { - if item.get("role").and_then(Value::as_str) != Some("developer") { - return None; - } - let content = item.get("content").and_then(Value::as_array)?; - let texts: Vec<&str> = content - .iter() - .filter_map(|entry| entry.get("text").and_then(Value::as_str)) - .collect(); - if texts.is_empty() { - None - } else { - Some(texts.join("\n")) - } - }) - .collect() + .and_then(|tools| { + tools.iter().find_map(|tool| { + if tool.get("name").and_then(Value::as_str) == Some(SEARCH_TOOL_BM25_TOOL_NAME) { + tool.get("description") + .and_then(Value::as_str) + .map(str::to_string) + } else { + None + } + }) }) - .unwrap_or_default() } fn search_tool_output_payload(request: &ResponsesRequest, call_id: &str) -> Value { let (content, _success) = request .function_call_output_content_and_success(call_id) - .expect("search_tool_bm25 function_call_output should be present"); - let content = content.expect("search_tool_bm25 output should include content"); - serde_json::from_str(&content).expect("search_tool_bm25 content should be valid JSON") + .unwrap_or_else(|| { + panic!("{SEARCH_TOOL_BM25_TOOL_NAME} function_call_output should be present") + }); + let content = content + .unwrap_or_else(|| panic!("{SEARCH_TOOL_BM25_TOOL_NAME} output should include content")); + serde_json::from_str(&content) + .unwrap_or_else(|_| panic!("{SEARCH_TOOL_BM25_TOOL_NAME} content should be valid JSON")) } fn active_selected_tools(payload: &Value) -> Vec { @@ -95,11 +106,80 @@ fn active_selected_tools(payload: &Value) -> Vec { .collect() } +fn search_result_tools(payload: &Value) -> Vec<&Value> { + payload + .get("tools") + .and_then(Value::as_array) + .map(Vec::as_slice) + .unwrap_or_default() + .iter() + .collect() +} + +fn rmcp_server_config(command: String) -> McpServerConfig { + McpServerConfig { + transport: McpServerTransportConfig::Stdio { + command, + args: Vec::new(), + env: None, + env_vars: Vec::new(), + cwd: None, + }, + enabled: true, + required: false, + disabled_reason: None, + startup_timeout_sec: Some(Duration::from_secs(10)), + tool_timeout_sec: None, + enabled_tools: None, + disabled_tools: None, + scopes: None, + } +} + +fn configure_apps_with_optional_rmcp( + config: &mut Config, + apps_base_url: &str, + rmcp_server_bin: Option, +) { + config.features.enable(Feature::Apps); + config.features.disable(Feature::AppsMcpGateway); + config.chatgpt_base_url = apps_base_url.to_string(); + if let Some(command) = rmcp_server_bin { + let mut servers = config.mcp_servers.get().clone(); + servers.insert("rmcp".to_string(), rmcp_server_config(command)); + config + .mcp_servers + .set(servers) + .expect("test mcp servers should accept any configuration"); + } +} + +fn configured_builder(apps_base_url: String, rmcp_server_bin: Option) -> TestCodexBuilder { + test_codex().with_config(move |config| { + configure_apps_with_optional_rmcp(config, apps_base_url.as_str(), rmcp_server_bin); + }) +} + +async fn submit_user_input(thread: &Arc, text: &str) -> Result<()> { + thread + .submit(Op::UserInput { + items: vec![UserInput::Text { + text: text.to_string(), + text_elements: Vec::new(), + }], + final_output_json_schema: None, + }) + .await?; + wait_for_event(thread, |event| matches!(event, EventMsg::TurnComplete(_))).await; + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn search_tool_flag_adds_tool() -> Result<()> { skip_if_no_network!(Ok(())); let server = start_mock_server().await; + let apps_server = AppsTestServer::mount(&server).await?; let mock = mount_sse_sequence( &server, vec![sse(vec![ @@ -110,9 +190,7 @@ async fn search_tool_flag_adds_tool() -> Result<()> { ) .await; - let mut builder = test_codex().with_config(|config| { - config.features.enable(Feature::Apps); - }); + let mut builder = configured_builder(apps_server.chatgpt_base_url.clone(), None); let test = builder.build(&server).await?; test.submit_turn_with_policies( @@ -125,18 +203,19 @@ async fn search_tool_flag_adds_tool() -> Result<()> { let body = mock.single_request().body_json(); let tools = tool_names(&body); assert!( - tools.iter().any(|name| name == "search_tool_bm25"), - "tools list should include search_tool_bm25 when enabled: {tools:?}" + tools.iter().any(|name| name == SEARCH_TOOL_BM25_TOOL_NAME), + "tools list should include {SEARCH_TOOL_BM25_TOOL_NAME} when enabled: {tools:?}" ); Ok(()) } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn search_tool_adds_developer_instructions() -> Result<()> { +async fn search_tool_adds_discovery_instructions_to_tool_description() -> Result<()> { skip_if_no_network!(Ok(())); let server = start_mock_server().await; + let apps_server = AppsTestServer::mount(&server).await?; let mock = mount_sse_sequence( &server, vec![sse(vec![ @@ -147,9 +226,7 @@ async fn search_tool_adds_developer_instructions() -> Result<()> { ) .await; - let mut builder = test_codex().with_config(|config| { - config.features.enable(Feature::Apps); - }); + let mut builder = configured_builder(apps_server.chatgpt_base_url.clone(), None); let test = builder.build(&server).await?; test.submit_turn_with_policies( @@ -160,24 +237,24 @@ async fn search_tool_adds_developer_instructions() -> Result<()> { .await?; let body = mock.single_request().body_json(); - let developer_texts = developer_messages(&body); + let description = search_tool_description(&body).expect("search tool description should exist"); assert!( - developer_texts.iter().any(|text| { - SEARCH_TOOL_INSTRUCTION_SNIPPETS - .iter() - .all(|snippet| text.contains(snippet)) - }), - "developer instructions should include search tool workflow: {developer_texts:?}" + SEARCH_TOOL_INSTRUCTION_SNIPPETS + .iter() + .all(|snippet| description.contains(snippet)), + "search tool description should include search tool workflow: {description:?}" ); Ok(()) } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn search_tool_hides_mcp_tools_without_search() -> Result<()> { +async fn search_tool_hides_apps_tools_without_search_but_keeps_non_app_tools_visible() -> Result<()> +{ skip_if_no_network!(Ok(())); let server = start_mock_server().await; + let apps_server = AppsTestServer::mount(&server).await?; let mock = mount_sse_sequence( &server, vec![sse(vec![ @@ -189,34 +266,10 @@ async fn search_tool_hides_mcp_tools_without_search() -> Result<()> { .await; let rmcp_test_server_bin = stdio_server_bin()?; - let mut builder = test_codex().with_config(move |config| { - config.features.enable(Feature::Apps); - let mut servers = config.mcp_servers.get().clone(); - servers.insert( - "rmcp".to_string(), - McpServerConfig { - transport: McpServerTransportConfig::Stdio { - command: rmcp_test_server_bin, - args: Vec::new(), - env: None, - env_vars: Vec::new(), - cwd: None, - }, - enabled: true, - required: false, - disabled_reason: None, - startup_timeout_sec: Some(Duration::from_secs(10)), - tool_timeout_sec: None, - enabled_tools: None, - disabled_tools: None, - scopes: None, - }, - ); - config - .mcp_servers - .set(servers) - .expect("test mcp servers should accept any configuration"); - }); + let mut builder = configured_builder( + apps_server.chatgpt_base_url.clone(), + Some(rmcp_test_server_bin), + ); let test = builder.build(&server).await?; test.submit_turn_with_policies( @@ -229,35 +282,96 @@ async fn search_tool_hides_mcp_tools_without_search() -> Result<()> { let body = mock.single_request().body_json(); let tools = tool_names(&body); assert!( - tools.iter().any(|name| name == "search_tool_bm25"), - "tools list should include search_tool_bm25 when enabled: {tools:?}" + tools.iter().any(|name| name == SEARCH_TOOL_BM25_TOOL_NAME), + "tools list should include {SEARCH_TOOL_BM25_TOOL_NAME}: {tools:?}" ); assert!( - !tools.iter().any(|name| name == "mcp__rmcp__echo"), - "tools list should not include MCP tools before search: {tools:?}" + tools.iter().any(|name| name == RMCP_ECHO_TOOL), + "non-app MCP tools should remain visible in Apps mode: {tools:?}" ); assert!( - !tools.iter().any(|name| name == "mcp__rmcp__image"), - "tools list should not include MCP tools before search: {tools:?}" + tools.iter().any(|name| name == RMCP_IMAGE_TOOL), + "non-app MCP tools should remain visible in Apps mode: {tools:?}" + ); + assert!( + !tools.iter().any(|name| name == CALENDAR_CREATE_TOOL), + "apps tools should stay hidden before search/mention: {tools:?}" + ); + assert!( + !tools.iter().any(|name| name == CALENDAR_LIST_TOOL), + "apps tools should stay hidden before search/mention: {tools:?}" ); Ok(()) } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn search_tool_selection_persists_within_turn_and_resets_next_turn() -> Result<()> { +async fn explicit_app_mentions_expose_apps_tools_without_search() -> Result<()> { skip_if_no_network!(Ok(())); let server = start_mock_server().await; + let apps_server = AppsTestServer::mount(&server).await?; + let mock = mount_sse_sequence( + &server, + vec![sse(vec![ + ev_response_created("resp-1"), + ev_assistant_message("msg-1", "done"), + ev_completed("resp-1"), + ])], + ) + .await; + + let rmcp_test_server_bin = stdio_server_bin()?; + let mut builder = configured_builder( + apps_server.chatgpt_base_url.clone(), + Some(rmcp_test_server_bin), + ); + let test = builder.build(&server).await?; + + test.submit_turn_with_policies( + "Use [$calendar](app://calendar) and then call tools.", + AskForApproval::Never, + SandboxPolicy::DangerFullAccess, + ) + .await?; + + let body = mock.single_request().body_json(); + let tools = tool_names(&body); + assert!( + tools.iter().any(|name| name == RMCP_ECHO_TOOL), + "non-app tools should remain visible: {tools:?}" + ); + assert!( + tools.iter().any(|name| name == CALENDAR_CREATE_TOOL), + "apps tools should be available after explicit app mention: {tools:?}" + ); + assert!( + tools.iter().any(|name| name == CALENDAR_LIST_TOOL), + "apps tools should be available after explicit app mention: {tools:?}" + ); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn search_tool_selection_persists_across_turns() -> Result<()> { + skip_if_no_network!(Ok(())); + + let server = start_mock_server().await; + let apps_server = AppsTestServer::mount(&server).await?; let call_id = "tool-search"; let args = json!({ - "query": "echo", + "query": CALENDAR_CREATE_QUERY, "limit": 1, }); let responses = vec![ sse(vec![ ev_response_created("resp-1"), - ev_function_call(call_id, "search_tool_bm25", &serde_json::to_string(&args)?), + ev_function_call( + call_id, + SEARCH_TOOL_BM25_TOOL_NAME, + &serde_json::to_string(&args)?, + ), ev_completed("resp-1"), ]), sse(vec![ @@ -272,38 +386,14 @@ async fn search_tool_selection_persists_within_turn_and_resets_next_turn() -> Re let mock = mount_sse_sequence(&server, responses).await; let rmcp_test_server_bin = stdio_server_bin()?; - let mut builder = test_codex().with_config(move |config| { - config.features.enable(Feature::Apps); - let mut servers = config.mcp_servers.get().clone(); - servers.insert( - "rmcp".to_string(), - McpServerConfig { - transport: McpServerTransportConfig::Stdio { - command: rmcp_test_server_bin, - args: Vec::new(), - env: None, - env_vars: Vec::new(), - cwd: None, - }, - enabled: true, - required: false, - disabled_reason: None, - startup_timeout_sec: Some(Duration::from_secs(10)), - tool_timeout_sec: None, - enabled_tools: None, - disabled_tools: None, - scopes: None, - }, - ); - config - .mcp_servers - .set(servers) - .expect("test mcp servers should accept any configuration"); - }); + let mut builder = configured_builder( + apps_server.chatgpt_base_url.clone(), + Some(rmcp_test_server_bin), + ); let test = builder.build(&server).await?; test.submit_turn_with_policies( - "find the echo tool", + "find calendar create tool", AskForApproval::Never, SandboxPolicy::DangerFullAccess, ) @@ -325,18 +415,12 @@ async fn search_tool_selection_persists_within_turn_and_resets_next_turn() -> Re let first_tools = tool_names(&requests[0].body_json()); assert!( - !first_tools.iter().any(|name| name == "mcp__rmcp__echo"), - "first request should not include MCP tools before search: {first_tools:?}" - ); - - let second_tools = tool_names(&requests[1].body_json()); - assert!( - second_tools.iter().any(|name| name == "mcp__rmcp__echo"), - "second request should include selected MCP tool: {second_tools:?}" + first_tools.iter().any(|name| name == RMCP_ECHO_TOOL), + "non-app MCP tools should be available before search: {first_tools:?}" ); assert!( - !second_tools.iter().any(|name| name == "mcp__rmcp__image"), - "second request should only include selected MCP tool: {second_tools:?}" + !first_tools.iter().any(|name| name == CALENDAR_CREATE_TOOL), + "apps tools should not be visible before search: {first_tools:?}" ); let search_output_payload = search_tool_output_payload(&requests[1], call_id); @@ -344,16 +428,51 @@ async fn search_tool_selection_persists_within_turn_and_resets_next_turn() -> Re search_output_payload.get("selected_tools").is_none(), "selected_tools should not be returned: {search_output_payload:?}" ); - assert_eq!( - active_selected_tools(&search_output_payload), - vec!["mcp__rmcp__echo".to_string()], + for tool in search_result_tools(&search_output_payload) { + assert_eq!( + tool.get("server").and_then(Value::as_str), + Some("codex_apps"), + "search results should only include codex_apps tools: {search_output_payload:?}" + ); + } + + let selected_tools = active_selected_tools(&search_output_payload); + assert!( + selected_tools + .iter() + .any(|tool| tool == CALENDAR_CREATE_TOOL), + "calendar create tool should be selected: {search_output_payload:?}" ); + assert!( + !selected_tools + .iter() + .any(|tool_name| tool_name.starts_with("mcp__rmcp__")), + "search should not add rmcp tools to active selection: {search_output_payload:?}" + ); + + let second_tools = tool_names(&requests[1].body_json()); + assert!( + second_tools.iter().any(|name| name == RMCP_ECHO_TOOL), + "non-app MCP tools should remain visible after search: {second_tools:?}" + ); + for selected_tool in &selected_tools { + assert!( + second_tools.iter().any(|name| name == selected_tool), + "follow-up request should include selected tool {selected_tool:?}: {second_tools:?}" + ); + } let third_tools = tool_names(&requests[2].body_json()); assert!( - !third_tools.iter().any(|name| name == "mcp__rmcp__echo"), - "third request should not include MCP tools after turn reset: {third_tools:?}" + third_tools.iter().any(|name| name == RMCP_ECHO_TOOL), + "non-app MCP tools should remain visible on later turns: {third_tools:?}" ); + for selected_tool in &selected_tools { + assert!( + third_tools.iter().any(|name| name == selected_tool), + "subsequent turn should include selected tool {selected_tool:?}: {third_tools:?}" + ); + } Ok(()) } @@ -363,14 +482,15 @@ async fn search_tool_selection_unions_results_within_turn() -> Result<()> { skip_if_no_network!(Ok(())); let server = start_mock_server().await; - let first_call_id = "tool-search-echo"; - let second_call_id = "tool-search-image"; + let apps_server = AppsTestServer::mount(&server).await?; + let first_call_id = "tool-search-create"; + let second_call_id = "tool-search-list"; let first_args = json!({ - "query": "echo", + "query": CALENDAR_CREATE_QUERY, "limit": 1, }); let second_args = json!({ - "query": "image", + "query": CALENDAR_LIST_QUERY, "limit": 1, }); let responses = vec![ @@ -378,7 +498,7 @@ async fn search_tool_selection_unions_results_within_turn() -> Result<()> { ev_response_created("resp-1"), ev_function_call( first_call_id, - "search_tool_bm25", + SEARCH_TOOL_BM25_TOOL_NAME, &serde_json::to_string(&first_args)?, ), ev_completed("resp-1"), @@ -387,7 +507,7 @@ async fn search_tool_selection_unions_results_within_turn() -> Result<()> { ev_response_created("resp-2"), ev_function_call( second_call_id, - "search_tool_bm25", + SEARCH_TOOL_BM25_TOOL_NAME, &serde_json::to_string(&second_args)?, ), ev_completed("resp-2"), @@ -400,38 +520,14 @@ async fn search_tool_selection_unions_results_within_turn() -> Result<()> { let mock = mount_sse_sequence(&server, responses).await; let rmcp_test_server_bin = stdio_server_bin()?; - let mut builder = test_codex().with_config(move |config| { - config.features.enable(Feature::Apps); - let mut servers = config.mcp_servers.get().clone(); - servers.insert( - "rmcp".to_string(), - McpServerConfig { - transport: McpServerTransportConfig::Stdio { - command: rmcp_test_server_bin, - args: Vec::new(), - env: None, - env_vars: Vec::new(), - cwd: None, - }, - enabled: true, - required: false, - disabled_reason: None, - startup_timeout_sec: Some(Duration::from_secs(10)), - tool_timeout_sec: None, - enabled_tools: None, - disabled_tools: None, - scopes: None, - }, - ); - config - .mcp_servers - .set(servers) - .expect("test mcp servers should accept any configuration"); - }); + let mut builder = configured_builder( + apps_server.chatgpt_base_url.clone(), + Some(rmcp_test_server_bin), + ); let test = builder.build(&server).await?; test.submit_turn_with_policies( - "find echo and image tools", + "find create and list calendar tools", AskForApproval::Never, SandboxPolicy::DangerFullAccess, ) @@ -447,28 +543,16 @@ async fn search_tool_selection_unions_results_within_turn() -> Result<()> { let first_tools = tool_names(&requests[0].body_json()); assert!( - !first_tools.iter().any(|name| name == "mcp__rmcp__echo"), - "first request should not include MCP tools before search: {first_tools:?}" - ); - - let second_tools = tool_names(&requests[1].body_json()); - assert!( - second_tools.iter().any(|name| name == "mcp__rmcp__echo"), - "second request should include echo after first search: {second_tools:?}" + first_tools.iter().any(|name| name == RMCP_ECHO_TOOL), + "non-app MCP tools should be visible before search: {first_tools:?}" ); assert!( - !second_tools.iter().any(|name| name == "mcp__rmcp__image"), - "second request should not include image before second search runs: {second_tools:?}" - ); - - let third_tools = tool_names(&requests[2].body_json()); - assert!( - third_tools.iter().any(|name| name == "mcp__rmcp__echo"), - "third request should still include echo: {third_tools:?}" + !first_tools.iter().any(|name| name == CALENDAR_CREATE_TOOL), + "apps tools should be hidden before search: {first_tools:?}" ); assert!( - third_tools.iter().any(|name| name == "mcp__rmcp__image"), - "third request should include image after second search: {third_tools:?}" + !first_tools.iter().any(|name| name == CALENDAR_LIST_TOOL), + "apps tools should be hidden before search: {first_tools:?}" ); let second_search_payload = search_tool_output_payload(&requests[2], second_call_id); @@ -476,12 +560,502 @@ async fn search_tool_selection_unions_results_within_turn() -> Result<()> { second_search_payload.get("selected_tools").is_none(), "selected_tools should not be returned: {second_search_payload:?}" ); + for tool in search_result_tools(&second_search_payload) { + assert_eq!( + tool.get("server").and_then(Value::as_str), + Some("codex_apps"), + "search results should only include codex_apps tools: {second_search_payload:?}" + ); + } + + let selected_tools = active_selected_tools(&second_search_payload); assert_eq!( - active_selected_tools(&second_search_payload), + selected_tools, vec![ - "mcp__rmcp__echo".to_string(), - "mcp__rmcp__image".to_string(), + CALENDAR_CREATE_TOOL.to_string(), + CALENDAR_LIST_TOOL.to_string(), ], + "two searches in one turn should union selected apps tools" + ); + + let third_tools = tool_names(&requests[2].body_json()); + assert!( + third_tools.iter().any(|name| name == RMCP_ECHO_TOOL), + "non-app MCP tools should remain visible after repeated search: {third_tools:?}" + ); + assert!( + third_tools.iter().any(|name| name == CALENDAR_CREATE_TOOL), + "calendar create should be available after repeated search: {third_tools:?}" + ); + assert!( + third_tools.iter().any(|name| name == CALENDAR_LIST_TOOL), + "calendar list should be available after repeated search: {third_tools:?}" + ); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn search_tool_selection_restores_when_resumed() -> Result<()> { + skip_if_no_network!(Ok(())); + + let server = start_mock_server().await; + let apps_server = AppsTestServer::mount(&server).await?; + let call_id = "tool-search"; + let args = json!({ + "query": CALENDAR_CREATE_QUERY, + "limit": 1, + }); + let responses = vec![ + sse(vec![ + ev_response_created("resp-1"), + ev_function_call( + call_id, + SEARCH_TOOL_BM25_TOOL_NAME, + &serde_json::to_string(&args)?, + ), + ev_completed("resp-1"), + ]), + sse(vec![ + ev_response_created("resp-2"), + ev_assistant_message("msg-1", "done"), + ev_completed("resp-2"), + ]), + sse(vec![ + ev_response_created("resp-3"), + ev_assistant_message("msg-2", "resumed done"), + ev_completed("resp-3"), + ]), + ]; + let mock = mount_sse_sequence(&server, responses).await; + + let rmcp_test_server_bin = stdio_server_bin()?; + let mut builder = configured_builder( + apps_server.chatgpt_base_url.clone(), + Some(rmcp_test_server_bin.clone()), + ); + let test = builder.build(&server).await?; + + let home = test.home.clone(); + let rollout_path = test + .session_configured + .rollout_path + .clone() + .expect("rollout path should be available for resume"); + + test.submit_turn_with_policies( + "find calendar tools", + AskForApproval::Never, + SandboxPolicy::DangerFullAccess, + ) + .await?; + + let requests = mock.requests(); + assert_eq!( + requests.len(), + 2, + "expected 2 requests after initial turn, got {}", + requests.len() + ); + let search_output_payload = search_tool_output_payload(&requests[1], call_id); + let selected_tools = active_selected_tools(&search_output_payload); + assert!( + selected_tools + .iter() + .any(|name| name == CALENDAR_CREATE_TOOL), + "search should select calendar create before resume: {search_output_payload:?}" + ); + + let mut resume_builder = configured_builder( + apps_server.chatgpt_base_url.clone(), + Some(rmcp_test_server_bin), + ); + let resumed = resume_builder.resume(&server, home, rollout_path).await?; + resumed + .submit_turn_with_policies( + "hello after resume", + AskForApproval::Never, + SandboxPolicy::DangerFullAccess, + ) + .await?; + + let requests = mock.requests(); + assert_eq!( + requests.len(), + 3, + "expected 3 requests after resumed turn, got {}", + requests.len() + ); + let resumed_tools = tool_names(&requests[2].body_json()); + assert!( + resumed_tools.iter().any(|name| name == RMCP_ECHO_TOOL), + "non-app tools should remain visible after resume: {resumed_tools:?}" + ); + for selected_tool in &selected_tools { + assert!( + resumed_tools.iter().any(|name| name == selected_tool), + "resumed request should include restored selected tool {selected_tool:?}: {resumed_tools:?}" + ); + } + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn search_tool_selection_union_restores_when_resumed_after_multiple_search_calls() +-> Result<()> { + skip_if_no_network!(Ok(())); + + let server = start_mock_server().await; + let apps_server = AppsTestServer::mount(&server).await?; + let first_call_id = "tool-search-create"; + let second_call_id = "tool-search-list"; + let first_args = json!({ + "query": CALENDAR_CREATE_QUERY, + "limit": 1, + }); + let second_args = json!({ + "query": CALENDAR_LIST_QUERY, + "limit": 1, + }); + let responses = vec![ + sse(vec![ + ev_response_created("resp-1"), + ev_function_call( + first_call_id, + SEARCH_TOOL_BM25_TOOL_NAME, + &serde_json::to_string(&first_args)?, + ), + ev_completed("resp-1"), + ]), + sse(vec![ + ev_response_created("resp-2"), + ev_assistant_message("msg-1", "first search done"), + ev_completed("resp-2"), + ]), + sse(vec![ + ev_response_created("resp-3"), + ev_function_call( + second_call_id, + SEARCH_TOOL_BM25_TOOL_NAME, + &serde_json::to_string(&second_args)?, + ), + ev_completed("resp-3"), + ]), + sse(vec![ + ev_response_created("resp-4"), + ev_assistant_message("msg-2", "second search done"), + ev_completed("resp-4"), + ]), + sse(vec![ + ev_response_created("resp-5"), + ev_assistant_message("msg-3", "resumed done"), + ev_completed("resp-5"), + ]), + ]; + let mock = mount_sse_sequence(&server, responses).await; + + let rmcp_test_server_bin = stdio_server_bin()?; + let mut builder = configured_builder( + apps_server.chatgpt_base_url.clone(), + Some(rmcp_test_server_bin.clone()), + ); + let test = builder.build(&server).await?; + + let home = test.home.clone(); + let rollout_path = test + .session_configured + .rollout_path + .clone() + .expect("rollout path should be available for resume"); + + test.submit_turn_with_policies( + "find create calendar tool", + AskForApproval::Never, + SandboxPolicy::DangerFullAccess, + ) + .await?; + test.submit_turn_with_policies( + "find list calendar tool", + AskForApproval::Never, + SandboxPolicy::DangerFullAccess, + ) + .await?; + + let requests = mock.requests(); + assert_eq!( + requests.len(), + 4, + "expected 4 requests before resume, got {}", + requests.len() + ); + + let first_search_payload = search_tool_output_payload(&requests[1], first_call_id); + let first_result_tools = search_result_tools(&first_search_payload); + assert_eq!( + first_result_tools.len(), + 1, + "first search should return exactly one tool: {first_search_payload:?}" + ); + assert_eq!( + first_result_tools[0].get("name").and_then(Value::as_str), + Some(CALENDAR_CREATE_TOOL), + "first search should return calendar create tool: {first_search_payload:?}" + ); + let first_selected_tools = active_selected_tools(&first_search_payload); + assert_eq!( + first_selected_tools, + vec![CALENDAR_CREATE_TOOL.to_string()], + "first search should only select create tool: {first_search_payload:?}" + ); + + let second_search_payload = search_tool_output_payload(&requests[3], second_call_id); + let second_result_tools = search_result_tools(&second_search_payload); + assert_eq!( + second_result_tools.len(), + 1, + "second search should return exactly one tool: {second_search_payload:?}" + ); + assert_eq!( + second_result_tools[0].get("name").and_then(Value::as_str), + Some(CALENDAR_LIST_TOOL), + "second search should return calendar list tool: {second_search_payload:?}" + ); + let second_selected_tools = active_selected_tools(&second_search_payload); + assert_eq!( + second_selected_tools, + vec![ + CALENDAR_CREATE_TOOL.to_string(), + CALENDAR_LIST_TOOL.to_string(), + ], + "multiple searches should persist union before resume: {second_search_payload:?}" + ); + + let mut resume_builder = configured_builder( + apps_server.chatgpt_base_url.clone(), + Some(rmcp_test_server_bin), + ); + let resumed = resume_builder.resume(&server, home, rollout_path).await?; + resumed + .submit_turn_with_policies( + "hello after resume with union", + AskForApproval::Never, + SandboxPolicy::DangerFullAccess, + ) + .await?; + + let requests = mock.requests(); + assert_eq!( + requests.len(), + 5, + "expected 5 requests after resumed turn, got {}", + requests.len() + ); + + let resumed_tools = tool_names(&requests[4].body_json()); + assert!( + resumed_tools.iter().any(|name| name == RMCP_ECHO_TOOL), + "non-app tools should remain visible after resume: {resumed_tools:?}" + ); + assert!( + resumed_tools + .iter() + .any(|name| name == CALENDAR_CREATE_TOOL), + "resumed turn should restore calendar create tool: {resumed_tools:?}" + ); + assert!( + resumed_tools.iter().any(|name| name == CALENDAR_LIST_TOOL), + "resumed turn should restore calendar list tool: {resumed_tools:?}" + ); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn search_tool_selection_restores_when_forked_with_full_history() -> Result<()> { + skip_if_no_network!(Ok(())); + + let server = start_mock_server().await; + let apps_server = AppsTestServer::mount(&server).await?; + let call_id = "tool-search"; + let args = json!({ + "query": CALENDAR_CREATE_QUERY, + "limit": 1, + }); + let responses = vec![ + sse(vec![ + ev_response_created("resp-1"), + ev_function_call( + call_id, + SEARCH_TOOL_BM25_TOOL_NAME, + &serde_json::to_string(&args)?, + ), + ev_completed("resp-1"), + ]), + sse(vec![ + ev_response_created("resp-2"), + ev_assistant_message("msg-1", "done"), + ev_completed("resp-2"), + ]), + sse(vec![ + ev_response_created("resp-3"), + ev_assistant_message("msg-2", "forked done"), + ev_completed("resp-3"), + ]), + ]; + let mock = mount_sse_sequence(&server, responses).await; + + let rmcp_test_server_bin = stdio_server_bin()?; + let mut builder = configured_builder( + apps_server.chatgpt_base_url.clone(), + Some(rmcp_test_server_bin), + ); + let test = builder.build(&server).await?; + + test.submit_turn_with_policies( + "find calendar tools", + AskForApproval::Never, + SandboxPolicy::DangerFullAccess, + ) + .await?; + + let requests = mock.requests(); + assert_eq!( + requests.len(), + 2, + "expected 2 requests after initial turn, got {}", + requests.len() + ); + let search_output_payload = search_tool_output_payload(&requests[1], call_id); + let selected_tools = active_selected_tools(&search_output_payload); + assert!( + selected_tools + .iter() + .any(|name| name == CALENDAR_CREATE_TOOL), + "search should select calendar create: {search_output_payload:?}" + ); + + let rollout_path = test + .codex + .rollout_path() + .expect("rollout path should exist for fork"); + let NewThread { thread: forked, .. } = test + .thread_manager + .fork_thread(usize::MAX, test.config.clone(), rollout_path, false) + .await?; + submit_user_input(&forked, "hello after fork").await?; + + let requests = mock.requests(); + assert_eq!( + requests.len(), + 3, + "expected 3 requests after forked turn, got {}", + requests.len() + ); + let forked_tools = tool_names(&requests[2].body_json()); + assert!( + forked_tools.iter().any(|name| name == RMCP_ECHO_TOOL), + "non-app tools should remain visible in forked thread: {forked_tools:?}" + ); + for selected_tool in &selected_tools { + assert!( + forked_tools.iter().any(|name| name == selected_tool), + "forked request should include restored selected tool {selected_tool:?}: {forked_tools:?}" + ); + } + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn search_tool_selection_drops_when_fork_excludes_search_turn() -> Result<()> { + skip_if_no_network!(Ok(())); + + let server = start_mock_server().await; + let apps_server = AppsTestServer::mount(&server).await?; + let call_id = "tool-search"; + let args = json!({ + "query": CALENDAR_CREATE_QUERY, + "limit": 1, + }); + let responses = vec![ + sse(vec![ + ev_response_created("resp-1"), + ev_function_call( + call_id, + SEARCH_TOOL_BM25_TOOL_NAME, + &serde_json::to_string(&args)?, + ), + ev_completed("resp-1"), + ]), + sse(vec![ + ev_response_created("resp-2"), + ev_assistant_message("msg-1", "done"), + ev_completed("resp-2"), + ]), + sse(vec![ + ev_response_created("resp-3"), + ev_assistant_message("msg-2", "forked done"), + ev_completed("resp-3"), + ]), + ]; + let mock = mount_sse_sequence(&server, responses).await; + + let rmcp_test_server_bin = stdio_server_bin()?; + let mut builder = configured_builder( + apps_server.chatgpt_base_url.clone(), + Some(rmcp_test_server_bin), + ); + let test = builder.build(&server).await?; + + test.submit_turn_with_policies( + "find calendar tools", + AskForApproval::Never, + SandboxPolicy::DangerFullAccess, + ) + .await?; + + let requests = mock.requests(); + assert_eq!( + requests.len(), + 2, + "expected 2 requests after initial turn, got {}", + requests.len() + ); + let search_output_payload = search_tool_output_payload(&requests[1], call_id); + let selected_tools = active_selected_tools(&search_output_payload); + assert!( + !selected_tools.is_empty(), + "search turn should produce selected tools: {search_output_payload:?}" + ); + + let rollout_path = test + .codex + .rollout_path() + .expect("rollout path should exist for fork"); + let NewThread { thread: forked, .. } = test + .thread_manager + .fork_thread(0, test.config.clone(), rollout_path, false) + .await?; + submit_user_input(&forked, "hello after fork").await?; + + let requests = mock.requests(); + assert_eq!( + requests.len(), + 3, + "expected 3 requests after forked turn, got {}", + requests.len() + ); + let forked_tools = tool_names(&requests[2].body_json()); + assert!( + forked_tools.iter().any(|name| name == RMCP_ECHO_TOOL), + "non-app tools should remain visible in forked thread: {forked_tools:?}" + ); + assert!( + !forked_tools + .iter() + .any(|name| name.starts_with("mcp__codex_apps__")), + "forked history without search turn should not restore apps tools: {forked_tools:?}" ); Ok(()) diff --git a/codex-rs/core/tests/suite/snapshots/all__suite__compact__manual_compact_with_history_shapes.snap b/codex-rs/core/tests/suite/snapshots/all__suite__compact__manual_compact_with_history_shapes.snap new file mode 100644 index 0000000000..bcca5d3575 --- /dev/null +++ b/codex-rs/core/tests/suite/snapshots/all__suite__compact__manual_compact_with_history_shapes.snap @@ -0,0 +1,21 @@ +--- +source: core/tests/suite/compact.rs +expression: "format_labeled_requests_snapshot(\"Manual /compact with prior user history compacts existing history and the follow-up turn includes the compact summary plus new user message.\",\n&[(\"Local Compaction Request\", &requests[1]),\n(\"Local Post-Compaction History Layout\", &requests[2]),])" +--- +Scenario: Manual /compact with prior user history compacts existing history and the follow-up turn includes the compact summary plus new user message. + +## Local Compaction Request +00:message/developer: +01:message/user: +02:message/user:> +03:message/user:first manual turn +04:message/assistant:FIRST_REPLY +05:message/user: + +## Local Post-Compaction History Layout +00:message/developer: +01:message/user: +02:message/user:> +03:message/user:first manual turn +04:message/user:\nFIRST_MANUAL_SUMMARY +05:message/user:second manual turn diff --git a/codex-rs/core/tests/suite/snapshots/all__suite__compact__manual_compact_without_prev_user_shapes.snap b/codex-rs/core/tests/suite/snapshots/all__suite__compact__manual_compact_without_prev_user_shapes.snap new file mode 100644 index 0000000000..bdb4fe9bae --- /dev/null +++ b/codex-rs/core/tests/suite/snapshots/all__suite__compact__manual_compact_without_prev_user_shapes.snap @@ -0,0 +1,18 @@ +--- +source: core/tests/suite/compact.rs +expression: "format_labeled_requests_snapshot(\"Manual /compact with no prior user turn currently still issues a compaction request; follow-up turn carries canonical context and the new user message.\",\n&[(\"Local Compaction Request\", &requests[0]),\n(\"Local Post-Compaction History Layout\", &requests[1]),])" +--- +Scenario: Manual /compact with no prior user turn currently still issues a compaction request; follow-up turn carries canonical context and the new user message. + +## Local Compaction Request +00:message/developer: +01:message/user: +02:message/user:> +03:message/user: + +## Local Post-Compaction History Layout +00:message/developer: +01:message/user: +02:message/user:> +03:message/user:\nMANUAL_EMPTY_SUMMARY +04:message/user:AFTER_MANUAL_EMPTY_COMPACT diff --git a/codex-rs/core/tests/suite/snapshots/all__suite__compact__mid_turn_compaction_shapes.snap b/codex-rs/core/tests/suite/snapshots/all__suite__compact__mid_turn_compaction_shapes.snap new file mode 100644 index 0000000000..c13f78aff9 --- /dev/null +++ b/codex-rs/core/tests/suite/snapshots/all__suite__compact__mid_turn_compaction_shapes.snap @@ -0,0 +1,22 @@ +--- +source: core/tests/suite/compact.rs +assertion_line: 2646 +expression: "format_labeled_requests_snapshot(\"True mid-turn continuation compaction after tool output: compact request includes tool artifacts, and the continuation request includes the summary in the same turn.\",\n&[(\"Local Compaction Request\", &auto_compact_mock.single_request()),\n(\"Local Post-Compaction History Layout\",\n&post_auto_compact_mock.single_request()),])" +--- +Scenario: True mid-turn continuation compaction after tool output: compact request includes tool artifacts, and the continuation request includes the summary in the same turn. + +## Local Compaction Request +00:message/developer: +01:message/user: +02:message/user:> +03:message/user:function call limit push +04:function_call/test_tool +05:function_call_output:unsupported call: test_tool +06:message/user: + +## Local Post-Compaction History Layout +00:message/developer: +01:message/user: +02:message/user:> +03:message/user:function call limit push +04:message/user:\nAUTO_SUMMARY diff --git a/codex-rs/core/tests/suite/snapshots/all__suite__compact__pre_sampling_model_switch_compaction_shapes.snap b/codex-rs/core/tests/suite/snapshots/all__suite__compact__pre_sampling_model_switch_compaction_shapes.snap new file mode 100644 index 0000000000..8d6c4d9b1a --- /dev/null +++ b/codex-rs/core/tests/suite/snapshots/all__suite__compact__pre_sampling_model_switch_compaction_shapes.snap @@ -0,0 +1,31 @@ +--- +source: core/tests/suite/compact.rs +assertion_line: 1773 +expression: "format_labeled_requests_snapshot(\"Pre-sampling compaction on model switch to a smaller context window: current behavior compacts using prior-turn history only (incoming user message excluded), and the follow-up request carries compacted history plus the new user message.\",\n&[(\"Initial Request (Previous Model)\", &requests[0]),\n(\"Pre-sampling Compaction Request\", &requests[1]),\n(\"Post-Compaction Follow-up Request (Next Model)\", &requests[2]),])" +--- +Scenario: Pre-sampling compaction on model switch to a smaller context window: current behavior compacts using prior-turn history only (incoming user message excluded), and the follow-up request carries compacted history plus the new user message. + +## Initial Request (Previous Model) +00:message/developer: +01:message/user: +02:message/user:> +03:message/developer: +04:message/user:before switch + +## Pre-sampling Compaction Request +00:message/developer: +01:message/user: +02:message/user:> +03:message/developer: +04:message/user:before switch +05:message/assistant:before switch +06:message/user: + +## Post-Compaction Follow-up Request (Next Model) +00:message/developer: +01:message/user: +02:message/user:> +03:message/user:before switch +04:message/user:\nPRE_SAMPLING_SUMMARY +05:message/developer:\nThe user was previously using a different model.... +06:message/user:after switch diff --git a/codex-rs/core/tests/suite/snapshots/all__suite__compact__pre_turn_compaction_context_window_exceeded_shapes.snap b/codex-rs/core/tests/suite/snapshots/all__suite__compact__pre_turn_compaction_context_window_exceeded_shapes.snap new file mode 100644 index 0000000000..da30b01bd7 --- /dev/null +++ b/codex-rs/core/tests/suite/snapshots/all__suite__compact__pre_turn_compaction_context_window_exceeded_shapes.snap @@ -0,0 +1,13 @@ +--- +source: core/tests/suite/compact.rs +expression: "format_labeled_requests_snapshot(\"Pre-turn auto-compaction context-window failure: compaction request excludes the incoming user message and the turn errors.\",\n&[(\"Local Compaction Request (Incoming User Excluded)\", &requests[1]),])" +--- +Scenario: Pre-turn auto-compaction context-window failure: compaction request excludes the incoming user message and the turn errors. + +## Local Compaction Request (Incoming User Excluded) +00:message/developer: +01:message/user: +02:message/user:> +03:message/user:USER_ONE +04:message/assistant:FIRST_REPLY +05:message/user: diff --git a/codex-rs/core/tests/suite/snapshots/all__suite__compact__pre_turn_compaction_including_incoming_shapes.snap b/codex-rs/core/tests/suite/snapshots/all__suite__compact__pre_turn_compaction_including_incoming_shapes.snap new file mode 100644 index 0000000000..e586c8521a --- /dev/null +++ b/codex-rs/core/tests/suite/snapshots/all__suite__compact__pre_turn_compaction_including_incoming_shapes.snap @@ -0,0 +1,25 @@ +--- +source: core/tests/suite/compact.rs +expression: "format_labeled_requests_snapshot(\"Pre-turn auto-compaction with a context override emits the context diff in the compact request while the incoming user message is still excluded.\",\n&[(\"Local Compaction Request\", &requests[2]),\n(\"Local Post-Compaction History Layout\", &requests[3]),])" +--- +Scenario: Pre-turn auto-compaction with a context override emits the context diff in the compact request while the incoming user message is still excluded. + +## Local Compaction Request +00:message/developer: +01:message/user: +02:message/user:> +03:message/user:USER_ONE +04:message/assistant:FIRST_REPLY +05:message/user:USER_TWO +06:message/assistant:SECOND_REPLY +07:message/user: +08:message/user: + +## Local Post-Compaction History Layout +00:message/developer: +01:message/user: +02:message/user: +03:message/user:USER_ONE +04:message/user:USER_TWO +05:message/user:\nPRE_TURN_SUMMARY +06:message/user: | | | USER_THREE diff --git a/codex-rs/core/tests/suite/snapshots/all__suite__compact_remote__remote_manual_compact_with_history_shapes.snap b/codex-rs/core/tests/suite/snapshots/all__suite__compact_remote__remote_manual_compact_with_history_shapes.snap new file mode 100644 index 0000000000..bc795a19f4 --- /dev/null +++ b/codex-rs/core/tests/suite/snapshots/all__suite__compact_remote__remote_manual_compact_with_history_shapes.snap @@ -0,0 +1,21 @@ +--- +source: core/tests/suite/compact_remote.rs +assertion_line: 178 +expression: "format_labeled_requests_snapshot(\"Remote manual /compact where remote compact output is summary-only: follow-up layout uses returned summary plus new user message.\",\n&[(\"Remote Compaction Request\", &compact_request),\n(\"Remote Post-Compaction History Layout\", follow_up_request),])" +--- +Scenario: Remote manual /compact where remote compact output is summary-only: follow-up layout uses returned summary plus new user message. + +## Remote Compaction Request +00:message/developer: +01:message/user: +02:message/user:> +03:message/user:hello remote compact +04:message/assistant:FIRST_REMOTE_REPLY + +## Remote Post-Compaction History Layout +00:message/developer: +01:message/user: +02:message/user:> +03:message/user:REMOTE_COMPACTED_SUMMARY +04:compaction:encrypted=true +05:message/user:after compact diff --git a/codex-rs/core/tests/suite/snapshots/all__suite__compact_remote__remote_manual_compact_without_prev_user_shapes.snap b/codex-rs/core/tests/suite/snapshots/all__suite__compact_remote__remote_manual_compact_without_prev_user_shapes.snap new file mode 100644 index 0000000000..ce62806d99 --- /dev/null +++ b/codex-rs/core/tests/suite/snapshots/all__suite__compact_remote__remote_manual_compact_without_prev_user_shapes.snap @@ -0,0 +1,16 @@ +--- +source: core/tests/suite/compact_remote.rs +expression: "format_labeled_requests_snapshot(\"Remote manual /compact with no prior user turn still issues a compact request; follow-up turn carries canonical context and new user message.\",\n&[(\"Remote Compaction Request\", &compact_request),\n(\"Remote Post-Compaction History Layout\", &follow_up_request),])" +--- +Scenario: Remote manual /compact with no prior user turn still issues a compact request; follow-up turn carries canonical context and new user message. + +## Remote Compaction Request +00:message/developer: +01:message/user: +02:message/user:> + +## Remote Post-Compaction History Layout +00:message/developer: +01:message/user: +02:message/user:> +03:message/user:USER_ONE diff --git a/codex-rs/core/tests/suite/snapshots/all__suite__compact_remote__remote_mid_turn_compaction_shapes.snap b/codex-rs/core/tests/suite/snapshots/all__suite__compact_remote__remote_mid_turn_compaction_shapes.snap new file mode 100644 index 0000000000..8ef8701673 --- /dev/null +++ b/codex-rs/core/tests/suite/snapshots/all__suite__compact_remote__remote_mid_turn_compaction_shapes.snap @@ -0,0 +1,20 @@ +--- +source: core/tests/suite/compact_remote.rs +expression: "format_labeled_requests_snapshot(\"Remote mid-turn continuation compaction after tool output: compact request includes tool artifacts and follow-up request includes the summary.\",\n&[(\"Remote Compaction Request\", &compact_request),\n(\"Remote Post-Compaction History Layout\", &requests[1]),])" +--- +Scenario: Remote mid-turn continuation compaction after tool output: compact request includes tool artifacts and follow-up request includes the summary. + +## Remote Compaction Request +00:message/developer: +01:message/user: +02:message/user:> +03:message/user:USER_ONE +04:function_call/test_tool +05:function_call_output:unsupported call: test_tool + +## Remote Post-Compaction History Layout +00:message/user:USER_ONE +01:message/developer: +02:message/user: +03:message/user:> +04:message/user:\nREMOTE_MID_TURN_SUMMARY diff --git a/codex-rs/core/tests/suite/snapshots/all__suite__compact_remote__remote_pre_turn_compaction_context_window_exceeded_shapes.snap b/codex-rs/core/tests/suite/snapshots/all__suite__compact_remote__remote_pre_turn_compaction_context_window_exceeded_shapes.snap new file mode 100644 index 0000000000..e718bfda88 --- /dev/null +++ b/codex-rs/core/tests/suite/snapshots/all__suite__compact_remote__remote_pre_turn_compaction_context_window_exceeded_shapes.snap @@ -0,0 +1,12 @@ +--- +source: core/tests/suite/compact_remote.rs +expression: "format_labeled_requests_snapshot(\"Remote pre-turn auto-compaction context-window failure: compaction request excludes the incoming user message and the turn errors.\",\n&[(\"Remote Compaction Request (Incoming User Excluded)\",\n&include_attempt_request),])" +--- +Scenario: Remote pre-turn auto-compaction context-window failure: compaction request excludes the incoming user message and the turn errors. + +## Remote Compaction Request (Incoming User Excluded) +00:message/developer: +01:message/user: +02:message/user:> +03:message/user:USER_ONE +04:message/assistant:REMOTE_FIRST_REPLY diff --git a/codex-rs/core/tests/suite/snapshots/all__suite__compact_remote__remote_pre_turn_compaction_failure_shapes.snap b/codex-rs/core/tests/suite/snapshots/all__suite__compact_remote__remote_pre_turn_compaction_failure_shapes.snap new file mode 100644 index 0000000000..f8aa74e9a1 --- /dev/null +++ b/codex-rs/core/tests/suite/snapshots/all__suite__compact_remote__remote_pre_turn_compaction_failure_shapes.snap @@ -0,0 +1,12 @@ +--- +source: core/tests/suite/compact_remote.rs +expression: "format_labeled_requests_snapshot(\"Remote pre-turn auto-compaction parse failure: compaction request excludes the incoming user message and the turn stops.\",\n&[(\"Remote Compaction Request (Incoming User Excluded)\",\n&first_compact_mock.single_request()),])" +--- +Scenario: Remote pre-turn auto-compaction parse failure: compaction request excludes the incoming user message and the turn stops. + +## Remote Compaction Request (Incoming User Excluded) +00:message/developer: +01:message/user: +02:message/user:> +03:message/user:turn that exceeds token threshold +04:message/assistant:initial turn complete diff --git a/codex-rs/core/tests/suite/snapshots/all__suite__compact_remote__remote_pre_turn_compaction_including_incoming_shapes.snap b/codex-rs/core/tests/suite/snapshots/all__suite__compact_remote__remote_pre_turn_compaction_including_incoming_shapes.snap new file mode 100644 index 0000000000..00aaaeaa92 --- /dev/null +++ b/codex-rs/core/tests/suite/snapshots/all__suite__compact_remote__remote_pre_turn_compaction_including_incoming_shapes.snap @@ -0,0 +1,24 @@ +--- +source: core/tests/suite/compact_remote.rs +expression: "format_labeled_requests_snapshot(\"Remote pre-turn auto-compaction with a context override emits the context diff in the compact request while excluding the incoming user message.\",\n&[(\"Remote Compaction Request\", &compact_request),\n(\"Remote Post-Compaction History Layout\", &requests[2]),])" +--- +Scenario: Remote pre-turn auto-compaction with a context override emits the context diff in the compact request while excluding the incoming user message. + +## Remote Compaction Request +00:message/developer: +01:message/user: +02:message/user:> +03:message/user:USER_ONE +04:message/assistant:REMOTE_FIRST_REPLY +05:message/user:USER_TWO +06:message/assistant:REMOTE_SECOND_REPLY +07:message/user: + +## Remote Post-Compaction History Layout +00:message/user:USER_ONE +01:message/user:USER_TWO +02:message/developer: +03:message/user: +04:message/user: +05:message/user:\nREMOTE_PRE_TURN_SUMMARY +06:message/user:USER_THREE diff --git a/codex-rs/core/tests/suite/turn_state.rs b/codex-rs/core/tests/suite/turn_state.rs index 833c2aad9b..62bd974393 100644 --- a/codex-rs/core/tests/suite/turn_state.rs +++ b/codex-rs/core/tests/suite/turn_state.rs @@ -16,6 +16,7 @@ use core_test_support::responses::start_websocket_server_with_headers; use core_test_support::skip_if_no_network; use core_test_support::test_codex::test_codex; use pretty_assertions::assert_eq; +use serde_json::Value; const TURN_STATE_HEADER: &str = "x-codex-turn-state"; @@ -65,6 +66,25 @@ async fn responses_turn_state_persists_within_turn_and_resets_after() -> Result< ); assert_eq!(requests[2].header(TURN_STATE_HEADER), None); + let parse_turn_id = |header: Option| { + let value = header?; + let parsed: Value = serde_json::from_str(&value).ok()?; + parsed + .get("turn_id") + .and_then(Value::as_str) + .map(str::to_string) + }; + + let first_turn_id = parse_turn_id(requests[0].header("x-codex-turn-metadata")) + .expect("first request should include turn metadata turn_id"); + let second_turn_id = parse_turn_id(requests[1].header("x-codex-turn-metadata")) + .expect("follow-up request should include turn metadata turn_id"); + let third_turn_id = parse_turn_id(requests[2].header("x-codex-turn-metadata")) + .expect("new turn request should include turn metadata turn_id"); + + assert_eq!(first_turn_id, second_turn_id); + assert_ne!(second_turn_id, third_turn_id); + Ok(()) } diff --git a/codex-rs/core/tests/suite/view_image.rs b/codex-rs/core/tests/suite/view_image.rs index e81592cc5c..bd5bd6a037 100644 --- a/codex-rs/core/tests/suite/view_image.rs +++ b/codex-rs/core/tests/suite/view_image.rs @@ -671,6 +671,7 @@ async fn view_image_tool_returns_unsupported_message_for_text_only_model() -> an supported_in_api: true, input_modalities: vec![InputModality::Text], prefer_websockets: false, + used_fallback_model_metadata: false, priority: 1, upgrade: None, base_instructions: "base instructions".to_string(), diff --git a/codex-rs/debug-client/src/client.rs b/codex-rs/debug-client/src/client.rs index cacb5a7732..7cf80e3d86 100644 --- a/codex-rs/debug-client/src/client.rs +++ b/codex-rs/debug-client/src/client.rs @@ -180,6 +180,7 @@ impl AppServerClient { model_providers: None, source_kinds: None, archived: None, + cwd: None, }, }; self.send(&request)?; diff --git a/codex-rs/default.nix b/codex-rs/default.nix index c9586128e6..fc7dd978ef 100644 --- a/codex-rs/default.nix +++ b/codex-rs/default.nix @@ -5,6 +5,7 @@ rustPlatform, pkg-config, lib, + version ? "0.0.0", ... }: rustPlatform.buildRustPackage (_: { @@ -12,10 +13,18 @@ rustPlatform.buildRustPackage (_: { PKG_CONFIG_PATH = "${openssl.dev}/lib/pkgconfig:$PKG_CONFIG_PATH"; }; pname = "codex-rs"; - version = "0.1.0"; + inherit version; cargoLock.lockFile = ./Cargo.lock; doCheck = false; src = ./.; + + # Patch the workspace Cargo.toml so that cargo embeds the correct version in + # CARGO_PKG_VERSION (which the binary reads via env!("CARGO_PKG_VERSION")). + # On release commits the Cargo.toml already contains the real version and + # this sed is a no-op. + postPatch = '' + sed -i 's/^version = "0\.0\.0"$/version = "${version}"/' Cargo.toml + ''; nativeBuildInputs = [ cmake llvmPackages.clang diff --git a/codex-rs/docs/bazel.md b/codex-rs/docs/bazel.md index 52c5d523cb..a124688a24 100644 --- a/codex-rs/docs/bazel.md +++ b/codex-rs/docs/bazel.md @@ -23,7 +23,20 @@ As of 1/9/2026, this setup is still experimental as we stabilize it. ## Evolving the setup When you add or change Rust dependencies, update the Cargo.toml/Cargo.lock as normal. -The Bazel build should automatically pick things up without any manual action needed. +Then refresh the Bzlmod lockfile from the repo root: + +```bash +just bazel-lock-update +``` + +This runs `bazel mod deps --lockfile_mode=update` and updates `MODULE.bazel.lock` if needed. +Commit the lockfile changes along with your Cargo lockfile update. + +To verify lockfile alignment locally (the same check CI runs), use: + +```bash +just bazel-lock-check +``` In some cases, an upstream crate may need a patch or a `crate.annotation` in `../MODULE.bzl` to have it build in Bazel's sandbox or make it cross-compilation-friendly. If you see issues, diff --git a/codex-rs/docs/codex_mcp_interface.md b/codex-rs/docs/codex_mcp_interface.md index 293ecb7ecf..73512c03c8 100644 --- a/codex-rs/docs/codex_mcp_interface.md +++ b/codex-rs/docs/codex_mcp_interface.md @@ -81,9 +81,9 @@ Interrupt a running turn: `interruptConversation`. List/resume/archive: `listConversations`, `resumeConversation`, `archiveConversation`. -For v2 threads, use `thread/list` with `archived: true` to list archived rollouts and -`thread/unarchive` to restore them to the active sessions directory (it returns the restored -thread summary). +For v2 threads, use `thread/list` with filters such as `archived: true` or `cwd: "/path"` to +narrow results, and `thread/unarchive` to restore archived rollouts to the active sessions +directory (it returns the restored thread summary). ## Models diff --git a/codex-rs/exec-server/README.md b/codex-rs/exec-server/README.md index a12eaa19af..17d4f26f33 100644 --- a/codex-rs/exec-server/README.md +++ b/codex-rs/exec-server/README.md @@ -2,7 +2,7 @@ This crate contains the code for two executables: -- `codex-exec-mcp-server` is an MCP server that provides a tool named `shell` that runs a shell command inside a sandboxed instance of Bash. Every resulting `execve(2)` call made within Bash is intercepted and run via the executable defined by the `BASH_EXEC_WRAPPER` environment variable within the Bash process. In practice, `BASH_EXEC_WRAPPER` is set to `codex-execve-wrapper`. +- `codex-exec-mcp-server` is an MCP server that provides a tool named `shell` that runs a shell command inside a sandboxed shell process. Every resulting `execve(2)` call made within that shell is intercepted and run via the executable defined by the `EXEC_WRAPPER` environment variable within the shell process. In practice, `EXEC_WRAPPER` is set to `codex-execve-wrapper`. - `codex-execve-wrapper` is the executable that takes the arguments to the `execve(2)` call and "escalates" it to the MCP server via a shared file descriptor (specified by the `CODEX_ESCALATE_SOCKET` environment variable) for consideration. Based on the [Codex `.rules`](https://developers.openai.com/codex/local-config#rules-preview), the MCP server replies with one of: - `Run`: `codex-execve-wrapper` should invoke `execve(2)` on itself to run the original command within Bash - `Escalate`: forward the file descriptors of the current process to the MCP server so the command can be run faithfully outside the sandbox. Because the MCP server will have the original FDs for `stdout` and `stderr`, it can write those directly. When the process completes, the MCP server forwards the exit code to `codex-execve-wrapper` so that it exits in a consistent manner. @@ -10,7 +10,7 @@ This crate contains the code for two executables: ## Patched Bash -We carry a small patch to `execute_cmd.c` (see `patches/bash-exec-wrapper.patch`) that adds support for `BASH_EXEC_WRAPPER`. The original commit message is “add support for BASH_EXEC_WRAPPER” and the patch applies cleanly to `a8a1c2fac029404d3f42cd39f5a20f24b6e4fe4b` from https://github.com/bminor/bash. To rebuild manually: +We carry a small patch to `execute_cmd.c` (see `patches/bash-exec-wrapper.patch`) that adds support for `EXEC_WRAPPER`. The original commit message is “add support for BASH_EXEC_WRAPPER” and the patch applies cleanly to `a8a1c2fac029404d3f42cd39f5a20f24b6e4fe4b` from https://github.com/bminor/bash. To rebuild manually: ```bash git clone https://github.com/bminor/bash diff --git a/codex-rs/exec-server/src/posix/escalate_client.rs b/codex-rs/exec-server/src/posix/escalate_client.rs index 0594cd90f4..1cb0c5908d 100644 --- a/codex-rs/exec-server/src/posix/escalate_client.rs +++ b/codex-rs/exec-server/src/posix/escalate_client.rs @@ -5,11 +5,12 @@ use std::os::fd::OwnedFd; use anyhow::Context as _; -use crate::posix::escalate_protocol::BASH_EXEC_WRAPPER_ENV_VAR; use crate::posix::escalate_protocol::ESCALATE_SOCKET_ENV_VAR; +use crate::posix::escalate_protocol::EXEC_WRAPPER_ENV_VAR; use crate::posix::escalate_protocol::EscalateAction; use crate::posix::escalate_protocol::EscalateRequest; use crate::posix::escalate_protocol::EscalateResponse; +use crate::posix::escalate_protocol::LEGACY_BASH_EXEC_WRAPPER_ENV_VAR; use crate::posix::escalate_protocol::SuperExecMessage; use crate::posix::escalate_protocol::SuperExecResult; use crate::posix::socket::AsyncDatagramSocket; @@ -38,7 +39,7 @@ pub(crate) async fn run(file: String, argv: Vec) -> anyhow::Result .filter(|(k, _)| { !matches!( k.as_str(), - ESCALATE_SOCKET_ENV_VAR | BASH_EXEC_WRAPPER_ENV_VAR + ESCALATE_SOCKET_ENV_VAR | EXEC_WRAPPER_ENV_VAR | LEGACY_BASH_EXEC_WRAPPER_ENV_VAR ) }) .collect(); diff --git a/codex-rs/exec-server/src/posix/escalate_protocol.rs b/codex-rs/exec-server/src/posix/escalate_protocol.rs index e3fc27d071..7245f88fbe 100644 --- a/codex-rs/exec-server/src/posix/escalate_protocol.rs +++ b/codex-rs/exec-server/src/posix/escalate_protocol.rs @@ -8,8 +8,11 @@ use serde::Serialize; /// 'exec-server escalate' reads this to find the inherited FD for the escalate socket. pub(super) const ESCALATE_SOCKET_ENV_VAR: &str = "CODEX_ESCALATE_SOCKET"; -/// The patched bash uses this to wrap exec() calls. -pub(super) const BASH_EXEC_WRAPPER_ENV_VAR: &str = "BASH_EXEC_WRAPPER"; +/// Patched shells use this to wrap exec() calls. +pub(super) const EXEC_WRAPPER_ENV_VAR: &str = "EXEC_WRAPPER"; + +/// Compatibility alias for older patched bash builds. +pub(super) const LEGACY_BASH_EXEC_WRAPPER_ENV_VAR: &str = "BASH_EXEC_WRAPPER"; /// The client sends this to the server to request an exec() call. #[derive(Clone, Serialize, Deserialize, Debug, PartialEq, Eq)] diff --git a/codex-rs/exec-server/src/posix/escalate_server.rs b/codex-rs/exec-server/src/posix/escalate_server.rs index 2b688e63db..6f9e4fbc69 100644 --- a/codex-rs/exec-server/src/posix/escalate_server.rs +++ b/codex-rs/exec-server/src/posix/escalate_server.rs @@ -15,11 +15,12 @@ use codex_core::sandboxing::SandboxPermissions; use tokio::process::Command; use tokio_util::sync::CancellationToken; -use crate::posix::escalate_protocol::BASH_EXEC_WRAPPER_ENV_VAR; use crate::posix::escalate_protocol::ESCALATE_SOCKET_ENV_VAR; +use crate::posix::escalate_protocol::EXEC_WRAPPER_ENV_VAR; use crate::posix::escalate_protocol::EscalateAction; use crate::posix::escalate_protocol::EscalateRequest; use crate::posix::escalate_protocol::EscalateResponse; +use crate::posix::escalate_protocol::LEGACY_BASH_EXEC_WRAPPER_ENV_VAR; use crate::posix::escalate_protocol::SuperExecMessage; use crate::posix::escalate_protocol::SuperExecResult; use crate::posix::escalation_policy::EscalationPolicy; @@ -63,7 +64,11 @@ impl EscalateServer { client_socket.as_raw_fd().to_string(), ); env.insert( - BASH_EXEC_WRAPPER_ENV_VAR.to_string(), + EXEC_WRAPPER_ENV_VAR.to_string(), + self.execve_wrapper.to_string_lossy().to_string(), + ); + env.insert( + LEGACY_BASH_EXEC_WRAPPER_ENV_VAR.to_string(), self.execve_wrapper.to_string_lossy().to_string(), ); @@ -88,6 +93,7 @@ impl EscalateServer { expiration: ExecExpiration::Cancellation(cancel_rx), env, network: None, + network_attempt_id: None, sandbox_permissions: SandboxPermissions::UseDefault, windows_sandbox_level: WindowsSandboxLevel::Disabled, justification: None, diff --git a/codex-rs/exec-server/tests/common/lib.rs b/codex-rs/exec-server/tests/common/lib.rs index ce9fa05216..57fce3a150 100644 --- a/codex-rs/exec-server/tests/common/lib.rs +++ b/codex-rs/exec-server/tests/common/lib.rs @@ -33,9 +33,6 @@ pub async fn create_transport

( where P: AsRef, { - let mcp_executable = codex_utils_cargo_bin::cargo_bin("codex-exec-mcp-server")?; - let execve_wrapper = codex_utils_cargo_bin::cargo_bin("codex-execve-wrapper")?; - // `bash` is a test resource rather than a binary target, so we must use // `find_resource!` to locate it instead of `cargo_bin()`. let bash = find_resource!("../suite/bash")?; @@ -51,8 +48,24 @@ where .await?; assert!(status.success(), "dotslash fetch failed: {status:?}"); + create_transport_with_shell_path(codex_home, dotslash_cache, bash).await +} + +pub async fn create_transport_with_shell_path( + codex_home: P, + dotslash_cache: Q, + shell_path: R, +) -> anyhow::Result +where + P: AsRef, + Q: AsRef, + R: AsRef, +{ + let mcp_executable = codex_utils_cargo_bin::cargo_bin("codex-exec-mcp-server")?; + let execve_wrapper = codex_utils_cargo_bin::cargo_bin("codex-execve-wrapper")?; + let transport = TokioChildProcess::new(Command::new(&mcp_executable).configure(|cmd| { - cmd.arg("--bash").arg(bash); + cmd.arg("--bash").arg(shell_path.as_ref()); cmd.arg("--execve").arg(&execve_wrapper); cmd.env("CODEX_HOME", codex_home.as_ref()); cmd.env("DOTSLASH_CACHE", dotslash_cache.as_ref()); diff --git a/codex-rs/exec-server/tests/suite/accept_elicitation.rs b/codex-rs/exec-server/tests/suite/accept_elicitation.rs index 365a93b837..32f30963ab 100644 --- a/codex-rs/exec-server/tests/suite/accept_elicitation.rs +++ b/codex-rs/exec-server/tests/suite/accept_elicitation.rs @@ -10,6 +10,7 @@ use anyhow::ensure; use codex_exec_server::ExecResult; use exec_server_test_support::InteractiveClient; use exec_server_test_support::create_transport; +use exec_server_test_support::create_transport_with_shell_path; use exec_server_test_support::notify_readable_sandbox; use exec_server_test_support::write_default_execpolicy; use maplit::hashset; @@ -54,7 +55,46 @@ prefix_rule( let dotslash_cache_temp_dir = TempDir::new()?; let dotslash_cache = dotslash_cache_temp_dir.path(); let transport = create_transport(codex_home.as_ref(), dotslash_cache).await?; + run_accept_elicitation_for_prompt_rule_with_transport(transport).await +} +/// Verify the same prompt/escalation flow works when the server is launched +/// with a patched zsh binary. +/// +/// Set CODEX_TEST_ZSH_PATH to enable this test locally or in CI. +#[tokio::test(flavor = "current_thread")] +async fn accept_elicitation_for_prompt_rule_with_zsh() -> Result<()> { + let Some(zsh_path) = std::env::var_os("CODEX_TEST_ZSH_PATH") else { + eprintln!("skipping zsh test: CODEX_TEST_ZSH_PATH is not set"); + return Ok(()); + }; + let zsh_path = PathBuf::from(zsh_path); + + let codex_home = TempDir::new()?; + write_default_execpolicy( + r#" +# Create a rule with `decision = "prompt"` to exercise the elicitation flow. +prefix_rule( + pattern = ["git", "init"], + decision = "prompt", + match = [ + "git init ." + ], +) +"#, + codex_home.as_ref(), + ) + .await?; + let dotslash_cache_temp_dir = TempDir::new()?; + let dotslash_cache = dotslash_cache_temp_dir.path(); + let transport = + create_transport_with_shell_path(codex_home.as_ref(), dotslash_cache, &zsh_path).await?; + run_accept_elicitation_for_prompt_rule_with_transport(transport).await +} + +async fn run_accept_elicitation_for_prompt_rule_with_transport( + transport: rmcp::transport::TokioChildProcess, +) -> Result<()> { // Create an MCP client that approves expected elicitation messages. let project_root = TempDir::new()?; let project_root_path = project_root.path().canonicalize().unwrap(); diff --git a/codex-rs/exec/src/lib.rs b/codex-rs/exec/src/lib.rs index 31b2ff8615..68c88f0000 100644 --- a/codex-rs/exec/src/lib.rs +++ b/codex-rs/exec/src/lib.rs @@ -20,6 +20,7 @@ use codex_core::NewThread; use codex_core::OLLAMA_OSS_PROVIDER_ID; use codex_core::ThreadManager; use codex_core::auth::enforce_login_restrictions; +use codex_core::check_execpolicy_for_warnings; use codex_core::config::Config; use codex_core::config::ConfigBuilder; use codex_core::config::ConfigOverrides; @@ -28,6 +29,7 @@ use codex_core::config::load_config_as_toml_with_cli_overrides; use codex_core::config::resolve_oss_provider; use codex_core::config_loader::ConfigLoadError; use codex_core::config_loader::format_config_error_with_source; +use codex_core::format_exec_policy_error_with_source; use codex_core::git_info::get_git_repo_root; use codex_core::models_manager::manager::RefreshStrategy; use codex_core::protocol::AskForApproval; @@ -267,6 +269,19 @@ pub async fn run_main(cli: Cli, codex_linux_sandbox_exe: Option) -> any .cloud_requirements(cloud_requirements) .build() .await?; + + #[allow(clippy::print_stderr)] + match check_execpolicy_for_warnings(&config.config_layer_stack).await { + Ok(None) => {} + Ok(Some(err)) | Err(err) => { + eprintln!( + "Error loading rules:\n{}", + format_exec_policy_error_with_source(&err) + ); + std::process::exit(1); + } + } + set_default_client_residency_requirement(config.enforce_residency.value()); if let Err(err) = enforce_login_restrictions(&config) { diff --git a/codex-rs/linux-sandbox/tests/suite/landlock.rs b/codex-rs/linux-sandbox/tests/suite/landlock.rs index 84fa1d2c3e..fa26a8b969 100644 --- a/codex-rs/linux-sandbox/tests/suite/landlock.rs +++ b/codex-rs/linux-sandbox/tests/suite/landlock.rs @@ -76,6 +76,7 @@ async fn run_cmd_result_with_writable_roots( expiration: timeout_ms.into(), env: create_env_from_core_vars(), network: None, + network_attempt_id: None, sandbox_permissions: SandboxPermissions::UseDefault, windows_sandbox_level: WindowsSandboxLevel::Disabled, justification: None, @@ -123,7 +124,7 @@ async fn should_skip_bwrap_tests() -> bool { .await { Ok(output) => is_bwrap_unavailable_output(&output), - Err(CodexErr::Sandbox(SandboxErr::Denied { output })) => { + Err(CodexErr::Sandbox(SandboxErr::Denied { output, .. })) => { is_bwrap_unavailable_output(&output) } // Probe timeouts are not actionable for the bwrap-specific assertions below; @@ -142,7 +143,7 @@ fn expect_denied( assert_ne!(output.exit_code, 0, "{context}: expected nonzero exit code"); output } - Err(CodexErr::Sandbox(SandboxErr::Denied { output })) => *output, + Err(CodexErr::Sandbox(SandboxErr::Denied { output, .. })) => *output, Err(err) => panic!("{context}: {err:?}"), } } @@ -236,6 +237,7 @@ async fn assert_network_blocked(cmd: &[&str]) { expiration: NETWORK_TIMEOUT_MS.into(), env: create_env_from_core_vars(), network: None, + network_attempt_id: None, sandbox_permissions: SandboxPermissions::UseDefault, windows_sandbox_level: WindowsSandboxLevel::Disabled, justification: None, @@ -257,7 +259,7 @@ async fn assert_network_blocked(cmd: &[&str]) { let output = match result { Ok(output) => output, - Err(CodexErr::Sandbox(SandboxErr::Denied { output })) => *output, + Err(CodexErr::Sandbox(SandboxErr::Denied { output, .. })) => *output, _ => { panic!("expected sandbox denied error, got: {result:?}"); } diff --git a/codex-rs/mcp-server/src/codex_tool_runner.rs b/codex-rs/mcp-server/src/codex_tool_runner.rs index 6c5b3c25a2..5af2191fcc 100644 --- a/codex-rs/mcp-server/src/codex_tool_runner.rs +++ b/codex-rs/mcp-server/src/codex_tool_runner.rs @@ -221,6 +221,7 @@ async fn run_codex_tool_session_inner( reason: _, proposed_execpolicy_amendment: _, parsed_cmd, + network_approval_context: _, }) => { handle_exec_approval_request( command, diff --git a/codex-rs/network-proxy/Cargo.toml b/codex-rs/network-proxy/Cargo.toml index 48a75c288d..cbcde383b2 100644 --- a/codex-rs/network-proxy/Cargo.toml +++ b/codex-rs/network-proxy/Cargo.toml @@ -14,6 +14,7 @@ workspace = true [dependencies] anyhow = { workspace = true } async-trait = { workspace = true } +base64 = { workspace = true } clap = { workspace = true, features = ["derive"] } codex-utils-absolute-path = { workspace = true } codex-utils-rustls-provider = { workspace = true } diff --git a/codex-rs/network-proxy/src/admin.rs b/codex-rs/network-proxy/src/admin.rs index 678972f810..02aebae1c0 100644 --- a/codex-rs/network-proxy/src/admin.rs +++ b/codex-rs/network-proxy/src/admin.rs @@ -89,7 +89,7 @@ async fn handle_admin_request( text_response(StatusCode::INTERNAL_SERVER_ERROR, "error") } }, - ("GET", "/blocked") => match state.drain_blocked().await { + ("GET", "/blocked") => match state.blocked_snapshot().await { Ok(blocked) => json_response(&BlockedResponse { blocked }), Err(err) => { error!("failed to read blocked queue: {err}"); diff --git a/codex-rs/network-proxy/src/http_proxy.rs b/codex-rs/network-proxy/src/http_proxy.rs index e59980cc5f..eed0ad9da6 100644 --- a/codex-rs/network-proxy/src/http_proxy.rs +++ b/codex-rs/network-proxy/src/http_proxy.rs @@ -1,4 +1,5 @@ use crate::config::NetworkMode; +use crate::metadata::attempt_id_from_proxy_authorization; use crate::network_policy::NetworkDecision; use crate::network_policy::NetworkDecisionSource; use crate::network_policy::NetworkPolicyDecider; @@ -16,7 +17,6 @@ use crate::responses::blocked_header_value; use crate::responses::blocked_message_with_policy; use crate::responses::blocked_text_response_with_policy; use crate::responses::json_response; -use crate::responses::policy_decision_prefix; use crate::runtime::unix_socket_permissions_supported; use crate::state::BlockedRequest; use crate::state::BlockedRequestArgs; @@ -36,11 +36,13 @@ use rama_core::layer::AddInputExtensionLayer; use rama_core::rt::Executor; use rama_core::service::service_fn; use rama_http::Body; +use rama_http::HeaderMap; +use rama_http::HeaderName; use rama_http::HeaderValue; use rama_http::Request; use rama_http::Response; use rama_http::StatusCode; -use rama_http::layer::remove_header::RemoveRequestHeaderLayer; +use rama_http::header; use rama_http::layer::remove_header::RemoveResponseHeaderLayer; use rama_http::matcher::MethodMatcher; use rama_http_backend::client::proxy::layer::HttpProxyConnector; @@ -119,7 +121,6 @@ async fn run_http_proxy_with_listener( service_fn(http_connect_proxy), ), RemoveResponseHeaderLayer::hop_by_hop(), - RemoveRequestHeaderLayer::hop_by_hop(), ) .into_layer(service_fn({ let policy_decider = policy_decider.clone(); @@ -159,6 +160,7 @@ async fn http_connect_accept( } let client = client_addr(&req); + let network_attempt_id = request_network_attempt_id(&req); let enabled = app_state .enabled() @@ -186,6 +188,7 @@ async fn http_connect_accept( method: Some("CONNECT".to_string()), command: None, exec_policy_hint: None, + attempt_id: network_attempt_id.clone(), }); match evaluate_host_policy(&app_state, policy_decider.as_ref(), &request).await { @@ -210,6 +213,10 @@ async fn http_connect_accept( method: Some("CONNECT".to_string()), mode: None, protocol: "http-connect".to_string(), + attempt_id: network_attempt_id.clone(), + decision: Some(details.decision.as_str().to_string()), + source: Some(details.source.as_str().to_string()), + port: Some(authority.port), })) .await; let client = client.as_deref().unwrap_or_default(); @@ -248,6 +255,10 @@ async fn http_connect_accept( method: Some("CONNECT".to_string()), mode: Some(NetworkMode::Limited), protocol: "http-connect".to_string(), + attempt_id: network_attempt_id, + decision: Some(details.decision.as_str().to_string()), + source: Some(details.source.as_str().to_string()), + port: Some(authority.port), })) .await; let client = client.as_deref().unwrap_or_default(); @@ -353,7 +364,7 @@ async fn forward_connect_tunnel( async fn http_plain_proxy( policy_decider: Option>, - req: Request, + mut req: Request, ) -> Result { let app_state = match req.extensions().get::>().cloned() { Some(state) => state, @@ -363,6 +374,7 @@ async fn http_plain_proxy( } }; let client = client_addr(&req); + let network_attempt_id = request_network_attempt_id(&req); let method_allowed = match app_state .method_allowed(req.method().as_str()) @@ -492,6 +504,7 @@ async fn http_plain_proxy( method: Some(req.method().as_str().to_string()), command: None, exec_policy_hint: None, + attempt_id: network_attempt_id.clone(), }); match evaluate_host_policy(&app_state, policy_decider.as_ref(), &request).await { @@ -516,6 +529,10 @@ async fn http_plain_proxy( method: Some(req.method().as_str().to_string()), mode: None, protocol: "http".to_string(), + attempt_id: network_attempt_id.clone(), + decision: Some(details.decision.as_str().to_string()), + source: Some(details.source.as_str().to_string()), + port: Some(port), })) .await; let client = client.as_deref().unwrap_or_default(); @@ -546,6 +563,10 @@ async fn http_plain_proxy( method: Some(req.method().as_str().to_string()), mode: Some(NetworkMode::Limited), protocol: "http".to_string(), + attempt_id: network_attempt_id, + decision: Some(details.decision.as_str().to_string()), + source: Some(details.source.as_str().to_string()), + port: Some(port), })) .await; let client = client.as_deref().unwrap_or_default(); @@ -578,6 +599,8 @@ async fn http_plain_proxy( UpstreamClient::direct() }; + // Strip hop-by-hop headers only after extracting metadata used for policy correlation. + remove_hop_by_hop_request_headers(req.headers_mut()); match client.serve(req).await { Ok(resp) => Ok(resp), Err(err) => { @@ -602,6 +625,7 @@ async fn proxy_via_unix_socket(req: Request, socket_path: &str) -> Result(input: &T) -> Option { .map(|info| info.peer_addr().to_string()) } +fn request_network_attempt_id(req: &Request) -> Option { + // Some HTTP stacks normalize proxy credentials into `authorization`; accept both. + attempt_id_from_proxy_authorization(req.headers().get("proxy-authorization")) + .or_else(|| attempt_id_from_proxy_authorization(req.headers().get("authorization"))) +} + +fn remove_hop_by_hop_request_headers(headers: &mut HeaderMap) { + while let Some(raw_connection) = headers.get(header::CONNECTION).cloned() { + headers.remove(header::CONNECTION); + if let Ok(raw_connection) = raw_connection.to_str() { + let connection_headers: Vec = raw_connection + .split(',') + .map(str::trim) + .filter(|token| !token.is_empty()) + .map(ToOwned::to_owned) + .collect(); + for token in connection_headers { + if let Ok(name) = HeaderName::from_bytes(token.as_bytes()) { + headers.remove(name); + } + } + } + } + for name in [ + &header::KEEP_ALIVE, + &header::PROXY_CONNECTION, + &header::PROXY_AUTHORIZATION, + &header::TRAILER, + &header::TRANSFER_ENCODING, + &header::UPGRADE, + ] { + headers.remove(name); + } + + // codespell:ignore te,TE + // 0x74,0x65 is ASCII "te" (the HTTP TE hop-by-hop header). + if let Ok(short_hop_header_name) = HeaderName::from_bytes(&[0x74, 0x65]) { + headers.remove(short_hop_header_name); + } +} + fn json_blocked(host: &str, reason: &str, details: Option<&PolicyDecisionDetails<'_>>) -> Response { - let (policy_decision_prefix, message) = details + let (message, decision, source, protocol, port) = details .map(|details| { ( - Some(policy_decision_prefix(details)), Some(blocked_message_with_policy(reason, details)), + Some(details.decision.as_str()), + Some(details.source.as_str()), + Some(details.protocol.as_policy_protocol()), + Some(details.port), ) }) - .unwrap_or((None, None)); + .unwrap_or((None, None, None, None, None)); let response = BlockedResponse { status: "blocked", host, reason, - policy_decision_prefix, + decision, + source, + protocol, + port, message, }; let mut resp = json_response(&response); @@ -667,6 +738,10 @@ async fn proxy_disabled_response( method, mode: None, protocol: protocol.as_policy_protocol().to_string(), + attempt_id: None, + decision: Some("deny".to_string()), + source: Some("proxy_state".to_string()), + port: Some(port), })) .await; @@ -703,7 +778,13 @@ struct BlockedResponse<'a> { host: &'a str, reason: &'a str, #[serde(skip_serializing_if = "Option::is_none")] - policy_decision_prefix: Option, + decision: Option<&'static str>, + #[serde(skip_serializing_if = "Option::is_none")] + source: Option<&'static str>, + #[serde(skip_serializing_if = "Option::is_none")] + protocol: Option<&'static str>, + #[serde(skip_serializing_if = "Option::is_none")] + port: Option, #[serde(skip_serializing_if = "Option::is_none")] message: Option, } @@ -715,6 +796,8 @@ mod tests { use crate::config::NetworkMode; use crate::config::NetworkProxySettings; use crate::runtime::network_proxy_state_for_policy; + use base64::Engine; + use base64::engine::general_purpose::STANDARD; use pretty_assertions::assert_eq; use rama_http::Method; use rama_http::Request; @@ -744,4 +827,67 @@ mod tests { "blocked-by-method-policy" ); } + + #[test] + fn request_network_attempt_id_reads_proxy_authorization_header() { + let encoded = STANDARD.encode("codex-net-attempt-attempt-1:"); + let req = Request::builder() + .method(Method::GET) + .uri("http://example.com") + .header("proxy-authorization", format!("Basic {encoded}")) + .body(Body::empty()) + .unwrap(); + assert_eq!( + request_network_attempt_id(&req), + Some("attempt-1".to_string()) + ); + } + + #[test] + fn request_network_attempt_id_reads_authorization_header_fallback() { + let encoded = STANDARD.encode("codex-net-attempt-attempt-2:"); + let req = Request::builder() + .method(Method::GET) + .uri("http://example.com") + .header("authorization", format!("Basic {encoded}")) + .body(Body::empty()) + .unwrap(); + assert_eq!( + request_network_attempt_id(&req), + Some("attempt-2".to_string()) + ); + } + + #[test] + fn remove_hop_by_hop_request_headers_keeps_forwarding_headers() { + let mut headers = HeaderMap::new(); + headers.insert( + header::CONNECTION, + HeaderValue::from_static("x-hop, keep-alive"), + ); + headers.insert("x-hop", HeaderValue::from_static("1")); + headers.insert( + header::PROXY_AUTHORIZATION, + HeaderValue::from_static("Basic abc"), + ); + headers.insert( + &header::X_FORWARDED_FOR, + HeaderValue::from_static("127.0.0.1"), + ); + headers.insert(header::HOST, HeaderValue::from_static("example.com")); + + remove_hop_by_hop_request_headers(&mut headers); + + assert_eq!(headers.get(header::CONNECTION), None); + assert_eq!(headers.get("x-hop"), None); + assert_eq!(headers.get(header::PROXY_AUTHORIZATION), None); + assert_eq!( + headers.get(&header::X_FORWARDED_FOR), + Some(&HeaderValue::from_static("127.0.0.1")) + ); + assert_eq!( + headers.get(header::HOST), + Some(&HeaderValue::from_static("example.com")) + ); + } } diff --git a/codex-rs/network-proxy/src/lib.rs b/codex-rs/network-proxy/src/lib.rs index 8ca0d969c8..536dd4bf6f 100644 --- a/codex-rs/network-proxy/src/lib.rs +++ b/codex-rs/network-proxy/src/lib.rs @@ -3,6 +3,7 @@ mod admin; mod config; mod http_proxy; +mod metadata; mod network_policy; mod policy; mod proxy; @@ -17,7 +18,9 @@ pub use config::NetworkMode; pub use config::NetworkProxyConfig; pub use config::host_and_port_from_network_addr; pub use network_policy::NetworkDecision; +pub use network_policy::NetworkDecisionSource; pub use network_policy::NetworkPolicyDecider; +pub use network_policy::NetworkPolicyDecision; pub use network_policy::NetworkPolicyRequest; pub use network_policy::NetworkPolicyRequestArgs; pub use network_policy::NetworkProtocol; @@ -32,6 +35,9 @@ pub use proxy::NetworkProxyHandle; pub use proxy::PROXY_URL_ENV_KEYS; pub use proxy::has_proxy_url_env_vars; pub use proxy::proxy_url_env_value; +pub use runtime::BlockedRequest; +pub use runtime::BlockedRequestArgs; +pub use runtime::BlockedRequestObserver; pub use runtime::ConfigReloader; pub use runtime::ConfigState; pub use runtime::NetworkProxyState; diff --git a/codex-rs/network-proxy/src/metadata.rs b/codex-rs/network-proxy/src/metadata.rs new file mode 100644 index 0000000000..2542125c35 --- /dev/null +++ b/codex-rs/network-proxy/src/metadata.rs @@ -0,0 +1,50 @@ +use base64::Engine; +use base64::engine::general_purpose::STANDARD; +use rama_http::HeaderValue; + +pub const NETWORK_ATTEMPT_USERNAME_PREFIX: &str = "codex-net-attempt-"; + +pub fn proxy_username_for_attempt_id(attempt_id: &str) -> String { + format!("{NETWORK_ATTEMPT_USERNAME_PREFIX}{attempt_id}") +} + +pub fn attempt_id_from_proxy_authorization(header: Option<&HeaderValue>) -> Option { + let header = header?; + let raw = header.to_str().ok()?; + let encoded = raw.strip_prefix("Basic ")?; + let decoded = STANDARD.decode(encoded.trim()).ok()?; + let decoded = String::from_utf8(decoded).ok()?; + let username = decoded + .split_once(':') + .map(|(user, _)| user) + .unwrap_or(decoded.as_str()); + let attempt_id = username.strip_prefix(NETWORK_ATTEMPT_USERNAME_PREFIX)?; + if attempt_id.is_empty() { + None + } else { + Some(attempt_id.to_string()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use base64::engine::general_purpose::STANDARD; + + #[test] + fn parses_attempt_id_from_proxy_authorization_header() { + let encoded = STANDARD.encode(format!("{NETWORK_ATTEMPT_USERNAME_PREFIX}abc123:")); + let header = HeaderValue::from_str(&format!("Basic {encoded}")).unwrap(); + assert_eq!( + attempt_id_from_proxy_authorization(Some(&header)), + Some("abc123".to_string()) + ); + } + + #[test] + fn ignores_non_attempt_proxy_authorization_header() { + let encoded = STANDARD.encode("normal-user:password"); + let header = HeaderValue::from_str(&format!("Basic {encoded}")).unwrap(); + assert_eq!(attempt_id_from_proxy_authorization(Some(&header)), None); + } +} diff --git a/codex-rs/network-proxy/src/network_policy.rs b/codex-rs/network-proxy/src/network_policy.rs index b70c3e0b17..fdbbe92819 100644 --- a/codex-rs/network-proxy/src/network_policy.rs +++ b/codex-rs/network-proxy/src/network_policy.rs @@ -26,7 +26,8 @@ impl NetworkProtocol { } } -#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[derive(Clone, Copy, Debug, serde::Deserialize, serde::Serialize, PartialEq, Eq)] +#[serde(rename_all = "lowercase")] pub enum NetworkPolicyDecision { Deny, Ask, @@ -41,7 +42,8 @@ impl NetworkPolicyDecision { } } -#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[derive(Clone, Copy, Debug, serde::Deserialize, serde::Serialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] pub enum NetworkDecisionSource { BaselinePolicy, ModeGuard, @@ -69,6 +71,7 @@ pub struct NetworkPolicyRequest { pub method: Option, pub command: Option, pub exec_policy_hint: Option, + pub attempt_id: Option, } pub struct NetworkPolicyRequestArgs { @@ -79,6 +82,7 @@ pub struct NetworkPolicyRequestArgs { pub method: Option, pub command: Option, pub exec_policy_hint: Option, + pub attempt_id: Option, } impl NetworkPolicyRequest { @@ -91,6 +95,7 @@ impl NetworkPolicyRequest { method, command, exec_policy_hint, + attempt_id, } = args; Self { protocol, @@ -100,6 +105,7 @@ impl NetworkPolicyRequest { method, command, exec_policy_hint, + attempt_id, } } } @@ -119,6 +125,10 @@ impl NetworkDecision { Self::deny_with_source(reason, NetworkDecisionSource::Decider) } + pub fn ask(reason: impl Into) -> Self { + Self::ask_with_source(reason, NetworkDecisionSource::Decider) + } + pub fn deny_with_source(reason: impl Into, source: NetworkDecisionSource) -> Self { let reason = reason.into(); let reason = if reason.is_empty() { @@ -216,9 +226,9 @@ fn map_decider_decision(decision: NetworkDecision) -> NetworkDecision { #[cfg(test)] mod tests { use super::*; - use crate::config::NetworkProxySettings; use crate::reasons::REASON_DENIED; + use crate::reasons::REASON_NOT_ALLOWED; use crate::reasons::REASON_NOT_ALLOWED_LOCAL; use crate::state::network_proxy_state_for_policy; use pretty_assertions::assert_eq; @@ -248,6 +258,7 @@ mod tests { method: Some("GET".to_string()), command: None, exec_policy_hint: None, + attempt_id: None, }); let decision = evaluate_host_policy(&state, Some(&decider), &request) @@ -281,6 +292,7 @@ mod tests { method: Some("GET".to_string()), command: None, exec_policy_hint: None, + attempt_id: None, }); let decision = evaluate_host_policy(&state, Some(&decider), &request) @@ -321,6 +333,7 @@ mod tests { method: Some("GET".to_string()), command: None, exec_policy_hint: None, + attempt_id: None, }); let decision = evaluate_host_policy(&state, Some(&decider), &request) @@ -336,4 +349,16 @@ mod tests { ); assert_eq!(calls.load(Ordering::SeqCst), 0); } + + #[test] + fn ask_uses_decider_source_and_ask_decision() { + assert_eq!( + NetworkDecision::ask(REASON_NOT_ALLOWED), + NetworkDecision::Deny { + reason: REASON_NOT_ALLOWED.to_string(), + source: NetworkDecisionSource::Decider, + decision: NetworkPolicyDecision::Ask, + } + ); + } } diff --git a/codex-rs/network-proxy/src/proxy.rs b/codex-rs/network-proxy/src/proxy.rs index 566ac0da25..55a0eaa165 100644 --- a/codex-rs/network-proxy/src/proxy.rs +++ b/codex-rs/network-proxy/src/proxy.rs @@ -1,7 +1,9 @@ use crate::admin; use crate::config; use crate::http_proxy; +use crate::metadata::proxy_username_for_attempt_id; use crate::network_policy::NetworkPolicyDecider; +use crate::runtime::BlockedRequestObserver; use crate::runtime::unix_socket_permissions_supported; use crate::socks5; use crate::state::NetworkProxyState; @@ -70,6 +72,7 @@ pub struct NetworkProxyBuilder { admin_addr: Option, managed_by_codex: bool, policy_decider: Option>, + blocked_request_observer: Option>, } impl Default for NetworkProxyBuilder { @@ -81,6 +84,7 @@ impl Default for NetworkProxyBuilder { admin_addr: None, managed_by_codex: true, policy_decider: None, + blocked_request_observer: None, } } } @@ -124,12 +128,31 @@ impl NetworkProxyBuilder { self } + pub fn blocked_request_observer(mut self, observer: O) -> Self + where + O: BlockedRequestObserver, + { + self.blocked_request_observer = Some(Arc::new(observer)); + self + } + + pub fn blocked_request_observer_arc( + mut self, + observer: Arc, + ) -> Self { + self.blocked_request_observer = Some(observer); + self + } + pub async fn build(self) -> Result { let state = self.state.ok_or_else(|| { anyhow::anyhow!( "NetworkProxyBuilder requires a state; supply one via builder.state(...)" ) })?; + state + .set_blocked_request_observer(self.blocked_request_observer.clone()) + .await; let current_cfg = state.current_cfg().await?; let (requested_http_addr, requested_socks_addr, requested_admin_addr, reserved_listeners) = if self.managed_by_codex { @@ -312,8 +335,12 @@ fn apply_proxy_env_overrides( socks_addr: SocketAddr, socks_enabled: bool, allow_local_binding: bool, + network_attempt_id: Option<&str>, ) { - let http_proxy_url = format!("http://{http_addr}"); + let http_proxy_url = network_attempt_id + .map(proxy_username_for_attempt_id) + .map(|username| format!("http://{username}@{http_addr}")) + .unwrap_or_else(|| format!("http://{http_addr}")); let socks_proxy_url = format!("socks5h://{socks_addr}"); env.insert( ALLOW_LOCAL_BINDING_ENV_KEY.to_string(), @@ -354,18 +381,25 @@ fn apply_proxy_env_overrides( env.insert("ELECTRON_GET_USE_PROXY".to_string(), "true".to_string()); - if socks_enabled { + // Keep HTTP_PROXY/HTTPS_PROXY as HTTP endpoints. A lot of clients break if + // those vars contain SOCKS URLs. We only switch ALL_PROXY here. + // + // For attempt-scoped runs, point ALL_PROXY at the HTTP proxy URL so the + // attempt metadata survives in proxy credentials for correlation. + if socks_enabled && network_attempt_id.is_none() { set_env_keys(env, ALL_PROXY_ENV_KEYS, &socks_proxy_url); set_env_keys(env, FTP_PROXY_ENV_KEYS, &socks_proxy_url); - #[cfg(target_os = "macos")] - { - // Preserve existing SSH wrappers (for example: Secretive/Teleport setups) - // and only provide a SOCKS ProxyCommand fallback when one is not present. - env.entry("GIT_SSH_COMMAND".to_string()) - .or_insert_with(|| format!("ssh -o ProxyCommand='nc -X 5 -x {socks_addr} %h %p'")); - } } else { set_env_keys(env, ALL_PROXY_ENV_KEYS, &http_proxy_url); + set_env_keys(env, FTP_PROXY_ENV_KEYS, &http_proxy_url); + } + + #[cfg(target_os = "macos")] + if socks_enabled { + // Preserve existing SSH wrappers (for example: Secretive/Teleport setups) + // and only provide a SOCKS ProxyCommand fallback when one is not present. + env.entry("GIT_SSH_COMMAND".to_string()) + .or_insert_with(|| format!("ssh -o ProxyCommand='nc -X 5 -x {socks_addr} %h %p'")); } } @@ -387,6 +421,14 @@ impl NetworkProxy { } pub fn apply_to_env(&self, env: &mut HashMap) { + self.apply_to_env_for_attempt(env, None); + } + + pub fn apply_to_env_for_attempt( + &self, + env: &mut HashMap, + network_attempt_id: Option<&str>, + ) { // Enforce proxying for child processes. We intentionally override existing values so // command-level environment cannot bypass the managed proxy endpoint. apply_proxy_env_overrides( @@ -395,6 +437,7 @@ impl NetworkProxy { self.socks_addr, self.socks_enabled, self.allow_local_binding, + network_attempt_id, ); } @@ -694,6 +737,7 @@ mod tests { SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 8081), true, false, + None, ); assert_eq!( @@ -736,6 +780,7 @@ mod tests { SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 8081), false, true, + None, ); assert_eq!( @@ -745,6 +790,39 @@ mod tests { assert_eq!(env.get(ALLOW_LOCAL_BINDING_ENV_KEY), Some(&"1".to_string())); } + #[test] + fn apply_proxy_env_overrides_embeds_attempt_id_in_http_proxy_url() { + let mut env = HashMap::new(); + apply_proxy_env_overrides( + &mut env, + SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 3128), + SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 8081), + true, + false, + Some("attempt-123"), + ); + + assert_eq!( + env.get("HTTP_PROXY"), + Some(&"http://codex-net-attempt-attempt-123@127.0.0.1:3128".to_string()) + ); + assert_eq!( + env.get("HTTPS_PROXY"), + Some(&"http://codex-net-attempt-attempt-123@127.0.0.1:3128".to_string()) + ); + assert_eq!( + env.get("ALL_PROXY"), + Some(&"http://codex-net-attempt-attempt-123@127.0.0.1:3128".to_string()) + ); + #[cfg(target_os = "macos")] + assert_eq!( + env.get("GIT_SSH_COMMAND"), + Some(&"ssh -o ProxyCommand='nc -X 5 -x 127.0.0.1:8081 %h %p'".to_string()) + ); + #[cfg(not(target_os = "macos"))] + assert_eq!(env.get("GIT_SSH_COMMAND"), None); + } + #[cfg(target_os = "macos")] #[test] fn apply_proxy_env_overrides_preserves_existing_git_ssh_command() { @@ -759,6 +837,7 @@ mod tests { SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 8081), true, false, + None, ); assert_eq!( diff --git a/codex-rs/network-proxy/src/responses.rs b/codex-rs/network-proxy/src/responses.rs index 354d4bbaf2..b848a00743 100644 --- a/codex-rs/network-proxy/src/responses.rs +++ b/codex-rs/network-proxy/src/responses.rs @@ -11,8 +11,6 @@ use rama_http::StatusCode; use serde::Serialize; use tracing::error; -const NETWORK_POLICY_DECISION_PREFIX: &str = "CODEX_NETWORK_POLICY_DECISION"; - pub struct PolicyDecisionDetails<'a> { pub decision: NetworkPolicyDecision, pub reason: &'a str, @@ -22,17 +20,6 @@ pub struct PolicyDecisionDetails<'a> { pub port: u16, } -#[derive(Serialize)] -#[serde(rename_all = "camelCase")] -struct PolicyDecisionPayload<'a> { - decision: &'a str, - reason: &'a str, - source: &'a str, - protocol: &'a str, - host: &'a str, - port: u16, -} - pub fn text_response(status: StatusCode, body: &str) -> Response { Response::builder() .status(status) @@ -70,7 +57,9 @@ pub fn blocked_header_value(reason: &str) -> &'static str { pub fn blocked_message(reason: &str) -> &'static str { match reason { - REASON_NOT_ALLOWED => "Codex blocked this request: domain not in allowlist.", + REASON_NOT_ALLOWED => { + "Codex blocked this request: domain not in allowlist (this is not a denylist block)." + } REASON_NOT_ALLOWED_LOCAL => { "Codex blocked this request: local/private addresses not allowed." } @@ -82,31 +71,9 @@ pub fn blocked_message(reason: &str) -> &'static str { } } -pub fn policy_decision_prefix(details: &PolicyDecisionDetails<'_>) -> String { - let payload = PolicyDecisionPayload { - decision: details.decision.as_str(), - reason: details.reason, - source: details.source.as_str(), - protocol: details.protocol.as_policy_protocol(), - host: details.host, - port: details.port, - }; - let payload_json = match serde_json::to_string(&payload) { - Ok(json) => json, - Err(err) => { - error!("failed to serialize policy decision payload: {err}"); - "{}".to_string() - } - }; - format!("{NETWORK_POLICY_DECISION_PREFIX} {payload_json}") -} - pub fn blocked_message_with_policy(reason: &str, details: &PolicyDecisionDetails<'_>) -> String { - format!( - "{}\n{}", - policy_decision_prefix(details), - blocked_message(reason) - ) + let _ = (details.reason, details.host); + blocked_message(reason).to_string() } pub fn blocked_text_response_with_policy( @@ -128,7 +95,7 @@ mod tests { use pretty_assertions::assert_eq; #[test] - fn policy_decision_prefix_serializes_expected_payload() { + fn blocked_message_with_policy_returns_human_message() { let details = PolicyDecisionDetails { decision: NetworkPolicyDecision::Ask, reason: REASON_NOT_ALLOWED, @@ -138,29 +105,10 @@ mod tests { port: 443, }; - let line = policy_decision_prefix(&details); - assert_eq!( - line, - r#"CODEX_NETWORK_POLICY_DECISION {"decision":"ask","reason":"not_allowed","source":"decider","protocol":"https_connect","host":"api.example.com","port":443}"# - ); - } - - #[test] - fn blocked_message_with_policy_includes_prefix_and_human_message() { - let details = PolicyDecisionDetails { - decision: NetworkPolicyDecision::Deny, - reason: REASON_NOT_ALLOWED, - source: NetworkDecisionSource::BaselinePolicy, - protocol: NetworkProtocol::Http, - host: "api.example.com", - port: 80, - }; - let message = blocked_message_with_policy(REASON_NOT_ALLOWED, &details); assert_eq!( message, - r#"CODEX_NETWORK_POLICY_DECISION {"decision":"deny","reason":"not_allowed","source":"baseline_policy","protocol":"http","host":"api.example.com","port":80} -Codex blocked this request: domain not in allowlist."# + "Codex blocked this request: domain not in allowlist (this is not a denylist block)." ); } } diff --git a/codex-rs/network-proxy/src/runtime.rs b/codex-rs/network-proxy/src/runtime.rs index 029c7b5f6c..7917747a8f 100644 --- a/codex-rs/network-proxy/src/runtime.rs +++ b/codex-rs/network-proxy/src/runtime.rs @@ -20,6 +20,7 @@ use globset::GlobSet; use serde::Serialize; use std::collections::HashSet; use std::collections::VecDeque; +use std::future::Future; use std::net::IpAddr; use std::path::Path; use std::sync::Arc; @@ -28,11 +29,13 @@ use time::OffsetDateTime; use tokio::net::lookup_host; use tokio::sync::RwLock; use tokio::time::timeout; +use tracing::debug; use tracing::info; use tracing::warn; const MAX_BLOCKED_EVENTS: usize = 200; const DNS_LOOKUP_TIMEOUT: Duration = Duration::from_secs(2); +const NETWORK_POLICY_VIOLATION_PREFIX: &str = "CODEX_NETWORK_POLICY_VIOLATION"; #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum HostBlockReason { @@ -71,6 +74,14 @@ pub struct BlockedRequest { pub method: Option, pub mode: Option, pub protocol: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub attempt_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub decision: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub source: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub port: Option, pub timestamp: i64, } @@ -81,6 +92,10 @@ pub struct BlockedRequestArgs { pub method: Option, pub mode: Option, pub protocol: String, + pub attempt_id: Option, + pub decision: Option, + pub source: Option, + pub port: Option, } impl BlockedRequest { @@ -92,6 +107,10 @@ impl BlockedRequest { method, mode, protocol, + attempt_id, + decision, + source, + port, } = args; Self { host, @@ -100,11 +119,28 @@ impl BlockedRequest { method, mode, protocol, + attempt_id, + decision, + source, + port, timestamp: unix_timestamp(), } } } +fn blocked_request_violation_log_line(entry: &BlockedRequest) -> String { + match serde_json::to_string(entry) { + Ok(json) => format!("{NETWORK_POLICY_VIOLATION_PREFIX} {json}"), + Err(err) => { + debug!("failed to serialize blocked request for violation log: {err}"); + format!( + "{NETWORK_POLICY_VIOLATION_PREFIX} host={} reason={}", + entry.host, entry.reason + ) + } + } +} + #[derive(Clone)] pub struct ConfigState { pub config: NetworkProxyConfig, @@ -112,6 +148,7 @@ pub struct ConfigState { pub deny_set: GlobSet, pub constraints: NetworkProxyConstraints, pub blocked: VecDeque, + pub blocked_total: u64, } #[async_trait] @@ -126,9 +163,33 @@ pub trait ConfigReloader: Send + Sync { async fn reload_now(&self) -> Result; } +#[async_trait] +pub trait BlockedRequestObserver: Send + Sync + 'static { + async fn on_blocked_request(&self, request: BlockedRequest); +} + +#[async_trait] +impl BlockedRequestObserver for Arc { + async fn on_blocked_request(&self, request: BlockedRequest) { + (**self).on_blocked_request(request).await + } +} + +#[async_trait] +impl BlockedRequestObserver for F +where + F: Fn(BlockedRequest) -> Fut + Send + Sync + 'static, + Fut: Future + Send, +{ + async fn on_blocked_request(&self, request: BlockedRequest) { + (self)(request).await + } +} + pub struct NetworkProxyState { state: Arc>, reloader: Arc, + blocked_request_observer: Arc>>>, } impl std::fmt::Debug for NetworkProxyState { @@ -144,18 +205,36 @@ impl Clone for NetworkProxyState { Self { state: self.state.clone(), reloader: self.reloader.clone(), + blocked_request_observer: self.blocked_request_observer.clone(), } } } impl NetworkProxyState { pub fn with_reloader(state: ConfigState, reloader: Arc) -> Self { + Self::with_reloader_and_blocked_observer(state, reloader, None) + } + + pub fn with_reloader_and_blocked_observer( + state: ConfigState, + reloader: Arc, + blocked_request_observer: Option>, + ) -> Self { Self { state: Arc::new(RwLock::new(state)), reloader, + blocked_request_observer: Arc::new(RwLock::new(blocked_request_observer)), } } + pub async fn set_blocked_request_observer( + &self, + blocked_request_observer: Option>, + ) { + let mut observer = self.blocked_request_observer.write().await; + *observer = blocked_request_observer; + } + pub async fn current_cfg(&self) -> Result { // Callers treat `NetworkProxyState` as a live view of policy. We reload-on-demand so edits to // `config.toml` (including Codex-managed writes) take effect without a restart. @@ -276,14 +355,52 @@ impl NetworkProxyState { pub async fn record_blocked(&self, entry: BlockedRequest) -> Result<()> { self.reload_if_needed().await?; + let blocked_for_observer = entry.clone(); + let blocked_request_observer = self.blocked_request_observer.read().await.clone(); + let violation_line = blocked_request_violation_log_line(&entry); let mut guard = self.state.write().await; + let host = entry.host.clone(); + let reason = entry.reason.clone(); + let decision = entry.decision.clone(); + let source = entry.source.clone(); + let protocol = entry.protocol.clone(); + let port = entry.port; + let attempt_id = entry.attempt_id.clone(); guard.blocked.push_back(entry); + guard.blocked_total = guard.blocked_total.saturating_add(1); + let total = guard.blocked_total; while guard.blocked.len() > MAX_BLOCKED_EVENTS { guard.blocked.pop_front(); } + debug!( + "recorded blocked request telemetry (total={}, host={}, reason={}, decision={:?}, source={:?}, protocol={}, port={:?}, attempt_id={:?}, buffered={})", + total, + host, + reason, + decision, + source, + protocol, + port, + attempt_id, + guard.blocked.len() + ); + debug!("{violation_line}"); + drop(guard); + + if let Some(observer) = blocked_request_observer { + observer.on_blocked_request(blocked_for_observer).await; + } Ok(()) } + /// Returns a snapshot of buffered blocked-request entries without consuming + /// them. + pub async fn blocked_snapshot(&self) -> Result> { + self.reload_if_needed().await?; + let guard = self.state.read().await; + Ok(guard.blocked.iter().cloned().collect()) + } + /// Drain and return the buffered blocked-request entries in FIFO order. pub async fn drain_blocked(&self) -> Result> { self.reload_if_needed().await?; @@ -380,12 +497,17 @@ impl NetworkProxyState { match self.reloader.maybe_reload().await? { None => Ok(()), Some(mut new_state) => { - let (previous_cfg, blocked) = { + let (previous_cfg, blocked, blocked_total) = { let guard = self.state.read().await; - (guard.config.clone(), guard.blocked.clone()) + ( + guard.config.clone(), + guard.blocked.clone(), + guard.blocked_total, + ) }; log_policy_changes(&previous_cfg, &new_state.config); new_state.blocked = blocked; + new_state.blocked_total = blocked_total; { let mut guard = self.state.write().await; *guard = new_state; @@ -566,6 +688,95 @@ mod tests { ); } + #[tokio::test] + async fn blocked_snapshot_does_not_consume_entries() { + let state = network_proxy_state_for_policy(NetworkProxySettings::default()); + + state + .record_blocked(BlockedRequest::new(BlockedRequestArgs { + host: "google.com".to_string(), + reason: "not_allowed".to_string(), + client: None, + method: Some("GET".to_string()), + mode: None, + protocol: "http".to_string(), + attempt_id: None, + decision: Some("ask".to_string()), + source: Some("decider".to_string()), + port: Some(80), + })) + .await + .expect("entry should be recorded"); + + let snapshot = state + .blocked_snapshot() + .await + .expect("snapshot should succeed"); + assert_eq!(snapshot.len(), 1); + assert_eq!(snapshot[0].host, "google.com"); + assert_eq!(snapshot[0].decision.as_deref(), Some("ask")); + + let drained = state + .drain_blocked() + .await + .expect("drain should include snapshot entry"); + assert_eq!(drained.len(), 1); + assert_eq!(drained[0].host, snapshot[0].host); + assert_eq!(drained[0].reason, snapshot[0].reason); + assert_eq!(drained[0].decision, snapshot[0].decision); + assert_eq!(drained[0].source, snapshot[0].source); + assert_eq!(drained[0].port, snapshot[0].port); + } + + #[tokio::test] + async fn drain_blocked_returns_buffered_window() { + let state = network_proxy_state_for_policy(NetworkProxySettings::default()); + + for idx in 0..(MAX_BLOCKED_EVENTS + 5) { + state + .record_blocked(BlockedRequest::new(BlockedRequestArgs { + host: format!("example{idx}.com"), + reason: "not_allowed".to_string(), + client: None, + method: Some("GET".to_string()), + mode: None, + protocol: "http".to_string(), + attempt_id: None, + decision: Some("ask".to_string()), + source: Some("decider".to_string()), + port: Some(80), + })) + .await + .expect("entry should be recorded"); + } + + let blocked = state.drain_blocked().await.expect("drain should succeed"); + assert_eq!(blocked.len(), MAX_BLOCKED_EVENTS); + assert_eq!(blocked[0].host, "example5.com"); + } + + #[test] + fn blocked_request_violation_log_line_serializes_payload() { + let entry = BlockedRequest { + host: "google.com".to_string(), + reason: "not_allowed".to_string(), + client: Some("127.0.0.1".to_string()), + method: Some("GET".to_string()), + mode: Some(NetworkMode::Full), + protocol: "http".to_string(), + attempt_id: Some("attempt-1".to_string()), + decision: Some("ask".to_string()), + source: Some("decider".to_string()), + port: Some(80), + timestamp: 1_735_689_600, + }; + + assert_eq!( + blocked_request_violation_log_line(&entry), + r#"CODEX_NETWORK_POLICY_VIOLATION {"host":"google.com","reason":"not_allowed","client":"127.0.0.1","method":"GET","mode":"full","protocol":"http","attempt_id":"attempt-1","decision":"ask","source":"decider","port":80,"timestamp":1735689600}"# + ); + } + #[tokio::test] async fn host_blocked_subdomain_wildcards_exclude_apex() { let state = network_proxy_state_for_policy(NetworkProxySettings { diff --git a/codex-rs/network-proxy/src/socks5.rs b/codex-rs/network-proxy/src/socks5.rs index eaf265a7b9..176055a5d2 100644 --- a/codex-rs/network-proxy/src/socks5.rs +++ b/codex-rs/network-proxy/src/socks5.rs @@ -168,6 +168,10 @@ async fn handle_socks5_tcp( method: None, mode: None, protocol: "socks5".to_string(), + attempt_id: None, + decision: Some(details.decision.as_str().to_string()), + source: Some(details.source.as_str().to_string()), + port: Some(port), })) .await; let client = client.as_deref().unwrap_or_default(); @@ -198,6 +202,10 @@ async fn handle_socks5_tcp( method: None, mode: Some(NetworkMode::Limited), protocol: "socks5".to_string(), + attempt_id: None, + decision: Some(details.decision.as_str().to_string()), + source: Some(details.source.as_str().to_string()), + port: Some(port), })) .await; let client = client.as_deref().unwrap_or_default(); @@ -221,6 +229,7 @@ async fn handle_socks5_tcp( method: None, command: None, exec_policy_hint: None, + attempt_id: None, }); match evaluate_host_policy(&app_state, policy_decider.as_ref(), &request).await { @@ -245,6 +254,10 @@ async fn handle_socks5_tcp( method: None, mode: None, protocol: "socks5".to_string(), + attempt_id: None, + decision: Some(details.decision.as_str().to_string()), + source: Some(details.source.as_str().to_string()), + port: Some(port), })) .await; let client = client.as_deref().unwrap_or_default(); @@ -305,6 +318,10 @@ async fn inspect_socks5_udp( method: None, mode: None, protocol: "socks5-udp".to_string(), + attempt_id: None, + decision: Some(details.decision.as_str().to_string()), + source: Some(details.source.as_str().to_string()), + port: Some(port), })) .await; let client = client.as_deref().unwrap_or_default(); @@ -335,6 +352,10 @@ async fn inspect_socks5_udp( method: None, mode: Some(NetworkMode::Limited), protocol: "socks5-udp".to_string(), + attempt_id: None, + decision: Some(details.decision.as_str().to_string()), + source: Some(details.source.as_str().to_string()), + port: Some(port), })) .await; return Err(policy_denied_error(REASON_METHOD_NOT_ALLOWED, &details)); @@ -354,6 +375,7 @@ async fn inspect_socks5_udp( method: None, command: None, exec_policy_hint: None, + attempt_id: None, }); match evaluate_host_policy(&state, policy_decider.as_ref(), &request).await { @@ -378,6 +400,10 @@ async fn inspect_socks5_udp( method: None, mode: None, protocol: "socks5-udp".to_string(), + attempt_id: None, + decision: Some(details.decision.as_str().to_string()), + source: Some(details.source.as_str().to_string()), + port: Some(port), })) .await; let client = client.as_deref().unwrap_or_default(); diff --git a/codex-rs/network-proxy/src/state.rs b/codex-rs/network-proxy/src/state.rs index 509ada760e..97dfcc4b5d 100644 --- a/codex-rs/network-proxy/src/state.rs +++ b/codex-rs/network-proxy/src/state.rs @@ -60,6 +60,7 @@ pub fn build_config_state( deny_set, constraints, blocked: std::collections::VecDeque::new(), + blocked_total: 0, }) } diff --git a/codex-rs/protocol/src/approvals.rs b/codex-rs/protocol/src/approvals.rs index 635cd5223d..4ee7bd8b30 100644 --- a/codex-rs/protocol/src/approvals.rs +++ b/codex-rs/protocol/src/approvals.rs @@ -37,6 +37,24 @@ impl From> for ExecPolicyAmendment { } } +#[derive(Debug, Clone, Copy, Deserialize, Serialize, PartialEq, Eq, JsonSchema, TS)] +#[serde(rename_all = "snake_case")] +pub enum NetworkApprovalProtocol { + // TODO(viyatb): Add websocket protocol variants when managed proxy policy + // decisions expose websocket traffic as a distinct approval context. + Http, + #[serde(alias = "https_connect", alias = "http-connect")] + Https, + Socks5Tcp, + Socks5Udp, +} + +#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq, JsonSchema, TS)] +pub struct NetworkApprovalContext { + pub host: String, + pub protocol: NetworkApprovalProtocol, +} + #[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, TS)] pub struct ExecApprovalRequestEvent { /// Identifier for the associated exec call, if available. @@ -52,6 +70,10 @@ pub struct ExecApprovalRequestEvent { /// Optional human-readable reason for the approval (e.g. retry without sandbox). #[serde(skip_serializing_if = "Option::is_none")] pub reason: Option, + /// Optional network context for a blocked request that can be approved. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional)] + pub network_approval_context: Option, /// Proposed execpolicy amendment that can be applied to allow future runs. #[serde(default, skip_serializing_if = "Option::is_none")] #[ts(optional)] diff --git a/codex-rs/protocol/src/models.rs b/codex-rs/protocol/src/models.rs index b5ce0f2b2c..eba748f2e4 100644 --- a/codex-rs/protocol/src/models.rs +++ b/codex-rs/protocol/src/models.rs @@ -720,15 +720,18 @@ impl From> for ResponseInputItem { .into_iter() .flat_map(|c| match c { UserInput::Text { text, .. } => vec![ContentItem::InputText { text }], - UserInput::Image { image_url } => vec![ - ContentItem::InputText { - text: image_open_tag_text(), - }, - ContentItem::InputImage { image_url }, - ContentItem::InputText { - text: image_close_tag_text(), - }, - ], + UserInput::Image { image_url } => { + image_index += 1; + vec![ + ContentItem::InputText { + text: image_open_tag_text(), + }, + ContentItem::InputImage { image_url }, + ContentItem::InputText { + text: image_close_tag_text(), + }, + ] + } UserInput::LocalImage { path } => { image_index += 1; local_image_content_items_with_label_number(&path, Some(image_index)) @@ -1609,6 +1612,65 @@ mod tests { Ok(()) } + #[test] + fn mixed_remote_and_local_images_share_label_sequence() -> Result<()> { + let image_url = "data:image/png;base64,abc".to_string(); + let dir = tempdir()?; + let local_path = dir.path().join("local.png"); + // A tiny valid PNG (1x1) so this test doesn't depend on cross-crate file paths, which + // break under Bazel sandboxing. + const TINY_PNG_BYTES: &[u8] = &[ + 137, 80, 78, 71, 13, 10, 26, 10, 0, 0, 0, 13, 73, 72, 68, 82, 0, 0, 0, 1, 0, 0, 0, 1, + 8, 6, 0, 0, 0, 31, 21, 196, 137, 0, 0, 0, 11, 73, 68, 65, 84, 120, 156, 99, 96, 0, 2, + 0, 0, 5, 0, 1, 122, 94, 171, 63, 0, 0, 0, 0, 73, 69, 78, 68, 174, 66, 96, 130, + ]; + std::fs::write(&local_path, TINY_PNG_BYTES)?; + + let item = ResponseInputItem::from(vec![ + UserInput::Image { + image_url: image_url.clone(), + }, + UserInput::LocalImage { path: local_path }, + ]); + + match item { + ResponseInputItem::Message { content, .. } => { + assert_eq!( + content.first(), + Some(&ContentItem::InputText { + text: image_open_tag_text(), + }) + ); + assert_eq!(content.get(1), Some(&ContentItem::InputImage { image_url })); + assert_eq!( + content.get(2), + Some(&ContentItem::InputText { + text: image_close_tag_text(), + }) + ); + assert_eq!( + content.get(3), + Some(&ContentItem::InputText { + text: local_image_open_tag_text(2), + }) + ); + assert!(matches!( + content.get(4), + Some(ContentItem::InputImage { .. }) + )); + assert_eq!( + content.get(5), + Some(&ContentItem::InputText { + text: image_close_tag_text(), + }) + ); + } + other => panic!("expected message response but got {other:?}"), + } + + Ok(()) + } + #[test] fn local_image_read_error_adds_placeholder() -> Result<()> { let dir = tempdir()?; diff --git a/codex-rs/protocol/src/openai_models.rs b/codex-rs/protocol/src/openai_models.rs index d9275e3830..e61860698a 100644 --- a/codex-rs/protocol/src/openai_models.rs +++ b/codex-rs/protocol/src/openai_models.rs @@ -253,6 +253,11 @@ pub struct ModelInfo { /// When true, this model should use websocket transport even when websocket features are off. #[serde(default)] pub prefer_websockets: bool, + /// Internal-only marker set by core when a model slug resolved to fallback metadata. + #[serde(default, skip_serializing, skip_deserializing)] + #[schemars(skip)] + #[ts(skip)] + pub used_fallback_model_metadata: bool, } impl ModelInfo { @@ -517,6 +522,7 @@ mod tests { experimental_supported_tools: vec![], input_modalities: default_input_modalities(), prefer_websockets: false, + used_fallback_model_metadata: false, } } diff --git a/codex-rs/protocol/src/protocol.rs b/codex-rs/protocol/src/protocol.rs index 745068b16e..8a7fa9c382 100644 --- a/codex-rs/protocol/src/protocol.rs +++ b/codex-rs/protocol/src/protocol.rs @@ -54,6 +54,8 @@ pub use crate::approvals::ApplyPatchApprovalRequestEvent; pub use crate::approvals::ElicitationAction; pub use crate::approvals::ExecApprovalRequestEvent; pub use crate::approvals::ExecPolicyAmendment; +pub use crate::approvals::NetworkApprovalContext; +pub use crate::approvals::NetworkApprovalProtocol; pub use crate::request_user_input::RequestUserInputEvent; /// Open/close tags for special user-input blocks. Used across crates to avoid @@ -260,6 +262,12 @@ pub enum Op { /// Request MCP servers to reinitialize and refresh cached tool lists. RefreshMcpServers { config: McpServerRefreshConfig }, + /// Reload user config layer overrides for the active session. + /// + /// This updates runtime config-derived behavior (for example app + /// enable/disable state) without restarting the thread. + ReloadUserConfig, + /// Request the list of available custom prompts. ListCustomPrompts, @@ -1819,6 +1827,7 @@ pub enum SubAgentSource { parent_thread_id: ThreadId, depth: i32, }, + MemoryConsolidation, Other(String), } @@ -1840,6 +1849,7 @@ impl fmt::Display for SubAgentSource { match self { SubAgentSource::Review => f.write_str("review"), SubAgentSource::Compact => f.write_str("compact"), + SubAgentSource::MemoryConsolidation => f.write_str("memory_consolidation"), SubAgentSource::ThreadSpawn { parent_thread_id, depth, @@ -1932,6 +1942,12 @@ impl From for ResponseItem { } } +#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq, JsonSchema, TS)] +pub struct TurnContextNetworkItem { + pub allowed_domains: Vec, + pub denied_domains: Vec, +} + #[derive(Serialize, Deserialize, Clone, Debug, JsonSchema, TS)] pub struct TurnContextItem { #[serde(default, skip_serializing_if = "Option::is_none")] @@ -1939,6 +1955,8 @@ pub struct TurnContextItem { pub cwd: PathBuf, pub approval_policy: AskForApproval, pub sandbox_policy: SandboxPolicy, + #[serde(skip_serializing_if = "Option::is_none")] + pub network: Option, pub model: String, #[serde(skip_serializing_if = "Option::is_none")] pub personality: Option, @@ -2975,6 +2993,53 @@ mod tests { Ok(()) } + #[test] + fn turn_context_item_deserializes_without_network() -> Result<()> { + let item: TurnContextItem = serde_json::from_value(json!({ + "cwd": "/tmp", + "approval_policy": "never", + "sandbox_policy": { "type": "danger-full-access" }, + "model": "gpt-5", + "summary": "auto", + }))?; + + assert_eq!(item.network, None); + Ok(()) + } + + #[test] + fn turn_context_item_serializes_network_when_present() -> Result<()> { + let item = TurnContextItem { + turn_id: None, + cwd: PathBuf::from("/tmp"), + approval_policy: AskForApproval::Never, + sandbox_policy: SandboxPolicy::DangerFullAccess, + network: Some(TurnContextNetworkItem { + allowed_domains: vec!["api.example.com".to_string()], + denied_domains: vec!["blocked.example.com".to_string()], + }), + model: "gpt-5".to_string(), + personality: None, + collaboration_mode: None, + effort: None, + summary: ReasoningSummaryConfig::Auto, + user_instructions: None, + developer_instructions: None, + final_output_json_schema: None, + truncation_policy: None, + }; + + let value = serde_json::to_value(item)?; + assert_eq!( + value["network"], + json!({ + "allowed_domains": ["api.example.com"], + "denied_domains": ["blocked.example.com"], + }) + ); + Ok(()) + } + /// Serialize Event to verify that its JSON representation has the expected /// amount of nesting. #[test] diff --git a/codex-rs/rmcp-client/src/auth_status.rs b/codex-rs/rmcp-client/src/auth_status.rs index c752aec788..7ab72088b8 100644 --- a/codex-rs/rmcp-client/src/auth_status.rs +++ b/codex-rs/rmcp-client/src/auth_status.rs @@ -7,6 +7,7 @@ use codex_protocol::protocol::McpAuthStatus; use reqwest::Client; use reqwest::StatusCode; use reqwest::Url; +use reqwest::header::AUTHORIZATION; use reqwest::header::HeaderMap; use serde::Deserialize; use tracing::debug; @@ -33,12 +34,15 @@ pub async fn determine_streamable_http_auth_status( return Ok(McpAuthStatus::BearerToken); } + let default_headers = build_default_headers(http_headers, env_http_headers)?; + if default_headers.contains_key(AUTHORIZATION) { + return Ok(McpAuthStatus::BearerToken); + } + if has_oauth_tokens(server_name, url, store_mode)? { return Ok(McpAuthStatus::OAuth); } - let default_headers = build_default_headers(http_headers, env_http_headers)?; - match supports_oauth_login_with_headers(url, &default_headers).await { Ok(true) => Ok(McpAuthStatus::NotLoggedIn), Ok(false) => Ok(McpAuthStatus::Unsupported), @@ -139,3 +143,84 @@ fn discovery_paths(base_path: &str) -> Vec { candidates } + +#[cfg(test)] +mod tests { + use super::*; + use pretty_assertions::assert_eq; + use serial_test::serial; + use std::collections::HashMap; + use std::ffi::OsString; + + struct EnvVarGuard { + key: String, + original: Option, + } + + impl EnvVarGuard { + fn set(key: &str, value: &str) -> Self { + let original = std::env::var_os(key); + unsafe { + std::env::set_var(key, value); + } + Self { + key: key.to_string(), + original, + } + } + } + + impl Drop for EnvVarGuard { + fn drop(&mut self) { + if let Some(value) = &self.original { + unsafe { + std::env::set_var(&self.key, value); + } + } else { + unsafe { + std::env::remove_var(&self.key); + } + } + } + } + + #[tokio::test] + async fn determine_auth_status_uses_bearer_token_when_authorization_header_present() { + let status = determine_streamable_http_auth_status( + "server", + "not-a-url", + None, + Some(HashMap::from([( + "Authorization".to_string(), + "Bearer token".to_string(), + )])), + None, + OAuthCredentialsStoreMode::Keyring, + ) + .await + .expect("status should compute"); + + assert_eq!(status, McpAuthStatus::BearerToken); + } + + #[tokio::test] + #[serial(auth_status_env)] + async fn determine_auth_status_uses_bearer_token_when_env_authorization_header_present() { + let _guard = EnvVarGuard::set("CODEX_RMCP_CLIENT_AUTH_STATUS_TEST_TOKEN", "Bearer token"); + let status = determine_streamable_http_auth_status( + "server", + "not-a-url", + None, + None, + Some(HashMap::from([( + "Authorization".to_string(), + "CODEX_RMCP_CLIENT_AUTH_STATUS_TEST_TOKEN".to_string(), + )])), + OAuthCredentialsStoreMode::Keyring, + ) + .await + .expect("status should compute"); + + assert_eq!(status, McpAuthStatus::BearerToken); + } +} diff --git a/codex-rs/rmcp-client/src/rmcp_client.rs b/codex-rs/rmcp-client/src/rmcp_client.rs index 0deccf3919..e1d7045968 100644 --- a/codex-rs/rmcp-client/src/rmcp_client.rs +++ b/codex-rs/rmcp-client/src/rmcp_client.rs @@ -11,6 +11,7 @@ use anyhow::anyhow; use futures::FutureExt; use futures::future::BoxFuture; use oauth2::TokenResponse; +use reqwest::header::AUTHORIZATION; use reqwest::header::HeaderMap; use rmcp::model::CallToolRequestParams; use rmcp::model::CallToolResult; @@ -244,16 +245,18 @@ impl RmcpClient { ) -> Result { let default_headers = build_default_headers(http_headers, env_http_headers)?; - let initial_oauth_tokens = match bearer_token { - Some(_) => None, - None => match load_oauth_tokens(server_name, url, store_mode) { - Ok(tokens) => tokens, - Err(err) => { - warn!("failed to read tokens for server `{server_name}`: {err}"); - None + let initial_oauth_tokens = + if bearer_token.is_none() && !default_headers.contains_key(AUTHORIZATION) { + match load_oauth_tokens(server_name, url, store_mode) { + Ok(tokens) => tokens, + Err(err) => { + warn!("failed to read tokens for server `{server_name}`: {err}"); + None + } } - }, - }; + } else { + None + }; let transport = if let Some(initial_tokens) = initial_oauth_tokens.clone() { match create_oauth_transport_and_runtime( diff --git a/codex-rs/shell-command/src/command_safety/is_dangerous_command.rs b/codex-rs/shell-command/src/command_safety/is_dangerous_command.rs index 3e2c669c44..33b12bea60 100644 --- a/codex-rs/shell-command/src/command_safety/is_dangerous_command.rs +++ b/codex-rs/shell-command/src/command_safety/is_dangerous_command.rs @@ -104,25 +104,6 @@ fn is_dangerous_to_call_with_exec(command: &[String]) -> bool { let cmd0 = command.first().map(String::as_str); match cmd0 { - Some(cmd) if cmd.ends_with("git") => { - let Some((subcommand_idx, subcommand)) = - find_git_subcommand(command, &["reset", "rm", "branch", "push", "clean"]) - else { - return false; - }; - - match subcommand { - "reset" | "rm" => true, - "branch" => git_branch_is_delete(&command[subcommand_idx + 1..]), - "push" => git_push_is_dangerous(&command[subcommand_idx + 1..]), - "clean" => git_clean_is_force(&command[subcommand_idx + 1..]), - other => { - debug_assert!(false, "unexpected git subcommand from matcher: {other}"); - false - } - } - } - Some("rm") => matches!(command.get(1).map(String::as_str), Some("-f" | "-rf")), // for sudo simply do the check for @@ -133,48 +114,6 @@ fn is_dangerous_to_call_with_exec(command: &[String]) -> bool { } } -fn git_branch_is_delete(branch_args: &[String]) -> bool { - // Git allows stacking short flags (for example, `-dv` or `-vd`). Treat any - // short-flag group containing `d`/`D` as a delete flag. - branch_args.iter().map(String::as_str).any(|arg| { - matches!(arg, "-d" | "-D" | "--delete") - || arg.starts_with("--delete=") - || short_flag_group_contains(arg, 'd') - || short_flag_group_contains(arg, 'D') - }) -} - -fn short_flag_group_contains(arg: &str, target: char) -> bool { - arg.starts_with('-') && !arg.starts_with("--") && arg.chars().skip(1).any(|c| c == target) -} - -fn git_push_is_dangerous(push_args: &[String]) -> bool { - push_args.iter().map(String::as_str).any(|arg| { - matches!( - arg, - "--force" | "--force-with-lease" | "--force-if-includes" | "--delete" | "-f" | "-d" - ) || arg.starts_with("--force-with-lease=") - || arg.starts_with("--force-if-includes=") - || arg.starts_with("--delete=") - || short_flag_group_contains(arg, 'f') - || short_flag_group_contains(arg, 'd') - || git_push_refspec_is_dangerous(arg) - }) -} - -fn git_push_refspec_is_dangerous(arg: &str) -> bool { - // `+` forces updates and `:` deletes remote refs. - (arg.starts_with('+') || arg.starts_with(':')) && arg.len() > 1 -} - -fn git_clean_is_force(clean_args: &[String]) -> bool { - clean_args.iter().map(String::as_str).any(|arg| { - matches!(arg, "--force" | "-f") - || arg.starts_with("--force=") - || short_flag_group_contains(arg, 'f') - }) -} - #[cfg(test)] mod tests { use super::*; @@ -183,193 +122,6 @@ mod tests { items.iter().map(std::string::ToString::to_string).collect() } - #[test] - fn git_reset_is_dangerous() { - assert!(command_might_be_dangerous(&vec_str(&["git", "reset"]))); - } - - #[test] - fn bash_git_reset_is_dangerous() { - assert!(command_might_be_dangerous(&vec_str(&[ - "bash", - "-lc", - "git reset --hard", - ]))); - } - - #[test] - fn zsh_git_reset_is_dangerous() { - assert!(command_might_be_dangerous(&vec_str(&[ - "zsh", - "-lc", - "git reset --hard", - ]))); - } - - #[test] - fn git_status_is_not_dangerous() { - assert!(!command_might_be_dangerous(&vec_str(&["git", "status"]))); - } - - #[test] - fn bash_git_status_is_not_dangerous() { - assert!(!command_might_be_dangerous(&vec_str(&[ - "bash", - "-lc", - "git status", - ]))); - } - - #[test] - fn sudo_git_reset_is_dangerous() { - assert!(command_might_be_dangerous(&vec_str(&[ - "sudo", "git", "reset", "--hard", - ]))); - } - - #[test] - fn usr_bin_git_is_dangerous() { - assert!(command_might_be_dangerous(&vec_str(&[ - "/usr/bin/git", - "reset", - "--hard", - ]))); - } - - #[test] - fn git_branch_delete_is_dangerous() { - assert!(command_might_be_dangerous(&vec_str(&[ - "git", "branch", "-d", "feature", - ]))); - assert!(command_might_be_dangerous(&vec_str(&[ - "git", "branch", "-D", "feature", - ]))); - assert!(command_might_be_dangerous(&vec_str(&[ - "bash", - "-lc", - "git branch --delete feature", - ]))); - } - - #[test] - fn git_branch_delete_with_stacked_short_flags_is_dangerous() { - assert!(command_might_be_dangerous(&vec_str(&[ - "git", "branch", "-dv", "feature", - ]))); - assert!(command_might_be_dangerous(&vec_str(&[ - "git", "branch", "-vd", "feature", - ]))); - assert!(command_might_be_dangerous(&vec_str(&[ - "git", "branch", "-vD", "feature", - ]))); - assert!(command_might_be_dangerous(&vec_str(&[ - "git", "branch", "-Dvv", "feature", - ]))); - } - - #[test] - fn git_branch_delete_with_global_options_is_dangerous() { - assert!(command_might_be_dangerous(&vec_str(&[ - "git", "-C", ".", "branch", "-d", "feature", - ]))); - assert!(command_might_be_dangerous(&vec_str(&[ - "git", - "-c", - "color.ui=false", - "branch", - "-D", - "feature", - ]))); - assert!(command_might_be_dangerous(&vec_str(&[ - "bash", - "-lc", - "git -C . branch -d feature", - ]))); - } - - #[test] - fn git_checkout_reset_is_not_dangerous() { - // The first non-option token is "checkout", so later positional args - // like branch names must not be treated as subcommands. - assert!(!command_might_be_dangerous(&vec_str(&[ - "git", "checkout", "reset", - ]))); - } - - #[test] - fn git_push_force_is_dangerous() { - assert!(command_might_be_dangerous(&vec_str(&[ - "git", "push", "--force", "origin", "main", - ]))); - assert!(command_might_be_dangerous(&vec_str(&[ - "git", "push", "-f", "origin", "main", - ]))); - assert!(command_might_be_dangerous(&vec_str(&[ - "git", - "-C", - ".", - "push", - "--force-with-lease", - "origin", - "main", - ]))); - } - - #[test] - fn git_push_plus_refspec_is_dangerous() { - assert!(command_might_be_dangerous(&vec_str(&[ - "git", "push", "origin", "+main", - ]))); - assert!(command_might_be_dangerous(&vec_str(&[ - "git", - "push", - "origin", - "+refs/heads/main:refs/heads/main", - ]))); - } - - #[test] - fn git_push_delete_flag_is_dangerous() { - assert!(command_might_be_dangerous(&vec_str(&[ - "git", "push", "--delete", "origin", "feature", - ]))); - assert!(command_might_be_dangerous(&vec_str(&[ - "git", "push", "-d", "origin", "feature", - ]))); - } - - #[test] - fn git_push_delete_refspec_is_dangerous() { - assert!(command_might_be_dangerous(&vec_str(&[ - "git", "push", "origin", ":feature", - ]))); - assert!(command_might_be_dangerous(&vec_str(&[ - "bash", - "-lc", - "git push origin :feature", - ]))); - } - - #[test] - fn git_push_without_force_is_not_dangerous() { - assert!(!command_might_be_dangerous(&vec_str(&[ - "git", "push", "origin", "main", - ]))); - } - - #[test] - fn git_clean_force_is_dangerous_even_when_f_is_not_first_flag() { - assert!(command_might_be_dangerous(&vec_str(&[ - "git", "clean", "-fdx", - ]))); - assert!(command_might_be_dangerous(&vec_str(&[ - "git", "clean", "-xdf", - ]))); - assert!(command_might_be_dangerous(&vec_str(&[ - "git", "clean", "--force", - ]))); - } - #[test] fn rm_rf_is_dangerous() { assert!(command_might_be_dangerous(&vec_str(&["rm", "-rf", "/"]))); diff --git a/codex-rs/state/migrations/0009_stage1_outputs_rollout_slug.sql b/codex-rs/state/migrations/0009_stage1_outputs_rollout_slug.sql new file mode 100644 index 0000000000..9b3a1e077d --- /dev/null +++ b/codex-rs/state/migrations/0009_stage1_outputs_rollout_slug.sql @@ -0,0 +1,2 @@ +ALTER TABLE stage1_outputs +ADD COLUMN rollout_slug TEXT; diff --git a/codex-rs/state/migrations/0010_logs_process_id.sql b/codex-rs/state/migrations/0010_logs_process_id.sql new file mode 100644 index 0000000000..fdefcd53a8 --- /dev/null +++ b/codex-rs/state/migrations/0010_logs_process_id.sql @@ -0,0 +1,3 @@ +ALTER TABLE logs ADD COLUMN process_uuid TEXT; + +CREATE INDEX idx_logs_process_uuid ON logs(process_uuid); diff --git a/codex-rs/state/src/log_db.rs b/codex-rs/state/src/log_db.rs index 345e90c6a8..64deeef2ad 100644 --- a/codex-rs/state/src/log_db.rs +++ b/codex-rs/state/src/log_db.rs @@ -20,6 +20,7 @@ use chrono::Duration as ChronoDuration; use chrono::Utc; +use std::sync::OnceLock; use std::time::Duration; use std::time::SystemTime; use std::time::UNIX_EPOCH; @@ -33,6 +34,7 @@ use tracing::span::Id; use tracing::span::Record; use tracing_subscriber::Layer; use tracing_subscriber::registry::LookupSpan; +use uuid::Uuid; use crate::LogEntry; use crate::StateRuntime; @@ -44,14 +46,19 @@ const LOG_RETENTION_DAYS: i64 = 90; pub struct LogDbLayer { sender: mpsc::Sender, + process_uuid: String, } pub fn start(state_db: std::sync::Arc) -> LogDbLayer { + let process_uuid = current_process_log_uuid().to_string(); let (sender, receiver) = mpsc::channel(LOG_QUEUE_CAPACITY); tokio::spawn(run_inserter(std::sync::Arc::clone(&state_db), receiver)); tokio::spawn(run_retention_cleanup(state_db)); - LogDbLayer { sender } + LogDbLayer { + sender, + process_uuid, + } } impl Layer for LogDbLayer @@ -118,6 +125,7 @@ where target: metadata.target().to_string(), message: visitor.message, thread_id, + process_uuid: Some(self.process_uuid.clone()), module_path: metadata.module_path().map(ToString::to_string), file: metadata.file().map(ToString::to_string), line: metadata.line().map(|line| line as i64), @@ -196,6 +204,15 @@ where thread_id } +fn current_process_log_uuid() -> &'static str { + static PROCESS_LOG_UUID: OnceLock = OnceLock::new(); + PROCESS_LOG_UUID.get_or_init(|| { + let pid = std::process::id(); + let process_uuid = Uuid::new_v4(); + format!("pid:{pid}:{process_uuid}") + }) +} + async fn run_inserter( state_db: std::sync::Arc, mut receiver: mpsc::Receiver, diff --git a/codex-rs/state/src/model/log.rs b/codex-rs/state/src/model/log.rs index 819abb5d22..2aeaf2d5c8 100644 --- a/codex-rs/state/src/model/log.rs +++ b/codex-rs/state/src/model/log.rs @@ -9,6 +9,7 @@ pub struct LogEntry { pub target: String, pub message: Option, pub thread_id: Option, + pub process_uuid: Option, pub module_path: Option, pub file: Option, pub line: Option, @@ -23,6 +24,7 @@ pub struct LogRow { pub target: String, pub message: Option, pub thread_id: Option, + pub process_uuid: Option, pub file: Option, pub line: Option, } diff --git a/codex-rs/state/src/model/memories.rs b/codex-rs/state/src/model/memories.rs index aff702d673..813c999395 100644 --- a/codex-rs/state/src/model/memories.rs +++ b/codex-rs/state/src/model/memories.rs @@ -15,6 +15,7 @@ pub struct Stage1Output { pub source_updated_at: DateTime, pub raw_memory: String, pub rollout_summary: String, + pub rollout_slug: Option, pub cwd: PathBuf, pub generated_at: DateTime, } @@ -25,6 +26,7 @@ pub(crate) struct Stage1OutputRow { source_updated_at: i64, raw_memory: String, rollout_summary: String, + rollout_slug: Option, cwd: String, generated_at: i64, } @@ -36,6 +38,7 @@ impl Stage1OutputRow { source_updated_at: row.try_get("source_updated_at")?, raw_memory: row.try_get("raw_memory")?, rollout_summary: row.try_get("rollout_summary")?, + rollout_slug: row.try_get("rollout_slug")?, cwd: row.try_get("cwd")?, generated_at: row.try_get("generated_at")?, }) @@ -51,6 +54,7 @@ impl TryFrom for Stage1Output { source_updated_at: epoch_seconds_to_datetime(row.source_updated_at)?, raw_memory: row.raw_memory, rollout_summary: row.rollout_summary, + rollout_slug: row.rollout_slug, cwd: PathBuf::from(row.cwd), generated_at: epoch_seconds_to_datetime(row.generated_at)?, }) diff --git a/codex-rs/state/src/runtime.rs b/codex-rs/state/src/runtime.rs index 77930f8001..3a743a9c70 100644 --- a/codex-rs/state/src/runtime.rs +++ b/codex-rs/state/src/runtime.rs @@ -360,7 +360,7 @@ FROM threads } let mut builder = QueryBuilder::::new( - "INSERT INTO logs (ts, ts_nanos, level, target, message, thread_id, module_path, file, line) ", + "INSERT INTO logs (ts, ts_nanos, level, target, message, thread_id, process_uuid, module_path, file, line) ", ); builder.push_values(entries, |mut row, entry| { row.push_bind(entry.ts) @@ -369,6 +369,7 @@ FROM threads .push_bind(&entry.target) .push_bind(&entry.message) .push_bind(&entry.thread_id) + .push_bind(&entry.process_uuid) .push_bind(&entry.module_path) .push_bind(&entry.file) .push_bind(entry.line); @@ -388,7 +389,7 @@ FROM threads /// Query logs with optional filters. pub async fn query_logs(&self, query: &LogQuery) -> anyhow::Result> { let mut builder = QueryBuilder::::new( - "SELECT id, ts, ts_nanos, level, target, message, thread_id, file, line FROM logs WHERE 1 = 1", + "SELECT id, ts, ts_nanos, level, target, message, thread_id, process_uuid, file, line FROM logs WHERE 1 = 1", ); push_log_filters(&mut builder, query); if query.descending { @@ -1143,7 +1144,14 @@ WHERE id = 1 assert!( runtime - .mark_stage1_job_succeeded(thread_id, ownership_token.as_str(), 100, "raw", "sum") + .mark_stage1_job_succeeded( + thread_id, + ownership_token.as_str(), + 100, + "raw", + "sum", + None, + ) .await .expect("mark stage1 succeeded"), "stage1 success should finalize for current token" @@ -1492,6 +1500,7 @@ WHERE id = 1 up_to_date.updated_at.timestamp(), "raw", "summary", + None, ) .await .expect("mark up-to-date thread succeeded"), @@ -1715,6 +1724,7 @@ WHERE kind = 'memory_stage1' claim.thread.updated_at.timestamp(), "raw", "summary", + None, ) .await .expect("mark first-batch stage1 success"), @@ -1766,7 +1776,14 @@ WHERE kind = 'memory_stage1' }; assert!( runtime - .mark_stage1_job_succeeded(thread_id, ownership_token.as_str(), 100, "raw", "sum") + .mark_stage1_job_succeeded( + thread_id, + ownership_token.as_str(), + 100, + "raw", + "sum", + None, + ) .await .expect("mark stage1 succeeded"), "mark stage1 succeeded should write stage1_outputs" @@ -2058,6 +2075,7 @@ WHERE kind = 'memory_stage1' 100, "raw memory a", "summary a", + None, ) .await .expect("mark stage1 succeeded a"), @@ -2080,6 +2098,7 @@ WHERE kind = 'memory_stage1' 101, "raw memory b", "summary b", + Some("rollout-b"), ) .await .expect("mark stage1 succeeded b"), @@ -2093,9 +2112,11 @@ WHERE kind = 'memory_stage1' assert_eq!(outputs.len(), 2); assert_eq!(outputs[0].thread_id, thread_id_b); assert_eq!(outputs[0].rollout_summary, "summary b"); + assert_eq!(outputs[0].rollout_slug.as_deref(), Some("rollout-b")); assert_eq!(outputs[0].cwd, codex_home.join("workspace-b")); assert_eq!(outputs[1].thread_id, thread_id_a); assert_eq!(outputs[1].rollout_summary, "summary a"); + assert_eq!(outputs[1].rollout_slug, None); assert_eq!(outputs[1].cwd, codex_home.join("workspace-a")); let _ = tokio::fs::remove_dir_all(codex_home).await; @@ -2208,7 +2229,14 @@ VALUES (?, ?, ?, ?, ?) }; assert!( runtime - .mark_stage1_job_succeeded(thread_a, token_a.as_str(), 100, "raw-a", "summary-a") + .mark_stage1_job_succeeded( + thread_a, + token_a.as_str(), + 100, + "raw-a", + "summary-a", + None, + ) .await .expect("mark stage1 succeeded a"), "stage1 success should persist output for thread a" @@ -2224,7 +2252,14 @@ VALUES (?, ?, ?, ?, ?) }; assert!( runtime - .mark_stage1_job_succeeded(thread_b, token_b.as_str(), 101, "raw-b", "summary-b") + .mark_stage1_job_succeeded( + thread_b, + token_b.as_str(), + 101, + "raw-b", + "summary-b", + None, + ) .await .expect("mark stage1 succeeded b"), "stage1 success should persist output for thread b" diff --git a/codex-rs/state/src/runtime/memories.rs b/codex-rs/state/src/runtime/memories.rs index 4d1f6ffd59..c2a7f1fe91 100644 --- a/codex-rs/state/src/runtime/memories.rs +++ b/codex-rs/state/src/runtime/memories.rs @@ -191,7 +191,13 @@ LEFT JOIN jobs let rows = sqlx::query( r#" -SELECT so.thread_id, so.source_updated_at, so.raw_memory, so.rollout_summary, so.generated_at +SELECT + so.thread_id, + so.source_updated_at, + so.raw_memory, + so.rollout_summary, + so.rollout_slug, + so.generated_at , COALESCE(t.cwd, '') AS cwd FROM stage1_outputs AS so LEFT JOIN threads AS t @@ -407,6 +413,7 @@ WHERE kind = ? AND job_key = ? /// - sets `status='done'` and `last_success_watermark = input_watermark` /// - upserts `stage1_outputs` for the thread, replacing existing output only /// when `source_updated_at` is newer or equal + /// - persists optional `rollout_slug` for rollout summary artifact naming /// - enqueues/advances the global phase-2 job watermark using /// `source_updated_at` pub async fn mark_stage1_job_succeeded( @@ -416,6 +423,7 @@ WHERE kind = ? AND job_key = ? source_updated_at: i64, raw_memory: &str, rollout_summary: &str, + rollout_slug: Option<&str>, ) -> anyhow::Result { let now = Utc::now().timestamp(); let thread_id = thread_id.to_string(); @@ -454,12 +462,14 @@ INSERT INTO stage1_outputs ( source_updated_at, raw_memory, rollout_summary, + rollout_slug, generated_at -) VALUES (?, ?, ?, ?, ?) +) VALUES (?, ?, ?, ?, ?, ?) ON CONFLICT(thread_id) DO UPDATE SET source_updated_at = excluded.source_updated_at, raw_memory = excluded.raw_memory, rollout_summary = excluded.rollout_summary, + rollout_slug = excluded.rollout_slug, generated_at = excluded.generated_at WHERE excluded.source_updated_at >= stage1_outputs.source_updated_at "#, @@ -468,6 +478,7 @@ WHERE excluded.source_updated_at >= stage1_outputs.source_updated_at .bind(source_updated_at) .bind(raw_memory) .bind(rollout_summary) + .bind(rollout_slug) .bind(now) .execute(&mut *tx) .await?; diff --git a/codex-rs/tui/Cargo.toml b/codex-rs/tui/Cargo.toml index 9f268fbd49..7c35e534b6 100644 --- a/codex-rs/tui/Cargo.toml +++ b/codex-rs/tui/Cargo.toml @@ -15,8 +15,6 @@ path = "src/lib.rs" [features] # Enable vt100-based tests (emulator) when running with `--features vt100-tests`. vt100-tests = [] -# Gate verbose debug logging inside the TUI implementation. -debug-logs = [] [lints] workspace = true @@ -46,6 +44,7 @@ codex-utils-elapsed = { workspace = true } codex-utils-fuzzy-match = { workspace = true } codex-utils-oss = { workspace = true } codex-utils-sandbox-summary = { workspace = true } +codex-utils-sleep-inhibitor = { workspace = true } color-eyre = { workspace = true } crossterm = { workspace = true, features = ["bracketed-paste", "event-stream"] } derive_more = { workspace = true, features = ["is_variant"] } diff --git a/codex-rs/tui/src/app.rs b/codex-rs/tui/src/app.rs index e30d490bbb..7bd105695d 100644 --- a/codex-rs/tui/src/app.rs +++ b/codex-rs/tui/src/app.rs @@ -3,8 +3,6 @@ use crate::app_event::AppEvent; use crate::app_event::ExitMode; #[cfg(target_os = "windows")] use crate::app_event::WindowsSandboxEnableMode; -#[cfg(target_os = "windows")] -use crate::app_event::WindowsSandboxFallbackReason; use crate::app_event_sender::AppEventSender; use crate::bottom_pane::ApprovalRequest; use crate::bottom_pane::FeedbackAudience; @@ -365,6 +363,13 @@ fn should_show_model_migration_prompt( return false; } + if !available_models + .iter() + .any(|preset| preset.model == target_model && preset.show_in_picker) + { + return false; + } + if available_models .iter() .any(|preset| preset.model == current_model && preset.upgrade.is_some()) @@ -401,7 +406,7 @@ fn target_preset_for_upgrade<'a>( ) -> Option<&'a ModelPreset> { available_models .iter() - .find(|preset| preset.model == target_model) + .find(|preset| preset.model == target_model && preset.show_in_picker) } async fn handle_model_migration_prompt_if_needed( @@ -551,9 +556,21 @@ pub(crate) struct App { /// Set when the user confirms an update; propagated on exit. pub(crate) pending_update_action: Option, - /// Ignore the next ShutdownComplete event when we're intentionally - /// stopping a thread (e.g., before starting a new one). + /// One-shot guard used while switching threads. + /// + /// We set this when intentionally stopping the current thread before moving + /// to another one, then ignore exactly one `ShutdownComplete` so it is not + /// misclassified as an unexpected sub-agent death. suppress_shutdown_complete: bool, + /// Tracks the thread we intentionally shut down while exiting the app. + /// + /// When this matches the active thread, its `ShutdownComplete` should lead to + /// process exit instead of being treated as an unexpected sub-agent death that + /// triggers failover to the primary thread. + /// + /// This is thread-scoped state (`Option`) instead of a global bool + /// so shutdown events from other threads still take the normal failover path. + pending_shutdown_exit_thread_id: Option, windows_sandbox: WindowsSandboxState, @@ -626,6 +643,13 @@ impl App { .wrap_err_with(|| format!("Failed to rebuild config for cwd {cwd_display}")) } + async fn refresh_in_memory_config_from_disk(&mut self) -> Result<()> { + let mut config = self.rebuild_config_for_cwd(self.config.cwd.clone()).await?; + self.apply_runtime_policy_overrides(&mut config); + self.config = config; + Ok(()) + } + fn apply_runtime_policy_overrides(&mut self, config: &mut Config) { if let Some(policy) = self.runtime_approval_policy_override.as_ref() && let Err(err) = config.permissions.approval_policy.set(*policy) @@ -921,6 +945,29 @@ impl App { Ok(()) } + /// Returns `(closed_thread_id, primary_thread_id)` when a non-primary active + /// thread has died and we should fail over to the primary thread. + /// + /// A user-requested shutdown (`ExitMode::ShutdownFirst`) sets + /// `pending_shutdown_exit_thread_id`; matching shutdown completions are ignored + /// here so Ctrl+C-like exits don't accidentally resurrect the main thread. + /// + /// Failover is only eligible when all of these are true: + /// 1. the event is `ShutdownComplete`; + /// 2. the active thread differs from the primary thread; + /// 3. the active thread is not the pending shutdown-exit thread. + fn active_non_primary_shutdown_target(&self, msg: &EventMsg) -> Option<(ThreadId, ThreadId)> { + if !matches!(msg, EventMsg::ShutdownComplete) { + return None; + } + let active_thread_id = self.active_thread_id?; + let primary_thread_id = self.primary_thread_id?; + if self.pending_shutdown_exit_thread_id == Some(active_thread_id) { + return None; + } + (active_thread_id != primary_thread_id).then_some((active_thread_id, primary_thread_id)) + } + fn replay_thread_snapshot(&mut self, snapshot: ThreadEventSnapshot) { if let Some(event) = snapshot.session_configured { self.handle_codex_event_replay(event); @@ -1141,6 +1188,7 @@ impl App { feedback_audience, pending_update_action: None, suppress_shutdown_complete: false, + pending_shutdown_exit_thread_id: None, windows_sandbox: WindowsSandboxState::default(), thread_event_channels: HashMap::new(), active_thread_id: None, @@ -1218,7 +1266,7 @@ impl App { } }, if app.active_thread_rx.is_some() => { if let Some(event) = active { - app.handle_active_thread_event(tui, event)?; + app.handle_active_thread_event(tui, event).await?; } else { app.clear_active_thread().await; } @@ -1570,8 +1618,15 @@ impl App { self.enqueue_primary_event(event).await?; } AppEvent::Exit(mode) => match mode { - ExitMode::ShutdownFirst => self.chat_widget.submit_op(Op::Shutdown), + ExitMode::ShutdownFirst => { + // Mark the thread we are explicitly shutting down for exit so + // its shutdown completion does not trigger agent failover. + self.pending_shutdown_exit_thread_id = + self.active_thread_id.or(self.chat_widget.thread_id()); + self.chat_widget.submit_op(Op::Shutdown); + } ExitMode::Immediate => { + self.pending_shutdown_exit_thread_id = None; return Ok(AppRunControl::Exit(ExitReason::UserRequested)); } }, @@ -1598,19 +1653,24 @@ impl App { tui.frame_requester().schedule_frame(); } AppEvent::OpenAppLink { + app_id, title, description, instructions, url, is_installed, + is_enabled, } => { - self.chat_widget.open_app_link_view( - title, - description, - instructions, - url, - is_installed, - ); + self.chat_widget + .open_app_link_view(crate::bottom_pane::AppLinkViewParams { + app_id, + title, + description, + instructions, + url, + is_installed, + is_enabled, + }); } AppEvent::OpenUrlInBrowser { url } => { self.open_url_in_browser(url); @@ -1688,7 +1748,7 @@ impl App { AppEvent::OpenWindowsSandboxEnablePrompt { preset } => { self.chat_widget.open_windows_sandbox_enable_prompt(preset); } - AppEvent::OpenWindowsSandboxFallbackPrompt { preset, reason } => { + AppEvent::OpenWindowsSandboxFallbackPrompt { preset } => { self.otel_manager .counter("codex.windows_sandbox.fallback_prompt_shown", 1, &[]); self.chat_widget.clear_windows_sandbox_setup_status(); @@ -1700,7 +1760,7 @@ impl App { ); } self.chat_widget - .open_windows_sandbox_fallback_prompt(preset, reason); + .open_windows_sandbox_fallback_prompt(preset); } AppEvent::BeginWindowsSandboxElevatedSetup { preset } => { #[cfg(target_os = "windows")] @@ -1776,10 +1836,7 @@ impl App { error = %err, "failed to run elevated Windows sandbox setup" ); - AppEvent::OpenWindowsSandboxFallbackPrompt { - preset, - reason: WindowsSandboxFallbackReason::ElevationFailed, - } + AppEvent::OpenWindowsSandboxFallbackPrompt { preset } } }; tx.send(event); @@ -1800,6 +1857,7 @@ impl App { std::env::vars().collect(); let codex_home = self.config.codex_home.clone(); let tx = self.app_event_tx.clone(); + let otel_manager = self.otel_manager.clone(); self.chat_widget.show_windows_sandbox_setup_status(); tokio::task::spawn_blocking(move || { @@ -1810,6 +1868,11 @@ impl App { &env_map, codex_home.as_path(), ) { + otel_manager.counter( + "codex.windows_sandbox.legacy_setup_preflight_failed", + 1, + &[], + ); tracing::warn!( error = %err, "failed to preflight non-admin Windows sandbox setup" @@ -2286,6 +2349,12 @@ impl App { { Ok(()) => { self.chat_widget.update_skill_enabled(path.clone(), enabled); + if let Err(err) = self.refresh_in_memory_config_from_disk().await { + tracing::warn!( + error = %err, + "failed to refresh config after skill toggle" + ); + } } Err(err) => { let path_display = path.display(); @@ -2295,6 +2364,55 @@ impl App { } } } + AppEvent::SetAppEnabled { id, enabled } => { + let edits = if enabled { + vec![ + ConfigEdit::ClearPath { + segments: vec!["apps".to_string(), id.clone(), "enabled".to_string()], + }, + ConfigEdit::ClearPath { + segments: vec![ + "apps".to_string(), + id.clone(), + "disabled_reason".to_string(), + ], + }, + ] + } else { + vec![ + ConfigEdit::SetPath { + segments: vec!["apps".to_string(), id.clone(), "enabled".to_string()], + value: false.into(), + }, + ConfigEdit::SetPath { + segments: vec![ + "apps".to_string(), + id.clone(), + "disabled_reason".to_string(), + ], + value: "user".into(), + }, + ] + }; + match ConfigEditsBuilder::new(&self.config.codex_home) + .with_edits(edits) + .apply() + .await + { + Ok(()) => { + self.chat_widget.update_connector_enabled(&id, enabled); + if let Err(err) = self.refresh_in_memory_config_from_disk().await { + tracing::warn!(error = %err, "failed to refresh config after app toggle"); + } + self.chat_widget.submit_op(Op::ReloadUserConfig); + } + Err(err) => { + self.chat_widget.add_error_message(format!( + "Failed to update app config for {id}: {err}" + )); + } + } + } AppEvent::OpenPermissionsPopup => { self.chat_widget.open_permissions_popup(); } @@ -2392,6 +2510,9 @@ impl App { event.msg, EventMsg::SessionConfigured(_) | EventMsg::TokenCount(_) ); + // This guard is only for intentional thread-switch shutdowns. + // App-exit shutdowns are tracked by `pending_shutdown_exit_thread_id` + // and resolved in `handle_active_thread_event`. if self.suppress_shutdown_complete && matches!(event.msg, EventMsg::ShutdownComplete) { self.suppress_shutdown_complete = false; return; @@ -2413,7 +2534,51 @@ impl App { self.chat_widget.handle_codex_event_replay(event); } - fn handle_active_thread_event(&mut self, tui: &mut tui::Tui, event: Event) -> Result<()> { + /// Handles an event emitted by the currently active thread. + /// + /// This function enforces shutdown intent routing: unexpected non-primary + /// thread shutdowns fail over to the primary thread, while user-requested + /// app exits consume only the tracked shutdown completion and then proceed. + async fn handle_active_thread_event(&mut self, tui: &mut tui::Tui, event: Event) -> Result<()> { + // Capture this before any potential thread switch: we only want to clear + // the exit marker when the currently active thread acknowledges shutdown. + let pending_shutdown_exit_completed = matches!(&event.msg, EventMsg::ShutdownComplete) + && self.pending_shutdown_exit_thread_id == self.active_thread_id; + + // Processing order matters: + // + // 1. handle unexpected non-primary shutdown failover first; + // 2. clear pending exit marker for matching shutdown; + // 3. forward the event through normal handling. + // + // This preserves the mental model that user-requested exits do not trigger + // failover, while true sub-agent deaths still do. + if let Some((closed_thread_id, primary_thread_id)) = + self.active_non_primary_shutdown_target(&event.msg) + { + self.thread_event_channels.remove(&closed_thread_id); + self.select_agent_thread(tui, primary_thread_id).await?; + if self.active_thread_id == Some(primary_thread_id) { + self.chat_widget.add_info_message( + format!( + "Agent thread {closed_thread_id} closed. Switched back to main thread." + ), + None, + ); + } else { + self.clear_active_thread().await; + self.chat_widget.add_error_message(format!( + "Agent thread {closed_thread_id} closed. Failed to switch back to main thread {primary_thread_id}.", + )); + } + return Ok(()); + } + + if pending_shutdown_exit_completed { + // Clear only after seeing the shutdown completion for the tracked + // thread, so unrelated shutdowns cannot consume this marker. + self.pending_shutdown_exit_thread_id = None; + } self.handle_codex_event_now(event); if self.backtrack_render_pending { tui.frame_requester().schedule_frame(); @@ -2697,6 +2862,7 @@ impl App { #[cfg(test)] mod tests { use super::*; + use crate::app_backtrack::BacktrackSelection; use crate::app_backtrack::BacktrackState; use crate::app_backtrack::user_count; use crate::chatwidget::tests::make_chatwidget_manual_with_sender; @@ -2719,6 +2885,8 @@ mod tests { use codex_otel::OtelManager; use codex_protocol::ThreadId; use codex_protocol::user_input::TextElement; + use codex_protocol::user_input::UserInput; + use crossterm::event::KeyModifiers; use insta::assert_snapshot; use pretty_assertions::assert_eq; use ratatui::prelude::Line; @@ -2801,6 +2969,85 @@ mod tests { Ok(()) } + #[tokio::test] + async fn active_non_primary_shutdown_target_returns_none_for_non_shutdown_event() -> Result<()> + { + let mut app = make_test_app().await; + app.active_thread_id = Some(ThreadId::new()); + app.primary_thread_id = Some(ThreadId::new()); + + assert_eq!( + app.active_non_primary_shutdown_target(&EventMsg::SkillsUpdateAvailable), + None + ); + Ok(()) + } + + #[tokio::test] + async fn active_non_primary_shutdown_target_returns_none_for_primary_thread_shutdown() + -> Result<()> { + let mut app = make_test_app().await; + let thread_id = ThreadId::new(); + app.active_thread_id = Some(thread_id); + app.primary_thread_id = Some(thread_id); + + assert_eq!( + app.active_non_primary_shutdown_target(&EventMsg::ShutdownComplete), + None + ); + Ok(()) + } + + #[tokio::test] + async fn active_non_primary_shutdown_target_returns_ids_for_non_primary_shutdown() -> Result<()> + { + let mut app = make_test_app().await; + let active_thread_id = ThreadId::new(); + let primary_thread_id = ThreadId::new(); + app.active_thread_id = Some(active_thread_id); + app.primary_thread_id = Some(primary_thread_id); + + assert_eq!( + app.active_non_primary_shutdown_target(&EventMsg::ShutdownComplete), + Some((active_thread_id, primary_thread_id)) + ); + Ok(()) + } + + #[tokio::test] + async fn active_non_primary_shutdown_target_returns_none_when_shutdown_exit_is_pending() + -> Result<()> { + let mut app = make_test_app().await; + let active_thread_id = ThreadId::new(); + let primary_thread_id = ThreadId::new(); + app.active_thread_id = Some(active_thread_id); + app.primary_thread_id = Some(primary_thread_id); + app.pending_shutdown_exit_thread_id = Some(active_thread_id); + + assert_eq!( + app.active_non_primary_shutdown_target(&EventMsg::ShutdownComplete), + None + ); + Ok(()) + } + + #[tokio::test] + async fn active_non_primary_shutdown_target_still_switches_for_other_pending_exit_thread() + -> Result<()> { + let mut app = make_test_app().await; + let active_thread_id = ThreadId::new(); + let primary_thread_id = ThreadId::new(); + app.active_thread_id = Some(active_thread_id); + app.primary_thread_id = Some(primary_thread_id); + app.pending_shutdown_exit_thread_id = Some(ThreadId::new()); + + assert_eq!( + app.active_non_primary_shutdown_target(&EventMsg::ShutdownComplete), + Some((active_thread_id, primary_thread_id)) + ); + Ok(()) + } + async fn make_test_app() -> App { let (chat_widget, app_event_tx, _rx, _op_rx) = make_chatwidget_manual_with_sender().await; let config = chat_widget.config_ref().clone(); @@ -2843,6 +3090,7 @@ mod tests { feedback_audience: FeedbackAudience::External, pending_update_action: None, suppress_shutdown_complete: false, + pending_shutdown_exit_thread_id: None, windows_sandbox: WindowsSandboxState::default(), thread_event_channels: HashMap::new(), active_thread_id: None, @@ -2900,6 +3148,7 @@ mod tests { feedback_audience: FeedbackAudience::External, pending_update_action: None, suppress_shutdown_complete: false, + pending_shutdown_exit_thread_id: None, windows_sandbox: WindowsSandboxState::default(), thread_event_channels: HashMap::new(), active_thread_id: None, @@ -2929,6 +3178,19 @@ mod tests { ) } + fn app_enabled_in_effective_config(config: &Config, app_id: &str) -> Option { + config + .config_layer_stack + .effective_config() + .as_table() + .and_then(|table| table.get("apps")) + .and_then(TomlValue::as_table) + .and_then(|apps| apps.get(app_id)) + .and_then(TomlValue::as_table) + .and_then(|app| app.get("enabled")) + .and_then(TomlValue::as_bool) + } + fn all_model_presets() -> Vec { codex_core::test_support::all_model_presets().clone() } @@ -2959,25 +3221,25 @@ mod tests { let seen = BTreeMap::new(); assert!(should_show_model_migration_prompt( "gpt-5", - "gpt-5.1", + "gpt-5.2-codex", &seen, &all_model_presets() )); assert!(should_show_model_migration_prompt( "gpt-5-codex", - "gpt-5.1-codex", + "gpt-5.2-codex", &seen, &all_model_presets() )); assert!(should_show_model_migration_prompt( "gpt-5-codex-mini", - "gpt-5.1-codex-mini", + "gpt-5.2-codex", &seen, &all_model_presets() )); assert!(should_show_model_migration_prompt( "gpt-5.1-codex", - "gpt-5.1-codex-max", + "gpt-5.2-codex", &seen, &all_model_presets() )); @@ -3008,7 +3270,7 @@ mod tests { } #[tokio::test] - async fn model_migration_prompt_skips_when_target_missing() { + async fn model_migration_prompt_skips_when_target_missing_or_hidden() { let mut available = all_model_presets(); let mut current = available .iter() @@ -3026,7 +3288,7 @@ mod tests { available.retain(|preset| preset.model != "gpt-5-codex"); available.push(current.clone()); - assert!(should_show_model_migration_prompt( + assert!(!should_show_model_migration_prompt( ¤t.model, "missing-target", &BTreeMap::new(), @@ -3034,6 +3296,21 @@ mod tests { )); assert!(target_preset_for_upgrade(&available, "missing-target").is_none()); + + let mut with_hidden_target = all_model_presets(); + let target = with_hidden_target + .iter_mut() + .find(|preset| preset.model == "gpt-5.2-codex") + .expect("target preset present"); + target.show_in_picker = false; + + assert!(!should_show_model_migration_prompt( + "gpt-5-codex", + "gpt-5.2-codex", + &BTreeMap::new(), + &with_hidden_target, + )); + assert!(target_preset_for_upgrade(&with_hidden_target, "gpt-5.2-codex").is_none()); } #[tokio::test] @@ -3108,18 +3385,59 @@ mod tests { ); } + #[tokio::test] + async fn refresh_in_memory_config_from_disk_loads_latest_apps_state() -> Result<()> { + let mut app = make_test_app().await; + let codex_home = tempdir()?; + app.config.codex_home = codex_home.path().to_path_buf(); + let app_id = "unit_test_refresh_in_memory_config_connector".to_string(); + + assert_eq!(app_enabled_in_effective_config(&app.config, &app_id), None); + + ConfigEditsBuilder::new(&app.config.codex_home) + .with_edits([ + ConfigEdit::SetPath { + segments: vec!["apps".to_string(), app_id.clone(), "enabled".to_string()], + value: false.into(), + }, + ConfigEdit::SetPath { + segments: vec![ + "apps".to_string(), + app_id.clone(), + "disabled_reason".to_string(), + ], + value: "user".into(), + }, + ]) + .apply() + .await + .expect("persist app toggle"); + + assert_eq!(app_enabled_in_effective_config(&app.config, &app_id), None); + + app.refresh_in_memory_config_from_disk().await?; + + assert_eq!( + app_enabled_in_effective_config(&app.config, &app_id), + Some(false) + ); + Ok(()) + } + #[tokio::test] async fn backtrack_selection_with_duplicate_history_targets_unique_turn() { let (mut app, _app_event_rx, mut op_rx) = make_test_app_with_channels().await; let user_cell = |text: &str, text_elements: Vec, - local_image_paths: Vec| + local_image_paths: Vec, + remote_image_urls: Vec| -> Arc { Arc::new(UserHistoryCell { message: text.to_string(), text_elements, local_image_paths, + remote_image_urls, }) as Arc }; let agent_cell = |text: &str| -> Arc { @@ -3165,17 +3483,18 @@ mod tests { // and an edited turn appended after a session header boundary. app.transcript_cells = vec![ make_header(true), - user_cell("first question", Vec::new(), Vec::new()), + user_cell("first question", Vec::new(), Vec::new(), Vec::new()), agent_cell("answer first"), - user_cell("follow-up", Vec::new(), Vec::new()), + user_cell("follow-up", Vec::new(), Vec::new(), Vec::new()), agent_cell("answer follow-up"), make_header(false), - user_cell("first question", Vec::new(), Vec::new()), + user_cell("first question", Vec::new(), Vec::new(), Vec::new()), agent_cell("answer first"), user_cell( &edited_text, edited_text_elements.clone(), edited_local_image_paths.clone(), + vec!["https://example.com/backtrack.png".to_string()], ), agent_cell("answer edited"), ]; @@ -3214,8 +3533,16 @@ mod tests { assert_eq!(selection.prefill, edited_text); assert_eq!(selection.text_elements, edited_text_elements); assert_eq!(selection.local_image_paths, edited_local_image_paths); + assert_eq!( + selection.remote_image_urls, + vec!["https://example.com/backtrack.png".to_string()] + ); app.apply_backtrack_rollback(selection); + assert_eq!( + app.chat_widget.remote_image_urls(), + vec!["https://example.com/backtrack.png".to_string()] + ); let mut rollback_turns = None; while let Ok(op) = op_rx.try_recv() { @@ -3227,6 +3554,104 @@ mod tests { assert_eq!(rollback_turns, Some(1)); } + #[tokio::test] + async fn backtrack_remote_image_only_selection_clears_existing_composer_draft() { + let (mut app, _app_event_rx, mut op_rx) = make_test_app_with_channels().await; + + app.transcript_cells = vec![Arc::new(UserHistoryCell { + message: "original".to_string(), + text_elements: Vec::new(), + local_image_paths: Vec::new(), + remote_image_urls: Vec::new(), + }) as Arc]; + app.chat_widget + .set_composer_text("stale draft".to_string(), Vec::new(), Vec::new()); + + let remote_image_url = "https://example.com/remote-only.png".to_string(); + app.apply_backtrack_rollback(BacktrackSelection { + nth_user_message: 0, + prefill: String::new(), + text_elements: Vec::new(), + local_image_paths: Vec::new(), + remote_image_urls: vec![remote_image_url.clone()], + }); + + assert_eq!(app.chat_widget.composer_text_with_pending(), ""); + assert_eq!(app.chat_widget.remote_image_urls(), vec![remote_image_url]); + + let mut rollback_turns = None; + while let Ok(op) = op_rx.try_recv() { + if let Op::ThreadRollback { num_turns } = op { + rollback_turns = Some(num_turns); + } + } + assert_eq!(rollback_turns, Some(1)); + } + + #[tokio::test] + async fn backtrack_resubmit_preserves_data_image_urls_in_user_turn() { + let (mut app, _app_event_rx, mut op_rx) = make_test_app_with_channels().await; + + let thread_id = ThreadId::new(); + app.chat_widget.handle_codex_event(Event { + id: String::new(), + msg: EventMsg::SessionConfigured(SessionConfiguredEvent { + session_id: thread_id, + forked_from_id: None, + thread_name: None, + model: "gpt-test".to_string(), + model_provider_id: "test-provider".to_string(), + approval_policy: AskForApproval::Never, + sandbox_policy: SandboxPolicy::new_read_only_policy(), + cwd: PathBuf::from("/home/user/project"), + reasoning_effort: None, + history_log_id: 0, + history_entry_count: 0, + initial_messages: None, + network_proxy: None, + rollout_path: Some(PathBuf::new()), + }), + }); + + let data_image_url = "data:image/png;base64,abc123".to_string(); + app.transcript_cells = vec![Arc::new(UserHistoryCell { + message: "please inspect this".to_string(), + text_elements: Vec::new(), + local_image_paths: Vec::new(), + remote_image_urls: vec![data_image_url.clone()], + }) as Arc]; + + app.apply_backtrack_rollback(BacktrackSelection { + nth_user_message: 0, + prefill: "please inspect this".to_string(), + text_elements: Vec::new(), + local_image_paths: Vec::new(), + remote_image_urls: vec![data_image_url.clone()], + }); + + app.chat_widget + .handle_key_event(KeyEvent::new(KeyCode::Enter, KeyModifiers::NONE)); + + let mut saw_rollback = false; + let mut submitted_items: Option> = None; + while let Ok(op) = op_rx.try_recv() { + match op { + Op::ThreadRollback { .. } => saw_rollback = true, + Op::UserTurn { items, .. } => submitted_items = Some(items), + _ => {} + } + } + + assert!(saw_rollback); + let items = submitted_items.expect("expected user turn after backtrack resubmit"); + assert!(items.iter().any(|item| { + matches!( + item, + UserInput::Image { image_url } if image_url == &data_image_url + ) + })); + } + #[tokio::test] async fn replayed_initial_messages_apply_rollback_in_queue_order() { let (mut app, mut app_event_rx, _op_rx) = make_test_app_with_channels().await; @@ -3389,6 +3814,7 @@ mod tests { message: "first".to_string(), text_elements: Vec::new(), local_image_paths: Vec::new(), + remote_image_urls: Vec::new(), }) as Arc, Arc::new(AgentMessageCell::new( vec![Line::from("after first")], @@ -3398,6 +3824,7 @@ mod tests { message: "second".to_string(), text_elements: Vec::new(), local_image_paths: Vec::new(), + remote_image_urls: Vec::new(), }) as Arc, Arc::new(AgentMessageCell::new( vec![Line::from("after second")], diff --git a/codex-rs/tui/src/app_backtrack.rs b/codex-rs/tui/src/app_backtrack.rs index 22b437ec53..b9b5be2b5f 100644 --- a/codex-rs/tui/src/app_backtrack.rs +++ b/codex-rs/tui/src/app_backtrack.rs @@ -85,6 +85,8 @@ pub(crate) struct BacktrackSelection { pub(crate) text_elements: Vec, /// Local image paths associated with the selected user message. pub(crate) local_image_paths: Vec, + /// Remote image URLs associated with the selected user message. + pub(crate) remote_image_urls: Vec, } /// An in-flight rollback requested from core. @@ -207,12 +209,19 @@ impl App { let prefill = selection.prefill.clone(); let text_elements = selection.text_elements.clone(); let local_image_paths = selection.local_image_paths.clone(); + let remote_image_urls = selection.remote_image_urls.clone(); + let has_remote_image_urls = !remote_image_urls.is_empty(); self.backtrack.pending_rollback = Some(PendingBacktrackRollback { selection, thread_id: self.chat_widget.thread_id(), }); self.chat_widget.submit_op(Op::ThreadRollback { num_turns }); - if !prefill.is_empty() || !text_elements.is_empty() || !local_image_paths.is_empty() { + self.chat_widget.set_remote_image_urls(remote_image_urls); + if !prefill.is_empty() + || !text_elements.is_empty() + || !local_image_paths.is_empty() + || has_remote_image_urls + { self.chat_widget .set_composer_text(prefill, text_elements, local_image_paths); } @@ -523,7 +532,7 @@ impl App { return None; } - let (prefill, text_elements, local_image_paths) = + let (prefill, text_elements, local_image_paths, remote_image_urls) = nth_user_position(&self.transcript_cells, nth_user_message) .and_then(|idx| self.transcript_cells.get(idx)) .and_then(|cell| cell.as_any().downcast_ref::()) @@ -532,15 +541,17 @@ impl App { cell.message.clone(), cell.text_elements.clone(), cell.local_image_paths.clone(), + cell.remote_image_urls.clone(), ) }) - .unwrap_or_else(|| (String::new(), Vec::new(), Vec::new())); + .unwrap_or_else(|| (String::new(), Vec::new(), Vec::new(), Vec::new())); Some(BacktrackSelection { nth_user_message, prefill, text_elements, local_image_paths, + remote_image_urls, }) } @@ -659,6 +670,7 @@ mod tests { message: "first user".to_string(), text_elements: Vec::new(), local_image_paths: Vec::new(), + remote_image_urls: Vec::new(), }) as Arc, Arc::new(AgentMessageCell::new(vec![Line::from("assistant")], true)) as Arc, @@ -677,6 +689,7 @@ mod tests { message: "first".to_string(), text_elements: Vec::new(), local_image_paths: Vec::new(), + remote_image_urls: Vec::new(), }) as Arc, Arc::new(AgentMessageCell::new(vec![Line::from("after")], false)) as Arc, @@ -707,6 +720,7 @@ mod tests { message: "first".to_string(), text_elements: Vec::new(), local_image_paths: Vec::new(), + remote_image_urls: Vec::new(), }) as Arc, Arc::new(AgentMessageCell::new(vec![Line::from("between")], false)) as Arc, @@ -714,6 +728,7 @@ mod tests { message: "second".to_string(), text_elements: Vec::new(), local_image_paths: Vec::new(), + remote_image_urls: Vec::new(), }) as Arc, Arc::new(AgentMessageCell::new(vec![Line::from("tail")], false)) as Arc, @@ -759,6 +774,7 @@ mod tests { message: "first".to_string(), text_elements: Vec::new(), local_image_paths: Vec::new(), + remote_image_urls: Vec::new(), }) as Arc, Arc::new(AgentMessageCell::new( vec![Line::from("after first")], @@ -768,6 +784,7 @@ mod tests { message: "second".to_string(), text_elements: Vec::new(), local_image_paths: Vec::new(), + remote_image_urls: Vec::new(), }) as Arc, Arc::new(AgentMessageCell::new( vec![Line::from("after second")], @@ -795,6 +812,7 @@ mod tests { message: "first".to_string(), text_elements: Vec::new(), local_image_paths: Vec::new(), + remote_image_urls: Vec::new(), }) as Arc, Arc::new(AgentMessageCell::new(vec![Line::from("after")], false)) as Arc, diff --git a/codex-rs/tui/src/app_event.rs b/codex-rs/tui/src/app_event.rs index 67db6d7c8b..8bf3ab3703 100644 --- a/codex-rs/tui/src/app_event.rs +++ b/codex-rs/tui/src/app_event.rs @@ -36,13 +36,8 @@ pub(crate) enum WindowsSandboxEnableMode { Legacy, } -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -#[cfg_attr(not(target_os = "windows"), allow(dead_code))] -pub(crate) enum WindowsSandboxFallbackReason { - ElevationFailed, -} - #[derive(Debug, Clone)] +#[cfg_attr(not(target_os = "windows"), allow(dead_code))] pub(crate) struct ConnectorsSnapshot { pub(crate) connectors: Vec, } @@ -107,11 +102,13 @@ pub(crate) enum AppEvent { /// Open the app link view in the bottom pane. OpenAppLink { + app_id: String, title: String, description: Option, instructions: String, url: String, is_installed: bool, + is_enabled: bool, }, /// Open the provided URL in the user's browser. @@ -203,7 +200,6 @@ pub(crate) enum AppEvent { #[cfg_attr(not(target_os = "windows"), allow(dead_code))] OpenWindowsSandboxFallbackPrompt { preset: ApprovalPreset, - reason: WindowsSandboxFallbackReason, }, /// Begin the elevated Windows sandbox setup flow. @@ -297,6 +293,12 @@ pub(crate) enum AppEvent { enabled: bool, }, + /// Enable or disable an app by connector ID. + SetAppEnabled { + id: String, + enabled: bool, + }, + /// Notify that the manage skills popup was closed. ManageSkillsClosed, diff --git a/codex-rs/tui/src/bottom_pane/app_link_view.rs b/codex-rs/tui/src/bottom_pane/app_link_view.rs index 05ef32bfa8..698825d6f7 100644 --- a/codex-rs/tui/src/bottom_pane/app_link_view.rs +++ b/codex-rs/tui/src/bottom_pane/app_link_view.rs @@ -32,12 +32,24 @@ enum AppLinkScreen { InstallConfirmation, } +pub(crate) struct AppLinkViewParams { + pub(crate) app_id: String, + pub(crate) title: String, + pub(crate) description: Option, + pub(crate) instructions: String, + pub(crate) url: String, + pub(crate) is_installed: bool, + pub(crate) is_enabled: bool, +} + pub(crate) struct AppLinkView { + app_id: String, title: String, description: Option, instructions: String, url: String, is_installed: bool, + is_enabled: bool, app_event_tx: AppEventSender, screen: AppLinkScreen, selected_action: usize, @@ -45,20 +57,24 @@ pub(crate) struct AppLinkView { } impl AppLinkView { - pub(crate) fn new( - title: String, - description: Option, - instructions: String, - url: String, - is_installed: bool, - app_event_tx: AppEventSender, - ) -> Self { - Self { + pub(crate) fn new(params: AppLinkViewParams, app_event_tx: AppEventSender) -> Self { + let AppLinkViewParams { + app_id, title, description, instructions, url, is_installed, + is_enabled, + } = params; + Self { + app_id, + title, + description, + instructions, + url, + is_installed, + is_enabled, app_event_tx, screen: AppLinkScreen::Link, selected_action: 0, @@ -66,16 +82,24 @@ impl AppLinkView { } } - fn action_labels(&self) -> [&'static str; 2] { + fn action_labels(&self) -> Vec<&'static str> { match self.screen { AppLinkScreen::Link => { if self.is_installed { - ["Manage on ChatGPT", "Back"] + vec![ + "Manage on ChatGPT", + if self.is_enabled { + "Disable app" + } else { + "Enable app" + }, + "Back", + ] } else { - ["Install on ChatGPT", "Back"] + vec!["Install on ChatGPT", "Back"] } } - AppLinkScreen::InstallConfirmation => ["I already Installed it", "Back"], + AppLinkScreen::InstallConfirmation => vec!["I already Installed it", "Back"], } } @@ -87,42 +111,47 @@ impl AppLinkView { self.selected_action = (self.selected_action + 1).min(self.action_labels().len() - 1); } - fn handle_primary_action(&mut self) { - match self.screen { - AppLinkScreen::Link => { - self.app_event_tx.send(AppEvent::OpenUrlInBrowser { - url: self.url.clone(), - }); - if !self.is_installed { - self.screen = AppLinkScreen::InstallConfirmation; - self.selected_action = 0; - } - } - AppLinkScreen::InstallConfirmation => { - self.app_event_tx.send(AppEvent::RefreshConnectors { - force_refetch: true, - }); - self.complete = true; - } + fn open_chatgpt_link(&mut self) { + self.app_event_tx.send(AppEvent::OpenUrlInBrowser { + url: self.url.clone(), + }); + if !self.is_installed { + self.screen = AppLinkScreen::InstallConfirmation; + self.selected_action = 0; } } - fn handle_secondary_action(&mut self) { - match self.screen { - AppLinkScreen::Link => { - self.complete = true; - } - AppLinkScreen::InstallConfirmation => { - self.screen = AppLinkScreen::Link; - self.selected_action = 0; - } - } + fn refresh_connectors_and_close(&mut self) { + self.app_event_tx.send(AppEvent::RefreshConnectors { + force_refetch: true, + }); + self.complete = true; + } + + fn back_to_link_screen(&mut self) { + self.screen = AppLinkScreen::Link; + self.selected_action = 0; + } + + fn toggle_enabled(&mut self) { + self.is_enabled = !self.is_enabled; + self.app_event_tx.send(AppEvent::SetAppEnabled { + id: self.app_id.clone(), + enabled: self.is_enabled, + }); } fn activate_selected_action(&mut self) { - match self.selected_action { - 0 => self.handle_primary_action(), - _ => self.handle_secondary_action(), + match self.screen { + AppLinkScreen::Link => match self.selected_action { + 0 => self.open_chatgpt_link(), + 1 if self.is_installed => self.toggle_enabled(), + _ => self.complete = true, + }, + AppLinkScreen::InstallConfirmation => match self.selected_action { + 0 => self.refresh_connectors_and_close(), + _ => self.back_to_link_screen(), + }, } } @@ -308,20 +337,19 @@ impl BottomPaneView for AppLinkView { .. } => self.move_selection_next(), KeyEvent { - code: KeyCode::Char('1'), + code: KeyCode::Char(c), modifiers: KeyModifiers::NONE, .. } => { - self.selected_action = 0; - self.activate_selected_action(); - } - KeyEvent { - code: KeyCode::Char('2'), - modifiers: KeyModifiers::NONE, - .. - } => { - self.selected_action = 1; - self.activate_selected_action(); + if let Some(index) = c + .to_digit(10) + .and_then(|digit| digit.checked_sub(1)) + .map(|index| index as usize) + && index < self.action_labels().len() + { + self.selected_action = index; + self.activate_selected_action(); + } } KeyEvent { code: KeyCode::Enter, @@ -402,3 +430,67 @@ impl crate::render::renderable::Renderable for AppLinkView { } } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::app_event::AppEvent; + use tokio::sync::mpsc::unbounded_channel; + + #[test] + fn installed_app_has_toggle_action() { + let (tx_raw, _rx) = unbounded_channel::(); + let tx = AppEventSender::new(tx_raw); + let view = AppLinkView::new( + AppLinkViewParams { + app_id: "connector_1".to_string(), + title: "Notion".to_string(), + description: None, + instructions: "Manage app".to_string(), + url: "https://example.test/notion".to_string(), + is_installed: true, + is_enabled: true, + }, + tx, + ); + + assert_eq!( + view.action_labels(), + vec!["Manage on ChatGPT", "Disable app", "Back"] + ); + } + + #[test] + fn toggle_action_sends_set_app_enabled_and_updates_label() { + let (tx_raw, mut rx) = unbounded_channel::(); + let tx = AppEventSender::new(tx_raw); + let mut view = AppLinkView::new( + AppLinkViewParams { + app_id: "connector_1".to_string(), + title: "Notion".to_string(), + description: None, + instructions: "Manage app".to_string(), + url: "https://example.test/notion".to_string(), + is_installed: true, + is_enabled: true, + }, + tx, + ); + + view.handle_key_event(KeyEvent::new(KeyCode::Char('2'), KeyModifiers::NONE)); + + match rx.try_recv() { + Ok(AppEvent::SetAppEnabled { id, enabled }) => { + assert_eq!(id, "connector_1"); + assert!(!enabled); + } + Ok(other) => panic!("unexpected app event: {other:?}"), + Err(err) => panic!("missing app event: {err}"), + } + + assert_eq!( + view.action_labels(), + vec!["Manage on ChatGPT", "Enable app", "Back"] + ); + } +} diff --git a/codex-rs/tui/src/bottom_pane/approval_overlay.rs b/codex-rs/tui/src/bottom_pane/approval_overlay.rs index 42abe174d4..567f18f332 100644 --- a/codex-rs/tui/src/bottom_pane/approval_overlay.rs +++ b/codex-rs/tui/src/bottom_pane/approval_overlay.rs @@ -20,6 +20,7 @@ use codex_core::features::Features; use codex_core::protocol::ElicitationAction; use codex_core::protocol::ExecPolicyAmendment; use codex_core::protocol::FileChange; +use codex_core::protocol::NetworkApprovalContext; use codex_core::protocol::Op; use codex_core::protocol::ReviewDecision; use codex_protocol::mcp::RequestId; @@ -42,6 +43,7 @@ pub(crate) enum ApprovalRequest { id: String, command: Vec, reason: Option, + network_approval_context: Option, proposed_execpolicy_amendment: Option, }, ApplyPatch { @@ -108,11 +110,23 @@ impl ApprovalOverlay { ) -> (Vec, SelectionViewParams) { let (options, title) = match &variant { ApprovalVariant::Exec { + network_approval_context, proposed_execpolicy_amendment, .. } => ( - exec_options(proposed_execpolicy_amendment.clone()), - "Would you like to run the following command?".to_string(), + exec_options( + proposed_execpolicy_amendment.clone(), + network_approval_context.as_ref(), + ), + network_approval_context.as_ref().map_or_else( + || "Would you like to run the following command?".to_string(), + |network_approval_context| { + format!( + "Do you want to approve access to \"{}\"?", + network_approval_context.host + ) + }, + ), ), ApprovalVariant::ApplyPatch { .. } => ( patch_options(), @@ -342,6 +356,7 @@ impl From for ApprovalRequestState { id, command, reason, + network_approval_context, proposed_execpolicy_amendment, } => { let mut header: Vec> = Vec::new(); @@ -359,6 +374,7 @@ impl From for ApprovalRequestState { variant: ApprovalVariant::Exec { id, command, + network_approval_context, proposed_execpolicy_amendment, }, header: Box::new(Paragraph::new(header).wrap(Wrap { trim: false })), @@ -414,6 +430,7 @@ enum ApprovalVariant { Exec { id: String, command: Vec, + network_approval_context: Option, proposed_execpolicy_amendment: Option, }, ApplyPatch { @@ -447,7 +464,33 @@ impl ApprovalOption { } } -fn exec_options(proposed_execpolicy_amendment: Option) -> Vec { +fn exec_options( + proposed_execpolicy_amendment: Option, + network_approval_context: Option<&NetworkApprovalContext>, +) -> Vec { + if network_approval_context.is_some() { + return vec![ + ApprovalOption { + label: "Yes, just this once".to_string(), + decision: ApprovalDecision::Review(ReviewDecision::Approved), + display_shortcut: None, + additional_shortcuts: vec![key_hint::plain(KeyCode::Char('y'))], + }, + ApprovalOption { + label: "Yes, and allow this host for this session".to_string(), + decision: ApprovalDecision::Review(ReviewDecision::ApprovedForSession), + display_shortcut: None, + additional_shortcuts: vec![key_hint::plain(KeyCode::Char('a'))], + }, + ApprovalOption { + label: "No, and tell Codex what to do differently".to_string(), + decision: ApprovalDecision::Review(ReviewDecision::Abort), + display_shortcut: Some(key_hint::plain(KeyCode::Esc)), + additional_shortcuts: vec![key_hint::plain(KeyCode::Char('n'))], + }, + ]; + } + vec![ApprovalOption { label: "Yes, proceed".to_string(), decision: ApprovalDecision::Review(ReviewDecision::Approved), @@ -531,6 +574,7 @@ fn elicitation_options() -> Vec { mod tests { use super::*; use crate::app_event::AppEvent; + use codex_core::protocol::NetworkApprovalProtocol; use pretty_assertions::assert_eq; use tokio::sync::mpsc::unbounded_channel; @@ -539,6 +583,7 @@ mod tests { id: "test".to_string(), command: vec!["echo".to_string(), "hi".to_string()], reason: Some("reason".to_string()), + network_approval_context: None, proposed_execpolicy_amendment: None, } } @@ -581,6 +626,7 @@ mod tests { id: "test".to_string(), command: vec!["echo".to_string()], reason: None, + network_approval_context: None, proposed_execpolicy_amendment: Some(ExecPolicyAmendment::new(vec![ "echo".to_string(), ])), @@ -619,6 +665,7 @@ mod tests { id: "test".into(), command, reason: None, + network_approval_context: None, proposed_execpolicy_amendment: None, }; @@ -641,6 +688,67 @@ mod tests { ); } + #[test] + fn network_exec_options_use_expected_labels_and_hide_execpolicy_amendment() { + let network_context = NetworkApprovalContext { + host: "example.com".to_string(), + protocol: NetworkApprovalProtocol::Https, + }; + let options = exec_options( + Some(ExecPolicyAmendment::new(vec!["curl".to_string()])), + Some(&network_context), + ); + + let labels: Vec = options.into_iter().map(|option| option.label).collect(); + assert_eq!( + labels, + vec![ + "Yes, just this once".to_string(), + "Yes, and allow this host for this session".to_string(), + "No, and tell Codex what to do differently".to_string(), + ] + ); + } + + #[test] + fn network_exec_prompt_title_includes_host() { + let (tx, _rx) = unbounded_channel::(); + let tx = AppEventSender::new(tx); + let exec_request = ApprovalRequest::Exec { + id: "test".into(), + command: vec!["curl".into(), "https://example.com".into()], + reason: Some("network request blocked".into()), + network_approval_context: Some(NetworkApprovalContext { + host: "example.com".to_string(), + protocol: NetworkApprovalProtocol::Https, + }), + proposed_execpolicy_amendment: Some(ExecPolicyAmendment::new(vec!["curl".into()])), + }; + + let view = ApprovalOverlay::new(exec_request, tx, Features::with_defaults()); + let mut buf = Buffer::empty(Rect::new(0, 0, 100, view.desired_height(100))); + view.render(Rect::new(0, 0, 100, view.desired_height(100)), &mut buf); + + let rendered: Vec = (0..buf.area.height) + .map(|row| { + (0..buf.area.width) + .map(|col| buf[(col, row)].symbol().to_string()) + .collect() + }) + .collect(); + + assert!( + rendered + .iter() + .any(|line| line.contains("Do you want to approve access to \"example.com\"?")), + "expected network title to include host, got {rendered:?}" + ); + assert!( + !rendered.iter().any(|line| line.contains("don't ask again")), + "network prompt should not show execpolicy option, got {rendered:?}" + ); + } + #[test] fn exec_history_cell_wraps_with_two_space_indent() { let command = vec![ diff --git a/codex-rs/tui/src/bottom_pane/chat_composer.rs b/codex-rs/tui/src/bottom_pane/chat_composer.rs index 3ceeb728c1..36142cbaed 100644 --- a/codex-rs/tui/src/bottom_pane/chat_composer.rs +++ b/codex-rs/tui/src/bottom_pane/chat_composer.rs @@ -21,9 +21,10 @@ //! The Up/Down history path is managed by [`ChatComposerHistory`]. It merges: //! //! - Persistent cross-session history (text-only; no element ranges or attachments). -//! - Local in-session history (full text + text elements + local image paths). +//! - Local in-session history (full text + text elements + local/remote image attachments). //! -//! When recalling a local entry, the composer rehydrates text elements and image attachments. +//! When recalling a local entry, the composer rehydrates text elements and both attachment kinds +//! (local image paths + remote image URLs). //! When recalling a persistent entry, only the text is restored. //! Recalled entries move the cursor to end-of-line so repeated Up/Down presses keep shell-like //! history traversal semantics instead of dropping to column 0. @@ -39,13 +40,33 @@ //! - Expands pending paste placeholders so element ranges align with the final text. //! - Trims whitespace and rebases text elements accordingly. //! - Expands `/prompts:` custom prompts (named or numeric args), preserving text elements. -//! - Prunes attached images so only placeholders that survive expansion are sent. +//! - Prunes local attached images so only placeholders that survive expansion are sent. +//! - Preserves remote image URLs as separate attachments even when text is empty. //! //! The numeric auto-submit path used by the slash popup performs the same pending-paste expansion //! and attachment pruning, and clears pending paste state on success. //! Slash commands with arguments (like `/plan` and `/review`) reuse the same preparation path so //! pasted content and text elements are preserved when extracting args. //! +//! # Remote Image Rows (Up/Down/Delete) +//! +//! Remote image URLs are rendered as non-editable `[Image #N]` rows above the textarea (inside the +//! same composer block). These rows represent image attachments rehydrated from app-server/backtrack +//! history; TUI users can remove them, but cannot type into that row region. +//! +//! Keyboard behavior: +//! +//! - `Up` at textarea cursor `0` enters remote-row selection at the last remote image. +//! - `Up`/`Down` move selection between remote rows. +//! - `Down` on the last row clears selection and returns control to the textarea. +//! - `Delete`/`Backspace` remove the selected remote image row. +//! +//! Placeholder numbering is unified across remote and local images: +//! +//! - Remote rows occupy `[Image #1]..[Image #M]`. +//! - Local placeholders are offset after that range (`[Image #M+1]..`). +//! - Deleting a remote row relabels local placeholders to keep numbering contiguous. +//! //! # Non-bracketed Paste Bursts //! //! On some terminals (especially on Windows), pastes arrive as a rapid sequence of @@ -107,6 +128,7 @@ use ratatui::style::Stylize; use ratatui::text::Line; use ratatui::text::Span; use ratatui::widgets::Block; +use ratatui::widgets::Paragraph; use ratatui::widgets::StatefulWidgetRef; use ratatui::widgets::WidgetRef; @@ -277,7 +299,8 @@ pub(crate) struct ChatComposer { pending_pastes: Vec<(String, String)>, large_paste_counters: HashMap, has_focus: bool, - /// Invariant: attached images are labeled `[Image #1]..[Image #N]` in vec order. + /// Invariant: attached images are labeled in vec order as + /// `[Image #M+1]..[Image #N]`, where `M` is the number of remote images. attached_images: Vec, placeholder_text: String, is_task_running: bool, @@ -291,6 +314,10 @@ pub(crate) struct ChatComposer { custom_prompts: Vec, footer_mode: FooterMode, footer_hint_override: Option>, + remote_image_urls: Vec, + /// Tracks keyboard selection for the remote-image rows so Up/Down + Delete/Backspace + /// can highlight and remove remote attachments from the composer UI. + selected_remote_image_index: Option, footer_flash: Option, context_window_percent: Option, context_window_used_tokens: Option, @@ -390,6 +417,8 @@ impl ChatComposer { custom_prompts: Vec::new(), footer_mode: FooterMode::ComposerEmpty, footer_hint_override: None, + remote_image_urls: Vec::new(), + selected_remote_image_index: None, footer_flash: None, context_window_percent: None, context_window_used_tokens: None, @@ -492,7 +521,7 @@ impl ChatComposer { pub fn set_windows_degraded_sandbox_active(&mut self, enabled: bool) { self.windows_degraded_sandbox_active = enabled; } - fn layout_areas(&self, area: Rect) -> [Rect; 3] { + fn layout_areas(&self, area: Rect) -> [Rect; 4] { let footer_props = self.footer_props(); let footer_hint_height = self .custom_footer_height() @@ -511,8 +540,24 @@ impl ChatComposer { }; let [composer_rect, popup_rect] = Layout::vertical([Constraint::Min(3), popup_constraint]).areas(area); - let textarea_rect = composer_rect.inset(Insets::tlbr(1, LIVE_PREFIX_COLS, 1, 1)); - [composer_rect, textarea_rect, popup_rect] + let mut textarea_rect = composer_rect.inset(Insets::tlbr(1, LIVE_PREFIX_COLS, 1, 1)); + let remote_images_height = self + .remote_images_lines(textarea_rect.width) + .len() + .try_into() + .unwrap_or(u16::MAX) + .min(textarea_rect.height.saturating_sub(1)); + let remote_images_separator = u16::from(remote_images_height > 0); + let consumed = remote_images_height.saturating_add(remote_images_separator); + let remote_images_rect = Rect { + x: textarea_rect.x, + y: textarea_rect.y, + width: textarea_rect.width, + height: remote_images_height, + }; + textarea_rect.y = textarea_rect.y.saturating_add(consumed); + textarea_rect.height = textarea_rect.height.saturating_sub(consumed); + [composer_rect, remote_images_rect, textarea_rect, popup_rect] } fn footer_spacing(footer_hint_height: u16) -> u16 { @@ -523,9 +568,11 @@ impl ChatComposer { } } - /// Returns true if the composer currently contains no user input. + /// Returns true if the composer currently contains no user-entered input. pub(crate) fn is_empty(&self) -> bool { self.textarea.is_empty() + && self.attached_images.is_empty() + && self.remote_image_urls.is_empty() } /// Record the history metadata advertised by `SessionConfiguredEvent` so @@ -649,8 +696,8 @@ impl ChatComposer { /// Replace the composer content with text from an external editor. /// Clears pending paste placeholders and keeps only attachments whose /// placeholder labels still appear in the new text. Image placeholders - /// are renumbered to `[Image #1]..[Image #N]`. Cursor is placed at the end - /// after rebuilding elements. + /// are renumbered to `[Image #M+1]..[Image #N]` (where `M` is the number of + /// remote images). Cursor is placed at the end after rebuilding elements. pub(crate) fn apply_external_edit(&mut self, text: String) { self.pending_pastes.clear(); @@ -712,7 +759,8 @@ impl ChatComposer { self.textarea.insert_str(&text[idx..]); } - // Keep image placeholders normalized to [Image #1].. in attachment order. + // Keep local image placeholders normalized in attachment order after the + // remote-image prefix. self.relabel_attached_images_and_update_placeholders(); self.textarea.set_cursor(self.textarea.text().len()); self.sync_popups(); @@ -746,6 +794,25 @@ impl ChatComposer { self.footer_hint_override = items; } + pub(crate) fn set_remote_image_urls(&mut self, urls: Vec) { + self.remote_image_urls = urls; + self.selected_remote_image_index = None; + self.relabel_attached_images_and_update_placeholders(); + self.sync_popups(); + } + + pub(crate) fn remote_image_urls(&self) -> Vec { + self.remote_image_urls.clone() + } + + pub(crate) fn take_remote_image_urls(&mut self) -> Vec { + let urls = std::mem::take(&mut self.remote_image_urls); + self.selected_remote_image_index = None; + self.relabel_attached_images_and_update_placeholders(); + self.sync_popups(); + urls + } + #[cfg(test)] pub(crate) fn show_footer_flash(&mut self, line: Line<'static>, duration: Duration) { let expires_at = Instant::now() @@ -807,20 +874,15 @@ impl ChatComposer { self.textarea.set_text_with_elements(&text, &text_elements); - let image_placeholders: HashSet = text_elements - .iter() - .filter_map(|elem| elem.placeholder(&text).map(str::to_string)) - .collect(); for (idx, path) in local_image_paths.into_iter().enumerate() { - let placeholder = local_image_label_text(idx + 1); - if image_placeholders.contains(&placeholder) { - self.attached_images - .push(AttachedImage { placeholder, path }); - } + let placeholder = local_image_label_text(self.remote_image_urls.len() + idx + 1); + self.attached_images + .push(AttachedImage { placeholder, path }); } self.bind_mentions_from_snapshot(mention_bindings); - + self.relabel_attached_images_and_update_placeholders(); + self.selected_remote_image_index = None; self.textarea.set_cursor(0); self.sync_popups(); } @@ -848,13 +910,17 @@ impl ChatComposer { .map(|img| img.path.clone()) .collect(); let pending_pastes = std::mem::take(&mut self.pending_pastes); + let remote_image_urls = self.remote_image_urls.clone(); let mention_bindings = self.snapshot_mention_bindings(); self.set_text_content(String::new(), Vec::new(), Vec::new()); + self.remote_image_urls.clear(); + self.selected_remote_image_index = None; self.history.reset_navigation(); self.history.record_local_submission(HistoryEntry { text: previous.clone(), text_elements, local_image_paths, + remote_image_urls, mention_bindings, pending_pastes, }); @@ -878,9 +944,11 @@ impl ChatComposer { text, text_elements, local_image_paths, + remote_image_urls, mention_bindings, pending_pastes, } = entry; + self.set_remote_image_urls(remote_image_urls); self.set_text_content_with_mention_bindings( text, text_elements, @@ -935,7 +1003,7 @@ impl ChatComposer { /// Insert an attachment placeholder and track it for the next submission. pub fn attach_image(&mut self, path: PathBuf) { - let image_number = self.attached_images.len() + 1; + let image_number = self.remote_image_urls.len() + self.attached_images.len() + 1; let placeholder = local_image_label_text(image_number); // Insert as an element to match large paste placeholder behavior: // styled distinctly and treated atomically for cursor/mutations. @@ -2091,11 +2159,15 @@ impl ChatComposer { // Custom prompt expansion can remove or rewrite image placeholders, so prune any // attachments that no longer have a corresponding placeholder in the expanded text. self.prune_attached_images_for_submission(&text, &text_elements); - if text.is_empty() && self.attached_images.is_empty() { + if text.is_empty() && self.attached_images.is_empty() && self.remote_image_urls.is_empty() { return None; } self.recent_submission_mention_bindings = original_mention_bindings.clone(); - if record_history && (!text.is_empty() || !self.attached_images.is_empty()) { + if record_history + && (!text.is_empty() + || !self.attached_images.is_empty() + || !self.remote_image_urls.is_empty()) + { let local_image_paths = self .attached_images .iter() @@ -2105,6 +2177,7 @@ impl ChatComposer { text: text.clone(), text_elements: text_elements.clone(), local_image_paths, + remote_image_urls: self.remote_image_urls.clone(), mention_bindings: original_mention_bindings, pending_pastes: Vec::new(), }); @@ -2345,8 +2418,95 @@ impl ChatComposer { .collect() } + fn remote_images_lines(&self, _width: u16) -> Vec> { + self.remote_image_urls + .iter() + .enumerate() + .map(|(idx, _)| { + let label = local_image_label_text(idx + 1); + if self.selected_remote_image_index == Some(idx) { + label.cyan().reversed().into() + } else { + label.cyan().into() + } + }) + .collect() + } + + fn clear_remote_image_selection(&mut self) { + self.selected_remote_image_index = None; + } + + fn remove_selected_remote_image(&mut self, selected_index: usize) { + if selected_index >= self.remote_image_urls.len() { + self.clear_remote_image_selection(); + return; + } + self.remote_image_urls.remove(selected_index); + self.selected_remote_image_index = if self.remote_image_urls.is_empty() { + None + } else { + Some(selected_index.min(self.remote_image_urls.len() - 1)) + }; + self.relabel_attached_images_and_update_placeholders(); + self.sync_popups(); + } + + fn handle_remote_image_selection_key( + &mut self, + key_event: &KeyEvent, + ) -> Option<(InputResult, bool)> { + if self.remote_image_urls.is_empty() + || key_event.modifiers != KeyModifiers::NONE + || key_event.kind != KeyEventKind::Press + { + return None; + } + + match key_event.code { + KeyCode::Up => { + if let Some(selected) = self.selected_remote_image_index { + self.selected_remote_image_index = Some(selected.saturating_sub(1)); + Some((InputResult::None, true)) + } else if self.textarea.cursor() == 0 { + self.selected_remote_image_index = Some(self.remote_image_urls.len() - 1); + Some((InputResult::None, true)) + } else { + None + } + } + KeyCode::Down => { + if let Some(selected) = self.selected_remote_image_index { + if selected + 1 < self.remote_image_urls.len() { + self.selected_remote_image_index = Some(selected + 1); + } else { + self.clear_remote_image_selection(); + } + Some((InputResult::None, true)) + } else { + None + } + } + KeyCode::Delete | KeyCode::Backspace => { + if let Some(selected) = self.selected_remote_image_index { + self.remove_selected_remote_image(selected); + Some((InputResult::None, true)) + } else { + None + } + } + _ => None, + } + } + /// Handle key event when no popup is visible. fn handle_key_event_without_popup(&mut self, key_event: KeyEvent) -> (InputResult, bool) { + if let Some((result, redraw)) = self.handle_remote_image_selection_key(&key_event) { + return (result, redraw); + } + if self.selected_remote_image_index.is_some() { + self.clear_remote_image_selection(); + } if self.handle_shortcut_overlay_key(&key_event) { return (InputResult::None, true); } @@ -2569,7 +2729,10 @@ impl ChatComposer { // For non-char inputs (or after flushing), handle normally. // Track element removals so we can drop any corresponding placeholders without scanning // the full text. (Placeholders are atomic elements; when deleted, the element disappears.) - let elements_before = if self.pending_pastes.is_empty() && self.attached_images.is_empty() { + let elements_before = if self.pending_pastes.is_empty() + && self.attached_images.is_empty() + && self.remote_image_urls.is_empty() + { None } else { Some(self.textarea.element_payloads()) @@ -2632,7 +2795,7 @@ impl ChatComposer { fn relabel_attached_images_and_update_placeholders(&mut self) { for idx in 0..self.attached_images.len() { - let expected = local_image_label_text(idx + 1); + let expected = local_image_label_text(self.remote_image_urls.len() + idx + 1); let current = self.attached_images[idx].placeholder.clone(); if current == expected { continue; @@ -3099,7 +3262,7 @@ impl ChatComposer { && let Some(snapshot) = self.connectors_snapshot.as_ref() { for connector in &snapshot.connectors { - if !connector.is_accessible { + if !connector.is_accessible || !connector.is_enabled { continue; } let display_name = connectors::connector_display_label(connector); @@ -3259,11 +3422,11 @@ fn find_next_mention_token_range(text: &str, token: &str, from: usize) -> Option impl Renderable for ChatComposer { fn cursor_pos(&self, area: Rect) -> Option<(u16, u16)> { - if !self.input_enabled { + if !self.input_enabled || self.selected_remote_image_index.is_some() { return None; } - let [_, textarea_rect, _] = self.layout_areas(area); + let [_, _, textarea_rect, _] = self.layout_areas(area); let state = *self.textarea_state.borrow(); self.textarea.cursor_pos_with_state(textarea_rect, state) } @@ -3276,8 +3439,16 @@ impl Renderable for ChatComposer { let footer_spacing = Self::footer_spacing(footer_hint_height); let footer_total_height = footer_hint_height + footer_spacing; const COLS_WITH_MARGIN: u16 = LIVE_PREFIX_COLS + 1; - self.textarea - .desired_height(width.saturating_sub(COLS_WITH_MARGIN)) + let inner_width = width.saturating_sub(COLS_WITH_MARGIN); + let remote_images_height: u16 = self + .remote_images_lines(inner_width) + .len() + .try_into() + .unwrap_or(u16::MAX); + let remote_images_separator = u16::from(remote_images_height > 0); + self.textarea.desired_height(inner_width) + + remote_images_height + + remote_images_separator + 2 + match &self.active_popup { ActivePopup::None => footer_total_height, @@ -3294,7 +3465,8 @@ impl Renderable for ChatComposer { impl ChatComposer { pub(crate) fn render_with_mask(&self, area: Rect, buf: &mut Buffer, mask_char: Option) { - let [composer_rect, textarea_rect, popup_rect] = self.layout_areas(area); + let [composer_rect, remote_images_rect, textarea_rect, popup_rect] = + self.layout_areas(area); match &self.active_popup { ActivePopup::Command(popup) => { popup.render_ref(popup_rect, buf); @@ -3515,6 +3687,11 @@ impl ChatComposer { } let style = user_message_style(); Block::default().style(style).render_ref(composer_rect, buf); + if !remote_images_rect.is_empty() { + Paragraph::new(self.remote_images_lines(remote_images_rect.width)) + .style(style) + .render_ref(remote_images_rect, buf); + } if !textarea_rect.is_empty() { let prompt = if self.input_enabled { "›".bold() @@ -4230,6 +4407,85 @@ mod tests { assert_eq!(composer.textarea.element_payloads(), vec![placeholder]); } + #[test] + fn clear_for_ctrl_c_preserves_remote_offset_image_labels() { + let (tx, _rx) = unbounded_channel::(); + let sender = AppEventSender::new(tx); + let mut composer = ChatComposer::new( + true, + sender, + false, + "Ask Codex to do anything".to_string(), + false, + ); + let remote_image_url = "https://example.com/one.png".to_string(); + composer.set_remote_image_urls(vec![remote_image_url.clone()]); + let text = "[Image #2] draft".to_string(); + let text_elements = vec![TextElement::new( + (0.."[Image #2]".len()).into(), + Some("[Image #2]".to_string()), + )]; + let local_image_path = PathBuf::from("/tmp/local-draft.png"); + composer.set_text_content(text, text_elements, vec![local_image_path.clone()]); + let expected_text = composer.current_text(); + let expected_elements = composer.text_elements(); + assert_eq!(expected_text, "[Image #2] draft"); + assert_eq!( + expected_elements[0].placeholder(&expected_text), + Some("[Image #2]") + ); + + assert_eq!(composer.clear_for_ctrl_c(), Some(expected_text.clone())); + + assert_eq!( + composer.history.navigate_up(&composer.app_event_tx), + Some(HistoryEntry::with_pending_and_remote( + expected_text, + expected_elements, + vec![local_image_path], + Vec::new(), + vec![remote_image_url], + )) + ); + } + + #[test] + fn apply_history_entry_preserves_local_placeholders_after_remote_prefix() { + let (tx, _rx) = unbounded_channel::(); + let sender = AppEventSender::new(tx); + let mut composer = ChatComposer::new( + true, + sender, + false, + "Ask Codex to do anything".to_string(), + false, + ); + + let remote_image_url = "https://example.com/one.png".to_string(); + let local_image_path = PathBuf::from("/tmp/local-draft.png"); + composer.apply_history_entry(HistoryEntry::with_pending_and_remote( + "[Image #2] draft".to_string(), + vec![TextElement::new( + (0.."[Image #2]".len()).into(), + Some("[Image #2]".to_string()), + )], + vec![local_image_path.clone()], + Vec::new(), + vec![remote_image_url.clone()], + )); + + let restored_text = composer.current_text(); + assert_eq!(restored_text, "[Image #2] draft"); + let restored_elements = composer.text_elements(); + assert_eq!(restored_elements.len(), 1); + assert_eq!( + restored_elements[0].placeholder(&restored_text), + Some("[Image #2]") + ); + assert_eq!(composer.local_image_paths(), vec![local_image_path]); + assert_eq!(composer.remote_image_urls(), vec![remote_image_url]); + } + /// Behavior: `?` toggles the shortcut overlay only when the composer is otherwise empty. After /// any typing has occurred, `?` should be inserted as a literal character. #[test] @@ -4342,6 +4598,7 @@ mod tests { distribution_channel: None, install_url: Some("https://example.test/notion".to_string()), is_accessible: true, + is_enabled: true, }]; composer.set_connector_mentions(Some(ConnectorsSnapshot { connectors })); @@ -4355,6 +4612,36 @@ mod tests { assert_eq!(mention.path, Some("app://connector_1".to_string())); } + #[test] + fn set_connector_mentions_excludes_disabled_apps_from_mention_popup() { + let (tx, _rx) = unbounded_channel::(); + let sender = AppEventSender::new(tx); + let mut composer = ChatComposer::new( + true, + sender, + false, + "Ask Codex to do anything".to_string(), + false, + ); + composer.set_connectors_enabled(true); + composer.set_text_content("$".to_string(), Vec::new(), Vec::new()); + + let connectors = vec![AppInfo { + id: "connector_1".to_string(), + name: "Notion".to_string(), + description: Some("Workspace docs".to_string()), + logo_url: None, + logo_url_dark: None, + distribution_channel: None, + install_url: Some("https://example.test/notion".to_string()), + is_accessible: true, + is_enabled: false, + }]; + composer.set_connector_mentions(Some(ConnectorsSnapshot { connectors })); + + assert!(matches!(composer.active_popup, ActivePopup::None)); + } + #[test] fn shortcut_overlay_persists_while_task_running() { use crossterm::event::KeyCode; @@ -5070,6 +5357,43 @@ mod tests { }); } + #[test] + fn remote_image_rows_snapshots() { + use crossterm::event::KeyCode; + use crossterm::event::KeyEvent; + use crossterm::event::KeyModifiers; + + snapshot_composer_state("remote_image_rows", false, |composer| { + composer.set_remote_image_urls(vec![ + "https://example.com/one.png".to_string(), + "https://example.com/two.png".to_string(), + ]); + composer.set_text_content("describe these".to_string(), Vec::new(), Vec::new()); + }); + + snapshot_composer_state("remote_image_rows_selected", false, |composer| { + composer.set_remote_image_urls(vec![ + "https://example.com/one.png".to_string(), + "https://example.com/two.png".to_string(), + ]); + composer.set_text_content("describe these".to_string(), Vec::new(), Vec::new()); + composer.textarea.set_cursor(0); + let _ = composer.handle_key_event(KeyEvent::new(KeyCode::Up, KeyModifiers::NONE)); + }); + + snapshot_composer_state("remote_image_rows_after_delete_first", false, |composer| { + composer.set_remote_image_urls(vec![ + "https://example.com/one.png".to_string(), + "https://example.com/two.png".to_string(), + ]); + composer.set_text_content("describe these".to_string(), Vec::new(), Vec::new()); + composer.textarea.set_cursor(0); + let _ = composer.handle_key_event(KeyEvent::new(KeyCode::Up, KeyModifiers::NONE)); + let _ = composer.handle_key_event(KeyEvent::new(KeyCode::Up, KeyModifiers::NONE)); + let _ = composer.handle_key_event(KeyEvent::new(KeyCode::Delete, KeyModifiers::NONE)); + }); + } + #[test] fn slash_popup_model_first_for_mo_ui() { use ratatui::Terminal; @@ -5962,7 +6286,7 @@ mod tests { } #[test] - fn history_navigation_restores_image_attachments() { + fn history_navigation_restores_remote_and_local_image_attachments() { let (tx, _rx) = unbounded_channel::(); let sender = AppEventSender::new(tx); let mut composer = ChatComposer::new( @@ -5973,6 +6297,8 @@ mod tests { false, ); composer.set_steer_enabled(true); + let remote_image_url = "https://example.com/remote.png".to_string(); + composer.set_remote_image_urls(vec![remote_image_url.clone()]); let path = PathBuf::from("/tmp/image1.png"); composer.attach_image(path.clone()); @@ -5980,17 +6306,50 @@ mod tests { composer.handle_key_event(KeyEvent::new(KeyCode::Enter, KeyModifiers::NONE)); assert!(matches!(result, InputResult::Submitted { .. })); + let _ = composer.take_remote_image_urls(); composer.set_text_content(String::new(), Vec::new(), Vec::new()); let _ = composer.handle_key_event(KeyEvent::new(KeyCode::Up, KeyModifiers::NONE)); let text = composer.current_text(); - assert_eq!(text, "[Image #1]"); + assert_eq!(text, "[Image #2]"); let text_elements = composer.text_elements(); assert_eq!(text_elements.len(), 1); - assert_eq!(text_elements[0].placeholder(&text), Some("[Image #1]")); + assert_eq!(text_elements[0].placeholder(&text), Some("[Image #2]")); assert_eq!(composer.local_image_paths(), vec![path]); - assert_eq!(composer.textarea.cursor(), composer.textarea.text().len()); + assert_eq!(composer.remote_image_urls(), vec![remote_image_url]); + } + + #[test] + fn history_navigation_restores_remote_only_submissions() { + let (tx, _rx) = unbounded_channel::(); + let sender = AppEventSender::new(tx); + let mut composer = ChatComposer::new( + true, + sender, + false, + "Ask Codex to do anything".to_string(), + false, + ); + let remote_image_urls = vec![ + "https://example.com/one.png".to_string(), + "https://example.com/two.png".to_string(), + ]; + composer.set_remote_image_urls(remote_image_urls.clone()); + + let (submitted_text, submitted_elements) = composer + .prepare_submission_text(true) + .expect("remote-only submission should be prepared"); + assert_eq!(submitted_text, ""); + assert!(submitted_elements.is_empty()); + + let _ = composer.take_remote_image_urls(); + composer.set_text_content(String::new(), Vec::new(), Vec::new()); + + let _ = composer.handle_key_event(KeyEvent::new(KeyCode::Up, KeyModifiers::NONE)); + assert_eq!(composer.current_text(), ""); + assert!(composer.text_elements().is_empty()); + assert_eq!(composer.remote_image_urls(), remote_image_urls); } #[test] @@ -7872,6 +8231,141 @@ mod tests { assert_eq!(composer.attached_images.len(), 1); } + #[test] + fn remote_images_do_not_modify_textarea_text_or_elements() { + let (tx, _rx) = unbounded_channel::(); + let sender = AppEventSender::new(tx); + let mut composer = ChatComposer::new( + true, + sender, + false, + "Ask Codex to do anything".to_string(), + false, + ); + + composer.set_remote_image_urls(vec![ + "https://example.com/one.png".to_string(), + "https://example.com/two.png".to_string(), + ]); + + assert_eq!(composer.current_text(), ""); + assert_eq!(composer.text_elements(), Vec::::new()); + } + + #[test] + fn attach_image_after_remote_prefix_uses_offset_label() { + let (tx, _rx) = unbounded_channel::(); + let sender = AppEventSender::new(tx); + let mut composer = ChatComposer::new( + true, + sender, + false, + "Ask Codex to do anything".to_string(), + false, + ); + + composer.set_remote_image_urls(vec![ + "https://example.com/one.png".to_string(), + "https://example.com/two.png".to_string(), + ]); + composer.attach_image(PathBuf::from("/tmp/local.png")); + + assert_eq!(composer.attached_images[0].placeholder, "[Image #3]"); + assert_eq!(composer.current_text(), "[Image #3]"); + } + + #[test] + fn prepare_submission_keeps_remote_offset_local_placeholder_numbering() { + let (tx, _rx) = unbounded_channel::(); + let sender = AppEventSender::new(tx); + let mut composer = ChatComposer::new( + true, + sender, + false, + "Ask Codex to do anything".to_string(), + false, + ); + + composer.set_remote_image_urls(vec!["https://example.com/one.png".to_string()]); + let base_text = "[Image #2] hello".to_string(); + let base_elements = vec![TextElement::new( + (0.."[Image #2]".len()).into(), + Some("[Image #2]".to_string()), + )]; + composer.set_text_content( + base_text, + base_elements, + vec![PathBuf::from("/tmp/local.png")], + ); + + let (submitted_text, submitted_elements) = composer + .prepare_submission_text(true) + .expect("remote+local submission should be generated"); + assert_eq!(submitted_text, "[Image #2] hello"); + assert_eq!( + submitted_elements, + vec![TextElement::new( + (0.."[Image #2]".len()).into(), + Some("[Image #2]".to_string()) + )] + ); + } + + #[test] + fn prepare_submission_with_only_remote_images_returns_empty_text() { + let (tx, _rx) = unbounded_channel::(); + let sender = AppEventSender::new(tx); + let mut composer = ChatComposer::new( + true, + sender, + false, + "Ask Codex to do anything".to_string(), + false, + ); + + composer.set_remote_image_urls(vec!["https://example.com/one.png".to_string()]); + let (submitted_text, submitted_elements) = composer + .prepare_submission_text(true) + .expect("remote-only submission should be generated"); + assert_eq!(submitted_text, ""); + assert!(submitted_elements.is_empty()); + } + + #[test] + fn delete_selected_remote_image_relabels_local_placeholders() { + let (tx, _rx) = unbounded_channel::(); + let sender = AppEventSender::new(tx); + let mut composer = ChatComposer::new( + true, + sender, + false, + "Ask Codex to do anything".to_string(), + false, + ); + + composer.set_remote_image_urls(vec![ + "https://example.com/one.png".to_string(), + "https://example.com/two.png".to_string(), + ]); + composer.attach_image(PathBuf::from("/tmp/local.png")); + composer.textarea.set_cursor(0); + + let _ = composer.handle_key_event(KeyEvent::new(KeyCode::Up, KeyModifiers::NONE)); + let _ = composer.handle_key_event(KeyEvent::new(KeyCode::Delete, KeyModifiers::NONE)); + assert_eq!( + composer.remote_image_urls(), + vec!["https://example.com/one.png".to_string()] + ); + assert_eq!(composer.current_text(), "[Image #2]"); + assert_eq!(composer.attached_images[0].placeholder, "[Image #2]"); + + let _ = composer.handle_key_event(KeyEvent::new(KeyCode::Up, KeyModifiers::NONE)); + let _ = composer.handle_key_event(KeyEvent::new(KeyCode::Delete, KeyModifiers::NONE)); + assert_eq!(composer.remote_image_urls(), Vec::::new()); + assert_eq!(composer.current_text(), "[Image #1]"); + assert_eq!(composer.attached_images[0].placeholder, "[Image #1]"); + } + #[test] fn input_disabled_ignores_keypresses_and_hides_cursor() { use crossterm::event::KeyCode; diff --git a/codex-rs/tui/src/bottom_pane/chat_composer_history.rs b/codex-rs/tui/src/bottom_pane/chat_composer_history.rs index e1c1d32ca1..604641d208 100644 --- a/codex-rs/tui/src/bottom_pane/chat_composer_history.rs +++ b/codex-rs/tui/src/bottom_pane/chat_composer_history.rs @@ -17,6 +17,8 @@ pub(crate) struct HistoryEntry { pub(crate) text_elements: Vec, /// Local image paths captured alongside `text_elements`. pub(crate) local_image_paths: Vec, + /// Remote image URLs restored with this draft. + pub(crate) remote_image_urls: Vec, /// Mention bindings for tool/app/skill references inside `text`. pub(crate) mention_bindings: Vec, /// Placeholder-to-payload pairs used to restore large paste content. @@ -30,6 +32,7 @@ impl HistoryEntry { text: decoded.text, text_elements: Vec::new(), local_image_paths: Vec::new(), + remote_image_urls: Vec::new(), mention_bindings: decoded .mentions .into_iter() @@ -53,6 +56,25 @@ impl HistoryEntry { text, text_elements, local_image_paths, + remote_image_urls: Vec::new(), + mention_bindings: Vec::new(), + pending_pastes, + } + } + + #[cfg(test)] + pub(crate) fn with_pending_and_remote( + text: String, + text_elements: Vec, + local_image_paths: Vec, + pending_pastes: Vec<(String, String)>, + remote_image_urls: Vec, + ) -> Self { + Self { + text, + text_elements, + local_image_paths, + remote_image_urls, mention_bindings: Vec::new(), pending_pastes, } @@ -70,7 +92,7 @@ pub(crate) struct ChatComposerHistory { history_entry_count: usize, /// Messages submitted by the user *during this UI session* (newest at END). - /// Local entries retain full draft state (text elements, image paths, pending pastes). + /// Local entries retain full draft state (text elements, image paths, pending pastes, remote image URLs). local_history: Vec, /// Cache of persistent history entries fetched on-demand (text-only). @@ -115,6 +137,7 @@ impl ChatComposerHistory { if entry.text.is_empty() && entry.text_elements.is_empty() && entry.local_image_paths.is_empty() + && entry.remote_image_urls.is_empty() && entry.mention_bindings.is_empty() && entry.pending_pastes.is_empty() { diff --git a/codex-rs/tui/src/bottom_pane/mod.rs b/codex-rs/tui/src/bottom_pane/mod.rs index 32029996ae..43aecb4940 100644 --- a/codex-rs/tui/src/bottom_pane/mod.rs +++ b/codex-rs/tui/src/bottom_pane/mod.rs @@ -44,6 +44,7 @@ mod multi_select_picker; mod request_user_input; mod status_line_setup; pub(crate) use app_link_view::AppLinkView; +pub(crate) use app_link_view::AppLinkViewParams; pub(crate) use approval_overlay::ApprovalOverlay; pub(crate) use approval_overlay::ApprovalRequest; pub(crate) use request_user_input::RequestUserInputOverlay; @@ -249,6 +250,7 @@ impl BottomPane { /// Clear pending attachments and mention bindings e.g. when a slash command doesn't submit text. pub(crate) fn drain_pending_submission_state(&mut self) { let _ = self.take_recent_submission_images_with_placeholders(); + let _ = self.take_remote_image_urls(); let _ = self.take_recent_submission_mention_bindings(); let _ = self.take_mention_bindings(); } @@ -519,6 +521,21 @@ impl BottomPane { self.request_redraw(); } + pub(crate) fn set_remote_image_urls(&mut self, urls: Vec) { + self.composer.set_remote_image_urls(urls); + self.request_redraw(); + } + + pub(crate) fn remote_image_urls(&self) -> Vec { + self.composer.remote_image_urls() + } + + pub(crate) fn take_remote_image_urls(&mut self) -> Vec { + let urls = self.composer.take_remote_image_urls(); + self.request_redraw(); + urls + } + /// Update the status indicator header (defaults to "Working") and details below it. /// /// Passing `None` clears any existing details. No-ops if the status indicator is not active. @@ -989,6 +1006,7 @@ mod tests { id: "1".to_string(), command: vec!["echo".into(), "ok".into()], reason: None, + network_approval_context: None, proposed_execpolicy_amendment: None, } } @@ -1314,6 +1332,58 @@ mod tests { ); } + #[test] + fn remote_images_render_above_composer_text() { + let (tx_raw, _rx) = unbounded_channel::(); + let tx = AppEventSender::new(tx_raw); + let mut pane = BottomPane::new(BottomPaneParams { + app_event_tx: tx, + frame_requester: FrameRequester::test_dummy(), + has_input_focus: true, + enhanced_keys_supported: false, + placeholder_text: "Ask Codex to do anything".to_string(), + disable_paste_burst: false, + animations_enabled: true, + skills: Some(Vec::new()), + }); + + pane.set_remote_image_urls(vec![ + "https://example.com/one.png".to_string(), + "data:image/png;base64,aGVsbG8=".to_string(), + ]); + + assert_eq!(pane.composer_text(), ""); + let width = 48; + let height = pane.desired_height(width); + let area = Rect::new(0, 0, width, height); + let snapshot = render_snapshot(&pane, area); + assert!(snapshot.contains("[Image #1]")); + assert!(snapshot.contains("[Image #2]")); + } + + #[test] + fn drain_pending_submission_state_clears_remote_image_urls() { + let (tx_raw, _rx) = unbounded_channel::(); + let tx = AppEventSender::new(tx_raw); + let mut pane = BottomPane::new(BottomPaneParams { + app_event_tx: tx, + frame_requester: FrameRequester::test_dummy(), + has_input_focus: true, + enhanced_keys_supported: false, + placeholder_text: "Ask Codex to do anything".to_string(), + disable_paste_burst: false, + animations_enabled: true, + skills: Some(Vec::new()), + }); + + pane.set_remote_image_urls(vec!["https://example.com/one.png".to_string()]); + assert_eq!(pane.remote_image_urls().len(), 1); + + pane.drain_pending_submission_state(); + + assert!(pane.remote_image_urls().is_empty()); + } + #[test] fn esc_with_skill_popup_does_not_interrupt_task() { let (tx_raw, mut rx) = unbounded_channel::(); @@ -1333,6 +1403,7 @@ mod tests { interface: None, dependencies: None, policy: None, + permissions: None, path: PathBuf::from("test-skill"), scope: SkillScope::User, }]), diff --git a/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__chat_composer__tests__remote_image_rows.snap b/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__chat_composer__tests__remote_image_rows.snap new file mode 100644 index 0000000000..cb00c404ba --- /dev/null +++ b/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__chat_composer__tests__remote_image_rows.snap @@ -0,0 +1,13 @@ +--- +source: tui/src/bottom_pane/chat_composer.rs +expression: terminal.backend() +--- +" " +" [Image #1] " +" [Image #2] " +" " +"› describe these " +" " +" " +" " +" 100% context left " diff --git a/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__chat_composer__tests__remote_image_rows_after_delete_first.snap b/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__chat_composer__tests__remote_image_rows_after_delete_first.snap new file mode 100644 index 0000000000..fd3cf1f6cf --- /dev/null +++ b/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__chat_composer__tests__remote_image_rows_after_delete_first.snap @@ -0,0 +1,13 @@ +--- +source: tui/src/bottom_pane/chat_composer.rs +expression: terminal.backend() +--- +" " +" [Image #1] " +" " +"› describe these " +" " +" " +" " +" " +" 100% context left " diff --git a/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__chat_composer__tests__remote_image_rows_selected.snap b/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__chat_composer__tests__remote_image_rows_selected.snap new file mode 100644 index 0000000000..cb00c404ba --- /dev/null +++ b/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__chat_composer__tests__remote_image_rows_selected.snap @@ -0,0 +1,13 @@ +--- +source: tui/src/bottom_pane/chat_composer.rs +expression: terminal.backend() +--- +" " +" [Image #1] " +" [Image #2] " +" " +"› describe these " +" " +" " +" " +" 100% context left " diff --git a/codex-rs/tui/src/bottom_pane/textarea.rs b/codex-rs/tui/src/bottom_pane/textarea.rs index 2caa45602f..753514b7d1 100644 --- a/codex-rs/tui/src/bottom_pane/textarea.rs +++ b/codex-rs/tui/src/bottom_pane/textarea.rs @@ -487,8 +487,8 @@ impl TextArea { self.move_cursor_to_end_of_line(true); } _o => { - #[cfg(feature = "debug-logs")] - tracing::debug!("Unhandled key event in TextArea: {:?}", _o); + #[cfg(debug_assertions)] + tracing::trace!("Unhandled key event in TextArea: {:?}", _o); } } } diff --git a/codex-rs/tui/src/chatwidget.rs b/codex-rs/tui/src/chatwidget.rs index 5dc8605f55..f72255b22b 100644 --- a/codex-rs/tui/src/chatwidget.rs +++ b/codex-rs/tui/src/chatwidget.rs @@ -131,6 +131,7 @@ use codex_protocol::parse_command::ParsedCommand; use codex_protocol::request_user_input::RequestUserInputEvent; use codex_protocol::user_input::TextElement; use codex_protocol::user_input::UserInput; +use codex_utils_sleep_inhibitor::SleepInhibitor; use crossterm::event::KeyCode; use crossterm::event::KeyEvent; use crossterm::event::KeyEventKind; @@ -162,7 +163,6 @@ use crate::app_event::ConnectorsSnapshot; use crate::app_event::ExitMode; #[cfg(target_os = "windows")] use crate::app_event::WindowsSandboxEnableMode; -use crate::app_event::WindowsSandboxFallbackReason; use crate::app_event_sender::AppEventSender; use crate::bottom_pane::ApprovalRequest; use crate::bottom_pane::BottomPane; @@ -521,6 +521,7 @@ pub(crate) struct ChatWidget { skills_initial_state: Option>, last_unified_wait: Option, unified_exec_wait_streak: Option, + turn_sleep_inhibitor: SleepInhibitor, task_complete_pending: bool, unified_exec_processes: Vec, /// Tracks whether codex-core currently considers an agent turn to be in progress. @@ -649,6 +650,12 @@ pub(crate) struct ActiveCellTranscriptKey { pub(crate) struct UserMessage { text: String, local_images: Vec, + /// Remote image attachments represented as URLs (for example data URLs) + /// provided by app-server clients. + /// + /// Unlike `local_images`, these are not created by TUI image attach/paste + /// flows. The TUI can restore and remove them while editing/backtracking. + remote_image_urls: Vec, text_elements: Vec, mention_bindings: Vec, } @@ -658,6 +665,7 @@ impl From for UserMessage { Self { text, local_images: Vec::new(), + remote_image_urls: Vec::new(), // Plain text conversion has no UI element ranges. text_elements: Vec::new(), mention_bindings: Vec::new(), @@ -670,6 +678,7 @@ impl From<&str> for UserMessage { Self { text: text.to_string(), local_images: Vec::new(), + remote_image_urls: Vec::new(), // Plain text conversion has no UI element ranges. text_elements: Vec::new(), mention_bindings: Vec::new(), @@ -697,6 +706,7 @@ pub(crate) fn create_initial_user_message( Some(UserMessage { text, local_images, + remote_image_urls: Vec::new(), text_elements, mention_bindings: Vec::new(), }) @@ -712,6 +722,7 @@ fn remap_placeholders_for_message(message: UserMessage, next_label: &mut usize) text, text_elements, local_images, + remote_image_urls, mention_bindings, } = message; if local_images.is_empty() { @@ -719,6 +730,7 @@ fn remap_placeholders_for_message(message: UserMessage, next_label: &mut usize) text, text_elements, local_images, + remote_image_urls, mention_bindings, }; } @@ -773,6 +785,7 @@ fn remap_placeholders_for_message(message: UserMessage, next_label: &mut usize) UserMessage { text: rebuilt, local_images: remapped_images, + remote_image_urls, text_elements: rebuilt_elements, mention_bindings, } @@ -1141,22 +1154,8 @@ impl ChatWidget { self.request_redraw(); } - pub(crate) fn open_app_link_view( - &mut self, - title: String, - description: Option, - instructions: String, - url: String, - is_installed: bool, - ) { - let view = crate::bottom_pane::AppLinkView::new( - title, - description, - instructions, - url, - is_installed, - self.app_event_tx.clone(), - ); + pub(crate) fn open_app_link_view(&mut self, params: crate::bottom_pane::AppLinkViewParams) { + let view = crate::bottom_pane::AppLinkView::new(params, self.app_event_tx.clone()); self.bottom_pane.show_view(Box::new(view)); self.request_redraw(); } @@ -1290,6 +1289,7 @@ impl ChatWidget { fn on_task_started(&mut self) { self.agent_turn_running = true; + self.turn_sleep_inhibitor.set_turn_running(true); self.saw_plan_update_this_turn = false; self.saw_plan_item_this_turn = false; self.plan_delta_buffer.clear(); @@ -1347,6 +1347,7 @@ impl ChatWidget { // Mark task stopped and request redraw now that all content is in history. self.pending_status_indicator_restore = false; self.agent_turn_running = false; + self.turn_sleep_inhibitor.set_turn_running(false); self.update_task_running_state(); self.running_commands.clear(); self.suppressed_exec_calls.clear(); @@ -1583,6 +1584,7 @@ impl ChatWidget { self.finalize_active_cell_as_failed(); // Reset running state and clear streaming buffers. self.agent_turn_running = false; + self.turn_sleep_inhibitor.set_turn_running(false); self.update_task_running_state(); self.running_commands.clear(); self.suppressed_exec_calls.clear(); @@ -1732,11 +1734,15 @@ impl ChatWidget { text: self.bottom_pane.composer_text(), text_elements: self.bottom_pane.composer_text_elements(), local_images: self.bottom_pane.composer_local_images(), + remote_image_urls: self.bottom_pane.remote_image_urls(), mention_bindings: self.bottom_pane.composer_mention_bindings(), }; let mut to_merge: Vec = self.queued_user_messages.drain(..).collect(); - if !existing_message.text.is_empty() || !existing_message.local_images.is_empty() { + if !existing_message.text.is_empty() + || !existing_message.local_images.is_empty() + || !existing_message.remote_image_urls.is_empty() + { to_merge.push(existing_message); } @@ -1744,10 +1750,15 @@ impl ChatWidget { text: String::new(), text_elements: Vec::new(), local_images: Vec::new(), + remote_image_urls: Vec::new(), mention_bindings: Vec::new(), }; let mut combined_offset = 0usize; - let mut next_image_label = 1usize; + let total_remote_images = to_merge + .iter() + .map(|message| message.remote_image_urls.len()) + .sum::(); + let mut next_image_label = total_remote_images + 1; for (idx, message) in to_merge.into_iter().enumerate() { if idx > 0 { @@ -1766,6 +1777,7 @@ impl ChatWidget { elem })); combined.local_images.extend(message.local_images); + combined.remote_image_urls.extend(message.remote_image_urls); combined.mention_bindings.extend(message.mention_bindings); } @@ -1776,10 +1788,12 @@ impl ChatWidget { let UserMessage { text, local_images, + remote_image_urls, text_elements, mention_bindings, } = user_message; let local_image_paths = local_images.into_iter().map(|img| img.path).collect(); + self.set_remote_image_urls(remote_image_urls); self.bottom_pane.set_composer_text_with_mention_bindings( text, text_elements, @@ -2380,6 +2394,7 @@ impl ChatWidget { id: ev.call_id, command: ev.command, reason: ev.reason, + network_approval_context: ev.network_approval_context, proposed_execpolicy_amendment: ev.proposed_execpolicy_amendment, }; self.bottom_pane @@ -2559,6 +2574,7 @@ impl ChatWidget { let model = model.filter(|m| !m.trim().is_empty()); let mut config = config; config.model = model.clone(); + let prevent_idle_sleep = config.features.enabled(Feature::PreventIdleSleep); let mut rng = rand::rng(); let placeholder = PLACEHOLDERS[rng.random_range(0..PLACEHOLDERS.len())].to_string(); let codex_op_tx = spawn_agent(config.clone(), app_event_tx.clone(), thread_manager); @@ -2626,6 +2642,7 @@ impl ChatWidget { suppressed_exec_calls: HashSet::new(), last_unified_wait: None, unified_exec_wait_streak: None, + turn_sleep_inhibitor: SleepInhibitor::new(prevent_idle_sleep), task_complete_pending: false, unified_exec_processes: Vec::new(), agent_turn_running: false, @@ -2725,6 +2742,7 @@ impl ChatWidget { let model = model.filter(|m| !m.trim().is_empty()); let mut config = config; config.model = model.clone(); + let prevent_idle_sleep = config.features.enabled(Feature::PreventIdleSleep); let mut rng = rand::rng(); let placeholder = PLACEHOLDERS[rng.random_range(0..PLACEHOLDERS.len())].to_string(); @@ -2791,6 +2809,7 @@ impl ChatWidget { suppressed_exec_calls: HashSet::new(), last_unified_wait: None, unified_exec_wait_streak: None, + turn_sleep_inhibitor: SleepInhibitor::new(prevent_idle_sleep), task_complete_pending: false, unified_exec_processes: Vec::new(), agent_turn_running: false, @@ -2877,6 +2896,7 @@ impl ChatWidget { otel_manager, } = common; let model = model.filter(|m| !m.trim().is_empty()); + let prevent_idle_sleep = config.features.enabled(Feature::PreventIdleSleep); let mut rng = rand::rng(); let placeholder = PLACEHOLDERS[rng.random_range(0..PLACEHOLDERS.len())].to_string(); @@ -2945,6 +2965,7 @@ impl ChatWidget { suppressed_exec_calls: HashSet::new(), last_unified_wait: None, unified_exec_wait_streak: None, + turn_sleep_inhibitor: SleepInhibitor::new(prevent_idle_sleep), task_complete_pending: false, unified_exec_processes: Vec::new(), agent_turn_running: false, @@ -3106,11 +3127,14 @@ impl ChatWidget { text, text_elements, } => { + let local_images = self + .bottom_pane + .take_recent_submission_images_with_placeholders(); + let remote_image_urls = self.take_remote_image_urls(); let user_message = UserMessage { text, - local_images: self - .bottom_pane - .take_recent_submission_images_with_placeholders(), + local_images, + remote_image_urls, text_elements, mention_bindings: self .bottom_pane @@ -3131,11 +3155,14 @@ impl ChatWidget { text, text_elements, } => { + let local_images = self + .bottom_pane + .take_recent_submission_images_with_placeholders(); + let remote_image_urls = self.take_remote_image_urls(); let user_message = UserMessage { text, - local_images: self - .bottom_pane - .take_recent_submission_images_with_placeholders(), + local_images, + remote_image_urls, text_elements, mention_bindings: self .bottom_pane @@ -3521,11 +3548,14 @@ impl ChatWidget { else { return; }; + let local_images = self + .bottom_pane + .take_recent_submission_images_with_placeholders(); + let remote_image_urls = self.take_remote_image_urls(); let user_message = UserMessage { text: prepared_args, - local_images: self - .bottom_pane - .take_recent_submission_images_with_placeholders(), + local_images, + remote_image_urls, text_elements: prepared_elements, mention_bindings: self.bottom_pane.take_recent_submission_mention_bindings(), }; @@ -3675,18 +3705,22 @@ impl ChatWidget { let UserMessage { text, local_images, + remote_image_urls, text_elements, mention_bindings, } = user_message; - if text.is_empty() && local_images.is_empty() { + if text.is_empty() && local_images.is_empty() && remote_image_urls.is_empty() { return; } - if !local_images.is_empty() && !self.current_model_supports_images() { + if (!local_images.is_empty() || !remote_image_urls.is_empty()) + && !self.current_model_supports_images() + { self.restore_blocked_image_submission( text, text_elements, local_images, mention_bindings, + remote_image_urls, ); return; } @@ -3711,6 +3745,12 @@ impl ChatWidget { return; } + for image_url in &remote_image_urls { + items.push(UserInput::Image { + image_url: image_url.clone(), + }); + } + for image in &local_images { items.push(UserInput::LocalImage { path: image.path.clone(), @@ -3781,7 +3821,7 @@ impl ChatWidget { if !selected_app_ids.insert(app_id.to_string()) { continue; } - if let Some(app) = apps.iter().find(|app| app.id == app_id) { + if let Some(app) = apps.iter().find(|app| app.id == app_id && app.is_enabled) { items.push(UserInput::Mention { name: app.name.clone(), path: binding.path.clone(), @@ -3850,13 +3890,21 @@ impl ChatWidget { }); } - // Only show the text portion in conversation history. + // Show replayable user content in conversation history. if !text.is_empty() { let local_image_paths = local_images.into_iter().map(|img| img.path).collect(); self.add_to_history(history_cell::new_user_prompt( text, text_elements, local_image_paths, + remote_image_urls, + )); + } else if !remote_image_urls.is_empty() { + self.add_to_history(history_cell::new_user_prompt( + String::new(), + Vec::new(), + Vec::new(), + remote_image_urls, )); } @@ -3876,9 +3924,11 @@ impl ChatWidget { text_elements: Vec, local_images: Vec, mention_bindings: Vec, + remote_image_urls: Vec, ) { // Preserve the user's composed payload so they can retry after changing models. let local_image_paths = local_images.iter().map(|img| img.path.clone()).collect(); + self.set_remote_image_urls(remote_image_urls); self.bottom_pane.set_composer_text_with_mention_bindings( text, text_elements, @@ -4157,11 +4207,16 @@ impl ChatWidget { } fn on_user_message_event(&mut self, event: UserMessageEvent) { - if !event.message.trim().is_empty() { + let remote_image_urls = event.images.unwrap_or_default(); + if !event.message.trim().is_empty() + || !event.text_elements.is_empty() + || !remote_image_urls.is_empty() + { self.add_to_history(history_cell::new_user_prompt( event.message, event.text_elements, event.local_images, + remote_image_urls, )); } @@ -4586,6 +4641,7 @@ impl ChatWidget { let connectors = connectors::merge_connectors_with_accessible( all_connectors, accessible_connectors, + true, ); Ok(ConnectorsSnapshot { connectors }) } @@ -5367,15 +5423,23 @@ impl ChatWidget { }); })] } else { - Self::approval_preset_actions(preset.approval, preset.sandbox.clone()) + Self::approval_preset_actions( + preset.approval, + preset.sandbox.clone(), + name.clone(), + ) } } #[cfg(not(target_os = "windows"))] { - Self::approval_preset_actions(preset.approval, preset.sandbox.clone()) + Self::approval_preset_actions( + preset.approval, + preset.sandbox.clone(), + name.clone(), + ) } } else { - Self::approval_preset_actions(preset.approval, preset.sandbox.clone()) + Self::approval_preset_actions(preset.approval, preset.sandbox.clone(), name.clone()) }; items.push(SelectionItem { name, @@ -5429,6 +5493,7 @@ impl ChatWidget { fn approval_preset_actions( approval: AskForApproval, sandbox: SandboxPolicy, + label: String, ) -> Vec { vec![Box::new(move |tx| { let sandbox_clone = sandbox.clone(); @@ -5445,6 +5510,9 @@ impl ChatWidget { })); tx.send(AppEvent::UpdateAskForApprovalPolicy(approval)); tx.send(AppEvent::UpdateSandboxPolicy(sandbox_clone)); + tx.send(AppEvent::InsertHistoryCell(Box::new( + history_cell::new_info_event(format!("Permissions updated to {label}"), None), + ))); })] } @@ -5491,6 +5559,7 @@ impl ChatWidget { preset: ApprovalPreset, return_to_permissions: bool, ) { + let selected_name = preset.label.to_string(); let approval = preset.approval; let sandbox = preset.sandbox; let mut header_children: Vec> = Vec::new(); @@ -5507,12 +5576,14 @@ impl ChatWidget { )); let header = ColumnRenderable::with(header_children); - let mut accept_actions = Self::approval_preset_actions(approval, sandbox.clone()); + let mut accept_actions = + Self::approval_preset_actions(approval, sandbox.clone(), selected_name.clone()); accept_actions.push(Box::new(|tx| { tx.send(AppEvent::UpdateFullAccessWarningAcknowledged(true)); })); - let mut accept_and_remember_actions = Self::approval_preset_actions(approval, sandbox); + let mut accept_and_remember_actions = + Self::approval_preset_actions(approval, sandbox, selected_name); accept_and_remember_actions.push(Box::new(|tx| { tx.send(AppEvent::UpdateFullAccessWarningAcknowledged(true)); tx.send(AppEvent::PersistFullAccessWarningAcknowledged); @@ -5622,7 +5693,11 @@ impl ChatWidget { })); } if let (Some(approval), Some(sandbox)) = (approval, sandbox.clone()) { - accept_actions.extend(Self::approval_preset_actions(approval, sandbox)); + accept_actions.extend(Self::approval_preset_actions( + approval, + sandbox, + mode_label.to_string(), + )); } let mut accept_and_remember_actions: Vec = Vec::new(); @@ -5631,7 +5706,11 @@ impl ChatWidget { tx.send(AppEvent::PersistWorldWritableWarningAcknowledged); })); if let (Some(approval), Some(sandbox)) = (approval, sandbox) { - accept_and_remember_actions.extend(Self::approval_preset_actions(approval, sandbox)); + accept_and_remember_actions.extend(Self::approval_preset_actions( + approval, + sandbox, + mode_label.to_string(), + )); } let items = vec![ @@ -5752,7 +5831,7 @@ impl ChatWidget { name: "Use non-admin sandbox (higher risk if prompt injected)".to_string(), description: None, actions: vec![Box::new(move |tx| { - legacy_otel.counter("codex.windows_sandbox.fallback_use_legacy", 1, &[]); + legacy_otel.counter("codex.windows_sandbox.elevated_prompt_use_legacy", 1, &[]); tx.send(AppEvent::BeginWindowsSandboxLegacySetup { preset: legacy_preset.clone(), }); @@ -5764,7 +5843,7 @@ impl ChatWidget { name: "Quit".to_string(), description: None, actions: vec![Box::new(move |tx| { - quit_otel.counter("codex.windows_sandbox.elevated_prompt_decline", 1, &[]); + quit_otel.counter("codex.windows_sandbox.elevated_prompt_quit", 1, &[]); tx.send(AppEvent::Exit(ExitMode::ShutdownFirst)); })], dismiss_on_select: true, @@ -5785,15 +5864,9 @@ impl ChatWidget { pub(crate) fn open_windows_sandbox_enable_prompt(&mut self, _preset: ApprovalPreset) {} #[cfg(target_os = "windows")] - pub(crate) fn open_windows_sandbox_fallback_prompt( - &mut self, - preset: ApprovalPreset, - reason: WindowsSandboxFallbackReason, - ) { + pub(crate) fn open_windows_sandbox_fallback_prompt(&mut self, preset: ApprovalPreset) { use ratatui_macros::line; - let _ = reason; - let mut lines = Vec::new(); lines.push(line![ "Couldn't set up your sandbox with Administrator permissions".bold() @@ -5849,7 +5922,7 @@ impl ChatWidget { name: "Quit".to_string(), description: None, actions: vec![Box::new(move |tx| { - quit_otel.counter("codex.windows_sandbox.fallback_stay_current", 1, &[]); + quit_otel.counter("codex.windows_sandbox.fallback_prompt_quit", 1, &[]); tx.send(AppEvent::Exit(ExitMode::ShutdownFirst)); })], dismiss_on_select: true, @@ -5867,12 +5940,7 @@ impl ChatWidget { } #[cfg(not(target_os = "windows"))] - pub(crate) fn open_windows_sandbox_fallback_prompt( - &mut self, - _preset: ApprovalPreset, - _reason: WindowsSandboxFallbackReason, - ) { - } + pub(crate) fn open_windows_sandbox_fallback_prompt(&mut self, _preset: ApprovalPreset) {} #[cfg(target_os = "windows")] pub(crate) fn maybe_prompt_windows_sandbox_enable(&mut self, show_now: bool) { @@ -5976,6 +6044,11 @@ impl ChatWidget { if feature == Feature::Personality { self.sync_personality_command_enabled(); } + if feature == Feature::PreventIdleSleep { + self.turn_sleep_inhibitor = SleepInhibitor::new(enabled); + self.turn_sleep_inhibitor + .set_turn_running(self.agent_turn_running); + } #[cfg(target_os = "windows")] if matches!( feature, @@ -6414,6 +6487,7 @@ impl ChatWidget { let connector_title = connector_label.clone(); let link_description = Self::connector_description(connector); let description = Self::connector_brief_description(connector); + let status_label = Self::connector_status_label(connector); let search_value = format!("{connector_label} {}", connector.id); let mut item = SelectionItem { name: connector_label, @@ -6422,42 +6496,47 @@ impl ChatWidget { ..Default::default() }; let is_installed = connector.is_accessible; - let (selected_label, missing_label, instructions) = if connector.is_accessible { - ( - "Press Enter to view the app link.", - "App link unavailable.", - "Manage this app in your browser.", + let selected_label = if is_installed { + format!( + "{status_label}. Press Enter to open the app page to install, manage, or enable/disable this app." ) } else { - ( - "Press Enter to view the install link.", - "Install link unavailable.", - "Install this app in your browser, then reload Codex.", - ) + format!("{status_label}. Press Enter to open the app page to install this app.") + }; + let missing_label = format!("{status_label}. App link unavailable."); + let instructions = if connector.is_accessible { + "Manage this app in your browser." + } else { + "Install this app in your browser, then reload Codex." }; if let Some(install_url) = connector.install_url.clone() { + let app_id = connector.id.clone(); + let is_enabled = connector.is_enabled; let title = connector_title.clone(); let instructions = instructions.to_string(); let description = link_description.clone(); item.actions = vec![Box::new(move |tx| { tx.send(AppEvent::OpenAppLink { + app_id: app_id.clone(), title: title.clone(), description: description.clone(), instructions: instructions.clone(), url: install_url.clone(), is_installed, + is_enabled, }); })]; item.dismiss_on_select = true; - item.selected_description = Some(selected_label.to_string()); + item.selected_description = Some(selected_label); } else { + let missing_label_for_action = missing_label.clone(); item.actions = vec![Box::new(move |tx| { tx.send(AppEvent::InsertHistoryCell(Box::new( - history_cell::new_info_event(missing_label.to_string(), None), + history_cell::new_info_event(missing_label_for_action.clone(), None), ))); })]; item.dismiss_on_select = true; - item.selected_description = Some(missing_label.to_string()); + item.selected_description = Some(missing_label); } items.push(item); } @@ -6490,17 +6569,25 @@ impl ChatWidget { } fn connector_brief_description(connector: &connectors::AppInfo) -> String { - let status_label = if connector.is_accessible { - "Connected" - } else { - "Can be installed" - }; + let status_label = Self::connector_status_label(connector); match Self::connector_description(connector) { Some(description) => format!("{status_label} · {description}"), None => status_label.to_string(), } } + fn connector_status_label(connector: &connectors::AppInfo) -> &'static str { + if connector.is_accessible { + if connector.is_enabled { + "Installed" + } else { + "Installed · Disabled" + } + } else { + "Can be installed" + } + } + fn connector_description(connector: &connectors::AppInfo) -> Option { connector .description @@ -6645,6 +6732,7 @@ impl ChatWidget { let user_message = UserMessage { text, local_images: Vec::new(), + remote_image_urls: Vec::new(), text_elements: Vec::new(), mention_bindings: Vec::new(), }; @@ -6677,6 +6765,19 @@ impl ChatWidget { .set_composer_text(text, text_elements, local_image_paths); } + pub(crate) fn set_remote_image_urls(&mut self, remote_image_urls: Vec) { + self.bottom_pane.set_remote_image_urls(remote_image_urls); + } + + fn take_remote_image_urls(&mut self) -> Vec { + self.bottom_pane.take_remote_image_urls() + } + + #[cfg(test)] + pub(crate) fn remote_image_urls(&self) -> Vec { + self.bottom_pane.remote_image_urls() + } + pub(crate) fn show_esc_backtrack_hint(&mut self) { self.bottom_pane.show_esc_backtrack_hint(); } @@ -6727,7 +6828,28 @@ impl ChatWidget { } match result { - Ok(snapshot) => { + Ok(mut snapshot) => { + if !is_final { + snapshot.connectors = connectors::merge_connectors_with_accessible( + Vec::new(), + snapshot.connectors, + false, + ); + } + snapshot.connectors = + connectors::with_app_enabled_state(snapshot.connectors, &self.config); + if let ConnectorsCacheState::Ready(existing_snapshot) = &self.connectors_cache { + let enabled_by_id: HashMap<&str, bool> = existing_snapshot + .connectors + .iter() + .map(|connector| (connector.id.as_str(), connector.is_enabled)) + .collect(); + for connector in &mut snapshot.connectors { + if let Some(is_enabled) = enabled_by_id.get(connector.id.as_str()) { + connector.is_enabled = *is_enabled; + } + } + } self.refresh_connectors_popup_if_open(&snapshot.connectors); if is_final || !matches!(self.connectors_cache, ConnectorsCacheState::Ready(_)) { self.connectors_cache = ConnectorsCacheState::Ready(snapshot.clone()); @@ -6746,6 +6868,29 @@ impl ChatWidget { } } + pub(crate) fn update_connector_enabled(&mut self, connector_id: &str, enabled: bool) { + let ConnectorsCacheState::Ready(mut snapshot) = self.connectors_cache.clone() else { + return; + }; + + let mut changed = false; + for connector in &mut snapshot.connectors { + if connector.id == connector_id { + changed = connector.is_enabled != enabled; + connector.is_enabled = enabled; + break; + } + } + + if !changed { + return; + } + + self.refresh_connectors_popup_if_open(&snapshot.connectors); + self.connectors_cache = ConnectorsCacheState::Ready(snapshot.clone()); + self.bottom_pane.set_connectors_snapshot(Some(snapshot)); + } + pub(crate) fn open_review_popup(&mut self) { let mut items: Vec = Vec::new(); diff --git a/codex-rs/tui/src/chatwidget/skills.rs b/codex-rs/tui/src/chatwidget/skills.rs index ba8dab416d..e7fadde1af 100644 --- a/codex-rs/tui/src/chatwidget/skills.rs +++ b/codex-rs/tui/src/chatwidget/skills.rs @@ -190,6 +190,7 @@ fn protocol_skill_to_core(skill: &ProtocolSkillMetadata) -> SkillMetadata { .collect(), }), policy: None, + permissions: None, path: skill.path.clone(), scope: skill.scope, } @@ -267,12 +268,12 @@ pub(crate) fn find_app_mentions( } let mut slug_counts: HashMap = HashMap::new(); - for app in apps { + for app in apps.iter().filter(|app| app.is_enabled) { let slug = connector_mention_slug(app); *slug_counts.entry(slug).or_insert(0) += 1; } - for app in apps { + for app in apps.iter().filter(|app| app.is_enabled) { let slug = connector_mention_slug(app); let slug_count = slug_counts.get(&slug).copied().unwrap_or(0); if mentions.names.contains(&slug) @@ -285,7 +286,7 @@ pub(crate) fn find_app_mentions( } apps.iter() - .filter(|app| selected_ids.contains(&app.id)) + .filter(|app| app.is_enabled && selected_ids.contains(&app.id)) .cloned() .collect() } diff --git a/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__permissions_selection_history_after_mode_switch.snap b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__permissions_selection_history_after_mode_switch.snap new file mode 100644 index 0000000000..9cc7afa1f5 --- /dev/null +++ b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__permissions_selection_history_after_mode_switch.snap @@ -0,0 +1,5 @@ +--- +source: tui/src/chatwidget/tests.rs +expression: "lines_to_single_string(&cells[0])" +--- +• Permissions updated to Full Access diff --git a/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__permissions_selection_history_full_access_to_default.snap b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__permissions_selection_history_full_access_to_default.snap new file mode 100644 index 0000000000..135e5b1bfa --- /dev/null +++ b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__permissions_selection_history_full_access_to_default.snap @@ -0,0 +1,5 @@ +--- +source: tui/src/chatwidget/tests.rs +expression: "lines_to_single_string(&cells[0])" +--- +• Permissions updated to Default diff --git a/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__permissions_selection_history_full_access_to_default@windows.snap b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__permissions_selection_history_full_access_to_default@windows.snap new file mode 100644 index 0000000000..eb08108564 --- /dev/null +++ b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__permissions_selection_history_full_access_to_default@windows.snap @@ -0,0 +1,5 @@ +--- +source: tui/src/chatwidget/tests.rs +expression: "lines_to_single_string(&cells[0])" +--- +• Permissions updated to Default (non-admin sandbox) diff --git a/codex-rs/tui/src/chatwidget/tests.rs b/codex-rs/tui/src/chatwidget/tests.rs index 82b22790f0..f069db9160 100644 --- a/codex-rs/tui/src/chatwidget/tests.rs +++ b/codex-rs/tui/src/chatwidget/tests.rs @@ -20,6 +20,8 @@ use codex_core::config::Config; use codex_core::config::ConfigBuilder; use codex_core::config::Constrained; use codex_core::config::ConstraintError; +#[cfg(target_os = "windows")] +use codex_core::config::types::WindowsSandboxModeToml; use codex_core::config_loader::RequirementSource; use codex_core::features::Feature; use codex_core::models_manager::manager::ModelsManager; @@ -250,16 +252,174 @@ async fn replayed_user_message_preserves_text_elements_and_local_images() { cell.message.clone(), cell.text_elements.clone(), cell.local_image_paths.clone(), + cell.remote_image_urls.clone(), )); break; } } - let (stored_message, stored_elements, stored_images) = + let (stored_message, stored_elements, stored_images, stored_remote_image_urls) = user_cell.expect("expected a replayed user history cell"); assert_eq!(stored_message, message); assert_eq!(stored_elements, text_elements); assert_eq!(stored_images, local_images); + assert!(stored_remote_image_urls.is_empty()); +} + +#[tokio::test] +async fn replayed_user_message_preserves_remote_image_urls() { + let (mut chat, mut rx, _ops) = make_chatwidget_manual(None).await; + + let message = "replayed with remote image".to_string(); + let remote_image_urls = vec!["https://example.com/image.png".to_string()]; + + let conversation_id = ThreadId::new(); + let rollout_file = NamedTempFile::new().unwrap(); + let configured = codex_core::protocol::SessionConfiguredEvent { + session_id: conversation_id, + forked_from_id: None, + thread_name: None, + model: "test-model".to_string(), + model_provider_id: "test-provider".to_string(), + approval_policy: AskForApproval::Never, + sandbox_policy: SandboxPolicy::new_read_only_policy(), + cwd: PathBuf::from("/home/user/project"), + reasoning_effort: Some(ReasoningEffortConfig::default()), + history_log_id: 0, + history_entry_count: 0, + initial_messages: Some(vec![EventMsg::UserMessage(UserMessageEvent { + message: message.clone(), + images: Some(remote_image_urls.clone()), + text_elements: Vec::new(), + local_images: Vec::new(), + })]), + network_proxy: None, + rollout_path: Some(rollout_file.path().to_path_buf()), + }; + + chat.handle_codex_event(Event { + id: "initial".into(), + msg: EventMsg::SessionConfigured(configured), + }); + + let mut user_cell = None; + while let Ok(ev) = rx.try_recv() { + if let AppEvent::InsertHistoryCell(cell) = ev + && let Some(cell) = cell.as_any().downcast_ref::() + { + user_cell = Some(( + cell.message.clone(), + cell.local_image_paths.clone(), + cell.remote_image_urls.clone(), + )); + break; + } + } + + let (stored_message, stored_local_images, stored_remote_image_urls) = + user_cell.expect("expected a replayed user history cell"); + assert_eq!(stored_message, message); + assert!(stored_local_images.is_empty()); + assert_eq!(stored_remote_image_urls, remote_image_urls); +} + +#[tokio::test] +async fn replayed_user_message_with_only_remote_images_renders_history_cell() { + let (mut chat, mut rx, _ops) = make_chatwidget_manual(None).await; + + let remote_image_urls = vec!["https://example.com/remote-only.png".to_string()]; + + let conversation_id = ThreadId::new(); + let rollout_file = NamedTempFile::new().unwrap(); + let configured = codex_core::protocol::SessionConfiguredEvent { + session_id: conversation_id, + forked_from_id: None, + thread_name: None, + model: "test-model".to_string(), + model_provider_id: "test-provider".to_string(), + approval_policy: AskForApproval::Never, + sandbox_policy: SandboxPolicy::new_read_only_policy(), + cwd: PathBuf::from("/home/user/project"), + reasoning_effort: Some(ReasoningEffortConfig::default()), + history_log_id: 0, + history_entry_count: 0, + initial_messages: Some(vec![EventMsg::UserMessage(UserMessageEvent { + message: String::new(), + images: Some(remote_image_urls.clone()), + text_elements: Vec::new(), + local_images: Vec::new(), + })]), + network_proxy: None, + rollout_path: Some(rollout_file.path().to_path_buf()), + }; + + chat.handle_codex_event(Event { + id: "initial".into(), + msg: EventMsg::SessionConfigured(configured), + }); + + let mut user_cell = None; + while let Ok(ev) = rx.try_recv() { + if let AppEvent::InsertHistoryCell(cell) = ev + && let Some(cell) = cell.as_any().downcast_ref::() + { + user_cell = Some((cell.message.clone(), cell.remote_image_urls.clone())); + break; + } + } + + let (stored_message, stored_remote_image_urls) = + user_cell.expect("expected a replayed remote-image-only user history cell"); + assert!(stored_message.is_empty()); + assert_eq!(stored_remote_image_urls, remote_image_urls); +} + +#[tokio::test] +async fn replayed_user_message_with_only_local_images_does_not_render_history_cell() { + let (mut chat, mut rx, _ops) = make_chatwidget_manual(None).await; + + let local_images = vec![PathBuf::from("/tmp/replay-local-only.png")]; + + let conversation_id = ThreadId::new(); + let rollout_file = NamedTempFile::new().unwrap(); + let configured = codex_core::protocol::SessionConfiguredEvent { + session_id: conversation_id, + forked_from_id: None, + thread_name: None, + model: "test-model".to_string(), + model_provider_id: "test-provider".to_string(), + approval_policy: AskForApproval::Never, + sandbox_policy: SandboxPolicy::new_read_only_policy(), + cwd: PathBuf::from("/home/user/project"), + reasoning_effort: Some(ReasoningEffortConfig::default()), + history_log_id: 0, + history_entry_count: 0, + initial_messages: Some(vec![EventMsg::UserMessage(UserMessageEvent { + message: String::new(), + images: None, + text_elements: Vec::new(), + local_images, + })]), + network_proxy: None, + rollout_path: Some(rollout_file.path().to_path_buf()), + }; + + chat.handle_codex_event(Event { + id: "initial".into(), + msg: EventMsg::SessionConfigured(configured), + }); + + let mut found_user_history_cell = false; + while let Ok(ev) = rx.try_recv() { + if let AppEvent::InsertHistoryCell(cell) = ev + && cell.as_any().downcast_ref::().is_some() + { + found_user_history_cell = true; + break; + } + } + + assert!(!found_user_history_cell); } #[tokio::test] @@ -394,16 +554,288 @@ async fn submission_preserves_text_elements_and_local_images() { cell.message.clone(), cell.text_elements.clone(), cell.local_image_paths.clone(), + cell.remote_image_urls.clone(), )); break; } } - let (stored_message, stored_elements, stored_images) = + let (stored_message, stored_elements, stored_images, stored_remote_image_urls) = user_cell.expect("expected submitted user history cell"); assert_eq!(stored_message, text); assert_eq!(stored_elements, text_elements); assert_eq!(stored_images, local_images); + assert!(stored_remote_image_urls.is_empty()); +} + +#[tokio::test] +async fn submission_with_remote_and_local_images_keeps_local_placeholder_numbering() { + let (mut chat, mut rx, mut op_rx) = make_chatwidget_manual(None).await; + + let conversation_id = ThreadId::new(); + let rollout_file = NamedTempFile::new().unwrap(); + let configured = codex_core::protocol::SessionConfiguredEvent { + session_id: conversation_id, + forked_from_id: None, + thread_name: None, + model: "test-model".to_string(), + model_provider_id: "test-provider".to_string(), + approval_policy: AskForApproval::Never, + sandbox_policy: SandboxPolicy::new_read_only_policy(), + cwd: PathBuf::from("/home/user/project"), + reasoning_effort: Some(ReasoningEffortConfig::default()), + history_log_id: 0, + history_entry_count: 0, + initial_messages: None, + network_proxy: None, + rollout_path: Some(rollout_file.path().to_path_buf()), + }; + chat.handle_codex_event(Event { + id: "initial".into(), + msg: EventMsg::SessionConfigured(configured), + }); + drain_insert_history(&mut rx); + + let remote_url = "https://example.com/remote.png".to_string(); + chat.set_remote_image_urls(vec![remote_url.clone()]); + + let placeholder = "[Image #2]"; + let text = format!("{placeholder} submit mixed"); + let text_elements = vec![TextElement::new( + (0..placeholder.len()).into(), + Some(placeholder.to_string()), + )]; + let local_images = vec![PathBuf::from("/tmp/submitted-mixed.png")]; + + chat.bottom_pane + .set_composer_text(text.clone(), text_elements.clone(), local_images.clone()); + assert_eq!(chat.bottom_pane.composer_text(), "[Image #2] submit mixed"); + chat.handle_key_event(KeyEvent::new(KeyCode::Enter, KeyModifiers::NONE)); + + let items = match next_submit_op(&mut op_rx) { + Op::UserTurn { items, .. } => items, + other => panic!("expected Op::UserTurn, got {other:?}"), + }; + assert_eq!(items.len(), 3); + assert_eq!( + items[0], + UserInput::Image { + image_url: remote_url.clone(), + } + ); + assert_eq!( + items[1], + UserInput::LocalImage { + path: local_images[0].clone(), + } + ); + assert_eq!( + items[2], + UserInput::Text { + text: text.clone(), + text_elements: text_elements.clone(), + } + ); + assert_eq!(text_elements[0].placeholder(&text), Some("[Image #2]")); + + let mut user_cell = None; + while let Ok(ev) = rx.try_recv() { + if let AppEvent::InsertHistoryCell(cell) = ev + && let Some(cell) = cell.as_any().downcast_ref::() + { + user_cell = Some(( + cell.message.clone(), + cell.text_elements.clone(), + cell.local_image_paths.clone(), + cell.remote_image_urls.clone(), + )); + break; + } + } + + let (stored_message, stored_elements, stored_images, stored_remote_image_urls) = + user_cell.expect("expected submitted user history cell"); + assert_eq!(stored_message, text); + assert_eq!(stored_elements, text_elements); + assert_eq!(stored_images, local_images); + assert_eq!(stored_remote_image_urls, vec![remote_url]); +} + +#[tokio::test] +async fn enter_with_only_remote_images_submits_user_turn() { + let (mut chat, mut rx, mut op_rx) = make_chatwidget_manual(None).await; + + let conversation_id = ThreadId::new(); + let rollout_file = NamedTempFile::new().unwrap(); + let configured = codex_core::protocol::SessionConfiguredEvent { + session_id: conversation_id, + forked_from_id: None, + thread_name: None, + model: "test-model".to_string(), + model_provider_id: "test-provider".to_string(), + approval_policy: AskForApproval::Never, + sandbox_policy: SandboxPolicy::new_read_only_policy(), + cwd: PathBuf::from("/home/user/project"), + reasoning_effort: Some(ReasoningEffortConfig::default()), + history_log_id: 0, + history_entry_count: 0, + initial_messages: None, + network_proxy: None, + rollout_path: Some(rollout_file.path().to_path_buf()), + }; + chat.handle_codex_event(Event { + id: "initial".into(), + msg: EventMsg::SessionConfigured(configured), + }); + drain_insert_history(&mut rx); + + let remote_url = "https://example.com/remote-only.png".to_string(); + chat.set_remote_image_urls(vec![remote_url.clone()]); + assert_eq!(chat.bottom_pane.composer_text(), ""); + + chat.handle_key_event(KeyEvent::new(KeyCode::Enter, KeyModifiers::NONE)); + + let items = match next_submit_op(&mut op_rx) { + Op::UserTurn { items, .. } => items, + other => panic!("expected Op::UserTurn, got {other:?}"), + }; + assert_eq!( + items, + vec![UserInput::Image { + image_url: remote_url.clone(), + }] + ); + assert!(chat.remote_image_urls().is_empty()); + + let mut user_cell = None; + while let Ok(ev) = rx.try_recv() { + if let AppEvent::InsertHistoryCell(cell) = ev + && let Some(cell) = cell.as_any().downcast_ref::() + { + user_cell = Some((cell.message.clone(), cell.remote_image_urls.clone())); + break; + } + } + + let (stored_message, stored_remote_image_urls) = + user_cell.expect("expected submitted user history cell"); + assert_eq!(stored_message, String::new()); + assert_eq!(stored_remote_image_urls, vec![remote_url]); +} + +#[tokio::test] +async fn shift_enter_with_only_remote_images_does_not_submit_user_turn() { + let (mut chat, mut rx, mut op_rx) = make_chatwidget_manual(None).await; + + let conversation_id = ThreadId::new(); + let rollout_file = NamedTempFile::new().unwrap(); + let configured = codex_core::protocol::SessionConfiguredEvent { + session_id: conversation_id, + forked_from_id: None, + thread_name: None, + model: "test-model".to_string(), + model_provider_id: "test-provider".to_string(), + approval_policy: AskForApproval::Never, + sandbox_policy: SandboxPolicy::new_read_only_policy(), + cwd: PathBuf::from("/home/user/project"), + reasoning_effort: Some(ReasoningEffortConfig::default()), + history_log_id: 0, + history_entry_count: 0, + initial_messages: None, + network_proxy: None, + rollout_path: Some(rollout_file.path().to_path_buf()), + }; + chat.handle_codex_event(Event { + id: "initial".into(), + msg: EventMsg::SessionConfigured(configured), + }); + drain_insert_history(&mut rx); + + let remote_url = "https://example.com/remote-only.png".to_string(); + chat.set_remote_image_urls(vec![remote_url.clone()]); + assert_eq!(chat.bottom_pane.composer_text(), ""); + + chat.handle_key_event(KeyEvent::new(KeyCode::Enter, KeyModifiers::SHIFT)); + + assert_no_submit_op(&mut op_rx); + assert_eq!(chat.remote_image_urls(), vec![remote_url]); +} + +#[tokio::test] +async fn enter_with_only_remote_images_does_not_submit_when_modal_is_active() { + let (mut chat, mut rx, mut op_rx) = make_chatwidget_manual(None).await; + + let conversation_id = ThreadId::new(); + let rollout_file = NamedTempFile::new().unwrap(); + let configured = codex_core::protocol::SessionConfiguredEvent { + session_id: conversation_id, + forked_from_id: None, + thread_name: None, + model: "test-model".to_string(), + model_provider_id: "test-provider".to_string(), + approval_policy: AskForApproval::Never, + sandbox_policy: SandboxPolicy::new_read_only_policy(), + cwd: PathBuf::from("/home/user/project"), + reasoning_effort: Some(ReasoningEffortConfig::default()), + history_log_id: 0, + history_entry_count: 0, + initial_messages: None, + network_proxy: None, + rollout_path: Some(rollout_file.path().to_path_buf()), + }; + chat.handle_codex_event(Event { + id: "initial".into(), + msg: EventMsg::SessionConfigured(configured), + }); + drain_insert_history(&mut rx); + + let remote_url = "https://example.com/remote-only.png".to_string(); + chat.set_remote_image_urls(vec![remote_url.clone()]); + + chat.open_review_popup(); + chat.handle_key_event(KeyEvent::new(KeyCode::Enter, KeyModifiers::NONE)); + + assert_eq!(chat.remote_image_urls(), vec![remote_url]); + assert_no_submit_op(&mut op_rx); +} + +#[tokio::test] +async fn enter_with_only_remote_images_does_not_submit_when_input_disabled() { + let (mut chat, mut rx, mut op_rx) = make_chatwidget_manual(None).await; + + let conversation_id = ThreadId::new(); + let rollout_file = NamedTempFile::new().unwrap(); + let configured = codex_core::protocol::SessionConfiguredEvent { + session_id: conversation_id, + forked_from_id: None, + thread_name: None, + model: "test-model".to_string(), + model_provider_id: "test-provider".to_string(), + approval_policy: AskForApproval::Never, + sandbox_policy: SandboxPolicy::new_read_only_policy(), + cwd: PathBuf::from("/home/user/project"), + reasoning_effort: Some(ReasoningEffortConfig::default()), + history_log_id: 0, + history_entry_count: 0, + initial_messages: None, + network_proxy: None, + rollout_path: Some(rollout_file.path().to_path_buf()), + }; + chat.handle_codex_event(Event { + id: "initial".into(), + msg: EventMsg::SessionConfigured(configured), + }); + drain_insert_history(&mut rx); + + let remote_url = "https://example.com/remote-only.png".to_string(); + chat.set_remote_image_urls(vec![remote_url.clone()]); + chat.bottom_pane + .set_composer_input_enabled(false, Some("Input disabled for test.".to_string())); + + chat.handle_key_event(KeyEvent::new(KeyCode::Enter, KeyModifiers::NONE)); + + assert_eq!(chat.remote_image_urls(), vec![remote_url]); + assert_no_submit_op(&mut op_rx); } #[tokio::test] @@ -444,6 +876,7 @@ async fn submission_prefers_selected_duplicate_skill_path() { interface: None, dependencies: None, policy: None, + permissions: None, path: repo_skill_path, scope: SkillScope::Repo, }, @@ -454,6 +887,7 @@ async fn submission_prefers_selected_duplicate_skill_path() { interface: None, dependencies: None, policy: None, + permissions: None, path: user_skill_path.clone(), scope: SkillScope::User, }, @@ -508,6 +942,7 @@ async fn blocked_image_restore_preserves_mention_bindings() { text_elements, local_images.clone(), mention_bindings.clone(), + Vec::new(), ); let mention_start = text.find("$file").expect("mention token exists"); @@ -537,6 +972,94 @@ async fn blocked_image_restore_preserves_mention_bindings() { ); } +#[tokio::test] +async fn blocked_image_restore_with_remote_images_keeps_local_placeholder_mapping() { + let (mut chat, _rx, _op_rx) = make_chatwidget_manual(None).await; + + let first_placeholder = "[Image #2]"; + let second_placeholder = "[Image #3]"; + let text = format!("{first_placeholder} first\n{second_placeholder} second"); + let second_start = text.find(second_placeholder).expect("second placeholder"); + let text_elements = vec![ + TextElement::new( + (0..first_placeholder.len()).into(), + Some(first_placeholder.to_string()), + ), + TextElement::new( + (second_start..second_start + second_placeholder.len()).into(), + Some(second_placeholder.to_string()), + ), + ]; + let local_images = vec![ + LocalImageAttachment { + placeholder: first_placeholder.to_string(), + path: PathBuf::from("/tmp/blocked-first.png"), + }, + LocalImageAttachment { + placeholder: second_placeholder.to_string(), + path: PathBuf::from("/tmp/blocked-second.png"), + }, + ]; + let remote_image_urls = vec!["https://example.com/blocked-remote.png".to_string()]; + + chat.restore_blocked_image_submission( + text.clone(), + text_elements.clone(), + local_images.clone(), + Vec::new(), + remote_image_urls.clone(), + ); + + assert_eq!(chat.bottom_pane.composer_text(), text); + assert_eq!(chat.bottom_pane.composer_text_elements(), text_elements); + assert_eq!(chat.bottom_pane.composer_local_images(), local_images); + assert_eq!(chat.remote_image_urls(), remote_image_urls); +} + +#[tokio::test] +async fn queued_restore_with_remote_images_keeps_local_placeholder_mapping() { + let (mut chat, _rx, _op_rx) = make_chatwidget_manual(None).await; + + let first_placeholder = "[Image #2]"; + let second_placeholder = "[Image #3]"; + let text = format!("{first_placeholder} first\n{second_placeholder} second"); + let second_start = text.find(second_placeholder).expect("second placeholder"); + let text_elements = vec![ + TextElement::new( + (0..first_placeholder.len()).into(), + Some(first_placeholder.to_string()), + ), + TextElement::new( + (second_start..second_start + second_placeholder.len()).into(), + Some(second_placeholder.to_string()), + ), + ]; + let local_images = vec![ + LocalImageAttachment { + placeholder: first_placeholder.to_string(), + path: PathBuf::from("/tmp/queued-first.png"), + }, + LocalImageAttachment { + placeholder: second_placeholder.to_string(), + path: PathBuf::from("/tmp/queued-second.png"), + }, + ]; + let remote_image_urls = vec!["https://example.com/queued-remote.png".to_string()]; + + chat.restore_user_message_to_composer(UserMessage { + text: text.clone(), + local_images: local_images.clone(), + remote_image_urls: remote_image_urls.clone(), + text_elements: text_elements.clone(), + mention_bindings: Vec::new(), + }); + + assert_eq!(chat.bottom_pane.composer_text(), text); + assert_eq!(chat.bottom_pane.composer_text_elements(), text_elements); + assert_eq!(chat.bottom_pane.composer_local_images(), local_images); + assert_eq!(chat.remote_image_urls(), remote_image_urls); +} + #[tokio::test] async fn interrupted_turn_restores_queued_messages_with_images_and_elements() { let (mut chat, _rx, _op_rx) = make_chatwidget_manual(None).await; @@ -571,6 +1094,7 @@ async fn interrupted_turn_restores_queued_messages_with_images_and_elements() { placeholder: first_placeholder.to_string(), path: first_images[0].clone(), }], + remote_image_urls: Vec::new(), text_elements: first_elements, mention_bindings: Vec::new(), }); @@ -580,6 +1104,7 @@ async fn interrupted_turn_restores_queued_messages_with_images_and_elements() { placeholder: second_placeholder.to_string(), path: second_images[0].clone(), }], + remote_image_urls: Vec::new(), text_elements: second_elements, mention_bindings: Vec::new(), }); @@ -649,6 +1174,7 @@ async fn interrupted_turn_restore_keeps_active_mode_for_resubmission() { chat.queued_user_messages.push_back(UserMessage { text: "Implement the plan.".to_string(), local_images: Vec::new(), + remote_image_urls: Vec::new(), text_elements: Vec::new(), mention_bindings: Vec::new(), }); @@ -711,6 +1237,7 @@ async fn remap_placeholders_uses_attachment_labels() { text, text_elements: elements, local_images: attachments, + remote_image_urls: vec!["https://example.com/a.png".to_string()], mention_bindings: Vec::new(), }; let mut next_label = 3usize; @@ -743,6 +1270,10 @@ async fn remap_placeholders_uses_attachment_labels() { }, ] ); + assert_eq!( + remapped.remote_image_urls, + vec!["https://example.com/a.png".to_string()] + ); } #[tokio::test] @@ -772,6 +1303,7 @@ async fn remap_placeholders_uses_byte_ranges_when_placeholder_missing() { text, text_elements: elements, local_images: attachments, + remote_image_urls: Vec::new(), mention_bindings: Vec::new(), }; let mut next_label = 3usize; @@ -1030,6 +1562,7 @@ async fn make_chatwidget_manual( if let Some(model) = model_override { cfg.model = Some(model.to_string()); } + let prevent_idle_sleep = cfg.features.enabled(Feature::PreventIdleSleep); let otel_manager = test_otel_manager(&cfg, resolved_model.as_str()); let mut bottom = BottomPane::new(BottomPaneParams { app_event_tx: app_event_tx.clone(), @@ -1086,6 +1619,7 @@ async fn make_chatwidget_manual( skills_initial_state: None, last_unified_wait: None, unified_exec_wait_streak: None, + turn_sleep_inhibitor: SleepInhibitor::new(prevent_idle_sleep), task_complete_pending: false, unified_exec_processes: Vec::new(), agent_turn_running: false, @@ -1148,6 +1682,15 @@ fn next_submit_op(op_rx: &mut tokio::sync::mpsc::UnboundedReceiver) -> Op { } } +fn assert_no_submit_op(op_rx: &mut tokio::sync::mpsc::UnboundedReceiver) { + while let Ok(op) = op_rx.try_recv() { + assert!( + !matches!(op, Op::UserTurn { .. }), + "unexpected submit op: {op:?}" + ); + } +} + fn set_chatgpt_auth(chat: &mut ChatWidget) { chat.auth_manager = codex_core::test_support::auth_manager_from_auth( CodexAuth::create_dummy_chatgpt_auth_for_testing(), @@ -1935,6 +2478,7 @@ async fn exec_approval_emits_proposed_command_and_decision_history() { reason: Some( "this is a test reason such as one that would be produced by the model".into(), ), + network_approval_context: None, proposed_execpolicy_amendment: None, parsed_cmd: vec![], }; @@ -1979,6 +2523,7 @@ async fn exec_approval_decision_truncates_multiline_and_long_commands() { reason: Some( "this is a test reason such as one that would be produced by the model".into(), ), + network_approval_context: None, proposed_execpolicy_amendment: None, parsed_cmd: vec![], }; @@ -2029,6 +2574,7 @@ async fn exec_approval_decision_truncates_multiline_and_long_commands() { command: vec!["bash".into(), "-lc".into(), long], cwd: std::env::current_dir().unwrap_or_else(|_| PathBuf::from(".")), reason: None, + network_approval_context: None, proposed_execpolicy_amendment: None, parsed_cmd: vec![], }; @@ -3866,11 +4412,13 @@ async fn apps_popup_refreshes_when_connectors_snapshot_updates() { let (mut chat, _rx, _op_rx) = make_chatwidget_manual(None).await; chat.config.features.enable(Feature::Apps); chat.bottom_pane.set_connectors_enabled(true); + let notion_id = "unit_test_apps_popup_refresh_connector_1"; + let linear_id = "unit_test_apps_popup_refresh_connector_2"; chat.on_connectors_loaded( Ok(ConnectorsSnapshot { connectors: vec![codex_chatgpt::connectors::AppInfo { - id: "connector_1".to_string(), + id: notion_id.to_string(), name: "Notion".to_string(), description: Some("Workspace docs".to_string()), logo_url: None, @@ -3878,6 +4426,7 @@ async fn apps_popup_refreshes_when_connectors_snapshot_updates() { distribution_channel: None, install_url: Some("https://example.test/notion".to_string()), is_accessible: true, + is_enabled: true, }], }), false, @@ -3893,12 +4442,16 @@ async fn apps_popup_refreshes_when_connectors_snapshot_updates() { before.contains("Installed 1 of 1 available apps."), "expected initial apps popup snapshot, got:\n{before}" ); + assert!( + before.contains("Installed. Press Enter to open the app page"), + "expected selected app description to explain the app page action, got:\n{before}" + ); chat.on_connectors_loaded( Ok(ConnectorsSnapshot { connectors: vec![ codex_chatgpt::connectors::AppInfo { - id: "connector_1".to_string(), + id: notion_id.to_string(), name: "Notion".to_string(), description: Some("Workspace docs".to_string()), logo_url: None, @@ -3906,9 +4459,10 @@ async fn apps_popup_refreshes_when_connectors_snapshot_updates() { distribution_channel: None, install_url: Some("https://example.test/notion".to_string()), is_accessible: true, + is_enabled: true, }, codex_chatgpt::connectors::AppInfo { - id: "connector_2".to_string(), + id: linear_id.to_string(), name: "Linear".to_string(), description: Some("Project tracking".to_string()), logo_url: None, @@ -3916,6 +4470,7 @@ async fn apps_popup_refreshes_when_connectors_snapshot_updates() { distribution_channel: None, install_url: Some("https://example.test/linear".to_string()), is_accessible: true, + is_enabled: true, }, ], }), @@ -3938,10 +4493,12 @@ async fn apps_refresh_failure_keeps_existing_full_snapshot() { let (mut chat, _rx, _op_rx) = make_chatwidget_manual(None).await; chat.config.features.enable(Feature::Apps); chat.bottom_pane.set_connectors_enabled(true); + let notion_id = "unit_test_apps_refresh_failure_connector_1"; + let linear_id = "unit_test_apps_refresh_failure_connector_2"; let full_connectors = vec![ codex_chatgpt::connectors::AppInfo { - id: "connector_1".to_string(), + id: notion_id.to_string(), name: "Notion".to_string(), description: Some("Workspace docs".to_string()), logo_url: None, @@ -3949,9 +4506,10 @@ async fn apps_refresh_failure_keeps_existing_full_snapshot() { distribution_channel: None, install_url: Some("https://example.test/notion".to_string()), is_accessible: true, + is_enabled: true, }, codex_chatgpt::connectors::AppInfo { - id: "connector_2".to_string(), + id: linear_id.to_string(), name: "Linear".to_string(), description: Some("Project tracking".to_string()), logo_url: None, @@ -3959,6 +4517,7 @@ async fn apps_refresh_failure_keeps_existing_full_snapshot() { distribution_channel: None, install_url: Some("https://example.test/linear".to_string()), is_accessible: false, + is_enabled: true, }, ]; chat.on_connectors_loaded( @@ -3971,7 +4530,7 @@ async fn apps_refresh_failure_keeps_existing_full_snapshot() { chat.on_connectors_loaded( Ok(ConnectorsSnapshot { connectors: vec![codex_chatgpt::connectors::AppInfo { - id: "connector_1".to_string(), + id: notion_id.to_string(), name: "Notion".to_string(), description: Some("Workspace docs".to_string()), logo_url: None, @@ -3979,6 +4538,7 @@ async fn apps_refresh_failure_keeps_existing_full_snapshot() { distribution_channel: None, install_url: Some("https://example.test/notion".to_string()), is_accessible: true, + is_enabled: true, }], }), false, @@ -3998,6 +4558,265 @@ async fn apps_refresh_failure_keeps_existing_full_snapshot() { ); } +#[tokio::test] +async fn apps_partial_refresh_uses_same_filtering_as_full_refresh() { + let (mut chat, _rx, _op_rx) = make_chatwidget_manual(None).await; + chat.config.features.enable(Feature::Apps); + chat.bottom_pane.set_connectors_enabled(true); + + let full_connectors = vec![ + codex_chatgpt::connectors::AppInfo { + id: "unit_test_connector_1".to_string(), + name: "Notion".to_string(), + description: Some("Workspace docs".to_string()), + logo_url: None, + logo_url_dark: None, + distribution_channel: None, + install_url: Some("https://example.test/notion".to_string()), + is_accessible: true, + is_enabled: true, + }, + codex_chatgpt::connectors::AppInfo { + id: "unit_test_connector_2".to_string(), + name: "Linear".to_string(), + description: Some("Project tracking".to_string()), + logo_url: None, + logo_url_dark: None, + distribution_channel: None, + install_url: Some("https://example.test/linear".to_string()), + is_accessible: false, + is_enabled: true, + }, + ]; + chat.on_connectors_loaded( + Ok(ConnectorsSnapshot { + connectors: full_connectors.clone(), + }), + true, + ); + chat.add_connectors_output(); + + chat.on_connectors_loaded( + Ok(ConnectorsSnapshot { + connectors: vec![ + codex_chatgpt::connectors::AppInfo { + id: "unit_test_connector_1".to_string(), + name: "Notion".to_string(), + description: Some("Workspace docs".to_string()), + logo_url: None, + logo_url_dark: None, + distribution_channel: None, + install_url: Some("https://example.test/notion".to_string()), + is_accessible: true, + is_enabled: true, + }, + codex_chatgpt::connectors::AppInfo { + id: "connector_openai_hidden".to_string(), + name: "Hidden OpenAI".to_string(), + description: Some("Should be filtered".to_string()), + logo_url: None, + logo_url_dark: None, + distribution_channel: None, + install_url: Some("https://example.test/hidden-openai".to_string()), + is_accessible: true, + is_enabled: true, + }, + ], + }), + false, + ); + + assert_matches!( + &chat.connectors_cache, + ConnectorsCacheState::Ready(snapshot) if snapshot.connectors == full_connectors + ); + + let popup = render_bottom_popup(&chat, 80); + assert!( + popup.contains("Installed 1 of 1 available apps."), + "expected partial refresh popup to use filtered connectors, got:\n{popup}" + ); + assert!( + !popup.contains("Hidden OpenAI"), + "expected disallowed connector to be filtered from partial refresh popup, got:\n{popup}" + ); +} + +#[tokio::test] +async fn apps_popup_shows_disabled_status_for_installed_but_disabled_apps() { + let (mut chat, _rx, _op_rx) = make_chatwidget_manual(None).await; + chat.config.features.enable(Feature::Apps); + chat.bottom_pane.set_connectors_enabled(true); + + chat.on_connectors_loaded( + Ok(ConnectorsSnapshot { + connectors: vec![codex_chatgpt::connectors::AppInfo { + id: "connector_1".to_string(), + name: "Notion".to_string(), + description: Some("Workspace docs".to_string()), + logo_url: None, + logo_url_dark: None, + distribution_channel: None, + install_url: Some("https://example.test/notion".to_string()), + is_accessible: true, + is_enabled: false, + }], + }), + true, + ); + + chat.add_connectors_output(); + let popup = render_bottom_popup(&chat, 80); + assert!( + popup.contains("Installed · Disabled. Press Enter to open the app page"), + "expected selected app description to include disabled status, got:\n{popup}" + ); + assert!( + popup.contains("enable/disable this app."), + "expected selected app description to mention enable/disable action, got:\n{popup}" + ); +} + +#[tokio::test] +async fn apps_initial_load_applies_enabled_state_from_config() { + let (mut chat, _rx, _op_rx) = make_chatwidget_manual(None).await; + chat.config.features.enable(Feature::Apps); + chat.bottom_pane.set_connectors_enabled(true); + + let temp = tempdir().expect("tempdir"); + let config_toml_path = + AbsolutePathBuf::try_from(temp.path().join("config.toml")).expect("absolute config path"); + let user_config = toml::from_str::( + "[apps.connector_1]\nenabled = false\ndisabled_reason = \"user\"\n", + ) + .expect("apps config"); + chat.config.config_layer_stack = chat + .config + .config_layer_stack + .with_user_config(&config_toml_path, user_config); + + chat.on_connectors_loaded( + Ok(ConnectorsSnapshot { + connectors: vec![codex_chatgpt::connectors::AppInfo { + id: "connector_1".to_string(), + name: "Notion".to_string(), + description: Some("Workspace docs".to_string()), + logo_url: None, + logo_url_dark: None, + distribution_channel: None, + install_url: Some("https://example.test/notion".to_string()), + is_accessible: true, + is_enabled: true, + }], + }), + true, + ); + + assert_matches!( + &chat.connectors_cache, + ConnectorsCacheState::Ready(snapshot) + if snapshot + .connectors + .iter() + .find(|connector| connector.id == "connector_1") + .is_some_and(|connector| !connector.is_enabled) + ); +} + +#[tokio::test] +async fn apps_refresh_preserves_toggled_enabled_state() { + let (mut chat, _rx, _op_rx) = make_chatwidget_manual(None).await; + chat.config.features.enable(Feature::Apps); + chat.bottom_pane.set_connectors_enabled(true); + + chat.on_connectors_loaded( + Ok(ConnectorsSnapshot { + connectors: vec![codex_chatgpt::connectors::AppInfo { + id: "connector_1".to_string(), + name: "Notion".to_string(), + description: Some("Workspace docs".to_string()), + logo_url: None, + logo_url_dark: None, + distribution_channel: None, + install_url: Some("https://example.test/notion".to_string()), + is_accessible: true, + is_enabled: true, + }], + }), + true, + ); + chat.update_connector_enabled("connector_1", false); + + chat.on_connectors_loaded( + Ok(ConnectorsSnapshot { + connectors: vec![codex_chatgpt::connectors::AppInfo { + id: "connector_1".to_string(), + name: "Notion".to_string(), + description: Some("Workspace docs".to_string()), + logo_url: None, + logo_url_dark: None, + distribution_channel: None, + install_url: Some("https://example.test/notion".to_string()), + is_accessible: true, + is_enabled: true, + }], + }), + true, + ); + + assert_matches!( + &chat.connectors_cache, + ConnectorsCacheState::Ready(snapshot) + if snapshot + .connectors + .iter() + .find(|connector| connector.id == "connector_1") + .is_some_and(|connector| !connector.is_enabled) + ); + + chat.add_connectors_output(); + let popup = render_bottom_popup(&chat, 80); + assert!( + popup.contains("Installed · Disabled. Press Enter to open the app page"), + "expected disabled status to persist after reload, got:\n{popup}" + ); +} + +#[tokio::test] +async fn apps_popup_for_not_installed_app_uses_install_only_selected_description() { + let (mut chat, _rx, _op_rx) = make_chatwidget_manual(None).await; + chat.config.features.enable(Feature::Apps); + chat.bottom_pane.set_connectors_enabled(true); + + chat.on_connectors_loaded( + Ok(ConnectorsSnapshot { + connectors: vec![codex_chatgpt::connectors::AppInfo { + id: "connector_2".to_string(), + name: "Linear".to_string(), + description: Some("Project tracking".to_string()), + logo_url: None, + logo_url_dark: None, + distribution_channel: None, + install_url: Some("https://example.test/linear".to_string()), + is_accessible: false, + is_enabled: true, + }], + }), + true, + ); + + chat.add_connectors_output(); + let popup = render_bottom_popup(&chat, 80); + assert!( + popup.contains("Can be installed. Press Enter to open the app page to install"), + "expected selected app description to be install-only for not-installed apps, got:\n{popup}" + ); + assert!( + !popup.contains("enable/disable this app."), + "did not expect enable/disable text for not-installed apps, got:\n{popup}" + ); +} + #[tokio::test] async fn experimental_features_popup_snapshot() { let (mut chat, _rx, _op_rx) = make_chatwidget_manual(None).await; @@ -4613,6 +5432,196 @@ async fn approvals_popup_navigation_skips_disabled() { ); } +#[tokio::test] +async fn permissions_selection_emits_history_cell_when_selection_changes() { + let (mut chat, mut rx, _op_rx) = make_chatwidget_manual(None).await; + #[cfg(target_os = "windows")] + { + chat.config.notices.hide_world_writable_warning = Some(true); + chat.set_windows_sandbox_mode(Some(WindowsSandboxModeToml::Unelevated)); + } + chat.config.notices.hide_full_access_warning = Some(true); + + chat.open_permissions_popup(); + chat.handle_key_event(KeyEvent::from(KeyCode::Down)); + chat.handle_key_event(KeyEvent::from(KeyCode::Enter)); + + let cells = drain_insert_history(&mut rx); + assert_eq!( + cells.len(), + 1, + "expected one permissions selection history cell" + ); + let rendered = lines_to_single_string(&cells[0]); + assert!( + rendered.contains("Permissions updated to"), + "expected permissions selection history message, got: {rendered}" + ); +} + +#[tokio::test] +async fn permissions_selection_history_snapshot_after_mode_switch() { + let (mut chat, mut rx, _op_rx) = make_chatwidget_manual(None).await; + #[cfg(target_os = "windows")] + { + chat.config.notices.hide_world_writable_warning = Some(true); + chat.set_windows_sandbox_mode(Some(WindowsSandboxModeToml::Unelevated)); + } + chat.config.notices.hide_full_access_warning = Some(true); + + chat.open_permissions_popup(); + chat.handle_key_event(KeyEvent::from(KeyCode::Down)); + #[cfg(target_os = "windows")] + chat.handle_key_event(KeyEvent::from(KeyCode::Down)); + chat.handle_key_event(KeyEvent::from(KeyCode::Enter)); + + let cells = drain_insert_history(&mut rx); + assert_eq!(cells.len(), 1, "expected one mode-switch history cell"); + assert_snapshot!( + "permissions_selection_history_after_mode_switch", + lines_to_single_string(&cells[0]) + ); +} + +#[tokio::test] +async fn permissions_selection_history_snapshot_full_access_to_default() { + let (mut chat, mut rx, _op_rx) = make_chatwidget_manual(None).await; + #[cfg(target_os = "windows")] + { + chat.config.notices.hide_world_writable_warning = Some(true); + chat.set_windows_sandbox_mode(Some(WindowsSandboxModeToml::Unelevated)); + } + chat.config.notices.hide_full_access_warning = Some(true); + chat.config + .permissions + .approval_policy + .set(AskForApproval::Never) + .expect("set approval policy"); + chat.config + .permissions + .sandbox_policy + .set(SandboxPolicy::DangerFullAccess) + .expect("set sandbox policy"); + + chat.open_permissions_popup(); + chat.handle_key_event(KeyEvent::from(KeyCode::Up)); + chat.handle_key_event(KeyEvent::from(KeyCode::Enter)); + + let cells = drain_insert_history(&mut rx); + assert_eq!(cells.len(), 1, "expected one mode-switch history cell"); + let rendered = lines_to_single_string(&cells[0]); + #[cfg(target_os = "windows")] + insta::with_settings!({ snapshot_suffix => "windows" }, { + assert_snapshot!("permissions_selection_history_full_access_to_default", rendered); + }); + #[cfg(not(target_os = "windows"))] + assert_snapshot!( + "permissions_selection_history_full_access_to_default", + rendered + ); +} + +#[tokio::test] +async fn permissions_selection_emits_history_cell_when_current_is_selected() { + let (mut chat, mut rx, _op_rx) = make_chatwidget_manual(None).await; + #[cfg(target_os = "windows")] + { + chat.config.notices.hide_world_writable_warning = Some(true); + chat.set_windows_sandbox_mode(Some(WindowsSandboxModeToml::Unelevated)); + } + chat.config + .permissions + .approval_policy + .set(AskForApproval::OnRequest) + .expect("set approval policy"); + chat.config + .permissions + .sandbox_policy + .set(SandboxPolicy::new_workspace_write_policy()) + .expect("set sandbox policy"); + + chat.open_permissions_popup(); + chat.handle_key_event(KeyEvent::from(KeyCode::Enter)); + + let cells = drain_insert_history(&mut rx); + assert_eq!( + cells.len(), + 1, + "expected history cell even when selecting current permissions" + ); + let rendered = lines_to_single_string(&cells[0]); + assert!( + rendered.contains("Permissions updated to"), + "expected permissions update history message, got: {rendered}" + ); +} + +#[tokio::test] +async fn permissions_full_access_history_cell_emitted_only_after_confirmation() { + let (mut chat, mut rx, _op_rx) = make_chatwidget_manual(None).await; + #[cfg(target_os = "windows")] + { + chat.config.notices.hide_world_writable_warning = Some(true); + chat.set_windows_sandbox_mode(Some(WindowsSandboxModeToml::Unelevated)); + } + chat.config.notices.hide_full_access_warning = None; + + chat.open_permissions_popup(); + chat.handle_key_event(KeyEvent::from(KeyCode::Down)); + #[cfg(target_os = "windows")] + chat.handle_key_event(KeyEvent::from(KeyCode::Down)); + chat.handle_key_event(KeyEvent::from(KeyCode::Enter)); + + let mut open_confirmation_event = None; + let mut cells_before_confirmation = Vec::new(); + while let Ok(event) = rx.try_recv() { + match event { + AppEvent::InsertHistoryCell(cell) => { + cells_before_confirmation.push(cell.display_lines(80)); + } + AppEvent::OpenFullAccessConfirmation { + preset, + return_to_permissions, + } => { + open_confirmation_event = Some((preset, return_to_permissions)); + } + _ => {} + } + } + if cfg!(not(target_os = "windows")) { + assert!( + cells_before_confirmation.is_empty(), + "did not expect history cell before confirming full access" + ); + } + let (preset, return_to_permissions) = + open_confirmation_event.expect("expected full access confirmation event"); + chat.open_full_access_confirmation(preset, return_to_permissions); + + let popup = render_bottom_popup(&chat, 80); + assert!( + popup.contains("Enable full access?"), + "expected full access confirmation popup, got: {popup}" + ); + + chat.handle_key_event(KeyEvent::from(KeyCode::Enter)); + let cells_after_confirmation = drain_insert_history(&mut rx); + let total_history_cells = cells_before_confirmation.len() + cells_after_confirmation.len(); + assert_eq!( + total_history_cells, 1, + "expected one full access history cell total" + ); + let rendered = if !cells_before_confirmation.is_empty() { + lines_to_single_string(&cells_before_confirmation[0]) + } else { + lines_to_single_string(&cells_after_confirmation[0]) + }; + assert!( + rendered.contains("Permissions updated to Full Access"), + "expected full access update history message, got: {rendered}" + ); +} + // // Snapshot test: command approval modal // @@ -4636,6 +5645,7 @@ async fn approval_modal_exec_snapshot() -> anyhow::Result<()> { reason: Some( "this is a test reason such as one that would be produced by the model".into(), ), + network_approval_context: None, proposed_execpolicy_amendment: Some(ExecPolicyAmendment::new(vec![ "echo".into(), "hello".into(), @@ -4692,6 +5702,7 @@ async fn approval_modal_exec_without_reason_snapshot() -> anyhow::Result<()> { command: vec!["bash".into(), "-lc".into(), "echo hello world".into()], cwd: std::env::current_dir().unwrap_or_else(|_| PathBuf::from(".")), reason: None, + network_approval_context: None, proposed_execpolicy_amendment: Some(ExecPolicyAmendment::new(vec![ "echo".into(), "hello".into(), @@ -4739,6 +5750,7 @@ async fn approval_modal_exec_multiline_prefix_hides_execpolicy_option_snapshot() command: command.clone(), cwd: std::env::current_dir().unwrap_or_else(|_| PathBuf::from(".")), reason: None, + network_approval_context: None, proposed_execpolicy_amendment: Some(ExecPolicyAmendment::new(command)), parsed_cmd: vec![], }; @@ -5098,6 +6110,7 @@ async fn status_widget_and_approval_modal_snapshot() { reason: Some( "this is a test reason such as one that would be produced by the model".into(), ), + network_approval_context: None, proposed_execpolicy_amendment: Some(ExecPolicyAmendment::new(vec![ "echo".into(), "hello world".into(), diff --git a/codex-rs/tui/src/history_cell.rs b/codex-rs/tui/src/history_cell.rs index 360d481aad..8789196946 100644 --- a/codex-rs/tui/src/history_cell.rs +++ b/codex-rs/tui/src/history_cell.rs @@ -49,6 +49,7 @@ use codex_protocol::account::PlanType; use codex_protocol::mcp::Resource; use codex_protocol::mcp::ResourceTemplate; use codex_protocol::models::WebSearchAction; +use codex_protocol::models::local_image_label_text; use codex_protocol::openai_models::ReasoningEffort as ReasoningEffortConfig; use codex_protocol::plan_tool::PlanItemArg; use codex_protocol::plan_tool::StepStatus; @@ -168,6 +169,7 @@ pub(crate) struct UserHistoryCell { pub text_elements: Vec, #[allow(dead_code)] pub local_image_paths: Vec, + pub remote_image_urls: Vec, } /// Build logical lines for a user message with styled text elements. @@ -236,10 +238,22 @@ fn build_user_message_lines_with_elements( raw_lines } +fn remote_image_display_line(style: Style, index: usize) -> Line<'static> { + Line::from(local_image_label_text(index)).style(style) +} + +fn trim_trailing_blank_lines(mut lines: Vec>) -> Vec> { + while lines + .last() + .is_some_and(|line| line.spans.iter().all(|span| span.content.trim().is_empty())) + { + lines.pop(); + } + lines +} + impl HistoryCell for UserHistoryCell { fn display_lines(&self, width: u16) -> Vec> { - let mut lines: Vec> = Vec::new(); - let wrap_width = width .saturating_sub( LIVE_PREFIX_COLS + 1, /* keep a one-column right margin for wrapping */ @@ -249,13 +263,35 @@ impl HistoryCell for UserHistoryCell { let style = user_message_style(); let element_style = style.fg(Color::Cyan); - let wrapped = if self.text_elements.is_empty() { - word_wrap_lines( - self.message.split('\n').map(|l| Line::from(l).style(style)), + let wrapped_remote_images = if self.remote_image_urls.is_empty() { + None + } else { + Some(word_wrap_lines( + self.remote_image_urls + .iter() + .enumerate() + .map(|(idx, _url)| { + remote_image_display_line(element_style, idx.saturating_add(1)) + }), + RtOptions::new(usize::from(wrap_width)) + .wrap_algorithm(textwrap::WrapAlgorithm::FirstFit), + )) + }; + + let wrapped_message = if self.message.is_empty() && self.text_elements.is_empty() { + None + } else if self.text_elements.is_empty() { + let message_without_trailing_newlines = self.message.trim_end_matches(['\r', '\n']); + let wrapped = word_wrap_lines( + message_without_trailing_newlines + .split('\n') + .map(|line| Line::from(line).style(style)), // Wrap algorithm matches textarea.rs. RtOptions::new(usize::from(wrap_width)) .wrap_algorithm(textwrap::WrapAlgorithm::FirstFit), - ) + ); + let wrapped = trim_trailing_blank_lines(wrapped); + (!wrapped.is_empty()).then_some(wrapped) } else { let raw_lines = build_user_message_lines_with_elements( &self.message, @@ -263,18 +299,57 @@ impl HistoryCell for UserHistoryCell { style, element_style, ); - word_wrap_lines( + let wrapped = word_wrap_lines( raw_lines, RtOptions::new(usize::from(wrap_width)) .wrap_algorithm(textwrap::WrapAlgorithm::FirstFit), - ) + ); + let wrapped = trim_trailing_blank_lines(wrapped); + (!wrapped.is_empty()).then_some(wrapped) }; - lines.push(Line::from("").style(style)); - lines.extend(prefix_lines(wrapped, "› ".bold().dim(), " ".into())); + if wrapped_remote_images.is_none() && wrapped_message.is_none() { + return Vec::new(); + } + + let mut lines: Vec> = vec![Line::from("").style(style)]; + + if let Some(wrapped_remote_images) = wrapped_remote_images { + lines.extend(prefix_lines( + wrapped_remote_images, + " ".into(), + " ".into(), + )); + if wrapped_message.is_some() { + lines.push(Line::from("").style(style)); + } + } + + if let Some(wrapped_message) = wrapped_message { + lines.extend(prefix_lines( + wrapped_message, + "› ".bold().dim(), + " ".into(), + )); + } + lines.push(Line::from("").style(style)); lines } + + fn desired_height(&self, width: u16) -> u16 { + self.display_lines(width) + .len() + .try_into() + .unwrap_or(u16::MAX) + } + + fn desired_transcript_height(&self, width: u16) -> u16 { + self.display_lines(width) + .len() + .try_into() + .unwrap_or(u16::MAX) + } } #[derive(Debug)] @@ -1018,11 +1093,13 @@ pub(crate) fn new_user_prompt( message: String, text_elements: Vec, local_image_paths: Vec, + remote_image_urls: Vec, ) -> UserHistoryCell { UserHistoryCell { message, text_elements, local_image_paths, + remote_image_urls, } } @@ -3359,6 +3436,7 @@ mod tests { message: msg.to_string(), text_elements: Vec::new(), local_image_paths: Vec::new(), + remote_image_urls: Vec::new(), }; // Small width to force wrapping more clearly. Effective wrap width is width-2 due to the ▌ prefix and trailing space. @@ -3369,6 +3447,120 @@ mod tests { insta::assert_snapshot!(rendered); } + #[test] + fn user_history_cell_renders_remote_image_urls() { + let cell = UserHistoryCell { + message: "describe these".to_string(), + text_elements: Vec::new(), + local_image_paths: Vec::new(), + remote_image_urls: vec!["https://example.com/example.png".to_string()], + }; + + let rendered = render_lines(&cell.display_lines(80)).join("\n"); + + assert!(rendered.contains("[Image #1]")); + assert!(rendered.contains("describe these")); + insta::assert_snapshot!(rendered); + } + + #[test] + fn user_history_cell_summarizes_inline_data_urls() { + let cell = UserHistoryCell { + message: "describe inline image".to_string(), + text_elements: Vec::new(), + local_image_paths: Vec::new(), + remote_image_urls: vec!["data:image/png;base64,aGVsbG8=".to_string()], + }; + + let rendered = render_lines(&cell.display_lines(80)).join("\n"); + + assert!(rendered.contains("[Image #1]")); + assert!(rendered.contains("describe inline image")); + } + + #[test] + fn user_history_cell_numbers_multiple_remote_images() { + let cell = UserHistoryCell { + message: "describe both".to_string(), + text_elements: Vec::new(), + local_image_paths: Vec::new(), + remote_image_urls: vec![ + "https://example.com/one.png".to_string(), + "https://example.com/two.png".to_string(), + ], + }; + + let rendered = render_lines(&cell.display_lines(80)).join("\n"); + + assert!(rendered.contains("[Image #1]")); + assert!(rendered.contains("[Image #2]")); + insta::assert_snapshot!(rendered); + } + + #[test] + fn user_history_cell_height_matches_rendered_lines_with_remote_images() { + let cell = UserHistoryCell { + message: "line one\nline two".to_string(), + text_elements: Vec::new(), + local_image_paths: Vec::new(), + remote_image_urls: vec![ + "https://example.com/one.png".to_string(), + "https://example.com/two.png".to_string(), + ], + }; + + let width = 80; + let rendered_len: u16 = cell + .display_lines(width) + .len() + .try_into() + .unwrap_or(u16::MAX); + assert_eq!(cell.desired_height(width), rendered_len); + assert_eq!(cell.desired_transcript_height(width), rendered_len); + } + + #[test] + fn user_history_cell_trims_trailing_blank_message_lines() { + let cell = UserHistoryCell { + message: "line one\n\n \n\t \n".to_string(), + text_elements: Vec::new(), + local_image_paths: Vec::new(), + remote_image_urls: vec!["https://example.com/one.png".to_string()], + }; + + let rendered = render_lines(&cell.display_lines(80)); + let trailing_blank_count = rendered + .iter() + .rev() + .take_while(|line| line.trim().is_empty()) + .count(); + assert_eq!(trailing_blank_count, 1); + assert!(rendered.iter().any(|line| line.contains("line one"))); + } + + #[test] + fn user_history_cell_trims_trailing_blank_message_lines_with_text_elements() { + let message = "tokenized\n\n\n".to_string(); + let cell = UserHistoryCell { + message, + text_elements: vec![TextElement::new( + (0..8).into(), + Some("tokenized".to_string()), + )], + local_image_paths: Vec::new(), + remote_image_urls: vec!["https://example.com/one.png".to_string()], + }; + + let rendered = render_lines(&cell.display_lines(80)); + let trailing_blank_count = rendered + .iter() + .rev() + .take_while(|line| line.trim().is_empty()) + .count(); + assert_eq!(trailing_blank_count, 1); + assert!(rendered.iter().any(|line| line.contains("tokenized"))); + } + #[test] fn plan_update_with_note_and_wrapping_snapshot() { // Long explanation forces wrapping; include long step text to verify step wrapping and alignment. diff --git a/codex-rs/tui/src/lib.rs b/codex-rs/tui/src/lib.rs index df9ecdf761..639916a79f 100644 --- a/codex-rs/tui/src/lib.rs +++ b/codex-rs/tui/src/lib.rs @@ -15,6 +15,7 @@ use codex_core::RolloutRecorder; use codex_core::ThreadSortKey; use codex_core::auth::AuthMode; use codex_core::auth::enforce_login_restrictions; +use codex_core::check_execpolicy_for_warnings; use codex_core::config::Config; use codex_core::config::ConfigBuilder; use codex_core::config::ConfigOverrides; @@ -27,6 +28,7 @@ use codex_core::config_loader::format_config_error_with_source; use codex_core::default_client::set_default_client_residency_requirement; use codex_core::find_thread_path_by_id_str; use codex_core::find_thread_path_by_name_str; +use codex_core::format_exec_policy_error_with_source; use codex_core::path_utils; use codex_core::protocol::AskForApproval; use codex_core::read_session_meta_line; @@ -288,6 +290,19 @@ pub async fn run_main( cloud_requirements.clone(), ) .await; + + #[allow(clippy::print_stderr)] + match check_execpolicy_for_warnings(&config.config_layer_stack).await { + Ok(None) => {} + Ok(Some(err)) | Err(err) => { + eprintln!( + "Error loading rules:\n{}", + format_exec_policy_error_with_source(&err) + ); + std::process::exit(1); + } + } + set_default_client_residency_requirement(config.enforce_residency.value()); if let Some(warning) = @@ -1008,6 +1023,7 @@ mod tests { cwd, approval_policy: config.permissions.approval_policy.value(), sandbox_policy: config.permissions.sandbox_policy.get().clone(), + network: None, model, personality: None, collaboration_mode: None, diff --git a/codex-rs/tui/src/snapshots/codex_tui__history_cell__tests__user_history_cell_numbers_multiple_remote_images.snap b/codex-rs/tui/src/snapshots/codex_tui__history_cell__tests__user_history_cell_numbers_multiple_remote_images.snap new file mode 100644 index 0000000000..076df8fffb --- /dev/null +++ b/codex-rs/tui/src/snapshots/codex_tui__history_cell__tests__user_history_cell_numbers_multiple_remote_images.snap @@ -0,0 +1,9 @@ +--- +source: tui/src/history_cell.rs +expression: rendered +--- + + [Image #1] + [Image #2] + +› describe both diff --git a/codex-rs/tui/src/snapshots/codex_tui__history_cell__tests__user_history_cell_renders_remote_image_urls.snap b/codex-rs/tui/src/snapshots/codex_tui__history_cell__tests__user_history_cell_renders_remote_image_urls.snap new file mode 100644 index 0000000000..923ee953ab --- /dev/null +++ b/codex-rs/tui/src/snapshots/codex_tui__history_cell__tests__user_history_cell_renders_remote_image_urls.snap @@ -0,0 +1,8 @@ +--- +source: tui/src/history_cell.rs +expression: rendered +--- + + [Image #1] + +› describe these diff --git a/codex-rs/tui/src/tooltips.rs b/codex-rs/tui/src/tooltips.rs index a9dbd82e2f..0600bd4165 100644 --- a/codex-rs/tui/src/tooltips.rs +++ b/codex-rs/tui/src/tooltips.rs @@ -8,10 +8,9 @@ const ANNOUNCEMENT_TIP_URL: &str = const IS_MACOS: bool = cfg!(target_os = "macos"); -const PAID_TOOLTIP: &str = "*New* Try the **Codex App** with 2x rate limits until *April 2nd*. Run 'codex app' or visit https://chatgpt.com/codex"; +const PAID_TOOLTIP: &str = "*New* Try the **Codex App** with 2x rate limits until *April 2nd*. Run 'codex app' or visit https://chatgpt.com/codex?app-landing-page=true"; const PAID_TOOLTIP_NON_MAC: &str = "*New* 2x rate limits until *April 2nd*."; -const OTHER_TOOLTIP: &str = - "*New* Build faster with the **Codex App**. Run 'codex app' or visit https://chatgpt.com/codex"; +const OTHER_TOOLTIP: &str = "*New* Build faster with the **Codex App**. Run 'codex app' or visit https://chatgpt.com/codex?app-landing-page=true"; const OTHER_TOOLTIP_NON_MAC: &str = "*New* Build faster with Codex."; const FREE_GO_TOOLTIP: &str = "*New* Codex is included in your plan for free through *March 2nd* – let’s build together."; diff --git a/codex-rs/utils/sleep-inhibitor/BUILD.bazel b/codex-rs/utils/sleep-inhibitor/BUILD.bazel new file mode 100644 index 0000000000..4884593efe --- /dev/null +++ b/codex-rs/utils/sleep-inhibitor/BUILD.bazel @@ -0,0 +1,6 @@ +load("//:defs.bzl", "codex_rust_crate") + +codex_rust_crate( + name = "sleep-inhibitor", + crate_name = "codex_utils_sleep_inhibitor", +) diff --git a/codex-rs/utils/sleep-inhibitor/Cargo.toml b/codex-rs/utils/sleep-inhibitor/Cargo.toml new file mode 100644 index 0000000000..703433d819 --- /dev/null +++ b/codex-rs/utils/sleep-inhibitor/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "codex-utils-sleep-inhibitor" +version.workspace = true +edition.workspace = true +license.workspace = true + +[lints] +workspace = true + +[target.'cfg(target_os = "macos")'.dependencies] +core-foundation = "0.9" +libc = { workspace = true } +tracing = { workspace = true } diff --git a/codex-rs/utils/sleep-inhibitor/src/dummy.rs b/codex-rs/utils/sleep-inhibitor/src/dummy.rs new file mode 100644 index 0000000000..767091ea93 --- /dev/null +++ b/codex-rs/utils/sleep-inhibitor/src/dummy.rs @@ -0,0 +1,16 @@ +use crate::PlatformSleepInhibitor; + +#[derive(Debug, Default)] +pub(crate) struct DummySleepInhibitor; + +impl DummySleepInhibitor { + pub(crate) fn new() -> Self { + Self + } +} + +impl PlatformSleepInhibitor for DummySleepInhibitor { + fn acquire(&mut self) {} + + fn release(&mut self) {} +} diff --git a/codex-rs/utils/sleep-inhibitor/src/lib.rs b/codex-rs/utils/sleep-inhibitor/src/lib.rs new file mode 100644 index 0000000000..0f2f32df64 --- /dev/null +++ b/codex-rs/utils/sleep-inhibitor/src/lib.rs @@ -0,0 +1,94 @@ +//! Cross-platform helper for preventing idle sleep while a turn is running. +//! +//! On macOS this uses native IOKit power assertions instead of spawning +//! `caffeinate`, so assertion lifecycle is tied directly to Rust object lifetime. + +#[cfg(not(target_os = "macos"))] +mod dummy; +#[cfg(target_os = "macos")] +mod macos_inhibitor; + +use std::fmt::Debug; + +/// Keeps the machine awake while a turn is in progress when enabled. +#[derive(Debug)] +pub struct SleepInhibitor { + enabled: bool, + platform: Box, +} + +pub(crate) trait PlatformSleepInhibitor: Debug { + fn acquire(&mut self); + fn release(&mut self); +} + +impl SleepInhibitor { + pub fn new(enabled: bool) -> Self { + #[cfg(target_os = "macos")] + let platform: Box = + Box::new(macos_inhibitor::MacOsSleepInhibitor::new()); + #[cfg(not(target_os = "macos"))] + let platform: Box = Box::new(dummy::DummySleepInhibitor::new()); + + Self { enabled, platform } + } + + /// Update the active turn state; turns sleep prevention on/off as needed. + pub fn set_turn_running(&mut self, turn_running: bool) { + if !self.enabled { + self.release(); + return; + } + + if turn_running { + self.acquire(); + } else { + self.release(); + } + } + + fn acquire(&mut self) { + self.platform.acquire(); + } + + fn release(&mut self) { + self.platform.release(); + } +} + +#[cfg(test)] +mod tests { + use super::SleepInhibitor; + + #[test] + fn sleep_inhibitor_toggles_without_panicking() { + let mut inhibitor = SleepInhibitor::new(true); + inhibitor.set_turn_running(true); + inhibitor.set_turn_running(false); + } + + #[test] + fn sleep_inhibitor_disabled_does_not_panic() { + let mut inhibitor = SleepInhibitor::new(false); + inhibitor.set_turn_running(true); + inhibitor.set_turn_running(false); + } + + #[test] + fn sleep_inhibitor_multiple_true_calls_are_idempotent() { + let mut inhibitor = SleepInhibitor::new(true); + inhibitor.set_turn_running(true); + inhibitor.set_turn_running(true); + inhibitor.set_turn_running(true); + inhibitor.set_turn_running(false); + } + + #[test] + fn sleep_inhibitor_can_toggle_multiple_times() { + let mut inhibitor = SleepInhibitor::new(true); + inhibitor.set_turn_running(true); + inhibitor.set_turn_running(false); + inhibitor.set_turn_running(true); + inhibitor.set_turn_running(false); + } +} diff --git a/codex-rs/utils/sleep-inhibitor/src/macos_inhibitor.rs b/codex-rs/utils/sleep-inhibitor/src/macos_inhibitor.rs new file mode 100644 index 0000000000..e788705e1c --- /dev/null +++ b/codex-rs/utils/sleep-inhibitor/src/macos_inhibitor.rs @@ -0,0 +1,192 @@ +use crate::PlatformSleepInhibitor; +use core_foundation::base::TCFType; +use core_foundation::string::CFString; +use core_foundation::string::CFStringRef; +use std::fmt::Debug; +use std::sync::OnceLock; +use tracing::warn; + +const ASSERTION_REASON: &str = "Codex is running an active turn"; +const MACOS_IDLE_SLEEP_ASSERTION_TYPE: &str = "PreventUserIdleSystemSleep"; +const IOKIT_FRAMEWORK_BINARY: &[u8] = b"/System/Library/Frameworks/IOKit.framework/IOKit\0"; +const IOPM_ASSERTION_CREATE_WITH_NAME_SYMBOL: &[u8] = b"IOPMAssertionCreateWithName\0"; +const IOPM_ASSERTION_RELEASE_SYMBOL: &[u8] = b"IOPMAssertionRelease\0"; +const IOKIT_ASSERTION_API_UNAVAILABLE: &str = "IOKit power assertion APIs are unavailable"; + +type IOPMAssertionReleaseFn = unsafe extern "C" fn(assertion_id: IOPMAssertionID) -> IOReturn; +type IOPMAssertionID = u32; +type IOPMAssertionLevel = u32; +type IOReturn = i32; + +const K_IOPM_ASSERTION_LEVEL_ON: IOPMAssertionLevel = 255; +const K_IORETURN_SUCCESS: IOReturn = 0; + +#[derive(Debug, Default)] +pub(crate) struct MacOsSleepInhibitor { + assertion: Option, +} + +impl MacOsSleepInhibitor { + pub(crate) fn new() -> Self { + Self { + ..Default::default() + } + } +} + +impl PlatformSleepInhibitor for MacOsSleepInhibitor { + fn acquire(&mut self) { + if self.assertion.is_some() { + return; + } + + match MacSleepAssertion::create(ASSERTION_REASON) { + Ok(assertion) => { + self.assertion = Some(assertion); + } + Err(error) => match error { + MacSleepAssertionError::ApiUnavailable(reason) => { + warn!(reason, "Failed to create macOS sleep-prevention assertion"); + } + MacSleepAssertionError::Iokit(code) => { + warn!( + iokit_error = code, + "Failed to create macOS sleep-prevention assertion" + ); + } + }, + } + } + + fn release(&mut self) { + self.assertion = None; + } +} + +#[derive(Debug)] +struct MacSleepAssertion { + id: IOPMAssertionID, +} + +impl MacSleepAssertion { + fn create(name: &str) -> Result { + let Some(api) = MacSleepApi::get() else { + return Err(MacSleepAssertionError::ApiUnavailable( + IOKIT_ASSERTION_API_UNAVAILABLE, + )); + }; + + let assertion_type = CFString::new(MACOS_IDLE_SLEEP_ASSERTION_TYPE); + let assertion_name = CFString::new(name); + let mut id: IOPMAssertionID = 0; + let result = unsafe { + (api.create_with_name)( + assertion_type.as_concrete_TypeRef(), + K_IOPM_ASSERTION_LEVEL_ON, + assertion_name.as_concrete_TypeRef(), + &mut id, + ) + }; + if result == K_IORETURN_SUCCESS { + Ok(Self { id }) + } else { + Err(MacSleepAssertionError::Iokit(result)) + } + } +} + +impl Drop for MacSleepAssertion { + fn drop(&mut self) { + let Some(api) = MacSleepApi::get() else { + warn!( + reason = IOKIT_ASSERTION_API_UNAVAILABLE, + "Failed to release macOS sleep-prevention assertion" + ); + return; + }; + + let result = unsafe { (api.release)(self.id) }; + if result != K_IORETURN_SUCCESS { + warn!( + iokit_error = result, + "Failed to release macOS sleep-prevention assertion" + ); + } + } +} + +#[derive(Debug, Clone, Copy)] +enum MacSleepAssertionError { + ApiUnavailable(&'static str), + Iokit(IOReturn), +} + +type IOPMAssertionCreateWithNameFn = unsafe extern "C" fn( + assertion_type: CFStringRef, + assertion_level: IOPMAssertionLevel, + assertion_name: CFStringRef, + assertion_id: *mut IOPMAssertionID, +) -> IOReturn; + +struct MacSleepApi { + // Keep the dlopen handle alive for the lifetime of the loaded symbols. + // This prevents accidental dlclose while function pointers are in use. + _iokit_handle: usize, + create_with_name: IOPMAssertionCreateWithNameFn, + release: IOPMAssertionReleaseFn, +} + +impl MacSleepApi { + fn get() -> Option<&'static Self> { + static API: OnceLock> = OnceLock::new(); + API.get_or_init(Self::load).as_ref() + } + + fn load() -> Option { + let handle = unsafe { + libc::dlopen( + IOKIT_FRAMEWORK_BINARY.as_ptr().cast(), + libc::RTLD_LOCAL | libc::RTLD_LAZY, + ) + }; + if handle.is_null() { + warn!(framework = "IOKit", "Failed to open IOKit framework"); + return None; + } + + let create_with_name = unsafe { + libc::dlsym( + handle, + IOPM_ASSERTION_CREATE_WITH_NAME_SYMBOL.as_ptr().cast(), + ) + }; + if create_with_name.is_null() { + warn!( + symbol = "IOPMAssertionCreateWithName", + "Failed to load IOKit symbol" + ); + let _ = unsafe { libc::dlclose(handle) }; + return None; + } + + let release = unsafe { libc::dlsym(handle, IOPM_ASSERTION_RELEASE_SYMBOL.as_ptr().cast()) }; + if release.is_null() { + warn!( + symbol = "IOPMAssertionRelease", + "Failed to load IOKit symbol" + ); + let _ = unsafe { libc::dlclose(handle) }; + return None; + } + + let create_with_name: IOPMAssertionCreateWithNameFn = + unsafe { std::mem::transmute(create_with_name) }; + let release: IOPMAssertionReleaseFn = unsafe { std::mem::transmute(release) }; + + Some(Self { + _iokit_handle: handle as usize, + create_with_name, + release, + }) + } +} diff --git a/docs/tui-chat-composer.md b/docs/tui-chat-composer.md index e4cf0c1b0c..40e23df3ea 100644 --- a/docs/tui-chat-composer.md +++ b/docs/tui-chat-composer.md @@ -56,10 +56,11 @@ The solution is to detect paste-like _bursts_ and buffer them into a single expl Up/Down recall is handled by `ChatComposerHistory` and merges two sources: - **Persistent history** (cross-session, fetched from `~/.codex/history.jsonl`): text-only. It - does **not** carry text element ranges or local image attachments, so recalling one of these - entries only restores the text. + does **not** carry text element ranges or image attachments, so recalling one of these entries + only restores the text. - **Local history** (current session): stores the full submission payload, including text - elements and local image paths. Recalling a local entry rehydrates placeholders and attachments. + elements, local image paths, and remote image URLs. Recalling a local entry rehydrates + placeholders and attachments. This distinction keeps the on-disk history backward compatible and avoids persisting attachments, while still providing a richer recall experience for in-session edits. @@ -127,6 +128,23 @@ positional args, Enter auto-submits without calling `prepare_submission_text`. T - Prunes attachments based on expanded placeholders. - Clears pending pastes after a successful auto-submit. +## Remote image rows (selection/deletion flow) + +Remote image URLs are shown as `[Image #N]` rows above the textarea, inside the same composer box. +They are attachment rows, not editable textarea content. + +- TUI can remove these rows, but cannot type before/between them. +- Press `Up` at textarea cursor position `0` to select the last remote image row. +- While selected, `Up`/`Down` moves selection across remote image rows. +- Pressing `Down` on the last row exits remote-row selection and returns to textarea editing. +- `Delete` or `Backspace` removes the selected remote image row. + +Image numbering is unified: + +- Remote image rows always occupy `[Image #1]..[Image #M]`. +- Local attached image placeholders start after that offset (`[Image #M+1]..`). +- Removing remote rows relabels local placeholders so numbering stays contiguous. + ## History navigation (Up/Down) and backtrack prefill `ChatComposerHistory` merges two kinds of history: @@ -139,6 +157,7 @@ Local history entries capture: - raw text (including placeholders), - `TextElement` ranges for placeholders, - local image paths, +- remote image URLs, - pending large-paste payloads (for drafts). Persistent history entries only restore text. They intentionally do **not** rehydrate attachments @@ -150,17 +169,17 @@ line). This keeps multiline cursor movement intact while preserving shell-like h ### Draft recovery (Ctrl+C) -Ctrl+C clears the composer but stashes the full draft state (text elements, image paths, and -pending paste payloads) into local history. Pressing Up immediately restores that draft, including -image placeholders and large-paste placeholders with their payloads. +Ctrl+C clears the composer but stashes the full draft state (text elements, local image paths, +remote image URLs, and pending paste payloads) into local history. Pressing Up immediately restores +that draft, including image placeholders and large-paste placeholders with their payloads. ### Submitted message recall -After a successful submission, the local history entry stores the submitted text and any element -ranges and local image paths. Pending paste payloads are cleared during submission, so large-paste -placeholders are expanded into their full text before being recorded. This means: +After a successful submission, the local history entry stores the submitted text, element ranges, +local image paths, and remote image URLs. Pending paste payloads are cleared during submission, so +large-paste placeholders are expanded into their full text before being recorded. This means: -- Up/Down recall of a submitted message restores image placeholders and their local paths. +- Up/Down recall of a submitted message restores remote image rows plus local image placeholders. - Recalled entries place the cursor at end-of-line to match typical shell history editing. - Large-paste placeholders are not expected in recalled submitted history; the text is the expanded paste content. @@ -168,14 +187,15 @@ placeholders are expanded into their full text before being recorded. This means ### Backtrack prefill Backtrack selections read `UserHistoryCell` data from the transcript. The composer prefill now -reuses the selected message’s text elements and local image paths, so image placeholders and -attachments rehydrate when rolling back to a prior user message. +reuses the selected message’s text elements, local image paths, and remote image URLs, so image +placeholders and attachments rehydrate when rolling back to a prior user message. ### External editor edits When the composer content is replaced from an external editor, the composer rebuilds text elements and keeps only attachments whose placeholders still appear in the new text. Image placeholders are -then normalized to `[Image #1]..[Image #N]` to keep attachment mapping consistent after edits. +then normalized to `[Image #M]..[Image #N]`, where `M` starts after the number of remote image +rows, to keep attachment mapping consistent after edits. ## Paste burst: concepts and assumptions diff --git a/flake.nix b/flake.nix index 711172d737..1fd41c6f14 100644 --- a/flake.nix +++ b/flake.nix @@ -9,7 +9,7 @@ }; }; - outputs = { nixpkgs, rust-overlay, ... }: + outputs = { self, nixpkgs, rust-overlay, ... }: let systems = [ "x86_64-linux" @@ -18,6 +18,19 @@ "aarch64-darwin" ]; forAllSystems = f: nixpkgs.lib.genAttrs systems f; + + # Read the version from the workspace Cargo.toml (the single source of + # truth used by the release workflow). + cargoToml = builtins.fromTOML (builtins.readFile ./codex-rs/Cargo.toml); + cargoVersion = cargoToml.workspace.package.version; + + # When building from a release commit the Cargo.toml already carries the + # real version (e.g. "0.101.0"). On the main branch it is the placeholder + # "0.0.0", so we fall back to a dev version derived from the flake source. + version = + if cargoVersion != "0.0.0" + then cargoVersion + else "0.0.0-dev+${self.shortRev or "dirty"}"; in { packages = forAllSystems (system: @@ -27,6 +40,7 @@ overlays = [ rust-overlay.overlays.default ]; }; codex-rs = pkgs.callPackage ./codex-rs { + inherit version; rustPlatform = pkgs.makeRustPlatform { cargo = pkgs.rust-bin.stable.latest.minimal; rustc = pkgs.rust-bin.stable.latest.minimal; diff --git a/justfile b/justfile index 024a31ba2a..33d62d515b 100644 --- a/justfile +++ b/justfile @@ -51,6 +51,14 @@ test: bazel-codex *args: bazel run //codex-rs/cli:codex --run_under="cd $PWD &&" -- "$@" +[no-cd] +bazel-lock-update: + bazel mod deps --lockfile_mode=update + +[no-cd] +bazel-lock-check: + ./scripts/check-module-bazel-lock.sh + bazel-test: bazel test //... --keep_going diff --git a/scripts/check-module-bazel-lock.sh b/scripts/check-module-bazel-lock.sh new file mode 100755 index 0000000000..1a14889643 --- /dev/null +++ b/scripts/check-module-bazel-lock.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -euo pipefail + +if ! bazel mod deps --lockfile_mode=error; then + echo "MODULE.bazel.lock is out of date." + echo "Run 'just bazel-lock-update' and commit the updated lockfile." + exit 1 +fi diff --git a/shell-tool-mcp/README.md b/shell-tool-mcp/README.md index ccfd0bcfba..08460f0021 100644 --- a/shell-tool-mcp/README.md +++ b/shell-tool-mcp/README.md @@ -99,7 +99,7 @@ The Codex harness (used by the CLI and the VS Code extension) sends such request This package wraps the `codex-exec-mcp-server` binary and its helpers so that the shell MCP can be invoked via `npx -y @openai/codex-shell-tool-mcp`. It bundles: - `codex-exec-mcp-server` and `codex-execve-wrapper` built for macOS (arm64, x64) and Linux (musl arm64, musl x64). -- A patched Bash that honors `BASH_EXEC_WRAPPER`, built for multiple glibc baselines (Ubuntu 24.04/22.04/20.04, Debian 12/11, CentOS-like 9) and macOS (15/14/13). +- A patched Bash that honors `EXEC_WRAPPER`, built for multiple glibc baselines (Ubuntu 24.04/22.04/20.04, Debian 12/11, CentOS-like 9) and macOS (15/14/13). - A launcher (`bin/mcp-server.js`) that picks the correct binaries for the current `process.platform` / `process.arch`, specifying `--execve` and `--bash` for the MCP, as appropriate. See [the README in the Codex repo](https://github.com/openai/codex/blob/main/codex-rs/exec-server/README.md) for details. diff --git a/shell-tool-mcp/patches/bash-exec-wrapper.patch b/shell-tool-mcp/patches/bash-exec-wrapper.patch index 6a7fedbb8f..f4471f2c2d 100644 --- a/shell-tool-mcp/patches/bash-exec-wrapper.patch +++ b/shell-tool-mcp/patches/bash-exec-wrapper.patch @@ -6,7 +6,7 @@ index 070f5119..d20ad2b9 100644 char sample[HASH_BANG_BUFSIZ]; size_t larray; -+ char* exec_wrapper = getenv("BASH_EXEC_WRAPPER"); ++ char* exec_wrapper = getenv("EXEC_WRAPPER"); + if (exec_wrapper && *exec_wrapper && !whitespace (*exec_wrapper)) + { + char *orig_command = command; diff --git a/shell-tool-mcp/patches/zsh-exec-wrapper.patch b/shell-tool-mcp/patches/zsh-exec-wrapper.patch new file mode 100644 index 0000000000..99cdcadcb2 --- /dev/null +++ b/shell-tool-mcp/patches/zsh-exec-wrapper.patch @@ -0,0 +1,34 @@ +diff --git a/Src/exec.c b/Src/exec.c +index 27bca11..baea760 100644 +--- a/Src/exec.c ++++ b/Src/exec.c +@@ -507,7 +507,9 @@ zexecve(char *pth, char **argv, char **newenvp) + { + int eno; + static char buf[PATH_MAX * 2+1]; +- char **eep; ++ char **eep, **exec_argv; ++ char *orig_pth = pth; ++ char *exec_wrapper; + + unmetafy(pth, NULL); + for (eep = argv; *eep; eep++) +@@ -526,8 +528,17 @@ zexecve(char *pth, char **argv, char **newenvp) + + if (newenvp == NULL) + newenvp = environ; ++ exec_argv = argv; ++ if ((exec_wrapper = getenv("EXEC_WRAPPER")) && ++ *exec_wrapper && !inblank(*exec_wrapper)) { ++ exec_argv = argv - 2; ++ exec_argv[0] = exec_wrapper; ++ exec_argv[1] = orig_pth; ++ pth = exec_wrapper; ++ } + winch_unblock(); +- execve(pth, argv, newenvp); ++ execve(pth, exec_argv, newenvp); ++ pth = orig_pth; + + /* If the execve returns (which in general shouldn't happen), * + * then check for an errno equal to ENOEXEC. This errno is set *