Merge f30c01c83e into sapling-pr-archive-bolinfest

This commit is contained in:
Michael Bolin
2026-02-15 20:05:45 -08:00
committed by GitHub
247 changed files with 17259 additions and 3072 deletions
+3 -1
View File
@@ -1,3 +1,5 @@
iTerm
iTerm2
psuedo
psuedo
te
TE
+1 -1
View File
@@ -3,4 +3,4 @@
skip = .git*,vendor,*-lock.yaml,*.lock,.codespellrc,*test.ts,*.jsonl,frame*.txt,*.snap,*.snap.new,*meriyah.umd.min.js
check-hidden = true
ignore-regex = ^\s*"image/\S+": ".*|\b(afterAll)\b
ignore-words-list = ratatui,ser,iTerm,iterm2,iterm
ignore-words-list = ratatui,ser,iTerm,iterm2,iterm,te,TE
+7
View File
@@ -21,6 +21,13 @@ body:
label: What subscription do you have?
validations:
required: true
- type: input
id: platform
attributes:
label: What platform is your computer?
description: |
For macOS and Linux: copy the output of `uname -mprs`
For Windows: copy the output of `"$([Environment]::OSVersion | ForEach-Object VersionString) $(if ([Environment]::Is64BitOperatingSystem) { "x64" } else { "x86" })"` in the PowerShell console
- type: textarea
id: actual
attributes:
-18
View File
@@ -1,18 +0,0 @@
You are an assistant that triages new GitHub issues by identifying potential duplicates.
You will receive the following JSON files located in the current working directory:
- `codex-current-issue.json`: JSON object describing the newly created issue (fields: number, title, body).
- `codex-existing-issues.json`: JSON array of recent issues (each element includes number, title, body, createdAt).
Instructions:
- Load both files as JSON and review their contents carefully. The codex-existing-issues.json file is large, ensure you explore all of it.
- Compare the current issue against the existing issues to find up to five that appear to describe the same underlying problem or request.
- Only consider an issue a potential duplicate if there is a clear overlap in symptoms, feature requests, reproduction steps, or error messages.
- Prioritize newer issues when similarity is comparable.
- Ignore pull requests and issues whose similarity is tenuous.
- When unsure, prefer returning fewer matches.
Output requirements:
- Respond with a JSON array of issue numbers (integers), ordered from most likely duplicate to least.
- Include at most five numbers.
- If you find no plausible duplicates, respond with `[]`.
+5
View File
@@ -65,6 +65,11 @@ jobs:
- name: Set up Bazel
uses: bazelbuild/setup-bazelisk@v3
- name: Check MODULE.bazel.lock is up to date
if: matrix.os == 'ubuntu-24.04' && matrix.target == 'x86_64-unknown-linux-gnu'
shell: bash
run: ./scripts/check-module-bazel-lock.sh
# TODO(mbolin): Bring this back once we have caching working. Currently,
# we never seem to get a cache hit but we still end up paying the cost of
# uploading at the end of the build, which takes over a minute!
+232 -16
View File
@@ -15,34 +15,68 @@ jobs:
permissions:
contents: read
outputs:
codex_output: ${{ steps.codex.outputs.final-message }}
codex_output: ${{ steps.select-final.outputs.codex_output }}
steps:
- uses: actions/checkout@v6
- name: Prepare Codex inputs
env:
GH_TOKEN: ${{ github.token }}
REPO: ${{ github.repository }}
ISSUE_NUMBER: ${{ github.event.issue.number }}
run: |
set -eo pipefail
CURRENT_ISSUE_FILE=codex-current-issue.json
EXISTING_ISSUES_FILE=codex-existing-issues.json
EXISTING_ALL_FILE=codex-existing-issues-all.json
EXISTING_OPEN_FILE=codex-existing-issues-open.json
gh issue list --repo "${{ github.repository }}" \
--json number,title,body,createdAt \
gh issue list --repo "$REPO" \
--json number,title,body,createdAt,updatedAt,state,labels \
--limit 1000 \
--state all \
--search "sort:created-desc" \
| jq '.' \
> "$EXISTING_ISSUES_FILE"
| jq '[.[] | {
number,
title,
body: ((.body // "")[0:4000]),
createdAt,
updatedAt,
state,
labels: ((.labels // []) | map(.name))
}]' \
> "$EXISTING_ALL_FILE"
gh issue view "${{ github.event.issue.number }}" \
--repo "${{ github.repository }}" \
gh issue list --repo "$REPO" \
--json number,title,body,createdAt,updatedAt,state,labels \
--limit 1000 \
--state open \
--search "sort:created-desc" \
| jq '[.[] | {
number,
title,
body: ((.body // "")[0:4000]),
createdAt,
updatedAt,
state,
labels: ((.labels // []) | map(.name))
}]' \
> "$EXISTING_OPEN_FILE"
gh issue view "$ISSUE_NUMBER" \
--repo "$REPO" \
--json number,title,body \
| jq '.' \
| jq '{number, title, body: ((.body // "")[0:4000])}' \
> "$CURRENT_ISSUE_FILE"
- id: codex
echo "Prepared duplicate detection input files."
echo "all_issue_count=$(jq 'length' "$EXISTING_ALL_FILE")"
echo "open_issue_count=$(jq 'length' "$EXISTING_OPEN_FILE")"
# Prompt instructions are intentionally inline in this workflow. The old
# .github/prompts/issue-deduplicator.txt file is obsolete and removed.
- id: codex-all
name: Find duplicates (pass 1, all issues)
uses: openai/codex-action@main
with:
openai-api-key: ${{ secrets.CODEX_OPENAI_API_KEY }}
@@ -52,14 +86,17 @@ jobs:
You will receive the following JSON files located in the current working directory:
- `codex-current-issue.json`: JSON object describing the newly created issue (fields: number, title, body).
- `codex-existing-issues.json`: JSON array of recent issues (each element includes number, title, body, createdAt).
- `codex-existing-issues-all.json`: JSON array of recent issues with states, timestamps, and labels.
Instructions:
- Compare the current issue against the existing issues to find up to five that appear to describe the same underlying problem or request.
- Focus on the underlying intent and context of each issue—such as reported symptoms, feature requests, reproduction steps, or error messages—rather than relying solely on string similarity or synthetic metrics.
- After your analysis, validate your results in 1-2 lines explaining your decision to return the selected matches.
- When unsure, prefer returning fewer matches.
- Include at most five numbers.
- Prioritize concrete overlap in symptoms, reproduction details, error signatures, and user intent.
- Prefer active unresolved issues when confidence is similar.
- Closed issues can still be valid duplicates if they clearly match.
- Return fewer matches rather than speculative ones.
- If confidence is low, return an empty list.
- Include at most five issue numbers.
- After analysis, provide a short reason for your decision.
output-schema: |
{
@@ -77,6 +114,179 @@ jobs:
"additionalProperties": false
}
- id: normalize-all
name: Normalize pass 1 output
env:
CODEX_OUTPUT: ${{ steps.codex-all.outputs.final-message }}
CURRENT_ISSUE_NUMBER: ${{ github.event.issue.number }}
run: |
set -eo pipefail
raw=${CODEX_OUTPUT//$'\r'/}
parsed=false
issues='[]'
reason=''
if [ -n "$raw" ] && printf '%s' "$raw" | jq -e 'type == "object" and (.issues | type == "array")' >/dev/null 2>&1; then
parsed=true
issues=$(printf '%s' "$raw" | jq -c '[.issues[] | tostring]')
reason=$(printf '%s' "$raw" | jq -r '.reason // ""')
else
reason='Pass 1 output was empty or invalid JSON.'
fi
filtered=$(jq -cn --argjson issues "$issues" --arg current "$CURRENT_ISSUE_NUMBER" '[
$issues[]
| tostring
| select(. != $current)
] | reduce .[] as $issue ([]; if index($issue) then . else . + [$issue] end) | .[:5]')
has_matches=false
if [ "$(jq 'length' <<< "$filtered")" -gt 0 ]; then
has_matches=true
fi
echo "Pass 1 parsed: $parsed"
echo "Pass 1 matches after filtering: $(jq 'length' <<< "$filtered")"
echo "Pass 1 reason: $reason"
{
echo "issues_json=$filtered"
echo "reason<<EOF"
echo "$reason"
echo "EOF"
echo "has_matches=$has_matches"
} >> "$GITHUB_OUTPUT"
- id: codex-open
name: Find duplicates (pass 2, open issues)
if: ${{ steps.normalize-all.outputs.has_matches != 'true' }}
uses: openai/codex-action@main
with:
openai-api-key: ${{ secrets.CODEX_OPENAI_API_KEY }}
allow-users: "*"
prompt: |
You are an assistant that triages new GitHub issues by identifying potential duplicates.
This is a fallback pass because a broad search did not find convincing matches.
You will receive the following JSON files located in the current working directory:
- `codex-current-issue.json`: JSON object describing the newly created issue (fields: number, title, body).
- `codex-existing-issues-open.json`: JSON array of open issues only.
Instructions:
- Search only these active unresolved issues for duplicates of the current issue.
- Prioritize concrete overlap in symptoms, reproduction details, error signatures, and user intent.
- Prefer fewer, higher-confidence matches.
- If confidence is low, return an empty list.
- Include at most five issue numbers.
- After analysis, provide a short reason for your decision.
output-schema: |
{
"type": "object",
"properties": {
"issues": {
"type": "array",
"items": {
"type": "string"
}
},
"reason": { "type": "string" }
},
"required": ["issues", "reason"],
"additionalProperties": false
}
- id: normalize-open
name: Normalize pass 2 output
if: ${{ steps.normalize-all.outputs.has_matches != 'true' }}
env:
CODEX_OUTPUT: ${{ steps.codex-open.outputs.final-message }}
CURRENT_ISSUE_NUMBER: ${{ github.event.issue.number }}
run: |
set -eo pipefail
raw=${CODEX_OUTPUT//$'\r'/}
parsed=false
issues='[]'
reason=''
if [ -n "$raw" ] && printf '%s' "$raw" | jq -e 'type == "object" and (.issues | type == "array")' >/dev/null 2>&1; then
parsed=true
issues=$(printf '%s' "$raw" | jq -c '[.issues[] | tostring]')
reason=$(printf '%s' "$raw" | jq -r '.reason // ""')
else
reason='Pass 2 output was empty or invalid JSON.'
fi
filtered=$(jq -cn --argjson issues "$issues" --arg current "$CURRENT_ISSUE_NUMBER" '[
$issues[]
| tostring
| select(. != $current)
] | reduce .[] as $issue ([]; if index($issue) then . else . + [$issue] end) | .[:5]')
has_matches=false
if [ "$(jq 'length' <<< "$filtered")" -gt 0 ]; then
has_matches=true
fi
echo "Pass 2 parsed: $parsed"
echo "Pass 2 matches after filtering: $(jq 'length' <<< "$filtered")"
echo "Pass 2 reason: $reason"
{
echo "issues_json=$filtered"
echo "reason<<EOF"
echo "$reason"
echo "EOF"
echo "has_matches=$has_matches"
} >> "$GITHUB_OUTPUT"
- id: select-final
name: Select final duplicate set
env:
PASS1_ISSUES: ${{ steps.normalize-all.outputs.issues_json }}
PASS1_REASON: ${{ steps.normalize-all.outputs.reason }}
PASS2_ISSUES: ${{ steps.normalize-open.outputs.issues_json }}
PASS2_REASON: ${{ steps.normalize-open.outputs.reason }}
PASS1_HAS_MATCHES: ${{ steps.normalize-all.outputs.has_matches }}
PASS2_HAS_MATCHES: ${{ steps.normalize-open.outputs.has_matches }}
run: |
set -eo pipefail
selected_issues='[]'
selected_reason='No plausible duplicates found.'
selected_pass='none'
if [ "$PASS1_HAS_MATCHES" = "true" ]; then
selected_issues=${PASS1_ISSUES:-'[]'}
selected_reason=${PASS1_REASON:-'Pass 1 found duplicates.'}
selected_pass='all'
fi
if [ "$PASS2_HAS_MATCHES" = "true" ]; then
selected_issues=${PASS2_ISSUES:-'[]'}
selected_reason=${PASS2_REASON:-'Pass 2 found duplicates.'}
selected_pass='open-fallback'
fi
final_json=$(jq -cn \
--argjson issues "$selected_issues" \
--arg reason "$selected_reason" \
--arg pass "$selected_pass" \
'{issues: $issues, reason: $reason, pass: $pass}')
echo "Final pass used: $selected_pass"
echo "Final duplicate count: $(jq '.issues | length' <<< "$final_json")"
echo "Final reason: $(jq -r '.reason' <<< "$final_json")"
{
echo "codex_output<<EOF"
echo "$final_json"
echo "EOF"
} >> "$GITHUB_OUTPUT"
comment-on-issue:
name: Comment with potential duplicates
needs: gather-duplicates
@@ -105,11 +315,17 @@ jobs:
const issues = Array.isArray(parsed?.issues) ? parsed.issues : [];
const currentIssueNumber = String(context.payload.issue.number);
const passUsed = typeof parsed?.pass === 'string' ? parsed.pass : 'unknown';
const reason = typeof parsed?.reason === 'string' ? parsed.reason : '';
console.log(`Current issue number: ${currentIssueNumber}`);
console.log(`Pass used: ${passUsed}`);
if (reason) {
console.log(`Reason: ${reason}`);
}
console.log(issues);
const filteredIssues = issues.filter((value) => String(value) !== currentIssueNumber);
const filteredIssues = [...new Set(issues.map((value) => String(value)))].filter((value) => value !== currentIssueNumber).slice(0, 5);
if (filteredIssues.length === 0) {
core.info('Codex reported no potential duplicates.');
+4 -4
View File
@@ -59,7 +59,7 @@ jobs:
working-directory: codex-rs
steps:
- uses: actions/checkout@v6
- uses: dtolnay/rust-toolchain@1.93
- uses: dtolnay/rust-toolchain@1.93.0
with:
components: rustfmt
- name: cargo fmt
@@ -75,7 +75,7 @@ jobs:
working-directory: codex-rs
steps:
- uses: actions/checkout@v6
- uses: dtolnay/rust-toolchain@1.93
- uses: dtolnay/rust-toolchain@1.93.0
- uses: taiki-e/install-action@44c6d64aa62cd779e873306675c7a58e86d6d532 # v2
with:
tool: cargo-shear
@@ -196,7 +196,7 @@ jobs:
fi
sudo DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends "${packages[@]}"
fi
- uses: dtolnay/rust-toolchain@1.93
- uses: dtolnay/rust-toolchain@1.93.0
with:
targets: ${{ matrix.target }}
components: clippy
@@ -513,7 +513,7 @@ jobs:
- name: Install DotSlash
uses: facebook/install-dotslash@v2
- uses: dtolnay/rust-toolchain@1.93
- uses: dtolnay/rust-toolchain@1.93.0
with:
targets: ${{ matrix.target }}
+1 -1
View File
@@ -82,7 +82,7 @@ jobs:
Write-Host "Total RAM: $ramGiB GiB"
Write-Host "Disk usage:"
Get-PSDrive -PSProvider FileSystem | Format-Table -AutoSize Name, @{Name='Size(GB)';Expression={[math]::Round(($_.Used + $_.Free) / 1GB, 1)}}, @{Name='Free(GB)';Expression={[math]::Round($_.Free / 1GB, 1)}}
- uses: dtolnay/rust-toolchain@1.93
- uses: dtolnay/rust-toolchain@1.93.0
with:
targets: ${{ matrix.target }}
+1 -1
View File
@@ -123,7 +123,7 @@ jobs:
sudo apt-get update -y
sudo DEBIAN_FRONTEND=noninteractive apt-get install -y libubsan1
fi
- uses: dtolnay/rust-toolchain@1.93
- uses: dtolnay/rust-toolchain@1.93.0
with:
targets: ${{ matrix.target }}
+1 -1
View File
@@ -31,7 +31,7 @@ jobs:
node-version: 22
cache: pnpm
- uses: dtolnay/rust-toolchain@1.93
- uses: dtolnay/rust-toolchain@1.93.0
- name: build codex
run: cargo build --bin codex
+214 -5
View File
@@ -105,7 +105,7 @@ jobs:
sudo DEBIAN_FRONTEND=noninteractive apt-get install -y libubsan1
fi
- uses: dtolnay/rust-toolchain@1.93
- uses: dtolnay/rust-toolchain@1.93.0
with:
targets: ${{ matrix.target }}
@@ -251,11 +251,11 @@ jobs:
set -euo pipefail
if command -v apt-get >/dev/null 2>&1; then
apt-get update
DEBIAN_FRONTEND=noninteractive apt-get install -y git build-essential bison autoconf gettext
DEBIAN_FRONTEND=noninteractive apt-get install -y git build-essential bison autoconf gettext libncursesw5-dev
elif command -v dnf >/dev/null 2>&1; then
dnf install -y git gcc gcc-c++ make bison autoconf gettext
dnf install -y git gcc gcc-c++ make bison autoconf gettext ncurses-devel
elif command -v yum >/dev/null 2>&1; then
yum install -y git gcc gcc-c++ make bison autoconf gettext
yum install -y git gcc gcc-c++ make bison autoconf gettext ncurses-devel
else
echo "Unsupported package manager in container"
exit 1
@@ -329,6 +329,212 @@ jobs:
path: artifacts/**
if-no-files-found: error
zsh-linux:
name: Build zsh (Linux) - ${{ matrix.variant }} - ${{ matrix.target }}
needs: metadata
runs-on: ${{ matrix.runner }}
timeout-minutes: 30
container:
image: ${{ matrix.image }}
strategy:
fail-fast: false
matrix:
include:
- runner: ubuntu-24.04
target: x86_64-unknown-linux-musl
variant: ubuntu-24.04
image: ubuntu:24.04
- runner: ubuntu-24.04
target: x86_64-unknown-linux-musl
variant: ubuntu-22.04
image: ubuntu:22.04
- runner: ubuntu-24.04
target: x86_64-unknown-linux-musl
variant: debian-12
image: debian:12
- runner: ubuntu-24.04
target: x86_64-unknown-linux-musl
variant: debian-11
image: debian:11
- runner: ubuntu-24.04
target: x86_64-unknown-linux-musl
variant: centos-9
image: quay.io/centos/centos:stream9
- runner: ubuntu-24.04-arm
target: aarch64-unknown-linux-musl
variant: ubuntu-24.04
image: arm64v8/ubuntu:24.04
- runner: ubuntu-24.04-arm
target: aarch64-unknown-linux-musl
variant: ubuntu-22.04
image: arm64v8/ubuntu:22.04
- runner: ubuntu-24.04-arm
target: aarch64-unknown-linux-musl
variant: ubuntu-20.04
image: arm64v8/ubuntu:20.04
- runner: ubuntu-24.04-arm
target: aarch64-unknown-linux-musl
variant: debian-12
image: arm64v8/debian:12
- runner: ubuntu-24.04-arm
target: aarch64-unknown-linux-musl
variant: debian-11
image: arm64v8/debian:11
- runner: ubuntu-24.04-arm
target: aarch64-unknown-linux-musl
variant: centos-9
image: quay.io/centos/centos:stream9
steps:
- name: Install build prerequisites
shell: bash
run: |
set -euo pipefail
if command -v apt-get >/dev/null 2>&1; then
apt-get update
DEBIAN_FRONTEND=noninteractive apt-get install -y git build-essential bison autoconf gettext libncursesw5-dev
elif command -v dnf >/dev/null 2>&1; then
dnf install -y git gcc gcc-c++ make bison autoconf gettext ncurses-devel
elif command -v yum >/dev/null 2>&1; then
yum install -y git gcc gcc-c++ make bison autoconf gettext ncurses-devel
else
echo "Unsupported package manager in container"
exit 1
fi
- name: Checkout repository
uses: actions/checkout@v6
- name: Build patched zsh
shell: bash
run: |
set -euo pipefail
git clone --depth 1 https://git.code.sf.net/p/zsh/code /tmp/zsh
cd /tmp/zsh
git fetch --depth 1 origin 77045ef899e53b9598bebc5a41db93a548a40ca6
git checkout 77045ef899e53b9598bebc5a41db93a548a40ca6
git apply "${GITHUB_WORKSPACE}/shell-tool-mcp/patches/zsh-exec-wrapper.patch"
./Util/preconfig
./configure
cores="$(command -v nproc >/dev/null 2>&1 && nproc || getconf _NPROCESSORS_ONLN)"
make -j"${cores}"
dest="${GITHUB_WORKSPACE}/artifacts/vendor/${{ matrix.target }}/zsh/${{ matrix.variant }}"
mkdir -p "$dest"
cp Src/zsh "$dest/zsh"
- name: Smoke test zsh exec wrapper
shell: bash
run: |
set -euo pipefail
tmpdir="$(mktemp -d)"
cat > "$tmpdir/exec-wrapper" <<'EOF'
#!/usr/bin/env bash
set -euo pipefail
: "${CODEX_WRAPPER_LOG:?missing CODEX_WRAPPER_LOG}"
printf '%s\n' "$@" > "$CODEX_WRAPPER_LOG"
file="$1"
shift
if [[ "$#" -eq 0 ]]; then
exec "$file"
fi
arg0="$1"
shift
exec -a "$arg0" "$file" "$@"
EOF
chmod +x "$tmpdir/exec-wrapper"
CODEX_WRAPPER_LOG="$tmpdir/wrapper.log" \
EXEC_WRAPPER="$tmpdir/exec-wrapper" \
/tmp/zsh/Src/zsh -fc '/bin/echo smoke-zsh' > "$tmpdir/stdout.txt"
grep -Fx "smoke-zsh" "$tmpdir/stdout.txt"
grep -Fx "/bin/echo" "$tmpdir/wrapper.log"
- uses: actions/upload-artifact@v6
with:
name: shell-tool-mcp-zsh-${{ matrix.target }}-${{ matrix.variant }}
path: artifacts/**
if-no-files-found: error
zsh-darwin:
name: Build zsh (macOS) - ${{ matrix.variant }} - ${{ matrix.target }}
needs: metadata
runs-on: ${{ matrix.runner }}
timeout-minutes: 30
strategy:
fail-fast: false
matrix:
include:
- runner: macos-15-xlarge
target: aarch64-apple-darwin
variant: macos-15
- runner: macos-14
target: aarch64-apple-darwin
variant: macos-14
steps:
- name: Install build prerequisites
shell: bash
run: |
set -euo pipefail
if ! command -v autoconf >/dev/null 2>&1; then
brew install autoconf
fi
- name: Checkout repository
uses: actions/checkout@v6
- name: Build patched zsh
shell: bash
run: |
set -euo pipefail
git clone --depth 1 https://git.code.sf.net/p/zsh/code /tmp/zsh
cd /tmp/zsh
git fetch --depth 1 origin 77045ef899e53b9598bebc5a41db93a548a40ca6
git checkout 77045ef899e53b9598bebc5a41db93a548a40ca6
git apply "${GITHUB_WORKSPACE}/shell-tool-mcp/patches/zsh-exec-wrapper.patch"
./Util/preconfig
./configure
cores="$(getconf _NPROCESSORS_ONLN)"
make -j"${cores}"
dest="${GITHUB_WORKSPACE}/artifacts/vendor/${{ matrix.target }}/zsh/${{ matrix.variant }}"
mkdir -p "$dest"
cp Src/zsh "$dest/zsh"
- name: Smoke test zsh exec wrapper
shell: bash
run: |
set -euo pipefail
tmpdir="$(mktemp -d)"
cat > "$tmpdir/exec-wrapper" <<'EOF'
#!/usr/bin/env bash
set -euo pipefail
: "${CODEX_WRAPPER_LOG:?missing CODEX_WRAPPER_LOG}"
printf '%s\n' "$@" > "$CODEX_WRAPPER_LOG"
file="$1"
shift
if [[ "$#" -eq 0 ]]; then
exec "$file"
fi
arg0="$1"
shift
exec -a "$arg0" "$file" "$@"
EOF
chmod +x "$tmpdir/exec-wrapper"
CODEX_WRAPPER_LOG="$tmpdir/wrapper.log" \
EXEC_WRAPPER="$tmpdir/exec-wrapper" \
/tmp/zsh/Src/zsh -fc '/bin/echo smoke-zsh' > "$tmpdir/stdout.txt"
grep -Fx "smoke-zsh" "$tmpdir/stdout.txt"
grep -Fx "/bin/echo" "$tmpdir/wrapper.log"
- uses: actions/upload-artifact@v6
with:
name: shell-tool-mcp-zsh-${{ matrix.target }}-${{ matrix.variant }}
path: artifacts/**
if-no-files-found: error
package:
name: Package npm module
needs:
@@ -336,6 +542,8 @@ jobs:
- rust-binaries
- bash-linux
- bash-darwin
- zsh-linux
- zsh-darwin
runs-on: ubuntu-latest
env:
PACKAGE_VERSION: ${{ needs.metadata.outputs.version }}
@@ -409,7 +617,8 @@ jobs:
chmod +x \
"$staging"/vendor/*/codex-exec-mcp-server \
"$staging"/vendor/*/codex-execve-wrapper \
"$staging"/vendor/*/bash/*/bash
"$staging"/vendor/*/bash/*/bash \
"$staging"/vendor/*/zsh/*/zsh
- name: Create npm tarball
shell: bash
+4
View File
@@ -15,6 +15,10 @@ In the codex-rs folder where the rust code lives:
- When writing tests, prefer comparing the equality of entire objects over fields one by one.
- When making a change that adds or changes an API, ensure that the documentation in the `docs/` folder is up to date if applicable.
- If you change `ConfigToml` or nested config types, run `just write-config-schema` to update `codex-rs/core/config.schema.json`.
- If you change Rust dependencies (`Cargo.toml` or `Cargo.lock`), run `just bazel-lock-update` from the
repo root to refresh `MODULE.bazel.lock`, and include that lockfile update in the same change.
- After dependency changes, run `just bazel-lock-check` from the repo root so lockfile drift is caught
locally before CI.
- Do not create small helper methods that are referenced only once.
Run `just fmt` (in `codex-rs` directory) automatically after you have finished making Rust code changes; do not ask for approval to run it. Additionally, run the tests:
+2 -2
View File
File diff suppressed because one or more lines are too long
+14
View File
@@ -1420,6 +1420,8 @@ dependencies = [
"codex-protocol",
"serde",
"serde_json",
"tungstenite",
"url",
"uuid",
]
@@ -1710,6 +1712,7 @@ dependencies = [
"include_dir",
"indexmap 2.13.0",
"indoc",
"insta",
"keyring",
"landlock",
"libc",
@@ -2035,6 +2038,7 @@ version = "0.0.0"
dependencies = [
"anyhow",
"async-trait",
"base64 0.22.1",
"clap",
"codex-utils-absolute-path",
"codex-utils-rustls-provider",
@@ -2294,6 +2298,7 @@ dependencies = [
"codex-utils-oss",
"codex-utils-pty",
"codex-utils-sandbox-summary",
"codex-utils-sleep-inhibitor",
"codex-windows-sandbox",
"color-eyre",
"crossterm",
@@ -2493,6 +2498,15 @@ dependencies = [
"regex",
]
[[package]]
name = "codex-utils-sleep-inhibitor"
version = "0.0.0"
dependencies = [
"core-foundation 0.9.4",
"libc",
"tracing",
]
[[package]]
name = "codex-utils-string"
version = "0.0.0"
+2
View File
@@ -54,6 +54,7 @@ members = [
"utils/elapsed",
"utils/sandbox-summary",
"utils/sanitizer",
"utils/sleep-inhibitor",
"utils/approval-presets",
"utils/oss",
"utils/fuzzy-match",
@@ -131,6 +132,7 @@ codex-utils-readiness = { path = "utils/readiness" }
codex-utils-rustls-provider = { path = "utils/rustls-provider" }
codex-utils-sandbox-summary = { path = "utils/sandbox-summary" }
codex-utils-sanitizer = { path = "utils/sanitizer" }
codex-utils-sleep-inhibitor = { path = "utils/sleep-inhibitor" }
codex-utils-string = { path = "utils/string" }
codex-windows-sandbox = { path = "windows-sandbox-rs" }
core_test_support = { path = "core/tests/common" }
@@ -1128,6 +1128,13 @@
"null"
]
},
"includeHidden": {
"description": "When true, include models that are hidden from the default picker list.",
"type": [
"boolean",
"null"
]
},
"limit": {
"description": "Optional page size; defaults to a reasonable server-side value.",
"format": "uint32",
@@ -2565,6 +2572,13 @@
"null"
]
},
"cwd": {
"description": "Optional cwd filter; when set, only threads whose session cwd exactly matches this path are returned.",
"type": [
"string",
"null"
]
},
"limit": {
"description": "Optional page size; defaults to a reasonable server-side value.",
"format": "uint32",
@@ -1438,6 +1438,17 @@
"description": "The command's working directory.",
"type": "string"
},
"network_approval_context": {
"anyOf": [
{
"$ref": "#/definitions/NetworkApprovalContext"
},
{
"type": "null"
}
],
"description": "Optional network context for a blocked request that can be approved."
},
"parsed_cmd": {
"items": {
"$ref": "#/definitions/ParsedCommand"
@@ -3315,6 +3326,30 @@
],
"type": "string"
},
"NetworkApprovalContext": {
"properties": {
"host": {
"type": "string"
},
"protocol": {
"$ref": "#/definitions/NetworkApprovalProtocol"
}
},
"required": [
"host",
"protocol"
],
"type": "object"
},
"NetworkApprovalProtocol": {
"enum": [
"http",
"https",
"socks5_tcp",
"socks5_udp"
],
"type": "string"
},
"ParsedCommand": {
"oneOf": [
{
@@ -6308,6 +6343,17 @@
"description": "The command's working directory.",
"type": "string"
},
"network_approval_context": {
"anyOf": [
{
"$ref": "#/definitions/NetworkApprovalContext"
},
{
"type": "null"
}
],
"description": "Optional network context for a blocked request that can be approved."
},
"parsed_cmd": {
"items": {
"$ref": "#/definitions/ParsedCommand"
@@ -0,0 +1,13 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"properties": {
"sessionId": {
"type": "string"
}
},
"required": [
"sessionId"
],
"title": "FuzzyFileSearchSessionCompletedNotification",
"type": "object"
}
@@ -193,6 +193,11 @@
"default": false,
"type": "boolean"
},
"isEnabled": {
"default": true,
"description": "Whether this app is enabled in config.toml. Example: ```toml [apps.bad_app] enabled = false ```",
"type": "boolean"
},
"logoUrl": {
"type": [
"string",
@@ -2054,6 +2059,17 @@
"description": "The command's working directory.",
"type": "string"
},
"network_approval_context": {
"anyOf": [
{
"$ref": "#/definitions/NetworkApprovalContext"
},
{
"type": "null"
}
],
"description": "Optional network context for a blocked request that can be approved."
},
"parsed_cmd": {
"items": {
"$ref": "#/definitions/ParsedCommand"
@@ -3747,6 +3763,17 @@
],
"type": "object"
},
"FuzzyFileSearchSessionCompletedNotification": {
"properties": {
"sessionId": {
"type": "string"
}
},
"required": [
"sessionId"
],
"type": "object"
},
"FuzzyFileSearchSessionUpdatedNotification": {
"properties": {
"files": {
@@ -4191,6 +4218,30 @@
],
"type": "string"
},
"NetworkApprovalContext": {
"properties": {
"host": {
"type": "string"
},
"protocol": {
"$ref": "#/definitions/NetworkApprovalProtocol"
}
},
"required": [
"host",
"protocol"
],
"type": "object"
},
"NetworkApprovalProtocol": {
"enum": [
"http",
"https",
"socks5_tcp",
"socks5_udp"
],
"type": "string"
},
"ParsedCommand": {
"oneOf": [
{
@@ -5992,7 +6043,8 @@
{
"enum": [
"review",
"compact"
"compact",
"memory_consolidation"
],
"type": "string"
},
@@ -8284,6 +8336,26 @@
"title": "FuzzyFileSearch/sessionUpdatedNotification",
"type": "object"
},
{
"properties": {
"method": {
"enum": [
"fuzzyFileSearch/sessionCompleted"
],
"title": "FuzzyFileSearch/sessionCompletedNotificationMethod",
"type": "string"
},
"params": {
"$ref": "#/definitions/FuzzyFileSearchSessionCompletedNotification"
}
},
"required": [
"method",
"params"
],
"title": "FuzzyFileSearch/sessionCompletedNotification",
"type": "object"
},
{
"description": "Notifies the user of world-writable directories on Windows, which cannot be protected by the sandbox.",
"properties": {
@@ -3451,6 +3451,17 @@
"description": "The command's working directory.",
"type": "string"
},
"network_approval_context": {
"anyOf": [
{
"$ref": "#/definitions/NetworkApprovalContext"
},
{
"type": "null"
}
],
"description": "Optional network context for a blocked request that can be approved."
},
"parsed_cmd": {
"items": {
"$ref": "#/definitions/ParsedCommand"
@@ -5412,6 +5423,19 @@
],
"type": "object"
},
"FuzzyFileSearchSessionCompletedNotification": {
"$schema": "http://json-schema.org/draft-07/schema#",
"properties": {
"sessionId": {
"type": "string"
}
},
"required": [
"sessionId"
],
"title": "FuzzyFileSearchSessionCompletedNotification",
"type": "object"
},
"FuzzyFileSearchSessionUpdatedNotification": {
"$schema": "http://json-schema.org/draft-07/schema#",
"properties": {
@@ -6250,6 +6274,30 @@
],
"type": "string"
},
"NetworkApprovalContext": {
"properties": {
"host": {
"type": "string"
},
"protocol": {
"$ref": "#/definitions/NetworkApprovalProtocol"
}
},
"required": [
"host",
"protocol"
],
"type": "object"
},
"NetworkApprovalProtocol": {
"enum": [
"http",
"https",
"socks5_tcp",
"socks5_udp"
],
"type": "string"
},
"NewConversationParams": {
"properties": {
"approvalPolicy": {
@@ -8503,6 +8551,26 @@
"title": "FuzzyFileSearch/sessionUpdatedNotification",
"type": "object"
},
{
"properties": {
"method": {
"enum": [
"fuzzyFileSearch/sessionCompleted"
],
"title": "FuzzyFileSearch/sessionCompletedNotificationMethod",
"type": "string"
},
"params": {
"$ref": "#/definitions/FuzzyFileSearchSessionCompletedNotification"
}
},
"required": [
"method",
"params"
],
"title": "FuzzyFileSearch/sessionCompletedNotification",
"type": "object"
},
{
"description": "Notifies the user of world-writable directories on Windows, which cannot be protected by the sandbox.",
"properties": {
@@ -9126,7 +9194,8 @@
{
"enum": [
"review",
"compact"
"compact",
"memory_consolidation"
],
"type": "string"
},
@@ -10201,6 +10270,11 @@
"default": false,
"type": "boolean"
},
"isEnabled": {
"default": true,
"description": "Whether this app is enabled in config.toml. Example: ```toml [apps.bad_app] enabled = false ```",
"type": "boolean"
},
"logoUrl": {
"type": [
"string",
@@ -12572,6 +12646,9 @@
"displayName": {
"type": "string"
},
"hidden": {
"type": "boolean"
},
"id": {
"type": "string"
},
@@ -12612,6 +12689,7 @@
"defaultReasoningEffort",
"description",
"displayName",
"hidden",
"id",
"isDefault",
"model",
@@ -12629,6 +12707,13 @@
"null"
]
},
"includeHidden": {
"description": "When true, include models that are hidden from the default picker list.",
"type": [
"boolean",
"null"
]
},
"limit": {
"description": "Optional page size; defaults to a reasonable server-side value.",
"format": "uint32",
@@ -14475,7 +14560,8 @@
{
"enum": [
"review",
"compact"
"compact",
"memory_consolidation"
],
"type": "string"
},
@@ -15329,6 +15415,13 @@
"null"
]
},
"cwd": {
"description": "Optional cwd filter; when set, only threads whose session cwd exactly matches this path are returned.",
"type": [
"string",
"null"
]
},
"limit": {
"description": "Optional page size; defaults to a reasonable server-side value.",
"format": "uint32",
@@ -1438,6 +1438,17 @@
"description": "The command's working directory.",
"type": "string"
},
"network_approval_context": {
"anyOf": [
{
"$ref": "#/definitions/NetworkApprovalContext"
},
{
"type": "null"
}
],
"description": "Optional network context for a blocked request that can be approved."
},
"parsed_cmd": {
"items": {
"$ref": "#/definitions/ParsedCommand"
@@ -3315,6 +3326,30 @@
],
"type": "string"
},
"NetworkApprovalContext": {
"properties": {
"host": {
"type": "string"
},
"protocol": {
"$ref": "#/definitions/NetworkApprovalProtocol"
}
},
"required": [
"host",
"protocol"
],
"type": "object"
},
"NetworkApprovalProtocol": {
"enum": [
"http",
"https",
"socks5_tcp",
"socks5_udp"
],
"type": "string"
},
"ParsedCommand": {
"oneOf": [
{
@@ -113,7 +113,8 @@
{
"enum": [
"review",
"compact"
"compact",
"memory_consolidation"
],
"type": "string"
},
@@ -113,7 +113,8 @@
{
"enum": [
"review",
"compact"
"compact",
"memory_consolidation"
],
"type": "string"
},
@@ -1438,6 +1438,17 @@
"description": "The command's working directory.",
"type": "string"
},
"network_approval_context": {
"anyOf": [
{
"$ref": "#/definitions/NetworkApprovalContext"
},
{
"type": "null"
}
],
"description": "Optional network context for a blocked request that can be approved."
},
"parsed_cmd": {
"items": {
"$ref": "#/definitions/ParsedCommand"
@@ -3315,6 +3326,30 @@
],
"type": "string"
},
"NetworkApprovalContext": {
"properties": {
"host": {
"type": "string"
},
"protocol": {
"$ref": "#/definitions/NetworkApprovalProtocol"
}
},
"required": [
"host",
"protocol"
],
"type": "object"
},
"NetworkApprovalProtocol": {
"enum": [
"http",
"https",
"socks5_tcp",
"socks5_udp"
],
"type": "string"
},
"ParsedCommand": {
"oneOf": [
{
@@ -1438,6 +1438,17 @@
"description": "The command's working directory.",
"type": "string"
},
"network_approval_context": {
"anyOf": [
{
"$ref": "#/definitions/NetworkApprovalContext"
},
{
"type": "null"
}
],
"description": "Optional network context for a blocked request that can be approved."
},
"parsed_cmd": {
"items": {
"$ref": "#/definitions/ParsedCommand"
@@ -3315,6 +3326,30 @@
],
"type": "string"
},
"NetworkApprovalContext": {
"properties": {
"host": {
"type": "string"
},
"protocol": {
"$ref": "#/definitions/NetworkApprovalProtocol"
}
},
"required": [
"host",
"protocol"
],
"type": "object"
},
"NetworkApprovalProtocol": {
"enum": [
"http",
"https",
"socks5_tcp",
"socks5_udp"
],
"type": "string"
},
"ParsedCommand": {
"oneOf": [
{
@@ -29,6 +29,11 @@
"default": false,
"type": "boolean"
},
"isEnabled": {
"default": true,
"description": "Whether this app is enabled in config.toml. Example: ```toml [apps.bad_app] enabled = false ```",
"type": "boolean"
},
"logoUrl": {
"type": [
"string",
@@ -29,6 +29,11 @@
"default": false,
"type": "boolean"
},
"isEnabled": {
"default": true,
"description": "Whether this app is enabled in config.toml. Example: ```toml [apps.bad_app] enabled = false ```",
"type": "boolean"
},
"logoUrl": {
"type": [
"string",
@@ -8,6 +8,13 @@
"null"
]
},
"includeHidden": {
"description": "When true, include models that are hidden from the default picker list.",
"type": [
"boolean",
"null"
]
},
"limit": {
"description": "Optional page size; defaults to a reasonable server-side value.",
"format": "uint32",
@@ -31,6 +31,9 @@
"displayName": {
"type": "string"
},
"hidden": {
"type": "boolean"
},
"id": {
"type": "string"
},
@@ -71,6 +74,7 @@
"defaultReasoningEffort",
"description",
"displayName",
"hidden",
"id",
"isDefault",
"model",
@@ -666,7 +666,8 @@
{
"enum": [
"review",
"compact"
"compact",
"memory_consolidation"
],
"type": "string"
},
@@ -39,6 +39,13 @@
"null"
]
},
"cwd": {
"description": "Optional cwd filter; when set, only threads whose session cwd exactly matches this path are returned.",
"type": [
"string",
"null"
]
},
"limit": {
"description": "Optional page size; defaults to a reasonable server-side value.",
"format": "uint32",
@@ -472,7 +472,8 @@
{
"enum": [
"review",
"compact"
"compact",
"memory_consolidation"
],
"type": "string"
},
@@ -472,7 +472,8 @@
{
"enum": [
"review",
"compact"
"compact",
"memory_consolidation"
],
"type": "string"
},
@@ -666,7 +666,8 @@
{
"enum": [
"review",
"compact"
"compact",
"memory_consolidation"
],
"type": "string"
},
@@ -472,7 +472,8 @@
{
"enum": [
"review",
"compact"
"compact",
"memory_consolidation"
],
"type": "string"
},
@@ -666,7 +666,8 @@
{
"enum": [
"review",
"compact"
"compact",
"memory_consolidation"
],
"type": "string"
},
@@ -472,7 +472,8 @@
{
"enum": [
"review",
"compact"
"compact",
"memory_consolidation"
],
"type": "string"
},
@@ -472,7 +472,8 @@
{
"enum": [
"review",
"compact"
"compact",
"memory_consolidation"
],
"type": "string"
},
@@ -2,6 +2,7 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
import type { ExecPolicyAmendment } from "./ExecPolicyAmendment";
import type { NetworkApprovalContext } from "./NetworkApprovalContext";
import type { ParsedCommand } from "./ParsedCommand";
export type ExecApprovalRequestEvent = {
@@ -26,6 +27,10 @@ cwd: string,
* Optional human-readable reason for the approval (e.g. retry without sandbox).
*/
reason: string | null,
/**
* Optional network context for a blocked request that can be approved.
*/
network_approval_context?: NetworkApprovalContext,
/**
* Proposed execpolicy amendment that can be applied to allow future runs.
*/
@@ -0,0 +1,5 @@
// GENERATED CODE! DO NOT MODIFY BY HAND!
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
export type FuzzyFileSearchSessionCompletedNotification = { sessionId: string, };
@@ -0,0 +1,6 @@
// GENERATED CODE! DO NOT MODIFY BY HAND!
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
import type { NetworkApprovalProtocol } from "./NetworkApprovalProtocol";
export type NetworkApprovalContext = { host: string, protocol: NetworkApprovalProtocol, };
@@ -0,0 +1,5 @@
// GENERATED CODE! DO NOT MODIFY BY HAND!
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
export type NetworkApprovalProtocol = "http" | "https" | "socks5_tcp" | "socks5_udp";
@@ -2,6 +2,7 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
import type { AuthStatusChangeNotification } from "./AuthStatusChangeNotification";
import type { FuzzyFileSearchSessionCompletedNotification } from "./FuzzyFileSearchSessionCompletedNotification";
import type { FuzzyFileSearchSessionUpdatedNotification } from "./FuzzyFileSearchSessionUpdatedNotification";
import type { LoginChatGptCompleteNotification } from "./LoginChatGptCompleteNotification";
import type { SessionConfiguredNotification } from "./SessionConfiguredNotification";
@@ -38,4 +39,4 @@ import type { WindowsWorldWritableWarningNotification } from "./v2/WindowsWorldW
/**
* Notification sent from the server to the client.
*/
export type ServerNotification = { "method": "error", "params": ErrorNotification } | { "method": "thread/started", "params": ThreadStartedNotification } | { "method": "thread/name/updated", "params": ThreadNameUpdatedNotification } | { "method": "thread/tokenUsage/updated", "params": ThreadTokenUsageUpdatedNotification } | { "method": "turn/started", "params": TurnStartedNotification } | { "method": "turn/completed", "params": TurnCompletedNotification } | { "method": "turn/diff/updated", "params": TurnDiffUpdatedNotification } | { "method": "turn/plan/updated", "params": TurnPlanUpdatedNotification } | { "method": "item/started", "params": ItemStartedNotification } | { "method": "item/completed", "params": ItemCompletedNotification } | { "method": "rawResponseItem/completed", "params": RawResponseItemCompletedNotification } | { "method": "item/agentMessage/delta", "params": AgentMessageDeltaNotification } | { "method": "item/plan/delta", "params": PlanDeltaNotification } | { "method": "item/commandExecution/outputDelta", "params": CommandExecutionOutputDeltaNotification } | { "method": "item/commandExecution/terminalInteraction", "params": TerminalInteractionNotification } | { "method": "item/fileChange/outputDelta", "params": FileChangeOutputDeltaNotification } | { "method": "item/mcpToolCall/progress", "params": McpToolCallProgressNotification } | { "method": "mcpServer/oauthLogin/completed", "params": McpServerOauthLoginCompletedNotification } | { "method": "account/updated", "params": AccountUpdatedNotification } | { "method": "account/rateLimits/updated", "params": AccountRateLimitsUpdatedNotification } | { "method": "app/list/updated", "params": AppListUpdatedNotification } | { "method": "item/reasoning/summaryTextDelta", "params": ReasoningSummaryTextDeltaNotification } | { "method": "item/reasoning/summaryPartAdded", "params": ReasoningSummaryPartAddedNotification } | { "method": "item/reasoning/textDelta", "params": ReasoningTextDeltaNotification } | { "method": "thread/compacted", "params": ContextCompactedNotification } | { "method": "deprecationNotice", "params": DeprecationNoticeNotification } | { "method": "configWarning", "params": ConfigWarningNotification } | { "method": "fuzzyFileSearch/sessionUpdated", "params": FuzzyFileSearchSessionUpdatedNotification } | { "method": "windows/worldWritableWarning", "params": WindowsWorldWritableWarningNotification } | { "method": "account/login/completed", "params": AccountLoginCompletedNotification } | { "method": "authStatusChange", "params": AuthStatusChangeNotification } | { "method": "loginChatGptComplete", "params": LoginChatGptCompleteNotification } | { "method": "sessionConfigured", "params": SessionConfiguredNotification };
export type ServerNotification = { "method": "error", "params": ErrorNotification } | { "method": "thread/started", "params": ThreadStartedNotification } | { "method": "thread/name/updated", "params": ThreadNameUpdatedNotification } | { "method": "thread/tokenUsage/updated", "params": ThreadTokenUsageUpdatedNotification } | { "method": "turn/started", "params": TurnStartedNotification } | { "method": "turn/completed", "params": TurnCompletedNotification } | { "method": "turn/diff/updated", "params": TurnDiffUpdatedNotification } | { "method": "turn/plan/updated", "params": TurnPlanUpdatedNotification } | { "method": "item/started", "params": ItemStartedNotification } | { "method": "item/completed", "params": ItemCompletedNotification } | { "method": "rawResponseItem/completed", "params": RawResponseItemCompletedNotification } | { "method": "item/agentMessage/delta", "params": AgentMessageDeltaNotification } | { "method": "item/plan/delta", "params": PlanDeltaNotification } | { "method": "item/commandExecution/outputDelta", "params": CommandExecutionOutputDeltaNotification } | { "method": "item/commandExecution/terminalInteraction", "params": TerminalInteractionNotification } | { "method": "item/fileChange/outputDelta", "params": FileChangeOutputDeltaNotification } | { "method": "item/mcpToolCall/progress", "params": McpToolCallProgressNotification } | { "method": "mcpServer/oauthLogin/completed", "params": McpServerOauthLoginCompletedNotification } | { "method": "account/updated", "params": AccountUpdatedNotification } | { "method": "account/rateLimits/updated", "params": AccountRateLimitsUpdatedNotification } | { "method": "app/list/updated", "params": AppListUpdatedNotification } | { "method": "item/reasoning/summaryTextDelta", "params": ReasoningSummaryTextDeltaNotification } | { "method": "item/reasoning/summaryPartAdded", "params": ReasoningSummaryPartAddedNotification } | { "method": "item/reasoning/textDelta", "params": ReasoningTextDeltaNotification } | { "method": "thread/compacted", "params": ContextCompactedNotification } | { "method": "deprecationNotice", "params": DeprecationNoticeNotification } | { "method": "configWarning", "params": ConfigWarningNotification } | { "method": "fuzzyFileSearch/sessionUpdated", "params": FuzzyFileSearchSessionUpdatedNotification } | { "method": "fuzzyFileSearch/sessionCompleted", "params": FuzzyFileSearchSessionCompletedNotification } | { "method": "windows/worldWritableWarning", "params": WindowsWorldWritableWarningNotification } | { "method": "account/login/completed", "params": AccountLoginCompletedNotification } | { "method": "authStatusChange", "params": AuthStatusChangeNotification } | { "method": "loginChatGptComplete", "params": LoginChatGptCompleteNotification } | { "method": "sessionConfigured", "params": SessionConfiguredNotification };
@@ -3,4 +3,4 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
import type { ThreadId } from "./ThreadId";
export type SubAgentSource = "review" | "compact" | { "thread_spawn": { parent_thread_id: ThreadId, depth: number, } } | { "other": string };
export type SubAgentSource = "review" | "compact" | { "thread_spawn": { parent_thread_id: ThreadId, depth: number, } } | "memory_consolidation" | { "other": string };
@@ -78,6 +78,7 @@ export type { FunctionCallOutputPayload } from "./FunctionCallOutputPayload";
export type { FuzzyFileSearchParams } from "./FuzzyFileSearchParams";
export type { FuzzyFileSearchResponse } from "./FuzzyFileSearchResponse";
export type { FuzzyFileSearchResult } from "./FuzzyFileSearchResult";
export type { FuzzyFileSearchSessionCompletedNotification } from "./FuzzyFileSearchSessionCompletedNotification";
export type { FuzzyFileSearchSessionUpdatedNotification } from "./FuzzyFileSearchSessionUpdatedNotification";
export type { GetAuthStatusParams } from "./GetAuthStatusParams";
export type { GetAuthStatusResponse } from "./GetAuthStatusResponse";
@@ -125,6 +126,8 @@ export type { McpToolCallEndEvent } from "./McpToolCallEndEvent";
export type { MessagePhase } from "./MessagePhase";
export type { ModeKind } from "./ModeKind";
export type { NetworkAccess } from "./NetworkAccess";
export type { NetworkApprovalContext } from "./NetworkApprovalContext";
export type { NetworkApprovalProtocol } from "./NetworkApprovalProtocol";
export type { NewConversationParams } from "./NewConversationParams";
export type { NewConversationResponse } from "./NewConversationResponse";
export type { ParsedCommand } from "./ParsedCommand";
@@ -5,4 +5,13 @@
/**
* EXPERIMENTAL - app metadata returned by app-list APIs.
*/
export type AppInfo = { id: string, name: string, description: string | null, logoUrl: string | null, logoUrlDark: string | null, distributionChannel: string | null, installUrl: string | null, isAccessible: boolean, };
export type AppInfo = { id: string, name: string, description: string | null, logoUrl: string | null, logoUrlDark: string | null, distributionChannel: string | null, installUrl: string | null, isAccessible: boolean,
/**
* Whether this app is enabled in config.toml.
* Example:
* ```toml
* [apps.bad_app]
* enabled = false
* ```
*/
isEnabled: boolean, };
@@ -5,4 +5,4 @@ import type { InputModality } from "../InputModality";
import type { ReasoningEffort } from "../ReasoningEffort";
import type { ReasoningEffortOption } from "./ReasoningEffortOption";
export type Model = { id: string, model: string, upgrade: string | null, displayName: string, description: string, supportedReasoningEfforts: Array<ReasoningEffortOption>, defaultReasoningEffort: ReasoningEffort, inputModalities: Array<InputModality>, supportsPersonality: boolean, isDefault: boolean, };
export type Model = { id: string, model: string, upgrade: string | null, displayName: string, description: string, hidden: boolean, supportedReasoningEfforts: Array<ReasoningEffortOption>, defaultReasoningEffort: ReasoningEffort, inputModalities: Array<InputModality>, supportsPersonality: boolean, isDefault: boolean, };
@@ -10,4 +10,8 @@ cursor?: string | null,
/**
* Optional page size; defaults to a reasonable server-side value.
*/
limit?: number | null, };
limit?: number | null,
/**
* When true, include models that are hidden from the default picker list.
*/
includeHidden?: boolean | null, };
@@ -31,4 +31,9 @@ sourceKinds?: Array<ThreadSourceKind> | null,
* Optional archived filter; when set to true, only archived threads are returned.
* If false or null, only non-archived threads are returned.
*/
archived?: boolean | null, };
archived?: boolean | null,
/**
* Optional cwd filter; when set, only threads whose session cwd exactly
* matches this path are returned.
*/
cwd?: string | null, };
@@ -758,6 +758,13 @@ pub struct FuzzyFileSearchSessionUpdatedNotification {
pub files: Vec<FuzzyFileSearchResult>,
}
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)]
#[serde(rename_all = "camelCase")]
#[ts(rename_all = "camelCase")]
pub struct FuzzyFileSearchSessionCompletedNotification {
pub session_id: String,
}
server_notification_definitions! {
/// NEW NOTIFICATIONS
Error => "error" (v2::ErrorNotification),
@@ -791,6 +798,7 @@ server_notification_definitions! {
DeprecationNotice => "deprecationNotice" (v2::DeprecationNoticeNotification),
ConfigWarning => "configWarning" (v2::ConfigWarningNotification),
FuzzyFileSearchSessionUpdated => "fuzzyFileSearch/sessionUpdated" (FuzzyFileSearchSessionUpdatedNotification),
FuzzyFileSearchSessionCompleted => "fuzzyFileSearch/sessionCompleted" (FuzzyFileSearchSessionCompletedNotification),
/// Notifies the user of world-writable directories on Windows, which cannot be protected by the sandbox.
WindowsWorldWritableWarning => "windows/worldWritableWarning" (v2::WindowsWorldWritableWarningNotification),
@@ -1227,7 +1235,8 @@ mod tests {
"id": 6,
"params": {
"limit": null,
"cursor": null
"cursor": null,
"includeHidden": null
}
}),
serde_json::to_value(&request)?,
@@ -1110,6 +1110,9 @@ pub struct ModelListParams {
/// Optional page size; defaults to a reasonable server-side value.
#[ts(optional = nullable)]
pub limit: Option<u32>,
/// When true, include models that are hidden from the default picker list.
#[ts(optional = nullable)]
pub include_hidden: Option<bool>,
}
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)]
@@ -1121,6 +1124,7 @@ pub struct Model {
pub upgrade: Option<String>,
pub display_name: String,
pub description: String,
pub hidden: bool,
pub supported_reasoning_efforts: Vec<ReasoningEffortOption>,
pub default_reasoning_effort: ReasoningEffort,
#[serde(default = "default_input_modalities")]
@@ -1290,6 +1294,14 @@ pub struct AppInfo {
pub install_url: Option<String>,
#[serde(default)]
pub is_accessible: bool,
/// Whether this app is enabled in config.toml.
/// Example:
/// ```toml
/// [apps.bad_app]
/// enabled = false
/// ```
#[serde(default = "default_enabled")]
pub is_enabled: bool,
}
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)]
@@ -1700,6 +1712,10 @@ pub struct ThreadListParams {
/// If false or null, only non-archived threads are returned.
#[ts(optional = nullable)]
pub archived: Option<bool>,
/// Optional cwd filter; when set, only threads whose session cwd exactly
/// matches this path are returned.
#[ts(optional = nullable)]
pub cwd: Option<String>,
}
#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, JsonSchema, TS)]
@@ -14,4 +14,6 @@ codex-app-server-protocol = { workspace = true }
codex-protocol = { workspace = true }
serde = { workspace = true, features = ["derive"] }
serde_json = { workspace = true }
tungstenite = { workspace = true }
url = { workspace = true }
uuid = { workspace = true, features = ["v4"] }
+48 -1
View File
@@ -1,2 +1,49 @@
# App Server Test Client
Exercises simple `codex app-server` flows end-to-end, logging JSON-RPC messages sent between client and server to stdout.
Quickstart for running and hitting `codex app-server`.
## Quickstart
Run from `<reporoot>/codex-rs`.
```bash
# 1) Build debug codex binary
cargo build -p codex-cli --bin codex
# 2) Start websocket app-server in background
cargo run -p codex-app-server-test-client -- \
--codex-bin ./target/debug/codex \
serve --listen ws://127.0.0.1:4222 --kill
# 3) Call app-server (defaults to ws://127.0.0.1:4222)
cargo run -p codex-app-server-test-client -- model-list
```
## Testing Thread Rejoin Behavior
Build and start an app server using commands above. The app-server log is written to `/tmp/codex-app-server-test-client/app-server.log`
### 1) Get a thread id
Create at least one thread, then list threads:
```bash
cargo run -p codex-app-server-test-client -- send-message-v2 "seed thread for rejoin test"
cargo run -p codex-app-server-test-client -- thread-list --limit 5
```
Copy a thread id from the `thread-list` output.
### 2) Rejoin while a turn is in progress (two terminals)
Terminal A:
```bash
cargo run --bin codex-app-server-test-client -- \
resume-message-v2 <THREAD_ID> "respond with thorough docs on the rust core"
```
Terminal B (while Terminal A is still streaming):
```bash
cargo run --bin codex-app-server-test-client -- thread-resume <THREAD_ID>
```
+430 -88
View File
@@ -1,8 +1,10 @@
use std::collections::VecDeque;
use std::fs;
use std::fs::OpenOptions;
use std::io::BufRead;
use std::io::BufReader;
use std::io::Write;
use std::net::TcpStream;
use std::path::Path;
use std::path::PathBuf;
use std::process::Child;
@@ -53,6 +55,8 @@ use codex_app_server_protocol::SendUserMessageParams;
use codex_app_server_protocol::SendUserMessageResponse;
use codex_app_server_protocol::ServerNotification;
use codex_app_server_protocol::ServerRequest;
use codex_app_server_protocol::ThreadListParams;
use codex_app_server_protocol::ThreadListResponse;
use codex_app_server_protocol::ThreadResumeParams;
use codex_app_server_protocol::ThreadResumeResponse;
use codex_app_server_protocol::ThreadStartParams;
@@ -67,15 +71,28 @@ use codex_protocol::protocol::EventMsg;
use serde::Serialize;
use serde::de::DeserializeOwned;
use serde_json::Value;
use tungstenite::Message;
use tungstenite::WebSocket;
use tungstenite::connect;
use tungstenite::stream::MaybeTlsStream;
use url::Url;
use uuid::Uuid;
/// Minimal launcher that initializes the Codex app-server and logs the handshake.
#[derive(Parser)]
#[command(author = "Codex", version, about = "Bootstrap Codex app-server", long_about = None)]
struct Cli {
/// Path to the `codex` CLI binary.
#[arg(long, env = "CODEX_BIN", default_value = "codex")]
codex_bin: PathBuf,
/// Path to the `codex` CLI binary. When set, requests use stdio by
/// spawning `codex app-server` as a child process.
#[arg(long, env = "CODEX_BIN", global = true)]
codex_bin: Option<PathBuf>,
/// Existing websocket server URL to connect to.
///
/// If neither `--codex-bin` nor `--url` is provided, defaults to
/// `ws://127.0.0.1:4222`.
#[arg(long, env = "CODEX_APP_SERVER_URL", global = true)]
url: Option<String>,
/// Forwarded to the `codex` CLI as `--config key=value`. Repeatable.
///
@@ -105,6 +122,18 @@ struct Cli {
#[derive(Subcommand)]
enum CliCommand {
/// Start `codex app-server` on a websocket endpoint in the background.
///
/// Logs are written to:
/// `/tmp/codex-app-server-test-client/`
Serve {
/// WebSocket listen URL passed to `codex app-server --listen`.
#[arg(long, default_value = "ws://127.0.0.1:4222")]
listen: String,
/// Kill any process listening on the same port before starting.
#[arg(long, default_value_t = false)]
kill: bool,
},
/// Send a user message through the Codex app-server.
SendMessage {
/// User message to send to Codex.
@@ -122,6 +151,13 @@ enum CliCommand {
/// User message to send to Codex.
user_message: String,
},
/// Resume a V2 thread and continuously stream notifications/events.
///
/// This command does not auto-exit; stop it with SIGINT/SIGTERM/SIGKILL.
ThreadResume {
/// Existing thread id to resume.
thread_id: String,
},
/// Start a V2 turn that elicits an ExecCommand approval.
#[command(name = "trigger-cmd-approval")]
TriggerCmdApproval {
@@ -151,11 +187,19 @@ enum CliCommand {
/// List the available models from the Codex app-server.
#[command(name = "model-list")]
ModelList,
/// List stored threads from the Codex app-server.
#[command(name = "thread-list")]
ThreadList {
/// Number of threads to return.
#[arg(long, default_value_t = 20)]
limit: u32,
},
}
pub fn run() -> Result<()> {
let Cli {
codex_bin,
url,
config_overrides,
dynamic_tools,
command,
@@ -164,59 +208,222 @@ pub fn run() -> Result<()> {
let dynamic_tools = parse_dynamic_tools_arg(&dynamic_tools)?;
match command {
CliCommand::Serve { listen, kill } => {
ensure_dynamic_tools_unused(&dynamic_tools, "serve")?;
let codex_bin = codex_bin.unwrap_or_else(|| PathBuf::from("codex"));
serve(&codex_bin, &config_overrides, &listen, kill)
}
CliCommand::SendMessage { user_message } => {
ensure_dynamic_tools_unused(&dynamic_tools, "send-message")?;
send_message(&codex_bin, &config_overrides, user_message)
let endpoint = resolve_endpoint(codex_bin, url)?;
send_message(&endpoint, &config_overrides, user_message)
}
CliCommand::SendMessageV2 { user_message } => {
send_message_v2(&codex_bin, &config_overrides, user_message, &dynamic_tools)
let endpoint = resolve_endpoint(codex_bin, url)?;
send_message_v2_endpoint(&endpoint, &config_overrides, user_message, &dynamic_tools)
}
CliCommand::ResumeMessageV2 {
thread_id,
user_message,
} => resume_message_v2(
&codex_bin,
&config_overrides,
thread_id,
user_message,
&dynamic_tools,
),
} => {
let endpoint = resolve_endpoint(codex_bin, url)?;
resume_message_v2(
&endpoint,
&config_overrides,
thread_id,
user_message,
&dynamic_tools,
)
}
CliCommand::ThreadResume { thread_id } => {
ensure_dynamic_tools_unused(&dynamic_tools, "thread-resume")?;
let endpoint = resolve_endpoint(codex_bin, url)?;
thread_resume_follow(&endpoint, &config_overrides, thread_id)
}
CliCommand::TriggerCmdApproval { user_message } => {
trigger_cmd_approval(&codex_bin, &config_overrides, user_message, &dynamic_tools)
let endpoint = resolve_endpoint(codex_bin, url)?;
trigger_cmd_approval(&endpoint, &config_overrides, user_message, &dynamic_tools)
}
CliCommand::TriggerPatchApproval { user_message } => {
trigger_patch_approval(&codex_bin, &config_overrides, user_message, &dynamic_tools)
let endpoint = resolve_endpoint(codex_bin, url)?;
trigger_patch_approval(&endpoint, &config_overrides, user_message, &dynamic_tools)
}
CliCommand::NoTriggerCmdApproval => {
no_trigger_cmd_approval(&codex_bin, &config_overrides, &dynamic_tools)
let endpoint = resolve_endpoint(codex_bin, url)?;
no_trigger_cmd_approval(&endpoint, &config_overrides, &dynamic_tools)
}
CliCommand::SendFollowUpV2 {
first_message,
follow_up_message,
} => send_follow_up_v2(
&codex_bin,
&config_overrides,
first_message,
follow_up_message,
&dynamic_tools,
),
} => {
let endpoint = resolve_endpoint(codex_bin, url)?;
send_follow_up_v2(
&endpoint,
&config_overrides,
first_message,
follow_up_message,
&dynamic_tools,
)
}
CliCommand::TestLogin => {
ensure_dynamic_tools_unused(&dynamic_tools, "test-login")?;
test_login(&codex_bin, &config_overrides)
let endpoint = resolve_endpoint(codex_bin, url)?;
test_login(&endpoint, &config_overrides)
}
CliCommand::GetAccountRateLimits => {
ensure_dynamic_tools_unused(&dynamic_tools, "get-account-rate-limits")?;
get_account_rate_limits(&codex_bin, &config_overrides)
let endpoint = resolve_endpoint(codex_bin, url)?;
get_account_rate_limits(&endpoint, &config_overrides)
}
CliCommand::ModelList => {
ensure_dynamic_tools_unused(&dynamic_tools, "model-list")?;
model_list(&codex_bin, &config_overrides)
let endpoint = resolve_endpoint(codex_bin, url)?;
model_list(&endpoint, &config_overrides)
}
CliCommand::ThreadList { limit } => {
ensure_dynamic_tools_unused(&dynamic_tools, "thread-list")?;
let endpoint = resolve_endpoint(codex_bin, url)?;
thread_list(&endpoint, &config_overrides, limit)
}
}
}
fn send_message(codex_bin: &Path, config_overrides: &[String], user_message: String) -> Result<()> {
let mut client = CodexClient::spawn(codex_bin, config_overrides)?;
enum Endpoint {
SpawnCodex(PathBuf),
ConnectWs(String),
}
fn resolve_endpoint(codex_bin: Option<PathBuf>, url: Option<String>) -> Result<Endpoint> {
if codex_bin.is_some() && url.is_some() {
bail!("--codex-bin and --url are mutually exclusive");
}
if let Some(codex_bin) = codex_bin {
return Ok(Endpoint::SpawnCodex(codex_bin));
}
if let Some(url) = url {
return Ok(Endpoint::ConnectWs(url));
}
Ok(Endpoint::ConnectWs("ws://127.0.0.1:4222".to_string()))
}
fn serve(codex_bin: &Path, config_overrides: &[String], listen: &str, kill: bool) -> Result<()> {
let runtime_dir = PathBuf::from("/tmp/codex-app-server-test-client");
fs::create_dir_all(&runtime_dir)
.with_context(|| format!("failed to create runtime dir {}", runtime_dir.display()))?;
let log_path = runtime_dir.join("app-server.log");
if kill {
kill_listeners_on_same_port(listen)?;
}
let log_file = OpenOptions::new()
.create(true)
.append(true)
.open(&log_path)
.with_context(|| format!("failed to open log file {}", log_path.display()))?;
let log_file_stderr = log_file
.try_clone()
.with_context(|| format!("failed to clone log file handle {}", log_path.display()))?;
let mut cmdline = format!(
"tail -f /dev/null | RUST_BACKTRACE=full RUST_LOG=warn,codex_=trace {}",
shell_quote(&codex_bin.display().to_string())
);
for override_kv in config_overrides {
cmdline.push_str(&format!(" --config {}", shell_quote(override_kv)));
}
cmdline.push_str(&format!(" app-server --listen {}", shell_quote(listen)));
let child = Command::new("nohup")
.arg("sh")
.arg("-c")
.arg(cmdline)
.stdin(Stdio::null())
.stdout(Stdio::from(log_file))
.stderr(Stdio::from(log_file_stderr))
.spawn()
.with_context(|| format!("failed to start `{}` app-server", codex_bin.display()))?;
let pid = child.id();
println!("started codex app-server");
println!("listen: {listen}");
println!("pid: {pid} (launcher process)");
println!("log: {}", log_path.display());
Ok(())
}
fn kill_listeners_on_same_port(listen: &str) -> Result<()> {
let url = Url::parse(listen).with_context(|| format!("invalid --listen URL `{listen}`"))?;
let port = url
.port_or_known_default()
.with_context(|| format!("unable to infer port from --listen URL `{listen}`"))?;
let output = Command::new("lsof")
.arg("-nP")
.arg(format!("-tiTCP:{port}"))
.arg("-sTCP:LISTEN")
.output()
.with_context(|| format!("failed to run lsof for port {port}"))?;
if !output.status.success() {
return Ok(());
}
let pids: Vec<u32> = String::from_utf8_lossy(&output.stdout)
.lines()
.filter_map(|line| line.trim().parse::<u32>().ok())
.collect();
if pids.is_empty() {
return Ok(());
}
for pid in pids {
println!("killing listener pid {pid} on port {port}");
let pid_str = pid.to_string();
let term_status = Command::new("kill")
.arg(&pid_str)
.status()
.with_context(|| format!("failed to send SIGTERM to pid {pid}"))?;
if !term_status.success() {
continue;
}
}
thread::sleep(Duration::from_millis(300));
let output = Command::new("lsof")
.arg("-nP")
.arg(format!("-tiTCP:{port}"))
.arg("-sTCP:LISTEN")
.output()
.with_context(|| format!("failed to re-check listeners on port {port}"))?;
if !output.status.success() {
return Ok(());
}
let remaining: Vec<u32> = String::from_utf8_lossy(&output.stdout)
.lines()
.filter_map(|line| line.trim().parse::<u32>().ok())
.collect();
for pid in remaining {
println!("force killing remaining listener pid {pid} on port {port}");
let _ = Command::new("kill").arg("-9").arg(pid.to_string()).status();
}
Ok(())
}
fn shell_quote(input: &str) -> String {
format!("'{}'", input.replace('\'', "'\\''"))
}
fn send_message(
endpoint: &Endpoint,
config_overrides: &[String],
user_message: String,
) -> Result<()> {
let mut client = CodexClient::connect(endpoint, config_overrides)?;
let initialize = client.initialize()?;
println!("< initialize response: {initialize:?}");
@@ -242,9 +449,19 @@ pub fn send_message_v2(
config_overrides: &[String],
user_message: String,
dynamic_tools: &Option<Vec<DynamicToolSpec>>,
) -> Result<()> {
let endpoint = Endpoint::SpawnCodex(codex_bin.to_path_buf());
send_message_v2_endpoint(&endpoint, config_overrides, user_message, dynamic_tools)
}
fn send_message_v2_endpoint(
endpoint: &Endpoint,
config_overrides: &[String],
user_message: String,
dynamic_tools: &Option<Vec<DynamicToolSpec>>,
) -> Result<()> {
send_message_v2_with_policies(
codex_bin,
endpoint,
config_overrides,
user_message,
None,
@@ -254,7 +471,7 @@ pub fn send_message_v2(
}
fn resume_message_v2(
codex_bin: &Path,
endpoint: &Endpoint,
config_overrides: &[String],
thread_id: String,
user_message: String,
@@ -262,7 +479,7 @@ fn resume_message_v2(
) -> Result<()> {
ensure_dynamic_tools_unused(dynamic_tools, "resume-message-v2")?;
let mut client = CodexClient::spawn(codex_bin, config_overrides)?;
let mut client = CodexClient::connect(endpoint, config_overrides)?;
let initialize = client.initialize()?;
println!("< initialize response: {initialize:?}");
@@ -288,8 +505,28 @@ fn resume_message_v2(
Ok(())
}
fn thread_resume_follow(
endpoint: &Endpoint,
config_overrides: &[String],
thread_id: String,
) -> Result<()> {
let mut client = CodexClient::connect(endpoint, config_overrides)?;
let initialize = client.initialize()?;
println!("< initialize response: {initialize:?}");
let resume_response = client.thread_resume(ThreadResumeParams {
thread_id,
..Default::default()
})?;
println!("< thread/resume response: {resume_response:?}");
println!("< streaming notifications until process is terminated");
client.stream_notifications_forever()
}
fn trigger_cmd_approval(
codex_bin: &Path,
endpoint: &Endpoint,
config_overrides: &[String],
user_message: Option<String>,
dynamic_tools: &Option<Vec<DynamicToolSpec>>,
@@ -298,7 +535,7 @@ fn trigger_cmd_approval(
"Run `touch /tmp/should-trigger-approval` so I can confirm the file exists.";
let message = user_message.unwrap_or_else(|| default_prompt.to_string());
send_message_v2_with_policies(
codex_bin,
endpoint,
config_overrides,
message,
Some(AskForApproval::OnRequest),
@@ -310,7 +547,7 @@ fn trigger_cmd_approval(
}
fn trigger_patch_approval(
codex_bin: &Path,
endpoint: &Endpoint,
config_overrides: &[String],
user_message: Option<String>,
dynamic_tools: &Option<Vec<DynamicToolSpec>>,
@@ -319,7 +556,7 @@ fn trigger_patch_approval(
"Create a file named APPROVAL_DEMO.txt containing a short hello message using apply_patch.";
let message = user_message.unwrap_or_else(|| default_prompt.to_string());
send_message_v2_with_policies(
codex_bin,
endpoint,
config_overrides,
message,
Some(AskForApproval::OnRequest),
@@ -331,13 +568,13 @@ fn trigger_patch_approval(
}
fn no_trigger_cmd_approval(
codex_bin: &Path,
endpoint: &Endpoint,
config_overrides: &[String],
dynamic_tools: &Option<Vec<DynamicToolSpec>>,
) -> Result<()> {
let prompt = "Run `touch should_not_trigger_approval.txt`";
send_message_v2_with_policies(
codex_bin,
endpoint,
config_overrides,
prompt.to_string(),
None,
@@ -347,14 +584,14 @@ fn no_trigger_cmd_approval(
}
fn send_message_v2_with_policies(
codex_bin: &Path,
endpoint: &Endpoint,
config_overrides: &[String],
user_message: String,
approval_policy: Option<AskForApproval>,
sandbox_policy: Option<SandboxPolicy>,
dynamic_tools: &Option<Vec<DynamicToolSpec>>,
) -> Result<()> {
let mut client = CodexClient::spawn(codex_bin, config_overrides)?;
let mut client = CodexClient::connect(endpoint, config_overrides)?;
let initialize = client.initialize()?;
println!("< initialize response: {initialize:?}");
@@ -385,13 +622,13 @@ fn send_message_v2_with_policies(
}
fn send_follow_up_v2(
codex_bin: &Path,
endpoint: &Endpoint,
config_overrides: &[String],
first_message: String,
follow_up_message: String,
dynamic_tools: &Option<Vec<DynamicToolSpec>>,
) -> Result<()> {
let mut client = CodexClient::spawn(codex_bin, config_overrides)?;
let mut client = CodexClient::connect(endpoint, config_overrides)?;
let initialize = client.initialize()?;
println!("< initialize response: {initialize:?}");
@@ -431,8 +668,8 @@ fn send_follow_up_v2(
Ok(())
}
fn test_login(codex_bin: &Path, config_overrides: &[String]) -> Result<()> {
let mut client = CodexClient::spawn(codex_bin, config_overrides)?;
fn test_login(endpoint: &Endpoint, config_overrides: &[String]) -> Result<()> {
let mut client = CodexClient::connect(endpoint, config_overrides)?;
let initialize = client.initialize()?;
println!("< initialize response: {initialize:?}");
@@ -461,8 +698,8 @@ fn test_login(codex_bin: &Path, config_overrides: &[String]) -> Result<()> {
}
}
fn get_account_rate_limits(codex_bin: &Path, config_overrides: &[String]) -> Result<()> {
let mut client = CodexClient::spawn(codex_bin, config_overrides)?;
fn get_account_rate_limits(endpoint: &Endpoint, config_overrides: &[String]) -> Result<()> {
let mut client = CodexClient::connect(endpoint, config_overrides)?;
let initialize = client.initialize()?;
println!("< initialize response: {initialize:?}");
@@ -473,8 +710,8 @@ fn get_account_rate_limits(codex_bin: &Path, config_overrides: &[String]) -> Res
Ok(())
}
fn model_list(codex_bin: &Path, config_overrides: &[String]) -> Result<()> {
let mut client = CodexClient::spawn(codex_bin, config_overrides)?;
fn model_list(endpoint: &Endpoint, config_overrides: &[String]) -> Result<()> {
let mut client = CodexClient::connect(endpoint, config_overrides)?;
let initialize = client.initialize()?;
println!("< initialize response: {initialize:?}");
@@ -485,6 +722,26 @@ fn model_list(codex_bin: &Path, config_overrides: &[String]) -> Result<()> {
Ok(())
}
fn thread_list(endpoint: &Endpoint, config_overrides: &[String], limit: u32) -> Result<()> {
let mut client = CodexClient::connect(endpoint, config_overrides)?;
let initialize = client.initialize()?;
println!("< initialize response: {initialize:?}");
let response = client.thread_list(ThreadListParams {
cursor: None,
limit: Some(limit),
sort_key: None,
model_providers: None,
source_kinds: None,
archived: None,
cwd: None,
})?;
println!("< thread/list response: {response:?}");
Ok(())
}
fn ensure_dynamic_tools_unused(
dynamic_tools: &Option<Vec<DynamicToolSpec>>,
command: &str,
@@ -519,15 +776,32 @@ fn parse_dynamic_tools_arg(dynamic_tools: &Option<String>) -> Result<Option<Vec<
Ok(Some(tools))
}
enum ClientTransport {
Stdio {
child: Child,
stdin: Option<ChildStdin>,
stdout: BufReader<ChildStdout>,
},
WebSocket {
url: String,
socket: Box<WebSocket<MaybeTlsStream<TcpStream>>>,
},
}
struct CodexClient {
child: Child,
stdin: Option<ChildStdin>,
stdout: BufReader<ChildStdout>,
transport: ClientTransport,
pending_notifications: VecDeque<JSONRPCNotification>,
}
impl CodexClient {
fn spawn(codex_bin: &Path, config_overrides: &[String]) -> Result<Self> {
fn connect(endpoint: &Endpoint, config_overrides: &[String]) -> Result<Self> {
match endpoint {
Endpoint::SpawnCodex(codex_bin) => Self::spawn_stdio(codex_bin, config_overrides),
Endpoint::ConnectWs(url) => Self::connect_websocket(url),
}
}
fn spawn_stdio(codex_bin: &Path, config_overrides: &[String]) -> Result<Self> {
let codex_bin_display = codex_bin.display();
let mut cmd = Command::new(codex_bin);
for override_kv in config_overrides {
@@ -551,9 +825,27 @@ impl CodexClient {
.context("codex app-server stdout unavailable")?;
Ok(Self {
child: codex_app_server,
stdin: Some(stdin),
stdout: BufReader::new(stdout),
transport: ClientTransport::Stdio {
child: codex_app_server,
stdin: Some(stdin),
stdout: BufReader::new(stdout),
},
pending_notifications: VecDeque::new(),
})
}
fn connect_websocket(url: &str) -> Result<Self> {
let parsed = Url::parse(url).with_context(|| format!("invalid websocket URL `{url}`"))?;
let (socket, _response) = connect(parsed.as_str()).with_context(|| {
format!(
"failed to connect to websocket app-server at `{url}`; if no server is running, start one with `codex-app-server-test-client serve --listen {url}`"
)
})?;
Ok(Self {
transport: ClientTransport::WebSocket {
url: url.to_string(),
socket: Box::new(socket),
},
pending_notifications: VecDeque::new(),
})
}
@@ -575,7 +867,16 @@ impl CodexClient {
},
};
self.send_request(request, request_id, "initialize")
let response: InitializeResponse = self.send_request(request, request_id, "initialize")?;
// Complete the initialize handshake.
let initialized = JSONRPCMessage::Notification(JSONRPCNotification {
method: "initialized".to_string(),
params: None,
});
self.write_jsonrpc_message(initialized)?;
Ok(response)
}
fn start_thread(&mut self) -> Result<NewConversationResponse> {
@@ -701,6 +1002,16 @@ impl CodexClient {
self.send_request(request, request_id, "model/list")
}
fn thread_list(&mut self, params: ThreadListParams) -> Result<ThreadListResponse> {
let request_id = self.request_id();
let request = ClientRequest::ThreadList {
request_id: request_id.clone(),
params,
};
self.send_request(request, request_id, "thread/list")
}
fn stream_conversation(&mut self, conversation_id: &ThreadId) -> Result<()> {
loop {
let notification = self.next_notification()?;
@@ -835,6 +1146,12 @@ impl CodexClient {
Ok(())
}
fn stream_notifications_forever(&mut self) -> Result<()> {
loop {
let _ = self.next_notification()?;
}
}
fn extract_event(
&self,
notification: JSONRPCNotification,
@@ -882,17 +1199,7 @@ impl CodexClient {
let request_json = serde_json::to_string(request)?;
let request_pretty = serde_json::to_string_pretty(request)?;
print_multiline_with_prefix("> ", &request_pretty);
if let Some(stdin) = self.stdin.as_mut() {
writeln!(stdin, "{request_json}")?;
stdin
.flush()
.context("failed to flush request to codex app-server")?;
} else {
bail!("codex app-server stdin closed");
}
Ok(())
self.write_payload(&request_json)
}
fn wait_for_response<T>(&mut self, request_id: RequestId, method: &str) -> Result<T>
@@ -947,17 +1254,8 @@ impl CodexClient {
fn read_jsonrpc_message(&mut self) -> Result<JSONRPCMessage> {
loop {
let mut response_line = String::new();
let bytes = self
.stdout
.read_line(&mut response_line)
.context("failed to read from codex app-server")?;
if bytes == 0 {
bail!("codex app-server closed stdout");
}
let trimmed = response_line.trim();
let raw = self.read_payload()?;
let trimmed = raw.trim();
if trimmed.is_empty() {
continue;
}
@@ -1086,16 +1384,56 @@ impl CodexClient {
let payload = serde_json::to_string(&message)?;
let pretty = serde_json::to_string_pretty(&message)?;
print_multiline_with_prefix("> ", &pretty);
self.write_payload(&payload)
}
if let Some(stdin) = self.stdin.as_mut() {
writeln!(stdin, "{payload}")?;
stdin
.flush()
.context("failed to flush response to codex app-server")?;
return Ok(());
fn write_payload(&mut self, payload: &str) -> Result<()> {
match &mut self.transport {
ClientTransport::Stdio { stdin, .. } => {
if let Some(stdin) = stdin.as_mut() {
writeln!(stdin, "{payload}")?;
stdin
.flush()
.context("failed to flush payload to codex app-server")?;
return Ok(());
}
bail!("codex app-server stdin closed")
}
ClientTransport::WebSocket { socket, url } => {
socket
.send(Message::Text(payload.to_string().into()))
.with_context(|| format!("failed to write websocket message to `{url}`"))?;
Ok(())
}
}
}
bail!("codex app-server stdin closed")
fn read_payload(&mut self) -> Result<String> {
match &mut self.transport {
ClientTransport::Stdio { stdout, .. } => {
let mut response_line = String::new();
let bytes = stdout
.read_line(&mut response_line)
.context("failed to read from codex app-server")?;
if bytes == 0 {
bail!("codex app-server closed stdout");
}
Ok(response_line)
}
ClientTransport::WebSocket { socket, url } => loop {
let frame = socket
.read()
.with_context(|| format!("failed to read websocket message from `{url}`"))?;
match frame {
Message::Text(text) => return Ok(text.to_string()),
Message::Binary(_) | Message::Ping(_) | Message::Pong(_) => continue,
Message::Close(_) => {
bail!("websocket app-server at `{url}` closed the connection")
}
Message::Frame(_) => continue,
}
},
}
}
}
@@ -1107,21 +1445,25 @@ fn print_multiline_with_prefix(prefix: &str, payload: &str) {
impl Drop for CodexClient {
fn drop(&mut self) {
let _ = self.stdin.take();
let ClientTransport::Stdio { child, stdin, .. } = &mut self.transport else {
return;
};
if let Ok(Some(status)) = self.child.try_wait() {
let _ = stdin.take();
if let Ok(Some(status)) = child.try_wait() {
println!("[codex app-server exited: {status}]");
return;
}
thread::sleep(Duration::from_millis(100));
if let Ok(Some(status)) = self.child.try_wait() {
if let Ok(Some(status)) = child.try_wait() {
println!("[codex app-server exited: {status}]");
return;
}
let _ = self.child.kill();
let _ = self.child.wait();
let _ = child.kill();
let _ = child.wait();
}
}
+15 -5
View File
@@ -117,7 +117,7 @@ Example with notification opt-out:
- `thread/start` — create a new thread; emits `thread/started` and auto-subscribes you to turn/item events for that thread.
- `thread/resume` — reopen an existing thread by id so subsequent `turn/start` calls append to it.
- `thread/fork` — fork an existing thread into a new thread id by copying the stored history; emits `thread/started` and auto-subscribes you to turn/item events for the new thread.
- `thread/list` — page through stored rollouts; supports cursor-based pagination and optional `modelProviders` filtering.
- `thread/list` — page through stored rollouts; supports cursor-based pagination and optional `modelProviders`, `sourceKinds`, `archived`, and `cwd` filters.
- `thread/loaded/list` — list the thread ids currently loaded in memory.
- `thread/read` — read a stored thread by id without resuming it; optionally include turns via `includeTurns`.
- `thread/archive` — move a thread’s rollout file into the archived directory; returns `{}` on success.
@@ -131,7 +131,7 @@ Example with notification opt-out:
- `turn/interrupt` — request cancellation of an in-flight turn by `(thread_id, turn_id)`; success is an empty `{}` response and the turn finishes with `status: "interrupted"`.
- `review/start` — kick off Codex’s automated reviewer for a thread; responds like `turn/start` and emits `item/started`/`item/completed` notifications with `enteredReviewMode` and `exitedReviewMode` items, plus a final assistant `agentMessage` containing the review.
- `command/exec` — run a single command under the server sandbox without starting a thread/turn (handy for utilities and validation).
- `model/list` — list available models (with reasoning effort options and optional `upgrade` model ids).
- `model/list` — list available models (set `includeHidden: true` to include entries with `hidden: true`), with reasoning effort options and optional `upgrade` model ids.
- `experimentalFeature/list` — list feature flags with stage metadata (`beta`, `underDevelopment`, `stable`, etc.), enabled/default-enabled state, and cursor pagination. For non-beta flags, `displayName`/`description`/`announcement` are `null`.
- `collaborationMode/list` — list available collaboration mode presets (experimental, no pagination).
- `skills/list` — list skills for one or more `cwd` values (optional `forceReload`).
@@ -221,6 +221,7 @@ Experimental API: `thread/start`, `thread/resume`, and `thread/fork` accept `per
- `modelProviders` — restrict results to specific providers; unset, null, or an empty array will include all providers.
- `sourceKinds` — restrict results to specific sources; omit or pass `[]` for interactive sessions only (`cli`, `vscode`).
- `archived` — when `true`, list archived threads only. When `false` or `null`, list non-archived threads (default).
- `cwd` — restrict results to threads whose session cwd exactly matches this path.
Example:
@@ -534,6 +535,13 @@ Examples:
- Opt out of legacy session setup event: `codex/event/session_configured`
- Opt out of streamed agent text deltas: `item/agentMessage/delta`
### Fuzzy file search events (experimental)
The fuzzy file search session API emits per-query notifications:
- `fuzzyFileSearch/sessionUpdated` — `{ sessionId, query, files }` with the current matching files for the active query.
- `fuzzyFileSearch/sessionCompleted` — `{ sessionId, query }` once indexing/matching for that query has completed.
### Turn events
The app-server streams JSON-RPC notifications while a turn is running. Each turn starts with `turn/started` (initial `turn`) and ends with `turn/completed` (final `turn` status). Token usage events stream separately via `thread/tokenUsage/updated`. Clients subscribe to the events they care about, rendering each item incrementally as updates arrive. The per-item lifecycle is always: `item/started` → zero or more item-specific deltas → `item/completed`.
@@ -763,7 +771,7 @@ To enable or disable a skill by path:
## Apps
Use `app/list` to fetch available apps (connectors). Each entry includes metadata like the app `id`, display `name`, `installUrl`, and whether it is currently accessible.
Use `app/list` to fetch available apps (connectors). Each entry includes metadata like the app `id`, display `name`, `installUrl`, whether it is currently accessible, and whether it is enabled in config.
```json
{ "method": "app/list", "id": 50, "params": {
@@ -782,7 +790,8 @@ Use `app/list` to fetch available apps (connectors). Each entry includes metadat
"logoUrlDark": null,
"distributionChannel": null,
"installUrl": "https://chatgpt.com/apps/demo-app/demo-app",
"isAccessible": true
"isAccessible": true,
"isEnabled": true
}
],
"nextCursor": null
@@ -808,7 +817,8 @@ The server also emits `app/list/updated` notifications whenever either source (a
"logoUrlDark": null,
"distributionChannel": null,
"installUrl": "https://chatgpt.com/apps/demo-app/demo-app",
"isAccessible": true
"isAccessible": true,
"isEnabled": true
}
]
}
@@ -206,6 +206,7 @@ pub(crate) async fn apply_bespoke_event_handling(
reason,
proposed_execpolicy_amendment,
parsed_cmd,
..
}) => match api_version {
ApiVersion::V1 => {
let params = ExecCommandApprovalParams {
+465 -122
View File
@@ -266,6 +266,13 @@ use crate::thread_state::ThreadStateManager;
const THREAD_LIST_DEFAULT_LIMIT: usize = 25;
const THREAD_LIST_MAX_LIMIT: usize = 100;
struct ThreadListFilters {
model_providers: Option<Vec<String>>,
source_kinds: Option<Vec<ThreadSourceKind>>,
archived: bool,
cwd: Option<PathBuf>,
}
// Duration before a ChatGPT login attempt is abandoned.
const LOGIN_CHATGPT_TIMEOUT: Duration = Duration::from_secs(10 * 60);
const APP_LIST_LOAD_TIMEOUT: Duration = Duration::from_secs(90);
@@ -1681,8 +1688,18 @@ impl CodexMessageProcessor {
let timeout_ms = params
.timeout_ms
.and_then(|timeout_ms| u64::try_from(timeout_ms).ok());
let managed_network_requirements_enabled =
self.config.managed_network_requirements_enabled();
let started_network_proxy = match self.config.permissions.network.as_ref() {
Some(spec) => match spec.start_proxy().await {
Some(spec) => match spec
.start_proxy(
self.config.permissions.sandbox_policy.get(),
None,
None,
managed_network_requirements_enabled,
)
.await
{
Ok(started) => Some(started),
Err(err) => {
let error = JSONRPCErrorError {
@@ -1705,6 +1722,7 @@ impl CodexMessageProcessor {
network: started_network_proxy
.as_ref()
.map(codex_core::config::StartedNetworkProxy::proxy),
network_attempt_id: None,
sandbox_permissions: SandboxPermissions::UseDefault,
windows_sandbox_level,
justification: None,
@@ -2430,6 +2448,7 @@ impl CodexMessageProcessor {
model_providers,
source_kinds,
archived,
cwd,
} = params;
let requested_page_size = limit
@@ -2444,10 +2463,13 @@ impl CodexMessageProcessor {
.list_threads_common(
requested_page_size,
cursor,
model_providers,
source_kinds,
core_sort_key,
archived.unwrap_or(false),
ThreadListFilters {
model_providers,
source_kinds,
archived: archived.unwrap_or(false),
cwd: cwd.map(PathBuf::from),
},
)
.await
{
@@ -2678,6 +2700,13 @@ impl CodexMessageProcessor {
}
async fn thread_resume(&mut self, request_id: ConnectionRequestId, params: ThreadResumeParams) {
if self
.resume_running_thread(request_id.clone(), &params)
.await
{
return;
}
let ThreadResumeParams {
thread_id,
history,
@@ -2695,77 +2724,21 @@ impl CodexMessageProcessor {
} = params;
let thread_history = if let Some(history) = history {
if history.is_empty() {
self.send_invalid_request_error(
request_id,
"history must not be empty".to_string(),
)
.await;
let Some(thread_history) = self
.resume_thread_from_history(request_id.clone(), history.as_slice())
.await
else {
return;
}
InitialHistory::Forked(history.into_iter().map(RolloutItem::ResponseItem).collect())
} else if let Some(path) = path {
match RolloutRecorder::get_rollout_history(&path).await {
Ok(initial_history) => initial_history,
Err(err) => {
self.send_invalid_request_error(
request_id,
format!("failed to load rollout `{}`: {err}", path.display()),
)
.await;
return;
}
}
};
thread_history
} else {
let existing_thread_id = match ThreadId::from_string(&thread_id) {
Ok(id) => id,
Err(err) => {
let error = JSONRPCErrorError {
code: INVALID_REQUEST_ERROR_CODE,
message: format!("invalid thread id: {err}"),
data: None,
};
self.outgoing.send_error(request_id, error).await;
return;
}
let Some(thread_history) = self
.resume_thread_from_rollout(request_id.clone(), &thread_id, path.as_ref())
.await
else {
return;
};
let path = match find_thread_path_by_id_str(
&self.config.codex_home,
&existing_thread_id.to_string(),
)
.await
{
Ok(Some(p)) => p,
Ok(None) => {
self.send_invalid_request_error(
request_id,
format!("no rollout found for thread id {existing_thread_id}"),
)
.await;
return;
}
Err(err) => {
self.send_invalid_request_error(
request_id,
format!("failed to locate thread id {existing_thread_id}: {err}"),
)
.await;
return;
}
};
match RolloutRecorder::get_rollout_history(&path).await {
Ok(initial_history) => initial_history,
Err(err) => {
self.send_invalid_request_error(
request_id,
format!("failed to load rollout `{}`: {err}", path.display()),
)
.await;
return;
}
}
thread_history
};
let history_cwd = thread_history.session_cwd();
@@ -2846,41 +2819,17 @@ impl CodexMessageProcessor {
);
}
let mut thread = match read_summary_from_rollout(
rollout_path.as_path(),
fallback_model_provider.as_str(),
)
.await
{
Ok(summary) => summary_to_thread(summary),
Err(err) => {
self.send_internal_error(
request_id,
format!(
"failed to load rollout `{}` for thread {thread_id}: {err}",
rollout_path.display()
),
)
.await;
return;
}
let Some(thread) = self
.load_thread_from_rollout_or_send_internal(
request_id.clone(),
thread_id,
rollout_path.as_path(),
fallback_model_provider.as_str(),
)
.await
else {
return;
};
match read_rollout_items_from_rollout(rollout_path.as_path()).await {
Ok(items) => {
thread.turns = build_turns_from_rollout_items(&items);
}
Err(err) => {
self.send_internal_error(
request_id,
format!(
"failed to load rollout `{}` for thread {thread_id}: {err}",
rollout_path.display()
),
)
.await;
return;
}
}
let response = ThreadResumeResponse {
thread,
@@ -2905,6 +2854,278 @@ impl CodexMessageProcessor {
}
}
async fn resume_running_thread(
&mut self,
request_id: ConnectionRequestId,
params: &ThreadResumeParams,
) -> bool {
if let Ok(existing_thread_id) = ThreadId::from_string(&params.thread_id)
&& let Ok(existing_thread) = self.thread_manager.get_thread(existing_thread_id).await
{
if params.history.is_some() {
self.send_invalid_request_error(
request_id,
format!(
"cannot resume thread {existing_thread_id} with history while it is already running"
),
)
.await;
return true;
}
let rollout_path = if let Some(path) = existing_thread.rollout_path() {
if path.exists() {
path
} else {
match find_thread_path_by_id_str(
&self.config.codex_home,
&existing_thread_id.to_string(),
)
.await
{
Ok(Some(path)) => path,
Ok(None) => {
self.send_invalid_request_error(
request_id,
format!("no rollout found for thread id {existing_thread_id}"),
)
.await;
return true;
}
Err(err) => {
self.send_invalid_request_error(
request_id,
format!("failed to locate thread id {existing_thread_id}: {err}"),
)
.await;
return true;
}
}
}
} else {
match find_thread_path_by_id_str(
&self.config.codex_home,
&existing_thread_id.to_string(),
)
.await
{
Ok(Some(path)) => path,
Ok(None) => {
self.send_invalid_request_error(
request_id,
format!("no rollout found for thread id {existing_thread_id}"),
)
.await;
return true;
}
Err(err) => {
self.send_invalid_request_error(
request_id,
format!("failed to locate thread id {existing_thread_id}: {err}"),
)
.await;
return true;
}
}
};
if let Some(requested_path) = params.path.as_ref()
&& requested_path != &rollout_path
{
self.send_invalid_request_error(
request_id,
format!(
"cannot resume running thread {existing_thread_id} with mismatched path: requested `{}`, active `{}`",
requested_path.display(),
rollout_path.display()
),
)
.await;
return true;
}
if let Err(err) = self
.ensure_conversation_listener(
existing_thread_id,
request_id.connection_id,
false,
ApiVersion::V2,
)
.await
{
tracing::warn!(
"failed to attach listener for thread {}: {}",
existing_thread_id,
err.message
);
}
let config_snapshot = existing_thread.config_snapshot().await;
let mismatch_details = collect_resume_override_mismatches(params, &config_snapshot);
if !mismatch_details.is_empty() {
tracing::warn!(
"thread/resume overrides ignored for running thread {}: {}",
existing_thread_id,
mismatch_details.join("; ")
);
}
let Some(thread) = self
.load_thread_from_rollout_or_send_internal(
request_id.clone(),
existing_thread_id,
rollout_path.as_path(),
config_snapshot.model_provider_id.as_str(),
)
.await
else {
return true;
};
let ThreadConfigSnapshot {
model,
model_provider_id,
approval_policy,
sandbox_policy,
cwd,
reasoning_effort,
..
} = config_snapshot;
let response = ThreadResumeResponse {
thread,
model,
model_provider: model_provider_id,
cwd,
approval_policy: approval_policy.into(),
sandbox: sandbox_policy.into(),
reasoning_effort,
};
self.outgoing.send_response(request_id, response).await;
return true;
}
false
}
async fn resume_thread_from_history(
&self,
request_id: ConnectionRequestId,
history: &[ResponseItem],
) -> Option<InitialHistory> {
if history.is_empty() {
self.send_invalid_request_error(request_id, "history must not be empty".to_string())
.await;
return None;
}
Some(InitialHistory::Forked(
history
.iter()
.cloned()
.map(RolloutItem::ResponseItem)
.collect(),
))
}
async fn resume_thread_from_rollout(
&self,
request_id: ConnectionRequestId,
thread_id: &str,
path: Option<&PathBuf>,
) -> Option<InitialHistory> {
let rollout_path = if let Some(path) = path {
path.clone()
} else {
let existing_thread_id = match ThreadId::from_string(thread_id) {
Ok(id) => id,
Err(err) => {
let error = JSONRPCErrorError {
code: INVALID_REQUEST_ERROR_CODE,
message: format!("invalid thread id: {err}"),
data: None,
};
self.outgoing.send_error(request_id, error).await;
return None;
}
};
match find_thread_path_by_id_str(
&self.config.codex_home,
&existing_thread_id.to_string(),
)
.await
{
Ok(Some(path)) => path,
Ok(None) => {
self.send_invalid_request_error(
request_id,
format!("no rollout found for thread id {existing_thread_id}"),
)
.await;
return None;
}
Err(err) => {
self.send_invalid_request_error(
request_id,
format!("failed to locate thread id {existing_thread_id}: {err}"),
)
.await;
return None;
}
}
};
match RolloutRecorder::get_rollout_history(&rollout_path).await {
Ok(initial_history) => Some(initial_history),
Err(err) => {
self.send_invalid_request_error(
request_id,
format!("failed to load rollout `{}`: {err}", rollout_path.display()),
)
.await;
None
}
}
}
async fn load_thread_from_rollout_or_send_internal(
&self,
request_id: ConnectionRequestId,
thread_id: ThreadId,
rollout_path: &Path,
fallback_provider: &str,
) -> Option<Thread> {
let mut thread = match read_summary_from_rollout(rollout_path, fallback_provider).await {
Ok(summary) => summary_to_thread(summary),
Err(err) => {
self.send_internal_error(
request_id,
format!(
"failed to load rollout `{}` for thread {thread_id}: {err}",
rollout_path.display()
),
)
.await;
return None;
}
};
match read_rollout_items_from_rollout(rollout_path).await {
Ok(items) => {
thread.turns = build_turns_from_rollout_items(&items);
Some(thread)
}
Err(err) => {
self.send_internal_error(
request_id,
format!(
"failed to load rollout `{}` for thread {thread_id}: {err}",
rollout_path.display()
),
)
.await;
None
}
}
}
async fn thread_fork(&mut self, request_id: ConnectionRequestId, params: ThreadForkParams) {
let ThreadForkParams {
thread_id,
@@ -3221,10 +3442,13 @@ impl CodexMessageProcessor {
.list_threads_common(
requested_page_size,
cursor,
model_providers,
None,
CoreThreadSortKey::UpdatedAt,
false,
ThreadListFilters {
model_providers,
source_kinds: None,
archived: false,
cwd: None,
},
)
.await
{
@@ -3242,11 +3466,15 @@ impl CodexMessageProcessor {
&self,
requested_page_size: usize,
cursor: Option<String>,
model_providers: Option<Vec<String>>,
source_kinds: Option<Vec<ThreadSourceKind>>,
sort_key: CoreThreadSortKey,
archived: bool,
filters: ThreadListFilters,
) -> Result<(Vec<ConversationSummary>, Option<String>), JSONRPCErrorError> {
let ThreadListFilters {
model_providers,
source_kinds,
archived,
cwd,
} = filters;
let mut cursor_obj: Option<RolloutCursor> = match cursor.as_ref() {
Some(cursor_str) => {
Some(parse_cursor(cursor_str).ok_or_else(|| JSONRPCErrorError {
@@ -3327,6 +3555,9 @@ impl CodexMessageProcessor {
if source_kind_filter
.as_ref()
.is_none_or(|filter| source_kind_matches(&summary.source, filter))
&& cwd
.as_ref()
.is_none_or(|expected_cwd| &summary.cwd == expected_cwd)
{
filtered.push(summary);
if filtered.len() >= remaining {
@@ -3372,10 +3603,15 @@ impl CodexMessageProcessor {
request_id: ConnectionRequestId,
params: ModelListParams,
) {
let ModelListParams { limit, cursor } = params;
let ModelListParams {
limit,
cursor,
include_hidden,
} = params;
let mut config = (*config).clone();
config.features.enable(Feature::RemoteModels);
let models = supported_models(thread_manager, &config).await;
let models =
supported_models(thread_manager, &config, include_hidden.unwrap_or(false)).await;
let total = models.len();
if total == 0 {
@@ -4589,6 +4825,11 @@ impl CodexMessageProcessor {
None => 0,
};
let (mut accessible_connectors, mut all_connectors) = tokio::join!(
connectors::list_cached_accessible_connectors_from_mcp_tools(&config),
connectors::list_cached_all_connectors(&config)
);
let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel();
let accessible_config = config.clone();
@@ -4603,16 +4844,17 @@ impl CodexMessageProcessor {
let _ = accessible_tx.send(AppListLoadResult::Accessible(result));
});
let all_config = config.clone();
tokio::spawn(async move {
let result = connectors::list_all_connectors_with_options(&config, force_refetch)
let result = connectors::list_all_connectors_with_options(&all_config, force_refetch)
.await
.map_err(|err| format!("failed to list apps: {err}"));
let _ = tx.send(AppListLoadResult::Directory(result));
});
let mut accessible_connectors: Option<Vec<AppInfo>> = None;
let mut all_connectors: Option<Vec<AppInfo>> = None;
let app_list_deadline = tokio::time::Instant::now() + APP_LIST_LOAD_TIMEOUT;
let mut accessible_loaded = false;
let mut all_loaded = false;
loop {
let result = match tokio::time::timeout_at(app_list_deadline, rx.recv()).await {
@@ -4643,6 +4885,7 @@ impl CodexMessageProcessor {
match result {
AppListLoadResult::Accessible(Ok(connectors)) => {
accessible_connectors = Some(connectors);
accessible_loaded = true;
}
AppListLoadResult::Accessible(Err(err)) => {
let error = JSONRPCErrorError {
@@ -4655,6 +4898,7 @@ impl CodexMessageProcessor {
}
AppListLoadResult::Directory(Ok(connectors)) => {
all_connectors = Some(connectors);
all_loaded = true;
}
AppListLoadResult::Directory(Err(err)) => {
let error = JSONRPCErrorError {
@@ -4667,13 +4911,16 @@ impl CodexMessageProcessor {
}
}
let merged = Self::merge_loaded_apps(
all_connectors.as_deref(),
accessible_connectors.as_deref(),
let merged = connectors::with_app_enabled_state(
Self::merge_loaded_apps(
all_connectors.as_deref(),
accessible_connectors.as_deref(),
),
&config,
);
Self::send_app_list_updated_notification(&outgoing, merged.clone()).await;
if accessible_connectors.is_some() && all_connectors.is_some() {
if accessible_loaded && all_loaded {
match Self::paginate_apps(merged.as_slice(), start, limit) {
Ok(response) => {
outgoing.send_response(request_id, response).await;
@@ -4692,9 +4939,10 @@ impl CodexMessageProcessor {
all_connectors: Option<&[AppInfo]>,
accessible_connectors: Option<&[AppInfo]>,
) -> Vec<AppInfo> {
let all_connectors_loaded = all_connectors.is_some();
let all = all_connectors.map_or_else(Vec::new, <[AppInfo]>::to_vec);
let accessible = accessible_connectors.map_or_else(Vec::new, <[AppInfo]>::to_vec);
connectors::merge_connectors_with_accessible(all, accessible)
connectors::merge_connectors_with_accessible(all, accessible, all_connectors_loaded)
}
fn paginate_apps(
@@ -5775,6 +6023,101 @@ impl CodexMessageProcessor {
}
}
fn collect_resume_override_mismatches(
request: &ThreadResumeParams,
config_snapshot: &ThreadConfigSnapshot,
) -> Vec<String> {
let mut mismatch_details = Vec::new();
if let Some(requested_model) = request.model.as_deref()
&& requested_model != config_snapshot.model
{
mismatch_details.push(format!(
"model requested={requested_model} active={}",
config_snapshot.model
));
}
if let Some(requested_provider) = request.model_provider.as_deref()
&& requested_provider != config_snapshot.model_provider_id
{
mismatch_details.push(format!(
"model_provider requested={requested_provider} active={}",
config_snapshot.model_provider_id
));
}
if let Some(requested_cwd) = request.cwd.as_deref() {
let requested_cwd_path = std::path::PathBuf::from(requested_cwd);
if requested_cwd_path != config_snapshot.cwd {
mismatch_details.push(format!(
"cwd requested={} active={}",
requested_cwd_path.display(),
config_snapshot.cwd.display()
));
}
}
if let Some(requested_approval) = request.approval_policy.as_ref() {
let active_approval: AskForApproval = config_snapshot.approval_policy.into();
if requested_approval != &active_approval {
mismatch_details.push(format!(
"approval_policy requested={requested_approval:?} active={active_approval:?}"
));
}
}
if let Some(requested_sandbox) = request.sandbox.as_ref() {
let sandbox_matches = matches!(
(requested_sandbox, &config_snapshot.sandbox_policy),
(
SandboxMode::ReadOnly,
codex_protocol::protocol::SandboxPolicy::ReadOnly { .. }
) | (
SandboxMode::WorkspaceWrite,
codex_protocol::protocol::SandboxPolicy::WorkspaceWrite { .. }
) | (
SandboxMode::DangerFullAccess,
codex_protocol::protocol::SandboxPolicy::DangerFullAccess
) | (
SandboxMode::DangerFullAccess,
codex_protocol::protocol::SandboxPolicy::ExternalSandbox { .. }
)
);
if !sandbox_matches {
mismatch_details.push(format!(
"sandbox requested={requested_sandbox:?} active={:?}",
config_snapshot.sandbox_policy
));
}
}
if let Some(requested_personality) = request.personality.as_ref()
&& config_snapshot.personality.as_ref() != Some(requested_personality)
{
mismatch_details.push(format!(
"personality requested={requested_personality:?} active={:?}",
config_snapshot.personality
));
}
if request.config.is_some() {
mismatch_details
.push("config overrides were provided and ignored while running".to_string());
}
if request.base_instructions.is_some() {
mismatch_details
.push("baseInstructions override was provided and ignored while running".to_string());
}
if request.developer_instructions.is_some() {
mismatch_details.push(
"developerInstructions override was provided and ignored while running".to_string(),
);
}
if request.persist_extended_history {
mismatch_details.push(
"persistExtendedHistory override was provided and ignored while running".to_string(),
);
}
mismatch_details
}
fn skills_to_info(
skills: &[codex_core::skills::SkillMetadata],
disabled_paths: &std::collections::HashSet<PathBuf>,
+18 -1
View File
@@ -6,6 +6,7 @@ use std::sync::atomic::AtomicBool;
use std::sync::atomic::Ordering;
use codex_app_server_protocol::FuzzyFileSearchResult;
use codex_app_server_protocol::FuzzyFileSearchSessionCompletedNotification;
use codex_app_server_protocol::FuzzyFileSearchSessionUpdatedNotification;
use codex_app_server_protocol::ServerNotification;
use codex_file_search as file_search;
@@ -195,6 +196,20 @@ impl SessionReporterImpl {
outgoing.send_server_notification(notification).await;
});
}
fn send_complete(&self) {
if self.shared.canceled.load(Ordering::Relaxed) {
return;
}
let session_id = self.shared.session_id.clone();
let outgoing = self.shared.outgoing.clone();
self.shared.runtime.spawn(async move {
let notification = ServerNotification::FuzzyFileSearchSessionCompleted(
FuzzyFileSearchSessionCompletedNotification { session_id },
);
outgoing.send_server_notification(notification).await;
});
}
}
impl file_search::SessionReporter for SessionReporterImpl {
@@ -202,7 +217,9 @@ impl file_search::SessionReporter for SessionReporterImpl {
self.send_snapshot(snapshot);
}
fn on_complete(&self) {}
fn on_complete(&self) {
self.send_complete();
}
}
fn collect_files(snapshot: &file_search::FileSearchSnapshot) -> Vec<FuzzyFileSearchResult> {
+7 -2
View File
@@ -8,12 +8,16 @@ use codex_core::models_manager::manager::RefreshStrategy;
use codex_protocol::openai_models::ModelPreset;
use codex_protocol::openai_models::ReasoningEffortPreset;
pub async fn supported_models(thread_manager: Arc<ThreadManager>, config: &Config) -> Vec<Model> {
pub async fn supported_models(
thread_manager: Arc<ThreadManager>,
config: &Config,
include_hidden: bool,
) -> Vec<Model> {
thread_manager
.list_models(config, RefreshStrategy::OnlineIfUncached)
.await
.into_iter()
.filter(|preset| preset.show_in_picker)
.filter(|preset| include_hidden || preset.show_in_picker)
.map(model_from_preset)
.collect()
}
@@ -25,6 +29,7 @@ fn model_from_preset(preset: ModelPreset) -> Model {
upgrade: preset.upgrade.map(|upgrade| upgrade.id),
display_name: preset.display_name.to_string(),
description: preset.description.to_string(),
hidden: !preset.show_in_picker,
supported_reasoning_efforts: reasoning_efforts_from_preset(
preset.supported_reasoning_efforts,
),
@@ -41,6 +41,7 @@ fn preset_to_info(preset: &ModelPreset, priority: i32) -> ModelInfo {
experimental_supported_tools: Vec::new(),
input_modalities: default_input_modalities(),
prefer_websockets: false,
used_fallback_model_metadata: false,
}
}
@@ -1,6 +1,7 @@
use anyhow::Result;
use anyhow::anyhow;
use app_test_support::McpProcess;
use codex_app_server_protocol::FuzzyFileSearchSessionCompletedNotification;
use codex_app_server_protocol::FuzzyFileSearchSessionUpdatedNotification;
use codex_app_server_protocol::JSONRPCResponse;
use codex_app_server_protocol::RequestId;
@@ -13,6 +14,7 @@ const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs
const SHORT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_millis(500);
const STOP_GRACE_PERIOD: std::time::Duration = std::time::Duration::from_millis(250);
const SESSION_UPDATED_METHOD: &str = "fuzzyFileSearch/sessionUpdated";
const SESSION_COMPLETED_METHOD: &str = "fuzzyFileSearch/sessionCompleted";
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum FileExpectation {
@@ -60,6 +62,29 @@ async fn wait_for_session_updated(
);
}
async fn wait_for_session_completed(
mcp: &mut McpProcess,
session_id: &str,
) -> Result<FuzzyFileSearchSessionCompletedNotification> {
for _ in 0..20 {
let notification = timeout(
DEFAULT_READ_TIMEOUT,
mcp.read_stream_until_notification_message(SESSION_COMPLETED_METHOD),
)
.await??;
let params = notification
.params
.ok_or_else(|| anyhow!("missing notification params"))?;
let payload =
serde_json::from_value::<FuzzyFileSearchSessionCompletedNotification>(params)?;
if payload.session_id == session_id {
return Ok(payload);
}
}
anyhow::bail!("did not receive expected session completion for sessionId={session_id}");
}
async fn assert_update_request_fails_for_missing_session(
mcp: &mut McpProcess,
session_id: &str,
@@ -271,12 +296,41 @@ async fn test_fuzzy_file_search_session_streams_updates() -> Result<()> {
assert_eq!(payload.files.len(), 1);
assert_eq!(payload.files[0].root, root_path);
assert_eq!(payload.files[0].path, "alpha.txt");
let completed = wait_for_session_completed(&mut mcp, session_id).await?;
assert_eq!(completed.session_id, session_id);
mcp.stop_fuzzy_file_search_session(session_id).await?;
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_fuzzy_file_search_session_no_updates_after_complete_until_query_edited() -> Result<()>
{
let codex_home = TempDir::new()?;
let root = TempDir::new()?;
std::fs::write(root.path().join("alpha.txt"), "contents")?;
let mut mcp = initialized_mcp(&codex_home).await?;
let root_path = root.path().to_string_lossy().to_string();
let session_id = "session-complete-invariant";
mcp.start_fuzzy_file_search_session(session_id, vec![root_path])
.await?;
mcp.update_fuzzy_file_search_session(session_id, "alp")
.await?;
wait_for_session_updated(&mut mcp, session_id, "alp", FileExpectation::NonEmpty).await?;
wait_for_session_completed(&mut mcp, session_id).await?;
assert_no_session_updates_for(&mut mcp, session_id, STOP_GRACE_PERIOD, SHORT_READ_TIMEOUT)
.await?;
mcp.update_fuzzy_file_search_session(session_id, "alpha")
.await?;
wait_for_session_updated(&mut mcp, session_id, "alpha", FileExpectation::NonEmpty).await?;
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_fuzzy_file_search_session_update_before_start_errors() -> Result<()> {
let codex_home = TempDir::new()?;
@@ -345,6 +399,7 @@ async fn test_fuzzy_file_search_session_multiple_query_updates_work() -> Result<
alp_payload.files.iter().all(|file| file.root == root_path),
true
);
wait_for_session_completed(&mut mcp, session_id).await?;
mcp.update_fuzzy_file_search_session(session_id, "zzzz")
.await?;
@@ -352,6 +407,7 @@ async fn test_fuzzy_file_search_session_multiple_query_updates_work() -> Result<
wait_for_session_updated(&mut mcp, session_id, "zzzz", FileExpectation::Any).await?;
assert_eq!(zzzz_payload.query, "zzzz");
assert_eq!(zzzz_payload.files.is_empty(), true);
wait_for_session_completed(&mut mcp, session_id).await?;
Ok(())
}
@@ -564,6 +564,7 @@ fn append_rollout_turn_context(path: &Path, timestamp: &str, model: &str) -> std
cwd: PathBuf::from("/"),
approval_policy: AskForApproval::Never,
sandbox_policy: SandboxPolicy::DangerFullAccess,
network: None,
model: model.to_string(),
personality: None,
collaboration_mode: None,
+345 -8
View File
@@ -1,5 +1,6 @@
use std::borrow::Cow;
use std::sync::Arc;
use std::sync::Mutex as StdMutex;
use std::time::Duration;
use anyhow::Result;
@@ -86,6 +87,7 @@ async fn list_apps_uses_thread_feature_flag_when_thread_id_is_provided() -> Resu
distribution_channel: None,
install_url: None,
is_accessible: false,
is_enabled: true,
}];
let tools = vec![connector_tool("beta", "Beta App")?];
let (server_url, server_handle) =
@@ -173,6 +175,78 @@ connectors = false
Ok(())
}
#[tokio::test]
async fn list_apps_reports_is_enabled_from_config() -> Result<()> {
let connectors = vec![AppInfo {
id: "beta".to_string(),
name: "Beta".to_string(),
description: Some("Beta connector".to_string()),
logo_url: None,
logo_url_dark: None,
distribution_channel: None,
install_url: None,
is_accessible: false,
is_enabled: true,
}];
let tools = vec![connector_tool("beta", "Beta App")?];
let (server_url, server_handle) =
start_apps_server_with_delays(connectors, tools, Duration::ZERO, Duration::ZERO).await?;
let codex_home = TempDir::new()?;
std::fs::write(
codex_home.path().join("config.toml"),
format!(
r#"
chatgpt_base_url = "{server_url}"
[features]
connectors = true
[apps.beta]
enabled = false
"#
),
)?;
write_chatgpt_auth(
codex_home.path(),
ChatGptAuthFixture::new("chatgpt-token")
.account_id("account-123")
.chatgpt_user_id("user-123")
.chatgpt_account_id("account-123"),
AuthCredentialsStoreMode::File,
)?;
let mut mcp = McpProcess::new(codex_home.path()).await?;
timeout(DEFAULT_TIMEOUT, mcp.initialize()).await??;
let request_id = mcp
.send_apps_list_request(AppsListParams {
limit: None,
cursor: None,
thread_id: None,
force_refetch: false,
})
.await?;
let response: JSONRPCResponse = timeout(
DEFAULT_TIMEOUT,
mcp.read_stream_until_response_message(RequestId::Integer(request_id)),
)
.await??;
let AppsListResponse {
data: response_data,
next_cursor,
} = to_response(response)?;
assert!(next_cursor.is_none());
assert_eq!(response_data.len(), 1);
assert_eq!(response_data[0].id, "beta");
assert!(!response_data[0].is_enabled);
server_handle.abort();
let _ = server_handle.await;
Ok(())
}
#[tokio::test]
async fn list_apps_emits_updates_and_returns_after_both_lists_load() -> Result<()> {
let connectors = vec![
@@ -185,6 +259,7 @@ async fn list_apps_emits_updates_and_returns_after_both_lists_load() -> Result<(
distribution_channel: None,
install_url: None,
is_accessible: false,
is_enabled: true,
},
AppInfo {
id: "beta".to_string(),
@@ -195,6 +270,7 @@ async fn list_apps_emits_updates_and_returns_after_both_lists_load() -> Result<(
distribution_channel: None,
install_url: None,
is_accessible: false,
is_enabled: true,
},
];
@@ -239,6 +315,7 @@ async fn list_apps_emits_updates_and_returns_after_both_lists_load() -> Result<(
distribution_channel: None,
install_url: Some("https://chatgpt.com/apps/beta-app/beta".to_string()),
is_accessible: true,
is_enabled: true,
}];
let first_update = read_app_list_updated_notification(&mut mcp).await?;
@@ -254,6 +331,7 @@ async fn list_apps_emits_updates_and_returns_after_both_lists_load() -> Result<(
distribution_channel: None,
install_url: Some("https://chatgpt.com/apps/beta/beta".to_string()),
is_accessible: true,
is_enabled: true,
},
AppInfo {
id: "alpha".to_string(),
@@ -264,6 +342,7 @@ async fn list_apps_emits_updates_and_returns_after_both_lists_load() -> Result<(
distribution_channel: None,
install_url: Some("https://chatgpt.com/apps/alpha/alpha".to_string()),
is_accessible: false,
is_enabled: true,
},
];
@@ -300,6 +379,7 @@ async fn list_apps_returns_connectors_with_accessible_flags() -> Result<()> {
distribution_channel: None,
install_url: None,
is_accessible: false,
is_enabled: true,
},
AppInfo {
id: "beta".to_string(),
@@ -310,6 +390,7 @@ async fn list_apps_returns_connectors_with_accessible_flags() -> Result<()> {
distribution_channel: None,
install_url: None,
is_accessible: false,
is_enabled: true,
},
];
@@ -358,6 +439,7 @@ async fn list_apps_returns_connectors_with_accessible_flags() -> Result<()> {
distribution_channel: None,
install_url: Some("https://chatgpt.com/apps/alpha/alpha".to_string()),
is_accessible: false,
is_enabled: true,
},
AppInfo {
id: "beta".to_string(),
@@ -368,6 +450,7 @@ async fn list_apps_returns_connectors_with_accessible_flags() -> Result<()> {
distribution_channel: None,
install_url: Some("https://chatgpt.com/apps/beta/beta".to_string()),
is_accessible: false,
is_enabled: true,
},
]
);
@@ -382,6 +465,7 @@ async fn list_apps_returns_connectors_with_accessible_flags() -> Result<()> {
distribution_channel: None,
install_url: Some("https://chatgpt.com/apps/beta/beta".to_string()),
is_accessible: true,
is_enabled: true,
},
AppInfo {
id: "alpha".to_string(),
@@ -392,6 +476,7 @@ async fn list_apps_returns_connectors_with_accessible_flags() -> Result<()> {
distribution_channel: None,
install_url: Some("https://chatgpt.com/apps/alpha/alpha".to_string()),
is_accessible: false,
is_enabled: true,
},
];
@@ -423,6 +508,7 @@ async fn list_apps_paginates_results() -> Result<()> {
distribution_channel: None,
install_url: None,
is_accessible: false,
is_enabled: true,
},
AppInfo {
id: "beta".to_string(),
@@ -433,6 +519,7 @@ async fn list_apps_paginates_results() -> Result<()> {
distribution_channel: None,
install_url: None,
is_accessible: false,
is_enabled: true,
},
];
@@ -486,6 +573,7 @@ async fn list_apps_paginates_results() -> Result<()> {
distribution_channel: None,
install_url: Some("https://chatgpt.com/apps/beta/beta".to_string()),
is_accessible: true,
is_enabled: true,
}];
assert_eq!(first_page, expected_first);
@@ -525,6 +613,7 @@ async fn list_apps_paginates_results() -> Result<()> {
distribution_channel: None,
install_url: Some("https://chatgpt.com/apps/alpha/alpha".to_string()),
is_accessible: false,
is_enabled: true,
}];
assert_eq!(second_page, expected_second);
@@ -545,6 +634,7 @@ async fn list_apps_force_refetch_preserves_previous_cache_on_failure() -> Result
distribution_channel: None,
install_url: None,
is_accessible: false,
is_enabled: true,
}];
let tools = vec![connector_tool("beta", "Beta App")?];
let (server_url, server_handle) =
@@ -633,6 +723,201 @@ async fn list_apps_force_refetch_preserves_previous_cache_on_failure() -> Result
Ok(())
}
#[tokio::test]
async fn list_apps_force_refetch_patches_updates_from_cached_snapshots() -> Result<()> {
let initial_connectors = vec![
AppInfo {
id: "alpha".to_string(),
name: "Alpha".to_string(),
description: Some("Alpha v1".to_string()),
logo_url: None,
logo_url_dark: None,
distribution_channel: None,
install_url: None,
is_accessible: false,
is_enabled: true,
},
AppInfo {
id: "beta".to_string(),
name: "Beta App".to_string(),
description: Some("Beta v1".to_string()),
logo_url: None,
logo_url_dark: None,
distribution_channel: None,
install_url: None,
is_accessible: false,
is_enabled: true,
},
];
let initial_tools = vec![connector_tool("beta", "Beta App")?];
let (server_url, server_handle, server_control) = start_apps_server_with_delays_and_control(
initial_connectors,
initial_tools,
Duration::from_millis(300),
Duration::ZERO,
)
.await?;
let codex_home = TempDir::new()?;
write_connectors_config(codex_home.path(), &server_url)?;
write_chatgpt_auth(
codex_home.path(),
ChatGptAuthFixture::new("chatgpt-token")
.account_id("account-123")
.chatgpt_user_id("user-123")
.chatgpt_account_id("account-123"),
AuthCredentialsStoreMode::File,
)?;
let mut mcp = McpProcess::new(codex_home.path()).await?;
timeout(DEFAULT_TIMEOUT, mcp.initialize()).await??;
let warm_request = mcp
.send_apps_list_request(AppsListParams {
limit: None,
cursor: None,
thread_id: None,
force_refetch: false,
})
.await?;
let warm_first_update = read_app_list_updated_notification(&mut mcp).await?;
assert_eq!(
warm_first_update.data,
vec![AppInfo {
id: "beta".to_string(),
name: "Beta App".to_string(),
description: None,
logo_url: None,
logo_url_dark: None,
distribution_channel: None,
install_url: Some("https://chatgpt.com/apps/beta-app/beta".to_string()),
is_accessible: true,
is_enabled: true,
}]
);
let warm_second_update = read_app_list_updated_notification(&mut mcp).await?;
assert_eq!(
warm_second_update.data,
vec![
AppInfo {
id: "beta".to_string(),
name: "Beta App".to_string(),
description: Some("Beta v1".to_string()),
logo_url: None,
logo_url_dark: None,
distribution_channel: None,
install_url: Some("https://chatgpt.com/apps/beta-app/beta".to_string()),
is_accessible: true,
is_enabled: true,
},
AppInfo {
id: "alpha".to_string(),
name: "Alpha".to_string(),
description: Some("Alpha v1".to_string()),
logo_url: None,
logo_url_dark: None,
distribution_channel: None,
install_url: Some("https://chatgpt.com/apps/alpha/alpha".to_string()),
is_accessible: false,
is_enabled: true,
},
]
);
let warm_response: JSONRPCResponse = timeout(
DEFAULT_TIMEOUT,
mcp.read_stream_until_response_message(RequestId::Integer(warm_request)),
)
.await??;
let AppsListResponse {
data: warm_data,
next_cursor: warm_next_cursor,
} = to_response(warm_response)?;
assert_eq!(warm_data, warm_second_update.data);
assert!(warm_next_cursor.is_none());
server_control.set_connectors(vec![AppInfo {
id: "alpha".to_string(),
name: "Alpha".to_string(),
description: Some("Alpha v2".to_string()),
logo_url: None,
logo_url_dark: None,
distribution_channel: None,
install_url: None,
is_accessible: false,
is_enabled: true,
}]);
server_control.set_tools(Vec::new());
let refetch_request = mcp
.send_apps_list_request(AppsListParams {
limit: None,
cursor: None,
thread_id: None,
force_refetch: true,
})
.await?;
let first_update = read_app_list_updated_notification(&mut mcp).await?;
assert_eq!(
first_update.data,
vec![
AppInfo {
id: "alpha".to_string(),
name: "Alpha".to_string(),
description: Some("Alpha v1".to_string()),
logo_url: None,
logo_url_dark: None,
distribution_channel: None,
install_url: Some("https://chatgpt.com/apps/alpha/alpha".to_string()),
is_accessible: false,
is_enabled: true,
},
AppInfo {
id: "beta".to_string(),
name: "Beta App".to_string(),
description: Some("Beta v1".to_string()),
logo_url: None,
logo_url_dark: None,
distribution_channel: None,
install_url: Some("https://chatgpt.com/apps/beta-app/beta".to_string()),
is_accessible: false,
is_enabled: true,
},
]
);
let expected_final = vec![AppInfo {
id: "alpha".to_string(),
name: "Alpha".to_string(),
description: Some("Alpha v2".to_string()),
logo_url: None,
logo_url_dark: None,
distribution_channel: None,
install_url: Some("https://chatgpt.com/apps/alpha/alpha".to_string()),
is_accessible: false,
is_enabled: true,
}];
let second_update = read_app_list_updated_notification(&mut mcp).await?;
assert_eq!(second_update.data, expected_final);
let refetch_response: JSONRPCResponse = timeout(
DEFAULT_TIMEOUT,
mcp.read_stream_until_response_message(RequestId::Integer(refetch_request)),
)
.await??;
let AppsListResponse {
data: refetch_data,
next_cursor: refetch_next_cursor,
} = to_response(refetch_response)?;
assert_eq!(refetch_data, expected_final);
assert!(refetch_next_cursor.is_none());
server_handle.abort();
Ok(())
}
async fn read_app_list_updated_notification(
mcp: &mut McpProcess,
) -> Result<AppListUpdatedNotification> {
@@ -652,22 +937,46 @@ async fn read_app_list_updated_notification(
struct AppsServerState {
expected_bearer: String,
expected_account_id: String,
response: serde_json::Value,
response: Arc<StdMutex<serde_json::Value>>,
directory_delay: Duration,
}
#[derive(Clone)]
struct AppListMcpServer {
tools: Arc<Vec<Tool>>,
tools: Arc<StdMutex<Vec<Tool>>>,
tools_delay: Duration,
}
impl AppListMcpServer {
fn new(tools: Arc<Vec<Tool>>, tools_delay: Duration) -> Self {
fn new(tools: Arc<StdMutex<Vec<Tool>>>, tools_delay: Duration) -> Self {
Self { tools, tools_delay }
}
}
#[derive(Clone)]
struct AppsServerControl {
response: Arc<StdMutex<serde_json::Value>>,
tools: Arc<StdMutex<Vec<Tool>>>,
}
impl AppsServerControl {
fn set_connectors(&self, connectors: Vec<AppInfo>) {
let mut response_guard = self
.response
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
*response_guard = json!({ "apps": connectors, "next_token": null });
}
fn set_tools(&self, tools: Vec<Tool>) {
let mut tools_guard = self
.tools
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
*tools_guard = tools;
}
}
impl ServerHandler for AppListMcpServer {
fn get_info(&self) -> ServerInfo {
ServerInfo {
@@ -688,8 +997,12 @@ impl ServerHandler for AppListMcpServer {
if tools_delay > Duration::ZERO {
tokio::time::sleep(tools_delay).await;
}
let tools = tools
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.clone();
Ok(ListToolsResult {
tools: (*tools).clone(),
tools,
next_cursor: None,
meta: None,
})
@@ -703,14 +1016,33 @@ async fn start_apps_server_with_delays(
directory_delay: Duration,
tools_delay: Duration,
) -> Result<(String, JoinHandle<()>)> {
let (server_url, server_handle, _server_control) =
start_apps_server_with_delays_and_control(connectors, tools, directory_delay, tools_delay)
.await?;
Ok((server_url, server_handle))
}
async fn start_apps_server_with_delays_and_control(
connectors: Vec<AppInfo>,
tools: Vec<Tool>,
directory_delay: Duration,
tools_delay: Duration,
) -> Result<(String, JoinHandle<()>, AppsServerControl)> {
let response = Arc::new(StdMutex::new(
json!({ "apps": connectors, "next_token": null }),
));
let tools = Arc::new(StdMutex::new(tools));
let state = AppsServerState {
expected_bearer: "Bearer chatgpt-token".to_string(),
expected_account_id: "account-123".to_string(),
response: json!({ "apps": connectors, "next_token": null }),
response: response.clone(),
directory_delay,
};
let state = Arc::new(state);
let tools = Arc::new(tools);
let server_control = AppsServerControl {
response,
tools: tools.clone(),
};
let listener = TcpListener::bind("127.0.0.1:0").await?;
let addr = listener.local_addr()?;
@@ -737,7 +1069,7 @@ async fn start_apps_server_with_delays(
let _ = axum::serve(listener, router).await;
});
Ok((format!("http://{addr}"), handle))
Ok((format!("http://{addr}"), handle, server_control))
}
async fn list_directory_connectors(
@@ -758,7 +1090,12 @@ async fn list_directory_connectors(
.is_some_and(|value| value == state.expected_account_id);
if bearer_ok && account_ok {
Ok(Json(state.response.clone()))
let response = state
.response
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.clone();
Ok(Json(response))
} else {
Err(StatusCode::UNAUTHORIZED)
}
@@ -33,6 +33,7 @@ async fn list_models_returns_all_models_with_large_limit() -> Result<()> {
.send_list_models_request(ModelListParams {
limit: Some(100),
cursor: None,
include_hidden: None,
})
.await?;
@@ -54,6 +55,7 @@ async fn list_models_returns_all_models_with_large_limit() -> Result<()> {
upgrade: None,
display_name: "gpt-5.2-codex".to_string(),
description: "Latest frontier agentic coding model.".to_string(),
hidden: false,
supported_reasoning_efforts: vec![
ReasoningEffortOption {
reasoning_effort: ReasoningEffort::Low,
@@ -84,6 +86,7 @@ async fn list_models_returns_all_models_with_large_limit() -> Result<()> {
upgrade: Some("gpt-5.2-codex".to_string()),
display_name: "gpt-5.1-codex-max".to_string(),
description: "Codex-optimized flagship for deep and fast reasoning.".to_string(),
hidden: false,
supported_reasoning_efforts: vec![
ReasoningEffortOption {
reasoning_effort: ReasoningEffort::Low,
@@ -114,6 +117,7 @@ async fn list_models_returns_all_models_with_large_limit() -> Result<()> {
upgrade: Some("gpt-5.2-codex".to_string()),
display_name: "gpt-5.1-codex-mini".to_string(),
description: "Optimized for codex. Cheaper, faster, but less capable.".to_string(),
hidden: false,
supported_reasoning_efforts: vec![
ReasoningEffortOption {
reasoning_effort: ReasoningEffort::Medium,
@@ -138,6 +142,7 @@ async fn list_models_returns_all_models_with_large_limit() -> Result<()> {
description:
"Latest frontier model with improvements across knowledge, reasoning and coding"
.to_string(),
hidden: false,
supported_reasoning_efforts: vec![
ReasoningEffortOption {
reasoning_effort: ReasoningEffort::Low,
@@ -173,6 +178,38 @@ async fn list_models_returns_all_models_with_large_limit() -> Result<()> {
Ok(())
}
#[tokio::test]
async fn list_models_includes_hidden_models() -> Result<()> {
let codex_home = TempDir::new()?;
write_models_cache(codex_home.path())?;
let mut mcp = McpProcess::new(codex_home.path()).await?;
timeout(DEFAULT_TIMEOUT, mcp.initialize()).await??;
let request_id = mcp
.send_list_models_request(ModelListParams {
limit: Some(100),
cursor: None,
include_hidden: Some(true),
})
.await?;
let response: JSONRPCResponse = timeout(
DEFAULT_TIMEOUT,
mcp.read_stream_until_response_message(RequestId::Integer(request_id)),
)
.await??;
let ModelListResponse {
data: items,
next_cursor,
} = to_response::<ModelListResponse>(response)?;
assert!(items.iter().any(|item| item.hidden));
assert!(next_cursor.is_none());
Ok(())
}
#[tokio::test]
async fn list_models_pagination_works() -> Result<()> {
let codex_home = TempDir::new()?;
@@ -185,6 +222,7 @@ async fn list_models_pagination_works() -> Result<()> {
.send_list_models_request(ModelListParams {
limit: Some(1),
cursor: None,
include_hidden: None,
})
.await?;
@@ -207,6 +245,7 @@ async fn list_models_pagination_works() -> Result<()> {
.send_list_models_request(ModelListParams {
limit: Some(1),
cursor: Some(next_cursor.clone()),
include_hidden: None,
})
.await?;
@@ -229,6 +268,7 @@ async fn list_models_pagination_works() -> Result<()> {
.send_list_models_request(ModelListParams {
limit: Some(1),
cursor: Some(third_cursor.clone()),
include_hidden: None,
})
.await?;
@@ -251,6 +291,7 @@ async fn list_models_pagination_works() -> Result<()> {
.send_list_models_request(ModelListParams {
limit: Some(1),
cursor: Some(fourth_cursor.clone()),
include_hidden: None,
})
.await?;
@@ -283,6 +324,7 @@ async fn list_models_rejects_invalid_cursor() -> Result<()> {
.send_list_models_request(ModelListParams {
limit: None,
cursor: Some("invalid".to_string()),
include_hidden: None,
})
.await?;
@@ -17,6 +17,8 @@ use codex_app_server_protocol::ThreadSourceKind;
use codex_core::ARCHIVED_SESSIONS_SUBDIR;
use codex_protocol::ThreadId;
use codex_protocol::protocol::GitInfo as CoreGitInfo;
use codex_protocol::protocol::RolloutItem;
use codex_protocol::protocol::RolloutLine;
use codex_protocol::protocol::SessionSource as CoreSessionSource;
use codex_protocol::protocol::SubAgentSource;
use pretty_assertions::assert_eq;
@@ -66,6 +68,7 @@ async fn list_threads_with_sort(
model_providers: providers,
source_kinds,
archived,
cwd: None,
})
.await?;
let resp: JSONRPCResponse = timeout(
@@ -127,6 +130,26 @@ fn set_rollout_mtime(path: &Path, updated_at_rfc3339: &str) -> Result<()> {
Ok(())
}
fn set_rollout_cwd(path: &Path, cwd: &Path) -> Result<()> {
let content = fs::read_to_string(path)?;
let mut lines: Vec<String> = content.lines().map(str::to_string).collect();
let first_line = lines
.first_mut()
.ok_or_else(|| anyhow::anyhow!("rollout at {} is empty", path.display()))?;
let mut rollout_line: RolloutLine = serde_json::from_str(first_line)?;
let RolloutItem::SessionMeta(mut session_meta_line) = rollout_line.item else {
return Err(anyhow::anyhow!(
"rollout at {} does not start with session metadata",
path.display()
));
};
session_meta_line.meta.cwd = cwd.to_path_buf();
rollout_line.item = RolloutItem::SessionMeta(session_meta_line);
*first_line = serde_json::to_string(&rollout_line)?;
fs::write(path, lines.join("\n") + "\n")?;
Ok(())
}
#[tokio::test]
async fn thread_list_basic_empty() -> Result<()> {
let codex_home = TempDir::new()?;
@@ -300,6 +323,63 @@ async fn thread_list_respects_provider_filter() -> Result<()> {
Ok(())
}
#[tokio::test]
async fn thread_list_respects_cwd_filter() -> Result<()> {
let codex_home = TempDir::new()?;
create_minimal_config(codex_home.path())?;
let filtered_id = create_fake_rollout(
codex_home.path(),
"2025-01-02T10-00-00",
"2025-01-02T10:00:00Z",
"filtered",
Some("mock_provider"),
None,
)?;
let unfiltered_id = create_fake_rollout(
codex_home.path(),
"2025-01-02T11-00-00",
"2025-01-02T11:00:00Z",
"unfiltered",
Some("mock_provider"),
None,
)?;
let target_cwd = codex_home.path().join("target-cwd");
fs::create_dir_all(&target_cwd)?;
set_rollout_cwd(
rollout_path(codex_home.path(), "2025-01-02T10-00-00", &filtered_id).as_path(),
&target_cwd,
)?;
let mut mcp = init_mcp(codex_home.path()).await?;
let request_id = mcp
.send_thread_list_request(codex_app_server_protocol::ThreadListParams {
cursor: None,
limit: Some(10),
sort_key: None,
model_providers: Some(vec!["mock_provider".to_string()]),
source_kinds: None,
archived: None,
cwd: Some(target_cwd.to_string_lossy().into_owned()),
})
.await?;
let resp: JSONRPCResponse = timeout(
DEFAULT_READ_TIMEOUT,
mcp.read_stream_until_response_message(RequestId::Integer(request_id)),
)
.await??;
let ThreadListResponse { data, next_cursor } = to_response::<ThreadListResponse>(resp)?;
assert_eq!(next_cursor, None);
assert_eq!(data.len(), 1);
assert_eq!(data[0].id, filtered_id);
assert_ne!(data[0].id, unfiltered_id);
assert_eq!(data[0].cwd, target_cwd);
Ok(())
}
#[tokio::test]
async fn thread_list_empty_source_kinds_defaults_to_interactive_only() -> Result<()> {
let codex_home = TempDir::new()?;
@@ -1107,6 +1187,7 @@ async fn thread_list_invalid_cursor_returns_error() -> Result<()> {
model_providers: Some(vec!["mock_provider".to_string()]),
source_kinds: None,
archived: None,
cwd: None,
})
.await?;
let error: JSONRPCError = timeout(
@@ -209,18 +209,412 @@ async fn thread_resume_without_overrides_does_not_change_updated_at_or_mtime() -
Ok(())
}
#[tokio::test]
async fn thread_resume_keeps_in_flight_turn_streaming() -> Result<()> {
let server = create_mock_responses_server_repeating_assistant("Done").await;
let codex_home = TempDir::new()?;
create_config_toml(codex_home.path(), &server.uri())?;
let mut primary = McpProcess::new(codex_home.path()).await?;
timeout(DEFAULT_READ_TIMEOUT, primary.initialize()).await??;
let start_id = primary
.send_thread_start_request(ThreadStartParams {
model: Some("gpt-5.1-codex-max".to_string()),
..Default::default()
})
.await?;
let start_resp: JSONRPCResponse = timeout(
DEFAULT_READ_TIMEOUT,
primary.read_stream_until_response_message(RequestId::Integer(start_id)),
)
.await??;
let ThreadStartResponse { thread, .. } = to_response::<ThreadStartResponse>(start_resp)?;
let seed_turn_id = primary
.send_turn_start_request(TurnStartParams {
thread_id: thread.id.clone(),
input: vec![UserInput::Text {
text: "seed history".to_string(),
text_elements: Vec::new(),
}],
..Default::default()
})
.await?;
timeout(
DEFAULT_READ_TIMEOUT,
primary.read_stream_until_response_message(RequestId::Integer(seed_turn_id)),
)
.await??;
timeout(
DEFAULT_READ_TIMEOUT,
primary.read_stream_until_notification_message("turn/completed"),
)
.await??;
primary.clear_message_buffer();
let mut secondary = McpProcess::new(codex_home.path()).await?;
timeout(DEFAULT_READ_TIMEOUT, secondary.initialize()).await??;
let turn_id = primary
.send_turn_start_request(TurnStartParams {
thread_id: thread.id.clone(),
input: vec![UserInput::Text {
text: "respond with docs".to_string(),
text_elements: Vec::new(),
}],
..Default::default()
})
.await?;
timeout(
DEFAULT_READ_TIMEOUT,
primary.read_stream_until_response_message(RequestId::Integer(turn_id)),
)
.await??;
timeout(
DEFAULT_READ_TIMEOUT,
primary.read_stream_until_notification_message("turn/started"),
)
.await??;
let resume_id = secondary
.send_thread_resume_request(ThreadResumeParams {
thread_id: thread.id,
..Default::default()
})
.await?;
timeout(
DEFAULT_READ_TIMEOUT,
secondary.read_stream_until_response_message(RequestId::Integer(resume_id)),
)
.await??;
timeout(
DEFAULT_READ_TIMEOUT,
primary.read_stream_until_notification_message("turn/completed"),
)
.await??;
Ok(())
}
#[tokio::test]
async fn thread_resume_rejects_history_when_thread_is_running() -> Result<()> {
let server = responses::start_mock_server().await;
let first_body = responses::sse(vec![
responses::ev_response_created("resp-1"),
responses::ev_assistant_message("msg-1", "Done"),
responses::ev_completed("resp-1"),
]);
let second_body = responses::sse(vec![responses::ev_response_created("resp-2")]);
let _first_response_mock = responses::mount_sse_once(&server, first_body).await;
let _second_response_mock = responses::mount_sse_once(&server, second_body).await;
let codex_home = TempDir::new()?;
create_config_toml(codex_home.path(), &server.uri())?;
let mut primary = McpProcess::new(codex_home.path()).await?;
timeout(DEFAULT_READ_TIMEOUT, primary.initialize()).await??;
let start_id = primary
.send_thread_start_request(ThreadStartParams {
model: Some("gpt-5.1-codex-max".to_string()),
..Default::default()
})
.await?;
let start_resp: JSONRPCResponse = timeout(
DEFAULT_READ_TIMEOUT,
primary.read_stream_until_response_message(RequestId::Integer(start_id)),
)
.await??;
let ThreadStartResponse { thread, .. } = to_response::<ThreadStartResponse>(start_resp)?;
let seed_turn_id = primary
.send_turn_start_request(TurnStartParams {
thread_id: thread.id.clone(),
input: vec![UserInput::Text {
text: "seed history".to_string(),
text_elements: Vec::new(),
}],
..Default::default()
})
.await?;
timeout(
DEFAULT_READ_TIMEOUT,
primary.read_stream_until_response_message(RequestId::Integer(seed_turn_id)),
)
.await??;
timeout(
DEFAULT_READ_TIMEOUT,
primary.read_stream_until_notification_message("turn/completed"),
)
.await??;
primary.clear_message_buffer();
let running_turn_id = primary
.send_turn_start_request(TurnStartParams {
thread_id: thread.id.clone(),
input: vec![UserInput::Text {
text: "keep running".to_string(),
text_elements: Vec::new(),
}],
..Default::default()
})
.await?;
timeout(
DEFAULT_READ_TIMEOUT,
primary.read_stream_until_response_message(RequestId::Integer(running_turn_id)),
)
.await??;
timeout(
DEFAULT_READ_TIMEOUT,
primary.read_stream_until_notification_message("turn/started"),
)
.await??;
let resume_id = primary
.send_thread_resume_request(ThreadResumeParams {
thread_id: thread.id,
history: Some(vec![ResponseItem::Message {
id: None,
role: "user".to_string(),
content: vec![ContentItem::InputText {
text: "history override".to_string(),
}],
end_turn: None,
phase: None,
}]),
..Default::default()
})
.await?;
let resume_err: JSONRPCError = timeout(
DEFAULT_READ_TIMEOUT,
primary.read_stream_until_error_message(RequestId::Integer(resume_id)),
)
.await??;
assert!(
resume_err.error.message.contains("cannot resume thread")
&& resume_err.error.message.contains("with history")
&& resume_err.error.message.contains("running"),
"unexpected resume error: {}",
resume_err.error.message
);
Ok(())
}
#[tokio::test]
async fn thread_resume_rejects_mismatched_path_when_thread_is_running() -> Result<()> {
let server = responses::start_mock_server().await;
let first_body = responses::sse(vec![
responses::ev_response_created("resp-1"),
responses::ev_assistant_message("msg-1", "Done"),
responses::ev_completed("resp-1"),
]);
let second_body = responses::sse(vec![responses::ev_response_created("resp-2")]);
let _first_response_mock = responses::mount_sse_once(&server, first_body).await;
let _second_response_mock = responses::mount_sse_once(&server, second_body).await;
let codex_home = TempDir::new()?;
create_config_toml(codex_home.path(), &server.uri())?;
let mut primary = McpProcess::new(codex_home.path()).await?;
timeout(DEFAULT_READ_TIMEOUT, primary.initialize()).await??;
let start_id = primary
.send_thread_start_request(ThreadStartParams {
model: Some("gpt-5.1-codex-max".to_string()),
..Default::default()
})
.await?;
let start_resp: JSONRPCResponse = timeout(
DEFAULT_READ_TIMEOUT,
primary.read_stream_until_response_message(RequestId::Integer(start_id)),
)
.await??;
let ThreadStartResponse { thread, .. } = to_response::<ThreadStartResponse>(start_resp)?;
let seed_turn_id = primary
.send_turn_start_request(TurnStartParams {
thread_id: thread.id.clone(),
input: vec![UserInput::Text {
text: "seed history".to_string(),
text_elements: Vec::new(),
}],
..Default::default()
})
.await?;
timeout(
DEFAULT_READ_TIMEOUT,
primary.read_stream_until_response_message(RequestId::Integer(seed_turn_id)),
)
.await??;
timeout(
DEFAULT_READ_TIMEOUT,
primary.read_stream_until_notification_message("turn/completed"),
)
.await??;
primary.clear_message_buffer();
let running_turn_id = primary
.send_turn_start_request(TurnStartParams {
thread_id: thread.id.clone(),
input: vec![UserInput::Text {
text: "keep running".to_string(),
text_elements: Vec::new(),
}],
..Default::default()
})
.await?;
timeout(
DEFAULT_READ_TIMEOUT,
primary.read_stream_until_response_message(RequestId::Integer(running_turn_id)),
)
.await??;
timeout(
DEFAULT_READ_TIMEOUT,
primary.read_stream_until_notification_message("turn/started"),
)
.await??;
let resume_id = primary
.send_thread_resume_request(ThreadResumeParams {
thread_id: thread.id,
path: Some(PathBuf::from("/tmp/does-not-match-running-rollout.jsonl")),
..Default::default()
})
.await?;
let resume_err: JSONRPCError = timeout(
DEFAULT_READ_TIMEOUT,
primary.read_stream_until_error_message(RequestId::Integer(resume_id)),
)
.await??;
assert!(
resume_err.error.message.contains("mismatched path"),
"unexpected resume error: {}",
resume_err.error.message
);
Ok(())
}
#[tokio::test]
async fn thread_resume_rejoins_running_thread_even_with_override_mismatch() -> Result<()> {
let server = responses::start_mock_server().await;
let first_body = responses::sse(vec![
responses::ev_response_created("resp-1"),
responses::ev_assistant_message("msg-1", "Done"),
responses::ev_completed("resp-1"),
]);
let second_body = responses::sse(vec![responses::ev_response_created("resp-2")]);
let _response_mock =
responses::mount_sse_sequence(&server, vec![first_body, second_body]).await;
let codex_home = TempDir::new()?;
create_config_toml(codex_home.path(), &server.uri())?;
let mut primary = McpProcess::new(codex_home.path()).await?;
timeout(DEFAULT_READ_TIMEOUT, primary.initialize()).await??;
let start_id = primary
.send_thread_start_request(ThreadStartParams {
model: Some("gpt-5.1-codex-max".to_string()),
..Default::default()
})
.await?;
let start_resp: JSONRPCResponse = timeout(
DEFAULT_READ_TIMEOUT,
primary.read_stream_until_response_message(RequestId::Integer(start_id)),
)
.await??;
let ThreadStartResponse { thread, .. } = to_response::<ThreadStartResponse>(start_resp)?;
let seed_turn_id = primary
.send_turn_start_request(TurnStartParams {
thread_id: thread.id.clone(),
input: vec![UserInput::Text {
text: "seed history".to_string(),
text_elements: Vec::new(),
}],
..Default::default()
})
.await?;
timeout(
DEFAULT_READ_TIMEOUT,
primary.read_stream_until_response_message(RequestId::Integer(seed_turn_id)),
)
.await??;
timeout(
DEFAULT_READ_TIMEOUT,
primary.read_stream_until_notification_message("turn/completed"),
)
.await??;
primary.clear_message_buffer();
let running_turn_id = primary
.send_turn_start_request(TurnStartParams {
thread_id: thread.id.clone(),
input: vec![UserInput::Text {
text: "keep running".to_string(),
text_elements: Vec::new(),
}],
..Default::default()
})
.await?;
timeout(
DEFAULT_READ_TIMEOUT,
primary.read_stream_until_response_message(RequestId::Integer(running_turn_id)),
)
.await??;
timeout(
DEFAULT_READ_TIMEOUT,
primary.read_stream_until_notification_message("turn/started"),
)
.await??;
let resume_id = primary
.send_thread_resume_request(ThreadResumeParams {
thread_id: thread.id.clone(),
model: Some("not-the-running-model".to_string()),
cwd: Some("/tmp".to_string()),
..Default::default()
})
.await?;
let resume_resp: JSONRPCResponse = timeout(
DEFAULT_READ_TIMEOUT,
primary.read_stream_until_response_message(RequestId::Integer(resume_id)),
)
.await??;
let ThreadResumeResponse { model, .. } = to_response::<ThreadResumeResponse>(resume_resp)?;
assert_eq!(model, "gpt-5.1-codex-max");
timeout(
DEFAULT_READ_TIMEOUT,
primary.read_stream_until_notification_message("turn/completed"),
)
.await??;
Ok(())
}
#[tokio::test]
async fn thread_resume_with_overrides_defers_updated_at_until_turn_start() -> Result<()> {
let server = create_mock_responses_server_repeating_assistant("Done").await;
let codex_home = TempDir::new()?;
let rollout = setup_rollout_fixture(codex_home.path(), &server.uri())?;
create_config_toml(codex_home.path(), &server.uri())?;
let mut mcp = McpProcess::new(codex_home.path()).await?;
timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??;
let RestartedThreadFixture {
mut mcp,
thread_id,
rollout_file_path,
} = start_materialized_thread_and_restart(codex_home.path(), "materialize").await?;
let expected_updated_at_rfc3339 = "2025-01-07T00:00:00Z";
set_rollout_mtime(rollout_file_path.as_path(), expected_updated_at_rfc3339)?;
let before_modified = std::fs::metadata(&rollout_file_path)?.modified()?;
let expected_updated_at = chrono::DateTime::parse_from_rfc3339(expected_updated_at_rfc3339)?
.with_timezone(&Utc)
.timestamp();
let resume_id = mcp
.send_thread_resume_request(ThreadResumeParams {
thread_id: rollout.conversation_id.clone(),
thread_id,
model: Some("mock-model".to_string()),
..Default::default()
})
@@ -230,16 +624,19 @@ async fn thread_resume_with_overrides_defers_updated_at_until_turn_start() -> Re
mcp.read_stream_until_response_message(RequestId::Integer(resume_id)),
)
.await??;
let ThreadResumeResponse { thread, .. } = to_response::<ThreadResumeResponse>(resume_resp)?;
let ThreadResumeResponse {
thread: resumed_thread,
..
} = to_response::<ThreadResumeResponse>(resume_resp)?;
assert_eq!(thread.updated_at, rollout.expected_updated_at);
assert_eq!(resumed_thread.updated_at, expected_updated_at);
let after_resume_modified = std::fs::metadata(&rollout.rollout_file_path)?.modified()?;
assert_eq!(after_resume_modified, rollout.before_modified);
let after_resume_modified = std::fs::metadata(&rollout_file_path)?.modified()?;
assert_eq!(after_resume_modified, before_modified);
let turn_id = mcp
.send_turn_start_request(TurnStartParams {
thread_id: rollout.conversation_id,
thread_id: resumed_thread.id,
input: vec![UserInput::Text {
text: "Hello".to_string(),
text_elements: Vec::new(),
@@ -258,8 +655,8 @@ async fn thread_resume_with_overrides_defers_updated_at_until_turn_start() -> Re
)
.await??;
let after_turn_modified = std::fs::metadata(&rollout.rollout_file_path)?.modified()?;
assert!(after_turn_modified > rollout.before_modified);
let after_turn_modified = std::fs::metadata(&rollout_file_path)?.modified()?;
assert!(after_turn_modified > before_modified);
Ok(())
}
@@ -374,22 +771,9 @@ async fn thread_resume_supports_history_and_overrides() -> Result<()> {
let codex_home = TempDir::new()?;
create_config_toml(codex_home.path(), &server.uri())?;
let mut mcp = McpProcess::new(codex_home.path()).await?;
timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??;
// Start a thread.
let start_id = mcp
.send_thread_start_request(ThreadStartParams {
model: Some("gpt-5.1-codex-max".to_string()),
..Default::default()
})
.await?;
let start_resp: JSONRPCResponse = timeout(
DEFAULT_READ_TIMEOUT,
mcp.read_stream_until_response_message(RequestId::Integer(start_id)),
)
.await??;
let ThreadStartResponse { thread, .. } = to_response::<ThreadStartResponse>(start_resp)?;
let RestartedThreadFixture {
mut mcp, thread_id, ..
} = start_materialized_thread_and_restart(codex_home.path(), "seed history").await?;
let history_text = "Hello from history";
let history = vec![ResponseItem::Message {
@@ -405,7 +789,7 @@ async fn thread_resume_supports_history_and_overrides() -> Result<()> {
// Resume with explicit history and override the model.
let resume_id = mcp
.send_thread_resume_request(ThreadResumeParams {
thread_id: thread.id,
thread_id,
history: Some(history),
model: Some("mock-model".to_string()),
model_provider: Some("mock_provider".to_string()),
@@ -429,6 +813,70 @@ async fn thread_resume_supports_history_and_overrides() -> Result<()> {
Ok(())
}
struct RestartedThreadFixture {
mcp: McpProcess,
thread_id: String,
rollout_file_path: PathBuf,
}
async fn start_materialized_thread_and_restart(
codex_home: &Path,
seed_text: &str,
) -> Result<RestartedThreadFixture> {
let mut first_mcp = McpProcess::new(codex_home).await?;
timeout(DEFAULT_READ_TIMEOUT, first_mcp.initialize()).await??;
let start_id = first_mcp
.send_thread_start_request(ThreadStartParams {
model: Some("gpt-5.1-codex-max".to_string()),
..Default::default()
})
.await?;
let start_resp: JSONRPCResponse = timeout(
DEFAULT_READ_TIMEOUT,
first_mcp.read_stream_until_response_message(RequestId::Integer(start_id)),
)
.await??;
let ThreadStartResponse { thread, .. } = to_response::<ThreadStartResponse>(start_resp)?;
let materialize_turn_id = first_mcp
.send_turn_start_request(TurnStartParams {
thread_id: thread.id.clone(),
input: vec![UserInput::Text {
text: seed_text.to_string(),
text_elements: Vec::new(),
}],
..Default::default()
})
.await?;
timeout(
DEFAULT_READ_TIMEOUT,
first_mcp.read_stream_until_response_message(RequestId::Integer(materialize_turn_id)),
)
.await??;
timeout(
DEFAULT_READ_TIMEOUT,
first_mcp.read_stream_until_notification_message("turn/completed"),
)
.await??;
let thread_id = thread.id;
let rollout_file_path = thread
.path
.ok_or_else(|| anyhow::anyhow!("thread path missing from thread/start response"))?;
drop(first_mcp);
let mut second_mcp = McpProcess::new(codex_home).await?;
timeout(DEFAULT_READ_TIMEOUT, second_mcp.initialize()).await??;
Ok(RestartedThreadFixture {
mcp: second_mcp,
thread_id,
rollout_file_path,
})
}
#[tokio::test]
async fn thread_resume_accepts_personality_override() -> Result<()> {
skip_if_no_network!(Ok(()));
@@ -449,10 +897,10 @@ async fn thread_resume_accepts_personality_override() -> Result<()> {
let codex_home = TempDir::new()?;
create_config_toml(codex_home.path(), &server.uri())?;
let mut mcp = McpProcess::new(codex_home.path()).await?;
timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??;
let mut primary = McpProcess::new(codex_home.path()).await?;
timeout(DEFAULT_READ_TIMEOUT, primary.initialize()).await??;
let start_id = mcp
let start_id = primary
.send_thread_start_request(ThreadStartParams {
model: Some("gpt-5.2-codex".to_string()),
..Default::default()
@@ -460,12 +908,12 @@ async fn thread_resume_accepts_personality_override() -> Result<()> {
.await?;
let start_resp: JSONRPCResponse = timeout(
DEFAULT_READ_TIMEOUT,
mcp.read_stream_until_response_message(RequestId::Integer(start_id)),
primary.read_stream_until_response_message(RequestId::Integer(start_id)),
)
.await??;
let ThreadStartResponse { thread, .. } = to_response::<ThreadStartResponse>(start_resp)?;
let materialize_id = mcp
let materialize_id = primary
.send_turn_start_request(TurnStartParams {
thread_id: thread.id.clone(),
input: vec![UserInput::Text {
@@ -477,16 +925,19 @@ async fn thread_resume_accepts_personality_override() -> Result<()> {
.await?;
timeout(
DEFAULT_READ_TIMEOUT,
mcp.read_stream_until_response_message(RequestId::Integer(materialize_id)),
primary.read_stream_until_response_message(RequestId::Integer(materialize_id)),
)
.await??;
timeout(
DEFAULT_READ_TIMEOUT,
mcp.read_stream_until_notification_message("turn/completed"),
primary.read_stream_until_notification_message("turn/completed"),
)
.await??;
let resume_id = mcp
let mut secondary = McpProcess::new(codex_home.path()).await?;
timeout(DEFAULT_READ_TIMEOUT, secondary.initialize()).await??;
let resume_id = secondary
.send_thread_resume_request(ThreadResumeParams {
thread_id: thread.id,
model: Some("gpt-5.2-codex".to_string()),
@@ -496,12 +947,12 @@ async fn thread_resume_accepts_personality_override() -> Result<()> {
.await?;
let resume_resp: JSONRPCResponse = timeout(
DEFAULT_READ_TIMEOUT,
mcp.read_stream_until_response_message(RequestId::Integer(resume_id)),
secondary.read_stream_until_response_message(RequestId::Integer(resume_id)),
)
.await??;
let resume: ThreadResumeResponse = to_response::<ThreadResumeResponse>(resume_resp)?;
let turn_id = mcp
let turn_id = secondary
.send_turn_start_request(TurnStartParams {
thread_id: resume.thread.id,
input: vec![UserInput::Text {
@@ -513,13 +964,13 @@ async fn thread_resume_accepts_personality_override() -> Result<()> {
.await?;
timeout(
DEFAULT_READ_TIMEOUT,
mcp.read_stream_until_response_message(RequestId::Integer(turn_id)),
secondary.read_stream_until_response_message(RequestId::Integer(turn_id)),
)
.await??;
timeout(
DEFAULT_READ_TIMEOUT,
mcp.read_stream_until_notification_message("turn/completed"),
secondary.read_stream_until_notification_message("turn/completed"),
)
.await??;
+15 -1
View File
@@ -12,6 +12,8 @@ const LINUX_SANDBOX_ARG0: &str = "codex-linux-sandbox";
const APPLY_PATCH_ARG0: &str = "apply_patch";
const MISSPELLED_APPLY_PATCH_ARG0: &str = "applypatch";
const LOCK_FILENAME: &str = ".lock";
#[cfg(target_os = "windows")]
const WINDOWS_TOKIO_WORKER_STACK_SIZE_BYTES: usize = 16 * 1024 * 1024;
/// Keeps the per-session PATH entry alive and locked for the process lifetime.
pub struct Arg0PathEntryGuard {
@@ -112,7 +114,7 @@ where
// Regular invocation – create a Tokio runtime and execute the provided
// async entry-point.
let runtime = tokio::runtime::Runtime::new()?;
let runtime = build_runtime()?;
runtime.block_on(async move {
let codex_linux_sandbox_exe: Option<PathBuf> = if cfg!(target_os = "linux") {
std::env::current_exe().ok()
@@ -124,6 +126,18 @@ where
})
}
fn build_runtime() -> anyhow::Result<tokio::runtime::Runtime> {
let mut builder = tokio::runtime::Builder::new_multi_thread();
builder.enable_all();
#[cfg(target_os = "windows")]
{
// Defensive hardening: Windows worker threads have lower effective
// stack headroom, so use a larger stack for runtime workers.
builder.thread_stack_size(WINDOWS_TOKIO_WORKER_STACK_SIZE_BYTES);
}
Ok(builder.build()?)
}
const ILLEGAL_ENV_VAR_PREFIX: &str = "CODEX_";
/// Load env vars from ~/.codex/.env.
+75 -1
View File
@@ -1,4 +1,5 @@
use std::collections::HashMap;
use std::collections::HashSet;
use std::sync::LazyLock;
use std::sync::Mutex as StdMutex;
@@ -19,7 +20,9 @@ pub use codex_core::connectors::connector_display_label;
use codex_core::connectors::connector_install_url;
pub use codex_core::connectors::list_accessible_connectors_from_mcp_tools;
pub use codex_core::connectors::list_accessible_connectors_from_mcp_tools_with_options;
pub use codex_core::connectors::list_cached_accessible_connectors_from_mcp_tools;
use codex_core::connectors::merge_connectors;
pub use codex_core::connectors::with_app_enabled_state;
#[derive(Debug, Deserialize)]
struct DirectoryListResponse {
@@ -72,13 +75,32 @@ pub async fn list_connectors(config: &Config) -> anyhow::Result<Vec<AppInfo>> {
);
let connectors = connectors_result?;
let accessible = accessible_result?;
Ok(merge_connectors_with_accessible(connectors, accessible))
Ok(with_app_enabled_state(
merge_connectors_with_accessible(connectors, accessible, true),
config,
))
}
pub async fn list_all_connectors(config: &Config) -> anyhow::Result<Vec<AppInfo>> {
list_all_connectors_with_options(config, false).await
}
pub async fn list_cached_all_connectors(config: &Config) -> Option<Vec<AppInfo>> {
if !config.features.enabled(Feature::Apps) {
return Some(Vec::new());
}
if init_chatgpt_token_from_auth(&config.codex_home, config.cli_auth_credentials_store_mode)
.await
.is_err()
{
return None;
}
let token_data = get_chatgpt_token_data()?;
let cache_key = all_connectors_cache_key(config, &token_data);
read_cached_all_connectors(&cache_key)
}
pub async fn list_all_connectors_with_options(
config: &Config,
force_refetch: bool,
@@ -164,7 +186,20 @@ fn write_cached_all_connectors(cache_key: AllConnectorsCacheKey, connectors: &[A
pub fn merge_connectors_with_accessible(
connectors: Vec<AppInfo>,
accessible_connectors: Vec<AppInfo>,
all_connectors_loaded: bool,
) -> Vec<AppInfo> {
let accessible_connectors = if all_connectors_loaded {
let connector_ids: HashSet<&str> = connectors
.iter()
.map(|connector| connector.id.as_str())
.collect();
accessible_connectors
.into_iter()
.filter(|connector| connector_ids.contains(connector.id.as_str()))
.collect()
} else {
accessible_connectors
};
let merged = merge_connectors(connectors, accessible_connectors);
filter_disallowed_connectors(merged)
}
@@ -283,6 +318,7 @@ fn directory_app_to_app_info(app: DirectoryApp) -> AppInfo {
distribution_channel: app.distribution_channel,
install_url: None,
is_accessible: false,
is_enabled: true,
}
}
@@ -341,6 +377,7 @@ mod tests {
distribution_channel: None,
install_url: None,
is_accessible: false,
is_enabled: true,
}
}
@@ -383,4 +420,41 @@ mod tests {
]);
assert_eq!(filtered, vec![app("delta")]);
}
fn merged_app(id: &str, is_accessible: bool) -> AppInfo {
AppInfo {
id: id.to_string(),
name: id.to_string(),
description: None,
logo_url: None,
logo_url_dark: None,
distribution_channel: None,
install_url: Some(connector_install_url(id, id)),
is_accessible,
is_enabled: true,
}
}
#[test]
fn excludes_accessible_connectors_not_in_all_when_all_loaded() {
let merged = merge_connectors_with_accessible(
vec![app("alpha")],
vec![app("alpha"), app("beta")],
true,
);
assert_eq!(merged, vec![merged_app("alpha", true)]);
}
#[test]
fn keeps_accessible_connectors_not_in_all_while_all_loading() {
let merged = merge_connectors_with_accessible(
vec![app("alpha")],
vec![app("alpha"), app("beta")],
false,
);
assert_eq!(
merged,
vec![merged_app("alpha", true), merged_app("beta", true)]
);
}
}
+10 -3
View File
@@ -213,12 +213,19 @@ async fn run_command_under_sandbox(
#[cfg(not(target_os = "macos"))]
let _ = log_denials;
let managed_network_requirements_enabled = config.managed_network_requirements_enabled();
// This proxy should only live for the lifetime of the child process.
let network_proxy = match config.permissions.network.as_ref() {
Some(spec) => Some(
spec.start_proxy()
.await
.map_err(|err| anyhow::anyhow!("failed to start managed network proxy: {err}"))?,
spec.start_proxy(
config.permissions.sandbox_policy.get(),
None,
None,
managed_network_requirements_enabled,
)
.await
.map_err(|err| anyhow::anyhow!("failed to start managed network proxy: {err}"))?,
),
None => None,
};
+2 -2
View File
@@ -93,10 +93,10 @@ enum Subcommand {
/// Remove stored authentication credentials.
Logout(LogoutCommand),
/// [experimental] Run Codex as an MCP server and manage MCP servers.
/// Manage external MCP servers for Codex.
Mcp(McpCli),
/// [experimental] Run the Codex MCP server (stdio transport).
/// Start Codex as an MCP server (stdio).
McpServer,
/// [experimental] Run the app server or related tooling.
@@ -17,6 +17,9 @@ pub(crate) fn subagent_header(source: &Option<SessionSource>) -> Option<String>
match sub {
codex_protocol::protocol::SubAgentSource::Review => Some("review".to_string()),
codex_protocol::protocol::SubAgentSource::Compact => Some("compact".to_string()),
codex_protocol::protocol::SubAgentSource::MemoryConsolidation => {
Some("memory_consolidation".to_string())
}
codex_protocol::protocol::SubAgentSource::ThreadSpawn { .. } => {
Some("collab_spawn".to_string())
}
@@ -89,6 +89,7 @@ async fn models_client_hits_models_endpoint() {
experimental_supported_tools: Vec::new(),
input_modalities: default_input_modalities(),
prefer_websockets: false,
used_fallback_model_metadata: false,
}],
};
+1
View File
@@ -152,6 +152,7 @@ codex-utils-cargo-bin = { workspace = true }
core_test_support = { workspace = true }
ctor = { workspace = true }
image = { workspace = true, features = ["jpeg", "png"] }
insta = { workspace = true }
maplit = { workspace = true }
predicates = { workspace = true }
pretty_assertions = { workspace = true }
+57
View File
@@ -188,6 +188,9 @@
"apps": {
"type": "boolean"
},
"apps_mcp_gateway": {
"type": "boolean"
},
"child_agents_md": {
"type": "boolean"
},
@@ -236,6 +239,9 @@
"powershell_utf8": {
"type": "boolean"
},
"prevent_idle_sleep": {
"type": "boolean"
},
"remote_models": {
"type": "boolean"
},
@@ -431,6 +437,43 @@
}
]
},
"MemoriesToml": {
"additionalProperties": false,
"description": "Memories settings loaded from config.toml.",
"properties": {
"max_raw_memories_for_global": {
"description": "Maximum number of recent raw memories retained for global consolidation.",
"format": "uint",
"minimum": 0.0,
"type": "integer"
},
"max_rollout_age_days": {
"description": "Maximum age of the threads used for memories.",
"format": "int64",
"type": "integer"
},
"max_rollouts_per_startup": {
"description": "Maximum number of rollout candidates processed per pass.",
"format": "uint",
"minimum": 0.0,
"type": "integer"
},
"min_rollout_idle_hours": {
"description": "Minimum idle time between last thread activity and memory creation (hours). > 12h recommended.",
"format": "int64",
"type": "integer"
},
"phase_1_model": {
"description": "Model used for thread summarisation.",
"type": "string"
},
"phase_2_model": {
"description": "Model used for memory consolidation.",
"type": "string"
}
},
"type": "object"
},
"ModeKind": {
"description": "Initial collaboration mode to use when the TUI starts.",
"enum": [
@@ -1275,6 +1318,9 @@
"apps": {
"type": "boolean"
},
"apps_mcp_gateway": {
"type": "boolean"
},
"child_agents_md": {
"type": "boolean"
},
@@ -1323,6 +1369,9 @@
"powershell_utf8": {
"type": "boolean"
},
"prevent_idle_sleep": {
"type": "boolean"
},
"remote_models": {
"type": "boolean"
},
@@ -1475,6 +1524,14 @@
"description": "Definition for MCP servers that Codex can reach out to for tool calls.",
"type": "object"
},
"memories": {
"allOf": [
{
"$ref": "#/definitions/MemoriesToml"
}
],
"description": "Memories subsystem settings."
},
"model": {
"description": "Optional override of model selection.",
"type": "string"
+367 -38
View File
@@ -7,9 +7,11 @@ use codex_protocol::protocol::SkillScope;
use serde::Serialize;
use sha1::Digest;
use sha1::Sha1;
use std::collections::HashSet;
use std::path::Path;
use std::path::PathBuf;
use std::sync::Arc;
use std::sync::Mutex;
use std::time::Duration;
use tokio::sync::mpsc;
@@ -17,15 +19,18 @@ use tokio::sync::mpsc;
pub(crate) struct TrackEventsContext {
pub(crate) model_slug: String,
pub(crate) thread_id: String,
pub(crate) turn_id: String,
}
pub(crate) fn build_track_events_context(
model_slug: String,
thread_id: String,
turn_id: String,
) -> TrackEventsContext {
TrackEventsContext {
model_slug,
thread_id,
turn_id,
}
}
@@ -35,9 +40,16 @@ pub(crate) struct SkillInvocation {
pub(crate) skill_path: PathBuf,
}
pub(crate) struct AppInvocation {
pub(crate) connector_id: Option<String>,
pub(crate) app_name: Option<String>,
pub(crate) invoke_type: Option<String>,
}
#[derive(Clone)]
pub(crate) struct AnalyticsEventsQueue {
sender: mpsc::Sender<TrackEventsJob>,
app_used_emitted_keys: Arc<Mutex<HashSet<(String, String)>>>,
}
pub(crate) struct AnalyticsEventsClient {
@@ -50,18 +62,45 @@ impl AnalyticsEventsQueue {
let (sender, mut receiver) = mpsc::channel(ANALYTICS_EVENTS_QUEUE_SIZE);
tokio::spawn(async move {
while let Some(job) = receiver.recv().await {
send_track_skill_invocations(&auth_manager, job).await;
match job {
TrackEventsJob::SkillInvocations(job) => {
send_track_skill_invocations(&auth_manager, job).await;
}
TrackEventsJob::AppMentioned(job) => {
send_track_app_mentioned(&auth_manager, job).await;
}
TrackEventsJob::AppUsed(job) => {
send_track_app_used(&auth_manager, job).await;
}
}
}
});
Self { sender }
Self {
sender,
app_used_emitted_keys: Arc::new(Mutex::new(HashSet::new())),
}
}
fn try_send(&self, job: TrackEventsJob) {
if self.sender.try_send(job).is_err() {
//TODO: add a metric for this
tracing::warn!("dropping skill analytics events: queue is full");
tracing::warn!("dropping analytics events: queue is full");
}
}
fn should_enqueue_app_used(&self, tracking: &TrackEventsContext, app: &AppInvocation) -> bool {
let Some(connector_id) = app.connector_id.as_ref() else {
return true;
};
let mut emitted = self
.app_used_emitted_keys
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if emitted.len() >= ANALYTICS_APP_USED_DEDUPE_MAX_KEYS {
emitted.clear();
}
emitted.insert((tracking.turn_id.clone(), connector_id.clone()))
}
}
impl AnalyticsEventsClient {
@@ -84,32 +123,76 @@ impl AnalyticsEventsClient {
invocations,
);
}
pub(crate) fn track_app_mentioned(
&self,
tracking: TrackEventsContext,
mentions: Vec<AppInvocation>,
) {
track_app_mentioned(
&self.queue,
Arc::clone(&self.config),
Some(tracking),
mentions,
);
}
pub(crate) fn track_app_used(&self, tracking: TrackEventsContext, app: AppInvocation) {
track_app_used(&self.queue, Arc::clone(&self.config), Some(tracking), app);
}
}
struct TrackEventsJob {
enum TrackEventsJob {
SkillInvocations(TrackSkillInvocationsJob),
AppMentioned(TrackAppMentionedJob),
AppUsed(TrackAppUsedJob),
}
struct TrackSkillInvocationsJob {
config: Arc<Config>,
tracking: TrackEventsContext,
invocations: Vec<SkillInvocation>,
}
struct TrackAppMentionedJob {
config: Arc<Config>,
tracking: TrackEventsContext,
mentions: Vec<AppInvocation>,
}
struct TrackAppUsedJob {
config: Arc<Config>,
tracking: TrackEventsContext,
app: AppInvocation,
}
const ANALYTICS_EVENTS_QUEUE_SIZE: usize = 256;
const ANALYTICS_EVENTS_TIMEOUT: Duration = Duration::from_secs(10);
const ANALYTICS_APP_USED_DEDUPE_MAX_KEYS: usize = 4096;
#[derive(Serialize)]
struct TrackEventsRequest {
events: Vec<TrackEvent>,
events: Vec<TrackEventRequest>,
}
#[derive(Serialize)]
struct TrackEvent {
#[serde(untagged)]
enum TrackEventRequest {
SkillInvocation(SkillInvocationEventRequest),
AppMentioned(CodexAppMentionedEventRequest),
AppUsed(CodexAppUsedEventRequest),
}
#[derive(Serialize)]
struct SkillInvocationEventRequest {
event_type: &'static str,
skill_id: String,
skill_name: String,
event_params: TrackEventParams,
event_params: SkillInvocationEventParams,
}
#[derive(Serialize)]
struct TrackEventParams {
struct SkillInvocationEventParams {
product_client_id: Option<String>,
skill_scope: Option<String>,
repo_url: Option<String>,
@@ -118,6 +201,29 @@ struct TrackEventParams {
model_slug: Option<String>,
}
#[derive(Serialize)]
struct CodexAppMetadata {
connector_id: Option<String>,
thread_id: Option<String>,
turn_id: Option<String>,
app_name: Option<String>,
product_client_id: Option<String>,
invoke_type: Option<String>,
model_slug: Option<String>,
}
#[derive(Serialize)]
struct CodexAppMentionedEventRequest {
event_type: &'static str,
event_params: CodexAppMetadata,
}
#[derive(Serialize)]
struct CodexAppUsedEventRequest {
event_type: &'static str,
event_params: CodexAppMetadata,
}
pub(crate) fn track_skill_invocations(
queue: &AnalyticsEventsQueue,
config: Arc<Config>,
@@ -133,34 +239,66 @@ pub(crate) fn track_skill_invocations(
if invocations.is_empty() {
return;
}
let job = TrackEventsJob {
let job = TrackEventsJob::SkillInvocations(TrackSkillInvocationsJob {
config,
tracking,
invocations,
};
});
queue.try_send(job);
}
async fn send_track_skill_invocations(auth_manager: &AuthManager, job: TrackEventsJob) {
let TrackEventsJob {
pub(crate) fn track_app_mentioned(
queue: &AnalyticsEventsQueue,
config: Arc<Config>,
tracking: Option<TrackEventsContext>,
mentions: Vec<AppInvocation>,
) {
if config.analytics_enabled == Some(false) {
return;
}
let Some(tracking) = tracking else {
return;
};
if mentions.is_empty() {
return;
}
let job = TrackEventsJob::AppMentioned(TrackAppMentionedJob {
config,
tracking,
mentions,
});
queue.try_send(job);
}
pub(crate) fn track_app_used(
queue: &AnalyticsEventsQueue,
config: Arc<Config>,
tracking: Option<TrackEventsContext>,
app: AppInvocation,
) {
if config.analytics_enabled == Some(false) {
return;
}
let Some(tracking) = tracking else {
return;
};
if !queue.should_enqueue_app_used(&tracking, &app) {
return;
}
let job = TrackEventsJob::AppUsed(TrackAppUsedJob {
config,
tracking,
app,
});
queue.try_send(job);
}
async fn send_track_skill_invocations(auth_manager: &AuthManager, job: TrackSkillInvocationsJob) {
let TrackSkillInvocationsJob {
config,
tracking,
invocations,
} = job;
let Some(auth) = auth_manager.auth().await else {
return;
};
if !auth.is_chatgpt_auth() {
return;
}
let access_token = match auth.get_token() {
Ok(token) => token,
Err(_) => return,
};
let Some(account_id) = auth.get_account_id() else {
return;
};
let mut events = Vec::with_capacity(invocations.len());
for invocation in invocations {
let skill_scope = match invocation.skill_scope {
@@ -183,21 +321,95 @@ async fn send_track_skill_invocations(auth_manager: &AuthManager, job: TrackEven
invocation.skill_path.as_path(),
invocation.skill_name.as_str(),
);
events.push(TrackEvent {
event_type: "skill_invocation",
skill_id,
skill_name: invocation.skill_name.clone(),
event_params: TrackEventParams {
thread_id: Some(tracking.thread_id.clone()),
invoke_type: Some("explicit".to_string()),
model_slug: Some(tracking.model_slug.clone()),
product_client_id: Some(crate::default_client::originator().value),
repo_url,
skill_scope: Some(skill_scope.to_string()),
events.push(TrackEventRequest::SkillInvocation(
SkillInvocationEventRequest {
event_type: "skill_invocation",
skill_id,
skill_name: invocation.skill_name.clone(),
event_params: SkillInvocationEventParams {
thread_id: Some(tracking.thread_id.clone()),
invoke_type: Some("explicit".to_string()),
model_slug: Some(tracking.model_slug.clone()),
product_client_id: Some(crate::default_client::originator().value),
repo_url,
skill_scope: Some(skill_scope.to_string()),
},
},
});
));
}
send_track_events(auth_manager, config, events).await;
}
async fn send_track_app_mentioned(auth_manager: &AuthManager, job: TrackAppMentionedJob) {
let TrackAppMentionedJob {
config,
tracking,
mentions,
} = job;
let events = mentions
.into_iter()
.map(|mention| {
let event_params = codex_app_metadata(&tracking, mention);
TrackEventRequest::AppMentioned(CodexAppMentionedEventRequest {
event_type: "codex_app_mentioned",
event_params,
})
})
.collect::<Vec<_>>();
send_track_events(auth_manager, config, events).await;
}
async fn send_track_app_used(auth_manager: &AuthManager, job: TrackAppUsedJob) {
let TrackAppUsedJob {
config,
tracking,
app,
} = job;
let event_params = codex_app_metadata(&tracking, app);
let events = vec![TrackEventRequest::AppUsed(CodexAppUsedEventRequest {
event_type: "codex_app_used",
event_params,
})];
send_track_events(auth_manager, config, events).await;
}
fn codex_app_metadata(tracking: &TrackEventsContext, app: AppInvocation) -> CodexAppMetadata {
CodexAppMetadata {
connector_id: app.connector_id,
thread_id: Some(tracking.thread_id.clone()),
turn_id: Some(tracking.turn_id.clone()),
app_name: app.app_name,
product_client_id: Some(crate::default_client::originator().value),
invoke_type: app.invoke_type,
model_slug: Some(tracking.model_slug.clone()),
}
}
async fn send_track_events(
auth_manager: &AuthManager,
config: Arc<Config>,
events: Vec<TrackEventRequest>,
) {
if events.is_empty() {
return;
}
let Some(auth) = auth_manager.auth().await else {
return;
};
if !auth.is_chatgpt_auth() {
return;
}
let access_token = match auth.get_token() {
Ok(token) => token,
Err(_) => return,
};
let Some(account_id) = auth.get_account_id() else {
return;
};
let base_url = config.chatgpt_base_url.trim_end_matches('/');
let url = format!("{base_url}/codex/analytics-events/events");
let payload = TrackEventsRequest { events };
@@ -269,9 +481,21 @@ fn normalize_path_for_skill_id(
#[cfg(test)]
mod tests {
use super::AnalyticsEventsQueue;
use super::AppInvocation;
use super::CodexAppMentionedEventRequest;
use super::CodexAppUsedEventRequest;
use super::TrackEventRequest;
use super::TrackEventsContext;
use super::codex_app_metadata;
use super::normalize_path_for_skill_id;
use pretty_assertions::assert_eq;
use serde_json::json;
use std::collections::HashSet;
use std::path::PathBuf;
use std::sync::Arc;
use std::sync::Mutex;
use tokio::sync::mpsc;
fn expected_absolute_path(path: &PathBuf) -> String {
std::fs::canonicalize(path)
@@ -328,4 +552,109 @@ mod tests {
assert_eq!(path, expected);
}
#[test]
fn app_mentioned_event_serializes_expected_shape() {
let tracking = TrackEventsContext {
model_slug: "gpt-5".to_string(),
thread_id: "thread-1".to_string(),
turn_id: "turn-1".to_string(),
};
let event = TrackEventRequest::AppMentioned(CodexAppMentionedEventRequest {
event_type: "codex_app_mentioned",
event_params: codex_app_metadata(
&tracking,
AppInvocation {
connector_id: Some("calendar".to_string()),
app_name: Some("Calendar".to_string()),
invoke_type: Some("explicit".to_string()),
},
),
});
let payload = serde_json::to_value(&event).expect("serialize app mentioned event");
assert_eq!(
payload,
json!({
"event_type": "codex_app_mentioned",
"event_params": {
"connector_id": "calendar",
"thread_id": "thread-1",
"turn_id": "turn-1",
"app_name": "Calendar",
"product_client_id": crate::default_client::originator().value,
"invoke_type": "explicit",
"model_slug": "gpt-5"
}
})
);
}
#[test]
fn app_used_event_serializes_expected_shape() {
let tracking = TrackEventsContext {
model_slug: "gpt-5".to_string(),
thread_id: "thread-2".to_string(),
turn_id: "turn-2".to_string(),
};
let event = TrackEventRequest::AppUsed(CodexAppUsedEventRequest {
event_type: "codex_app_used",
event_params: codex_app_metadata(
&tracking,
AppInvocation {
connector_id: Some("drive".to_string()),
app_name: Some("Google Drive".to_string()),
invoke_type: Some("implicit".to_string()),
},
),
});
let payload = serde_json::to_value(&event).expect("serialize app used event");
assert_eq!(
payload,
json!({
"event_type": "codex_app_used",
"event_params": {
"connector_id": "drive",
"thread_id": "thread-2",
"turn_id": "turn-2",
"app_name": "Google Drive",
"product_client_id": crate::default_client::originator().value,
"invoke_type": "implicit",
"model_slug": "gpt-5"
}
})
);
}
#[test]
fn app_used_dedupe_is_keyed_by_turn_and_connector() {
let (sender, _receiver) = mpsc::channel(1);
let queue = AnalyticsEventsQueue {
sender,
app_used_emitted_keys: Arc::new(Mutex::new(HashSet::new())),
};
let app = AppInvocation {
connector_id: Some("calendar".to_string()),
app_name: Some("Calendar".to_string()),
invoke_type: Some("implicit".to_string()),
};
let turn_1 = TrackEventsContext {
model_slug: "gpt-5".to_string(),
thread_id: "thread-1".to_string(),
turn_id: "turn-1".to_string(),
};
let turn_2 = TrackEventsContext {
model_slug: "gpt-5".to_string(),
thread_id: "thread-1".to_string(),
turn_id: "turn-2".to_string(),
};
assert_eq!(queue.should_enqueue_app_used(&turn_1, &app), true);
assert_eq!(queue.should_enqueue_app_used(&turn_1, &app), false);
assert_eq!(queue.should_enqueue_app_used(&turn_2, &app), true);
}
}
+4 -2
View File
@@ -325,6 +325,9 @@ impl ModelClient {
let subagent = match sub {
crate::protocol::SubAgentSource::Review => "review".to_string(),
crate::protocol::SubAgentSource::Compact => "compact".to_string(),
crate::protocol::SubAgentSource::MemoryConsolidation => {
"memory_consolidation".to_string()
}
crate::protocol::SubAgentSource::ThreadSpawn { .. } => "collab_spawn".to_string(),
crate::protocol::SubAgentSource::Other(label) => label.clone(),
};
@@ -630,7 +633,6 @@ impl ModelClientSession {
&mut self,
otel_manager: &OtelManager,
model_info: &ModelInfo,
turn_metadata_header: Option<&str>,
) -> std::result::Result<(), ApiError> {
if !self.client.responses_websocket_enabled(model_info) || self.client.websockets_disabled()
{
@@ -653,7 +655,7 @@ impl ModelClientSession {
client_setup.api_provider,
client_setup.api_auth,
Some(Arc::clone(&self.turn_state)),
turn_metadata_header,
None,
)
.await?;
self.connection = Some(connection);
+661 -140
View File
File diff suppressed because it is too large Load Diff
+2
View File
@@ -316,6 +316,7 @@ async fn handle_exec_approval(
command,
cwd,
reason,
network_approval_context,
proposed_execpolicy_amendment,
..
} = event;
@@ -327,6 +328,7 @@ async fn handle_exec_approval(
command,
cwd,
reason,
network_approval_context,
proposed_execpolicy_amendment,
);
let decision =
+139 -17
View File
@@ -7,11 +7,11 @@ use crate::client_common::ResponseEvent;
use crate::codex::Session;
use crate::codex::TurnContext;
use crate::codex::get_last_assistant_message_from_turn;
use crate::context_manager::ContextManager;
use crate::error::CodexErr;
use crate::error::Result as CodexResult;
use crate::protocol::CompactedItem;
use crate::protocol::EventMsg;
use crate::protocol::TurnContextItem;
use crate::protocol::TurnStartedEvent;
use crate::protocol::WarningEvent;
use crate::truncate::TruncationPolicy;
@@ -36,6 +36,31 @@ pub(crate) fn should_use_remote_compact_task(provider: &ModelProviderInfo) -> bo
provider.is_openai()
}
pub(crate) fn extract_trailing_model_switch_update_for_compaction_request(
history: &mut ContextManager,
) -> Option<ResponseItem> {
let history_items = history.raw_items();
let last_user_turn_boundary_index = history_items
.iter()
.rposition(crate::context_manager::is_user_turn_boundary);
let model_switch_index = history_items
.iter()
.enumerate()
.rev()
.find_map(|(i, item)| {
let is_trailing = last_user_turn_boundary_index.is_none_or(|boundary| i > boundary);
if is_trailing && Session::is_model_switch_developer_message(item) {
Some(i)
} else {
None
}
})?;
let mut replacement = history_items.to_vec();
let model_switch_item = replacement.remove(model_switch_index);
history.replace(replacement);
Some(model_switch_item)
}
pub(crate) async fn run_inline_auto_compact_task(
sess: Arc<Session>,
turn_context: Arc<TurnContext>,
@@ -76,6 +101,10 @@ async fn run_compact_task_inner(
let initial_input_for_turn: ResponseInputItem = ResponseInputItem::from(input);
let mut history = sess.clone_history().await;
// Keep compaction prompts in-distribution: if a model-switch update was injected at the
// tail of history (between turns), exclude it from the compaction request payload.
let stripped_model_switch_item =
extract_trailing_model_switch_update_for_compaction_request(&mut history);
history.record_items(
&[initial_input_for_turn.into()],
turn_context.truncation_policy,
@@ -85,7 +114,6 @@ async fn run_compact_task_inner(
let max_retries = turn_context.provider.stream_max_retries();
let mut retries = 0;
let turn_metadata_header = turn_context.resolve_turn_metadata_header().await;
let mut client_session = sess.services.model_client.new_session();
// Reuse one client session so turn-scoped state (sticky routing, websocket append tracking)
// survives retries within this compact turn.
@@ -95,21 +123,8 @@ async fn run_compact_task_inner(
// duplicating model settings on TurnContext, but an Op after turn start could update the
// session config before this write occurs.
let collaboration_mode = sess.current_collaboration_mode().await;
let rollout_item = RolloutItem::TurnContext(TurnContextItem {
turn_id: Some(turn_context.sub_id.clone()),
cwd: turn_context.cwd.clone(),
approval_policy: turn_context.approval_policy,
sandbox_policy: turn_context.sandbox_policy.clone(),
model: turn_context.model_info.slug.clone(),
personality: turn_context.personality,
collaboration_mode: Some(collaboration_mode),
effort: turn_context.reasoning_effort,
summary: turn_context.reasoning_summary,
user_instructions: turn_context.user_instructions.clone(),
developer_instructions: turn_context.developer_instructions.clone(),
final_output_json_schema: turn_context.final_output_json_schema.clone(),
truncation_policy: Some(turn_context.truncation_policy.into()),
});
let rollout_item =
RolloutItem::TurnContext(turn_context.to_turn_context_item(collaboration_mode));
sess.persist_rollout_items(&[rollout_item]).await;
loop {
@@ -124,6 +139,7 @@ async fn run_compact_task_inner(
personality: turn_context.personality,
..Default::default()
};
let turn_metadata_header = turn_context.turn_metadata_state.current_header_value();
let attempt_result = drain_to_completed(
&sess,
turn_context.as_ref(),
@@ -194,6 +210,11 @@ async fn run_compact_task_inner(
let initial_context = sess.build_initial_context(turn_context.as_ref()).await;
let mut new_history = build_compacted_history(initial_context, &user_messages, &summary_text);
// Reattach the stripped model-switch update only after successful compaction so the model
// still sees the switch instructions on the next real sampling request.
if let Some(model_switch_item) = stripped_model_switch_item {
new_history.push(model_switch_item);
}
let ghost_snapshots: Vec<ResponseItem> = history_items
.iter()
.filter(|item| matches!(item, ResponseItem::GhostSnapshot { .. }))
@@ -457,6 +478,107 @@ mod tests {
assert_eq!(None, joined);
}
#[test]
fn extract_trailing_model_switch_update_for_compaction_request_removes_trailing_item() {
let mut history = ContextManager::new();
history.replace(vec![
ResponseItem::Message {
id: None,
role: "user".to_string(),
content: vec![ContentItem::InputText {
text: "USER_MESSAGE".to_string(),
}],
end_turn: None,
phase: None,
},
ResponseItem::Message {
id: None,
role: "assistant".to_string(),
content: vec![ContentItem::OutputText {
text: "ASSISTANT_REPLY".to_string(),
}],
end_turn: None,
phase: None,
},
ResponseItem::Message {
id: None,
role: "developer".to_string(),
content: vec![ContentItem::InputText {
text: "<model_switch>\nNEW_MODEL_INSTRUCTIONS".to_string(),
}],
end_turn: None,
phase: None,
},
]);
let model_switch_item =
extract_trailing_model_switch_update_for_compaction_request(&mut history);
assert_eq!(history.raw_items().len(), 2);
assert!(model_switch_item.is_some());
assert!(
history
.raw_items()
.iter()
.all(|item| !Session::is_model_switch_developer_message(item))
);
}
#[test]
fn extract_trailing_model_switch_update_for_compaction_request_keeps_historical_item() {
let mut history = ContextManager::new();
history.replace(vec![
ResponseItem::Message {
id: None,
role: "user".to_string(),
content: vec![ContentItem::InputText {
text: "FIRST_USER_MESSAGE".to_string(),
}],
end_turn: None,
phase: None,
},
ResponseItem::Message {
id: None,
role: "developer".to_string(),
content: vec![ContentItem::InputText {
text: "<model_switch>\nOLDER_MODEL_INSTRUCTIONS".to_string(),
}],
end_turn: None,
phase: None,
},
ResponseItem::Message {
id: None,
role: "assistant".to_string(),
content: vec![ContentItem::OutputText {
text: "ASSISTANT_REPLY".to_string(),
}],
end_turn: None,
phase: None,
},
ResponseItem::Message {
id: None,
role: "user".to_string(),
content: vec![ContentItem::InputText {
text: "SECOND_USER_MESSAGE".to_string(),
}],
end_turn: None,
phase: None,
},
]);
let model_switch_item =
extract_trailing_model_switch_update_for_compaction_request(&mut history);
assert_eq!(history.raw_items().len(), 4);
assert!(model_switch_item.is_none());
assert!(
history
.raw_items()
.iter()
.any(Session::is_model_switch_developer_message)
);
}
#[test]
fn collect_user_messages_extracts_user_text_only() {
let items = vec![
+10
View File
@@ -3,6 +3,7 @@ use std::sync::Arc;
use crate::Prompt;
use crate::codex::Session;
use crate::codex::TurnContext;
use crate::compact::extract_trailing_model_switch_update_for_compaction_request;
use crate::context_manager::ContextManager;
use crate::context_manager::TotalTokenUsageBreakdown;
use crate::context_manager::estimate_response_item_model_visible_bytes;
@@ -65,6 +66,10 @@ async fn run_remote_compact_task_inner_impl(
sess.emit_turn_item_started(turn_context, &compaction_item)
.await;
let mut history = sess.clone_history().await;
// Keep compaction prompts in-distribution: if a model-switch update was injected at the
// tail of history (between turns), exclude it from the compaction request payload.
let stripped_model_switch_item =
extract_trailing_model_switch_update_for_compaction_request(&mut history);
let base_instructions = sess.get_base_instructions().await;
let deleted_items = trim_function_call_history_to_fit_context_window(
&mut history,
@@ -120,6 +125,11 @@ async fn run_remote_compact_task_inner_impl(
new_history = sess
.process_compacted_history(turn_context, new_history)
.await;
// Reattach the stripped model-switch update only after successful compaction so the model
// still sees the switch instructions on the next real sampling request.
if let Some(model_switch_item) = stripped_model_switch_item {
new_history.push(model_switch_item);
}
if !ghost_snapshots.is_empty() {
new_history.extend(ghost_snapshots);
+75
View File
@@ -7,6 +7,8 @@ use crate::config::types::History;
use crate::config::types::McpServerConfig;
use crate::config::types::McpServerDisabledReason;
use crate::config::types::McpServerTransportConfig;
use crate::config::types::MemoriesConfig;
use crate::config::types::MemoriesToml;
use crate::config::types::Notice;
use crate::config::types::NotificationMethod;
use crate::config::types::Notifications;
@@ -47,6 +49,8 @@ use crate::project_doc::LOCAL_PROJECT_DOC_FILENAME;
use crate::protocol::AskForApproval;
use crate::protocol::ReadOnlyAccess;
use crate::protocol::SandboxPolicy;
#[cfg(target_os = "macos")]
use crate::seatbelt_permissions::MacOsSeatbeltProfileExtensions;
use crate::windows_sandbox::WindowsSandboxLevelExt;
use crate::windows_sandbox::resolve_windows_sandbox_mode;
use codex_app_server_protocol::Tools;
@@ -76,6 +80,8 @@ use std::path::Path;
use std::path::PathBuf;
#[cfg(test)]
use tempfile::tempdir;
#[cfg(not(target_os = "macos"))]
type MacOsSeatbeltProfileExtensions = ();
use crate::config::profile::ConfigProfile;
use toml::Value as TomlValue;
@@ -131,6 +137,9 @@ pub struct Permissions {
/// Effective Windows sandbox mode derived from `[windows].sandbox` or
/// legacy feature keys.
pub windows_sandbox_mode: Option<WindowsSandboxModeToml>,
/// Optional macOS seatbelt extension profile used to extend default
/// seatbelt permissions when running under seatbelt.
pub macos_seatbelt_profile_extensions: Option<MacOsSeatbeltProfileExtensions>,
}
/// Application configuration loaded from disk and merged with overrides.
@@ -289,6 +298,9 @@ pub struct Config {
/// Maximum number of agent threads that can be open concurrently.
pub agent_max_threads: Option<usize>,
/// Memories subsystem settings.
pub memories: MemoriesConfig,
/// Directory containing all Codex state (defaults to `~/.codex` but can be
/// overridden by the `CODEX_HOME` environment variable).
pub codex_home: PathBuf,
@@ -1006,6 +1018,9 @@ pub struct ConfigToml {
/// Agent-related settings (thread limits, etc.).
pub agents: Option<AgentsToml>,
/// Memories subsystem settings.
pub memories: Option<MemoriesToml>,
/// User-level skill config entries keyed by SKILL.md path.
pub skills: Option<SkillsConfig>,
@@ -1737,6 +1752,7 @@ impl Config {
network,
shell_environment_policy,
windows_sandbox_mode,
macos_seatbelt_profile_extensions: None,
},
enforce_residency: enforce_residency.value,
did_user_set_custom_approval_policy_or_sandbox_mode,
@@ -1771,6 +1787,7 @@ impl Config {
.collect(),
tool_output_token_limit: cfg.tool_output_token_limit,
agent_max_threads,
memories: cfg.memories.unwrap_or_default().into(),
codex_home,
log_dir,
config_layer_stack,
@@ -1933,6 +1950,13 @@ impl Config {
self.permissions.windows_sandbox_mode
};
}
pub fn managed_network_requirements_enabled(&self) -> bool {
self.config_layer_stack
.requirements_toml()
.network
.is_some()
}
}
pub(crate) fn uses_deprecated_instructions_file(config_layer_stack: &ConfigLayerStack) -> bool {
@@ -1985,6 +2009,8 @@ mod tests {
use crate::config::types::FeedbackConfigToml;
use crate::config::types::HistoryPersistence;
use crate::config::types::McpServerTransportConfig;
use crate::config::types::MemoriesConfig;
use crate::config::types::MemoriesToml;
use crate::config::types::NotificationMethod;
use crate::config::types::Notifications;
use crate::config_loader::RequirementSource;
@@ -2068,6 +2094,47 @@ persistence = "none"
}),
history_no_persistence_cfg.history
);
let memories = r#"
[memories]
max_raw_memories_for_global = 512
max_rollout_age_days = 42
max_rollouts_per_startup = 9
min_rollout_idle_hours = 24
phase_1_model = "gpt-5-mini"
phase_2_model = "gpt-5"
"#;
let memories_cfg =
toml::from_str::<ConfigToml>(memories).expect("TOML deserialization should succeed");
assert_eq!(
Some(MemoriesToml {
max_raw_memories_for_global: Some(512),
max_rollout_age_days: Some(42),
max_rollouts_per_startup: Some(9),
min_rollout_idle_hours: Some(24),
phase_1_model: Some("gpt-5-mini".to_string()),
phase_2_model: Some("gpt-5".to_string()),
}),
memories_cfg.memories
);
let config = Config::load_from_base_config_with_overrides(
memories_cfg,
ConfigOverrides::default(),
tempdir().expect("tempdir").path().to_path_buf(),
)
.expect("load config from memories settings");
assert_eq!(
config.memories,
MemoriesConfig {
max_raw_memories_for_global: 512,
max_rollout_age_days: 42,
max_rollouts_per_startup: 9,
min_rollout_idle_hours: 24,
phase_1_model: Some("gpt-5-mini".to_string()),
phase_2_model: Some("gpt-5".to_string()),
}
);
}
#[test]
@@ -4032,6 +4099,7 @@ model_verbosity = "high"
network: None,
shell_environment_policy: ShellEnvironmentPolicy::default(),
windows_sandbox_mode: None,
macos_seatbelt_profile_extensions: None,
},
enforce_residency: Constrained::allow_any(None),
did_user_set_custom_approval_policy_or_sandbox_mode: true,
@@ -4047,6 +4115,7 @@ model_verbosity = "high"
project_doc_fallback_filenames: Vec::new(),
tool_output_token_limit: None,
agent_max_threads: DEFAULT_AGENT_MAX_THREADS,
memories: MemoriesConfig::default(),
codex_home: fixture.codex_home(),
log_dir: fixture.codex_home().join("log"),
config_layer_stack: Default::default(),
@@ -4141,6 +4210,7 @@ model_verbosity = "high"
network: None,
shell_environment_policy: ShellEnvironmentPolicy::default(),
windows_sandbox_mode: None,
macos_seatbelt_profile_extensions: None,
},
enforce_residency: Constrained::allow_any(None),
did_user_set_custom_approval_policy_or_sandbox_mode: true,
@@ -4156,6 +4226,7 @@ model_verbosity = "high"
project_doc_fallback_filenames: Vec::new(),
tool_output_token_limit: None,
agent_max_threads: DEFAULT_AGENT_MAX_THREADS,
memories: MemoriesConfig::default(),
codex_home: fixture.codex_home(),
log_dir: fixture.codex_home().join("log"),
config_layer_stack: Default::default(),
@@ -4248,6 +4319,7 @@ model_verbosity = "high"
network: None,
shell_environment_policy: ShellEnvironmentPolicy::default(),
windows_sandbox_mode: None,
macos_seatbelt_profile_extensions: None,
},
enforce_residency: Constrained::allow_any(None),
did_user_set_custom_approval_policy_or_sandbox_mode: true,
@@ -4263,6 +4335,7 @@ model_verbosity = "high"
project_doc_fallback_filenames: Vec::new(),
tool_output_token_limit: None,
agent_max_threads: DEFAULT_AGENT_MAX_THREADS,
memories: MemoriesConfig::default(),
codex_home: fixture.codex_home(),
log_dir: fixture.codex_home().join("log"),
config_layer_stack: Default::default(),
@@ -4341,6 +4414,7 @@ model_verbosity = "high"
network: None,
shell_environment_policy: ShellEnvironmentPolicy::default(),
windows_sandbox_mode: None,
macos_seatbelt_profile_extensions: None,
},
enforce_residency: Constrained::allow_any(None),
did_user_set_custom_approval_policy_or_sandbox_mode: true,
@@ -4356,6 +4430,7 @@ model_verbosity = "high"
project_doc_fallback_filenames: Vec::new(),
tool_output_token_limit: None,
agent_max_threads: DEFAULT_AGENT_MAX_THREADS,
memories: MemoriesConfig::default(),
codex_home: fixture.codex_home(),
log_dir: fixture.codex_home().join("log"),
config_layer_stack: Default::default(),
+33 -8
View File
@@ -1,8 +1,11 @@
use crate::config;
use crate::config_loader::NetworkConstraints;
use async_trait::async_trait;
use codex_network_proxy::BlockedRequestObserver;
use codex_network_proxy::ConfigReloader;
use codex_network_proxy::ConfigState;
use codex_network_proxy::NetworkDecision;
use codex_network_proxy::NetworkPolicyDecider;
use codex_network_proxy::NetworkProxy;
use codex_network_proxy::NetworkProxyConfig;
use codex_network_proxy::NetworkProxyConstraints;
@@ -11,6 +14,7 @@ use codex_network_proxy::NetworkProxyState;
use codex_network_proxy::build_config_state;
use codex_network_proxy::host_and_port_from_network_addr;
use codex_network_proxy::validate_policy_against_constraints;
use codex_protocol::protocol::SandboxPolicy;
use std::sync::Arc;
#[derive(Debug, Clone, PartialEq, Eq)]
@@ -92,20 +96,41 @@ impl NetworkProxySpec {
})
}
pub async fn start_proxy(&self) -> std::io::Result<StartedNetworkProxy> {
pub async fn start_proxy(
&self,
sandbox_policy: &SandboxPolicy,
policy_decider: Option<Arc<dyn NetworkPolicyDecider>>,
blocked_request_observer: Option<Arc<dyn BlockedRequestObserver>>,
enable_network_approval_flow: bool,
) -> std::io::Result<StartedNetworkProxy> {
let state =
build_config_state(self.config.clone(), self.constraints.clone()).map_err(|err| {
std::io::Error::other(format!("failed to build network proxy state: {err}"))
})?;
let reloader = Arc::new(StaticNetworkProxyReloader::new(state.clone()));
let state = NetworkProxyState::with_reloader(state, reloader);
let proxy = NetworkProxy::builder()
.state(Arc::new(state))
.build()
.await
.map_err(|err| {
std::io::Error::other(format!("failed to build network proxy: {err}"))
})?;
let mut builder = NetworkProxy::builder().state(Arc::new(state));
if enable_network_approval_flow
&& matches!(
sandbox_policy,
SandboxPolicy::ReadOnly { .. } | SandboxPolicy::WorkspaceWrite { .. }
)
{
builder = match policy_decider {
Some(policy_decider) => builder.policy_decider_arc(policy_decider),
None => builder.policy_decider(|_request| async {
// In restricted sandbox modes, allowlist misses should ask for
// explicit network approval instead of hard-denying.
NetworkDecision::ask("not_allowed")
}),
};
}
if let Some(blocked_request_observer) = blocked_request_observer {
builder = builder.blocked_request_observer_arc(blocked_request_observer);
}
let proxy = builder.build().await.map_err(|err| {
std::io::Error::other(format!("failed to build network proxy: {err}"))
})?;
let handle = proxy
.run()
.await
+72
View File
@@ -23,6 +23,10 @@ use serde::Serialize;
use serde::de::Error as SerdeError;
pub const DEFAULT_OTEL_ENVIRONMENT: &str = "dev";
pub const DEFAULT_MEMORIES_MAX_ROLLOUTS_PER_STARTUP: usize = 8;
pub const DEFAULT_MEMORIES_MAX_ROLLOUT_AGE_DAYS: i64 = 30;
pub const DEFAULT_MEMORIES_MIN_ROLLOUT_IDLE_HOURS: i64 = 12;
pub const DEFAULT_MEMORIES_MAX_RAW_MEMORIES_FOR_GLOBAL: usize = 1_024;
#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, JsonSchema)]
#[serde(rename_all = "kebab-case")]
@@ -353,6 +357,74 @@ pub struct FeedbackConfigToml {
pub enabled: Option<bool>,
}
/// Memories settings loaded from config.toml.
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Default, JsonSchema)]
#[schemars(deny_unknown_fields)]
pub struct MemoriesToml {
/// Maximum number of recent raw memories retained for global consolidation.
pub max_raw_memories_for_global: Option<usize>,
/// Maximum age of the threads used for memories.
pub max_rollout_age_days: Option<i64>,
/// Maximum number of rollout candidates processed per pass.
pub max_rollouts_per_startup: Option<usize>,
/// Minimum idle time between last thread activity and memory creation (hours). > 12h recommended.
pub min_rollout_idle_hours: Option<i64>,
/// Model used for thread summarisation.
pub phase_1_model: Option<String>,
/// Model used for memory consolidation.
pub phase_2_model: Option<String>,
}
/// Effective memories settings after defaults are applied.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct MemoriesConfig {
pub max_raw_memories_for_global: usize,
pub max_rollout_age_days: i64,
pub max_rollouts_per_startup: usize,
pub min_rollout_idle_hours: i64,
pub phase_1_model: Option<String>,
pub phase_2_model: Option<String>,
}
impl Default for MemoriesConfig {
fn default() -> Self {
Self {
max_raw_memories_for_global: DEFAULT_MEMORIES_MAX_RAW_MEMORIES_FOR_GLOBAL,
max_rollout_age_days: DEFAULT_MEMORIES_MAX_ROLLOUT_AGE_DAYS,
max_rollouts_per_startup: DEFAULT_MEMORIES_MAX_ROLLOUTS_PER_STARTUP,
min_rollout_idle_hours: DEFAULT_MEMORIES_MIN_ROLLOUT_IDLE_HOURS,
phase_1_model: None,
phase_2_model: None,
}
}
}
impl From<MemoriesToml> for MemoriesConfig {
fn from(toml: MemoriesToml) -> Self {
let defaults = Self::default();
Self {
max_raw_memories_for_global: toml
.max_raw_memories_for_global
.unwrap_or(defaults.max_raw_memories_for_global)
.min(4096),
max_rollout_age_days: toml
.max_rollout_age_days
.unwrap_or(defaults.max_rollout_age_days)
.clamp(0, 90),
max_rollouts_per_startup: toml
.max_rollouts_per_startup
.unwrap_or(defaults.max_rollouts_per_startup)
.min(128),
min_rollout_idle_hours: toml
.min_rollout_idle_hours
.unwrap_or(defaults.min_rollout_idle_hours)
.clamp(1, 48),
phase_1_model: toml.phase_1_model,
phase_2_model: toml.phase_2_model,
}
}
}
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, JsonSchema)]
#[serde(rename_all = "snake_case")]
pub enum AppDisabledReason {
+32
View File
@@ -9,6 +9,7 @@ use std::time::Instant;
use async_channel::unbounded;
pub use codex_app_server_protocol::AppInfo;
use codex_protocol::protocol::SandboxPolicy;
use serde::Deserialize;
use tokio_util::sync::CancellationToken;
use tracing::warn;
@@ -16,6 +17,7 @@ use crate::AuthManager;
use crate::CodexAuth;
use crate::SandboxState;
use crate::config::Config;
use crate::config::types::AppsConfigToml;
use crate::features::Feature;
use crate::mcp::CODEX_APPS_MCP_SERVER_NAME;
use crate::mcp::auth::compute_auth_statuses;
@@ -50,6 +52,19 @@ pub async fn list_accessible_connectors_from_mcp_tools(
list_accessible_connectors_from_mcp_tools_with_options(config, false).await
}
pub async fn list_cached_accessible_connectors_from_mcp_tools(
config: &Config,
) -> Option<Vec<AppInfo>> {
if !config.features.enabled(Feature::Apps) {
return Some(Vec::new());
}
let auth_manager = auth_manager_from_config(config);
let auth = auth_manager.auth().await;
let cache_key = accessible_connectors_cache_key(config, auth.as_ref());
read_cached_accessible_connectors(&cache_key)
}
pub async fn list_accessible_connectors_from_mcp_tools_with_options(
config: &Config,
force_refetch: bool,
@@ -265,6 +280,22 @@ pub fn merge_connectors(
merged
}
pub fn with_app_enabled_state(mut connectors: Vec<AppInfo>, config: &Config) -> Vec<AppInfo> {
let apps = read_apps_config(config).map(|apps_config| apps_config.apps);
for connector in &mut connectors {
if let Some(app) = apps.as_ref().and_then(|apps| apps.get(&connector.id)) {
connector.is_enabled = app.enabled;
}
}
connectors
}
fn read_apps_config(config: &Config) -> Option<AppsConfigToml> {
let effective_config = config.config_layer_stack.effective_config();
let apps_config = effective_config.as_table()?.get("apps")?.clone();
AppsConfigToml::deserialize(apps_config).ok()
}
fn collect_accessible_connectors<I>(tools: I) -> Vec<AppInfo>
where
I: IntoIterator<Item = (String, Option<String>)>,
@@ -291,6 +322,7 @@ where
distribution_channel: None,
install_url: Some(connector_install_url(&connector_name, &connector_id)),
is_accessible: true,
is_enabled: true,
})
.collect();
accessible.sort_by(|left, right| {
+10 -2
View File
@@ -1,4 +1,5 @@
use crate::exec::ExecToolCallOutput;
use crate::network_policy_decision::NetworkPolicyDecisionPayload;
use crate::token_data::KnownPlan;
use crate::token_data::PlanType;
use crate::truncate::TruncationPolicy;
@@ -31,7 +32,10 @@ pub enum SandboxErr {
"sandbox denied exec error, exit code: {}, stdout: {}, stderr: {}",
.output.exit_code, .output.stdout.text, .output.stderr.text
)]
Denied { output: Box<ExecToolCallOutput> },
Denied {
output: Box<ExecToolCallOutput>,
network_policy_decision: Option<NetworkPolicyDecisionPayload>,
},
/// Error from linux seccomp filter setup
#[cfg(target_os = "linux")]
@@ -616,7 +620,7 @@ impl CodexErr {
pub fn get_error_message_ui(e: &CodexErr) -> String {
let message = match e {
CodexErr::Sandbox(SandboxErr::Denied { output }) => {
CodexErr::Sandbox(SandboxErr::Denied { output, .. }) => {
let aggregated = output.aggregated_output.text.trim();
if !aggregated.is_empty() {
output.aggregated_output.text.clone()
@@ -736,6 +740,7 @@ mod tests {
};
let err = CodexErr::Sandbox(SandboxErr::Denied {
output: Box::new(output),
network_policy_decision: None,
});
assert_eq!(get_error_message_ui(&err), "aggregate detail");
}
@@ -752,6 +757,7 @@ mod tests {
};
let err = CodexErr::Sandbox(SandboxErr::Denied {
output: Box::new(output),
network_policy_decision: None,
});
assert_eq!(get_error_message_ui(&err), "stderr detail\nstdout detail");
}
@@ -768,6 +774,7 @@ mod tests {
};
let err = CodexErr::Sandbox(SandboxErr::Denied {
output: Box::new(output),
network_policy_decision: None,
});
assert_eq!(get_error_message_ui(&err), "stdout only");
}
@@ -811,6 +818,7 @@ mod tests {
};
let err = CodexErr::Sandbox(SandboxErr::Denied {
output: Box::new(output),
network_policy_decision: None,
});
assert_eq!(
get_error_message_ui(&err),
+56 -5
View File
@@ -15,6 +15,7 @@ use tokio::io::AsyncReadExt;
use tokio::io::BufReader;
use tokio::process::Child;
use tokio_util::sync::CancellationToken;
use uuid::Uuid;
use crate::error::CodexErr;
use crate::error::Result;
@@ -66,6 +67,7 @@ pub struct ExecParams {
pub expiration: ExecExpiration,
pub env: HashMap<String, String>,
pub network: Option<NetworkProxy>,
pub network_attempt_id: Option<Uuid>,
pub sandbox_permissions: SandboxPermissions,
pub windows_sandbox_level: codex_protocol::config_types::WindowsSandboxLevel,
pub justification: Option<String>,
@@ -184,13 +186,15 @@ pub async fn process_exec_tool_call(
mut env,
expiration,
network,
network_attempt_id,
sandbox_permissions,
windows_sandbox_level,
justification,
arg0: _,
} = params;
let network_attempt_id = network_attempt_id.map(|attempt_id| attempt_id.to_string());
if let Some(network) = network.as_ref() {
network.apply_to_env(&mut env);
network.apply_to_env_for_attempt(&mut env, network_attempt_id.as_deref());
}
let (program, args) = command.split_first().ok_or_else(|| {
CodexErr::Io(io::Error::new(
@@ -238,6 +242,7 @@ pub(crate) async fn execute_exec_env(
cwd,
env,
network,
network_attempt_id,
expiration,
sandbox,
windows_sandbox_level,
@@ -246,12 +251,18 @@ pub(crate) async fn execute_exec_env(
arg0,
} = env;
let network_attempt_id = match network_attempt_id.as_deref() {
Some(attempt_id) => Uuid::parse_str(attempt_id).ok(),
None => network.as_ref().map(|_| Uuid::new_v4()),
};
let params = ExecParams {
command,
cwd,
expiration,
env,
network,
network: network.clone(),
network_attempt_id,
sandbox_permissions,
windows_sandbox_level,
justification,
@@ -345,12 +356,14 @@ async fn exec_windows_sandbox(
cwd,
mut env,
network,
network_attempt_id,
expiration,
windows_sandbox_level,
..
} = params;
let network_attempt_id = network_attempt_id.map(|attempt_id| attempt_id.to_string());
if let Some(network) = network.as_ref() {
network.apply_to_env(&mut env);
network.apply_to_env_for_attempt(&mut env, network_attempt_id.as_deref());
}
// TODO(iceweasel-oai): run_windows_sandbox_capture should support all
@@ -490,6 +503,7 @@ fn finalize_exec_result(
if is_likely_sandbox_denied(sandbox_type, &exec_output) {
return Err(CodexErr::Sandbox(SandboxErr::Denied {
output: Box::new(exec_output),
network_policy_decision: None,
}));
}
@@ -701,13 +715,19 @@ async fn exec(
let ExecParams {
command,
cwd,
env,
mut env,
network,
network_attempt_id,
arg0,
expiration,
windows_sandbox_level: _,
..
} = params;
let network_attempt_id = network_attempt_id.map(|attempt_id| attempt_id.to_string());
if let Some(network) = network.as_ref() {
network.apply_to_env_for_attempt(&mut env, network_attempt_id.as_deref());
}
let (program, args) = command.split_first().ok_or_else(|| {
CodexErr::Io(io::Error::new(
@@ -722,7 +742,10 @@ async fn exec(
arg0: arg0_ref,
cwd,
sandbox_policy,
network: network.as_ref(),
// The environment already has attempt-scoped proxy settings from
// apply_to_env_for_attempt above. Passing network here would reapply
// non-attempt proxy vars and drop attempt correlation metadata.
network: None,
stdio_policy: StdioPolicy::RedirectForShellTool,
env,
})
@@ -951,6 +974,17 @@ mod tests {
assert!(!is_likely_sandbox_denied(SandboxType::None, &output));
}
#[test]
fn sandbox_detection_ignores_network_policy_text_in_non_sandbox_mode() {
let output = make_exec_output(
0,
"",
"",
r#"CODEX_NETWORK_POLICY_DECISION {"decision":"ask","reason":"not_allowed","source":"decider","protocol":"http","host":"google.com","port":80}"#,
);
assert!(!is_likely_sandbox_denied(SandboxType::None, &output));
}
#[test]
fn sandbox_detection_uses_aggregated_output() {
let output = make_exec_output(
@@ -965,6 +999,21 @@ mod tests {
));
}
#[test]
fn sandbox_detection_ignores_network_policy_text_with_zero_exit_code() {
let output = make_exec_output(
0,
"",
"",
r#"CODEX_NETWORK_POLICY_DECISION {"decision":"ask","source":"decider","protocol":"http","host":"google.com","port":80}"#,
);
assert!(!is_likely_sandbox_denied(
SandboxType::LinuxSeccomp,
&output
));
}
#[tokio::test]
async fn read_capped_limits_retained_bytes() {
let (mut writer, reader) = tokio::io::duplex(1024);
@@ -1088,6 +1137,7 @@ mod tests {
expiration: 500.into(),
env,
network: None,
network_attempt_id: None,
sandbox_permissions: SandboxPermissions::UseDefault,
windows_sandbox_level: codex_protocol::config_types::WindowsSandboxLevel::Disabled,
justification: None,
@@ -1141,6 +1191,7 @@ mod tests {
expiration: ExecExpiration::Cancellation(cancel_token),
env,
network: None,
network_attempt_id: None,
sandbox_permissions: SandboxPermissions::UseDefault,
windows_sandbox_level: codex_protocol::config_types::WindowsSandboxLevel::Disabled,
justification: None,
+276 -2
View File
@@ -35,6 +35,54 @@ const PROMPT_CONFLICT_REASON: &str =
const RULES_DIR_NAME: &str = "rules";
const RULE_EXTENSION: &str = "rules";
const DEFAULT_POLICY_FILE: &str = "default.rules";
static BANNED_PREFIX_SUGGESTIONS: &[&[&str]] = &[
&["python3"],
&["python3", "-"],
&["python3", "-c"],
&["python"],
&["python", "-"],
&["python", "-c"],
&["py"],
&["py", "-3"],
&["pythonw"],
&["pyw"],
&["pypy"],
&["pypy3"],
&["git"],
&["bash"],
&["bash", "-lc"],
&["sh"],
&["sh", "-c"],
&["sh", "-lc"],
&["zsh"],
&["zsh", "-lc"],
&["/bin/zsh"],
&["/bin/zsh", "-lc"],
&["/bin/bash"],
&["/bin/bash", "-lc"],
&["pwsh"],
&["pwsh", "-Command"],
&["pwsh", "-c"],
&["powershell"],
&["powershell", "-Command"],
&["powershell", "-c"],
&["powershell.exe"],
&["powershell.exe", "-Command"],
&["powershell.exe", "-c"],
&["env"],
&["sudo"],
&["node"],
&["node", "-e"],
&["perl"],
&["perl", "-e"],
&["ruby"],
&["ruby", "-e"],
&["php"],
&["php", "-r"],
&["lua"],
&["lua", "-e"],
&["osascript"],
];
fn is_policy_match(rule_match: &RuleMatch) -> bool {
match rule_match {
@@ -217,6 +265,77 @@ pub async fn check_execpolicy_for_warnings(
Ok(warning)
}
fn exec_policy_message_for_display(source: &codex_execpolicy::Error) -> String {
let message = source.to_string();
if let Some(line) = message
.lines()
.find(|line| line.trim_start().starts_with("error: "))
{
return line.to_owned();
}
if let Some(first_line) = message.lines().next()
&& let Some((_, detail)) = first_line.rsplit_once(": starlark error: ")
{
return detail.trim().to_string();
}
message
.lines()
.next()
.unwrap_or_default()
.trim()
.to_string()
}
fn parse_starlark_line_from_message(message: &str) -> Option<(PathBuf, usize)> {
let first_line = message.lines().next()?.trim();
let (path_and_position, _) = first_line.rsplit_once(": starlark error:")?;
let mut parts = path_and_position.rsplitn(3, ':');
let _column = parts.next()?.parse::<usize>().ok()?;
let line = parts.next()?.parse::<usize>().ok()?;
let path = PathBuf::from(parts.next()?);
if line == 0 {
return None;
}
Some((path, line))
}
pub fn format_exec_policy_error_with_source(error: &ExecPolicyError) -> String {
match error {
ExecPolicyError::ParsePolicy { path, source } => {
let rendered_source = source.to_string();
let structured_location = source
.location()
.map(|location| (PathBuf::from(location.path), location.range.start.line));
let parsed_location = parse_starlark_line_from_message(&rendered_source);
let location = match (structured_location, parsed_location) {
(Some((_, 1)), Some((parsed_path, parsed_line))) if parsed_line > 1 => {
Some((parsed_path, parsed_line))
}
(Some(structured), _) => Some(structured),
(None, parsed) => parsed,
};
let message = exec_policy_message_for_display(source);
match location {
Some((path, line)) => {
format!(
"{}:{}: {} (problem is on or around line {})",
path.display(),
line,
message,
line
)
}
None => format!("{path}: {message}"),
}
}
_ => error.to_string(),
}
}
async fn load_exec_policy_with_warning(
config_stack: &ConfigLayerStack,
) -> Result<(Policy, Option<ExecPolicyError>), ExecPolicyError> {
@@ -240,6 +359,10 @@ pub async fn load_exec_policy(config_stack: &ConfigLayerStack) -> Result<Policy,
policy_paths.extend(layer_policy_paths);
}
}
tracing::trace!(
policy_paths = ?policy_paths,
"loaded exec policies"
);
let mut parser = PolicyParser::new();
for policy_path in &policy_paths {
@@ -261,6 +384,7 @@ pub async fn load_exec_policy(config_stack: &ConfigLayerStack) -> Result<Policy,
let policy = parser.build();
tracing::debug!("loaded rules from {} files", policy_paths.len());
tracing::trace!(rules = ?policy, "exec policy rules loaded");
let Some(requirements_policy) = config_stack.requirements().exec_policy.as_deref() else {
return Ok(policy);
@@ -418,6 +542,15 @@ fn derive_requested_execpolicy_amendment(
if prefix_rule.is_empty() {
return None;
}
if BANNED_PREFIX_SUGGESTIONS.iter().any(|banned| {
prefix_rule.len() == banned.len()
&& prefix_rule
.iter()
.map(String::as_str)
.eq(banned.iter().copied())
}) {
return None;
}
if matched_rules
.iter()
@@ -617,6 +750,51 @@ mod tests {
assert!(files.is_empty());
}
#[tokio::test]
async fn format_exec_policy_error_with_source_renders_range() {
let temp_dir = tempdir().expect("create temp dir");
let config_stack = config_stack_for_dot_codex_folder(temp_dir.path());
let policy_dir = temp_dir.path().join(RULES_DIR_NAME);
fs::create_dir_all(&policy_dir).expect("create policy dir");
let broken_path = policy_dir.join("broken.rules");
fs::write(
&broken_path,
r#"prefix_rule(
pattern = ["tmux capture-pane"],
decision = "allow",
match = ["tmux capture-pane -p"],
)"#,
)
.expect("write broken policy file");
let err = load_exec_policy(&config_stack)
.await
.expect_err("expected parse error");
let rendered = format_exec_policy_error_with_source(&err);
assert!(rendered.contains("broken.rules:1:"));
assert!(rendered.contains("on or around line 1"));
}
#[test]
fn parse_starlark_line_from_message_extracts_path_and_line() {
let parsed = parse_starlark_line_from_message(
"/tmp/default.rules:143:1: starlark error: error: Parse error: unexpected new line",
)
.expect("parse should succeed");
assert_eq!(parsed.0, PathBuf::from("/tmp/default.rules"));
assert_eq!(parsed.1, 143);
}
#[test]
fn parse_starlark_line_from_message_rejects_zero_line() {
let parsed = parse_starlark_line_from_message(
"/tmp/default.rules:0:1: starlark error: error: Parse error: unexpected new line",
);
assert_eq!(parsed, None);
}
#[tokio::test]
async fn loads_policies_from_policy_subdirectory() {
let temp_dir = tempdir().expect("create temp dir");
@@ -1384,9 +1562,105 @@ prefix_rule(
);
}
#[test]
fn derive_requested_execpolicy_amendment_returns_none_for_missing_prefix_rule() {
assert_eq!(None, derive_requested_execpolicy_amendment(None, &[]));
}
#[test]
fn derive_requested_execpolicy_amendment_returns_none_for_empty_prefix_rule() {
assert_eq!(
None,
derive_requested_execpolicy_amendment(Some(&Vec::new()), &[])
);
}
#[test]
fn derive_requested_execpolicy_amendment_returns_none_for_exact_banned_prefix_rule() {
assert_eq!(
None,
derive_requested_execpolicy_amendment(
Some(&vec!["python".to_string(), "-c".to_string()]),
&[],
)
);
}
#[test]
fn derive_requested_execpolicy_amendment_returns_none_for_windows_and_pypy_variants() {
for prefix_rule in [
vec!["py".to_string()],
vec!["py".to_string(), "-3".to_string()],
vec!["pythonw".to_string()],
vec!["pyw".to_string()],
vec!["pypy".to_string()],
vec!["pypy3".to_string()],
] {
assert_eq!(
None,
derive_requested_execpolicy_amendment(Some(&prefix_rule), &[])
);
}
}
#[test]
fn derive_requested_execpolicy_amendment_returns_none_for_shell_and_powershell_variants() {
for prefix_rule in [
vec!["bash".to_string(), "-lc".to_string()],
vec!["sh".to_string(), "-c".to_string()],
vec!["sh".to_string(), "-lc".to_string()],
vec!["zsh".to_string(), "-lc".to_string()],
vec!["/bin/bash".to_string(), "-lc".to_string()],
vec!["/bin/zsh".to_string(), "-lc".to_string()],
vec!["pwsh".to_string()],
vec!["pwsh".to_string(), "-Command".to_string()],
vec!["pwsh".to_string(), "-c".to_string()],
vec!["powershell".to_string()],
vec!["powershell".to_string(), "-Command".to_string()],
vec!["powershell".to_string(), "-c".to_string()],
vec!["powershell.exe".to_string()],
vec!["powershell.exe".to_string(), "-Command".to_string()],
vec!["powershell.exe".to_string(), "-c".to_string()],
] {
assert_eq!(
None,
derive_requested_execpolicy_amendment(Some(&prefix_rule), &[])
);
}
}
#[test]
fn derive_requested_execpolicy_amendment_allows_non_exact_banned_prefix_rule_match() {
let prefix_rule = vec![
"python".to_string(),
"-c".to_string(),
"print('hi')".to_string(),
];
assert_eq!(
Some(ExecPolicyAmendment::new(prefix_rule.clone())),
derive_requested_execpolicy_amendment(Some(&prefix_rule), &[])
);
}
#[test]
fn derive_requested_execpolicy_amendment_returns_none_when_policy_prompt_matches() {
let prefix_rule = vec!["cargo".to_string(), "build".to_string()];
let matched_rules = vec![RuleMatch::PrefixRuleMatch {
matched_prefix: vec!["cargo".to_string()],
decision: Decision::Prompt,
justification: None,
}];
assert_eq!(
None,
derive_requested_execpolicy_amendment(Some(&prefix_rule), &matched_rules)
);
}
#[tokio::test]
async fn dangerous_git_push_requires_approval_in_danger_full_access() {
let command = vec_str(&["git", "push", "origin", "+main"]);
async fn dangerous_rm_rf_requires_approval_in_danger_full_access() {
let command = vec_str(&["rm", "-rf", "/tmp/nonexistent"]);
let manager = ExecPolicyManager::default();
let requirement = manager
.create_exec_approval_requirement_for_command(ExecApprovalRequest {
+39
View File
@@ -121,6 +121,8 @@ pub enum Feature {
Collab,
/// Enable apps.
Apps,
/// Route apps MCP calls through the configured gateway.
AppsMcpGateway,
/// Allow prompting and installing missing MCP dependencies.
SkillMcpDependencyInstall,
/// Prompt for missing skill env var dependencies.
@@ -131,6 +133,8 @@ pub enum Feature {
CollaborationModes,
/// Enable personality selection in the TUI.
Personality,
/// Prevent idle system sleep while a turn is actively running.
PreventIdleSleep,
/// Use the Responses API WebSocket transport for OpenAI by default.
ResponsesWebsockets,
/// Enable Responses API websocket v2 mode.
@@ -566,6 +570,12 @@ pub const FEATURES: &[FeatureSpec] = &[
},
default_enabled: false,
},
FeatureSpec {
id: Feature::AppsMcpGateway,
key: "apps_mcp_gateway",
stage: Stage::UnderDevelopment,
default_enabled: false,
},
FeatureSpec {
id: Feature::SkillMcpDependencyInstall,
key: "skill_mcp_dependency_install",
@@ -596,6 +606,20 @@ pub const FEATURES: &[FeatureSpec] = &[
stage: Stage::Stable,
default_enabled: true,
},
FeatureSpec {
id: Feature::PreventIdleSleep,
key: "prevent_idle_sleep",
stage: if cfg!(target_os = "macos") {
Stage::Experimental {
name: "Prevent sleep while running",
menu_description: "Keep your computer awake while Codex is running a thread.",
announcement: "NEW: Prevent sleep while running is now available in /experimental.",
}
} else {
Stage::UnderDevelopment
},
default_enabled: false,
},
FeatureSpec {
id: Feature::ResponsesWebsockets,
key: "responses_websockets",
@@ -680,6 +704,21 @@ mod tests {
}
}
#[test]
fn default_enabled_features_are_stable() {
for spec in FEATURES {
if spec.default_enabled {
assert_eq!(
spec.stage,
Stage::Stable,
"feature `{}` is enabled by default but is not stable ({:?})",
spec.key,
spec.stage
);
}
}
}
#[cfg(target_os = "linux")]
#[test]
fn use_linux_sandbox_bwrap_is_experimental_on_linux() {
+45 -1
View File
@@ -150,6 +150,15 @@ pub async fn get_head_commit_hash(cwd: &Path) -> Option<String> {
}
}
pub async fn get_has_changes(cwd: &Path) -> Option<bool> {
let output = run_git_command_with_timeout(&["status", "--porcelain"], cwd).await?;
if !output.status.success() {
return None;
}
Some(!output.stdout.is_empty())
}
fn parse_git_remote_urls(stdout: &str) -> Option<BTreeMap<String, String>> {
let mut remotes = BTreeMap::new();
for line in stdout.lines() {
@@ -254,7 +263,11 @@ pub async fn git_diff_to_remote(cwd: &Path) -> Option<GitDiffToRemote> {
/// Run a git command with a timeout to prevent blocking on large repositories
async fn run_git_command_with_timeout(args: &[&str], cwd: &Path) -> Option<std::process::Output> {
let mut command = Command::new("git");
command.args(args).current_dir(cwd).kill_on_drop(true);
command
.env("GIT_OPTIONAL_LOCKS", "0")
.args(args)
.current_dir(cwd)
.kill_on_drop(true);
let result = timeout(GIT_COMMAND_TIMEOUT, command.output()).await;
match result {
@@ -962,6 +975,37 @@ mod tests {
assert_eq!(git_info.branch, Some("feature-branch".to_string()));
}
#[tokio::test]
async fn test_get_has_changes_non_git_directory_returns_none() {
let temp_dir = TempDir::new().expect("Failed to create temp dir");
assert_eq!(get_has_changes(temp_dir.path()).await, None);
}
#[tokio::test]
async fn test_get_has_changes_clean_repo_returns_false() {
let temp_dir = TempDir::new().expect("Failed to create temp dir");
let repo_path = create_test_git_repo(&temp_dir).await;
assert_eq!(get_has_changes(&repo_path).await, Some(false));
}
#[tokio::test]
async fn test_get_has_changes_with_tracked_change_returns_true() {
let temp_dir = TempDir::new().expect("Failed to create temp dir");
let repo_path = create_test_git_repo(&temp_dir).await;
fs::write(repo_path.join("test.txt"), "updated tracked file").expect("write tracked file");
assert_eq!(get_has_changes(&repo_path).await, Some(true));
}
#[tokio::test]
async fn test_get_has_changes_with_untracked_change_returns_true() {
let temp_dir = TempDir::new().expect("Failed to create temp dir");
let repo_path = create_test_git_repo(&temp_dir).await;
fs::write(repo_path.join("new_file.txt"), "untracked").expect("write untracked file");
assert_eq!(get_has_changes(&repo_path).await, Some(true));
}
#[tokio::test]
async fn test_get_git_working_tree_state_clean_repo() {
let temp_dir = TempDir::new().expect("Failed to create temp dir");
+2
View File
@@ -41,6 +41,7 @@ pub mod landlock;
pub mod mcp;
mod mcp_connection_manager;
pub mod models_manager;
mod network_policy_decision;
pub mod network_proxy_loader;
pub use mcp_connection_manager::MCP_SANDBOX_STATE_CAPABILITY;
pub use mcp_connection_manager::MCP_SANDBOX_STATE_METHOD;
@@ -144,6 +145,7 @@ pub use apply_patch::CODEX_APPLY_PATCH_ARG1;
pub use client::X_CODEX_TURN_METADATA_HEADER;
pub use exec_policy::ExecPolicyError;
pub use exec_policy::check_execpolicy_for_warnings;
pub use exec_policy::format_exec_policy_error_with_source;
pub use exec_policy::load_exec_policy;
pub use file_watcher::FileWatcherEvent;
pub use safety::get_platform_sandbox;
+159 -2
View File
@@ -30,6 +30,15 @@ const MCP_TOOL_NAME_PREFIX: &str = "mcp";
const MCP_TOOL_NAME_DELIMITER: &str = "__";
pub(crate) const CODEX_APPS_MCP_SERVER_NAME: &str = "codex_apps";
const CODEX_CONNECTORS_TOKEN_ENV_VAR: &str = "CODEX_CONNECTORS_TOKEN";
const OPENAI_CONNECTORS_MCP_BASE_URL: &str = "https://api.openai.com";
const OPENAI_CONNECTORS_MCP_PATH: &str = "/v1/connectors/mcp/";
// Legacy vs new MCP gateway
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum CodexAppsMcpGateway {
LegacyMCPGateway,
MCPGateway,
}
fn codex_apps_mcp_bearer_token_env_var() -> Option<String> {
match env::var(CODEX_CONNECTORS_TOKEN_ENV_VAR) {
@@ -65,7 +74,15 @@ fn codex_apps_mcp_http_headers(auth: Option<&CodexAuth>) -> Option<HashMap<Strin
}
}
fn codex_apps_mcp_url(base_url: &str) -> String {
fn selected_config_codex_apps_mcp_gateway(config: &Config) -> CodexAppsMcpGateway {
if config.features.enabled(Feature::AppsMcpGateway) {
CodexAppsMcpGateway::MCPGateway
} else {
CodexAppsMcpGateway::LegacyMCPGateway
}
}
fn normalize_codex_apps_base_url(base_url: &str) -> String {
let mut base_url = base_url.trim_end_matches('/').to_string();
if (base_url.starts_with("https://chatgpt.com")
|| base_url.starts_with("https://chat.openai.com"))
@@ -73,6 +90,15 @@ fn codex_apps_mcp_url(base_url: &str) -> String {
{
base_url = format!("{base_url}/backend-api");
}
base_url
}
fn codex_apps_mcp_url_for_gateway(base_url: &str, gateway: CodexAppsMcpGateway) -> String {
if gateway == CodexAppsMcpGateway::MCPGateway {
return format!("{OPENAI_CONNECTORS_MCP_BASE_URL}{OPENAI_CONNECTORS_MCP_PATH}");
}
let base_url = normalize_codex_apps_base_url(base_url);
if base_url.contains("/backend-api") {
format!("{base_url}/wham/apps")
} else if base_url.contains("/api/codex") {
@@ -82,6 +108,13 @@ fn codex_apps_mcp_url(base_url: &str) -> String {
}
}
pub(crate) fn codex_apps_mcp_url(config: &Config) -> String {
codex_apps_mcp_url_for_gateway(
&config.chatgpt_base_url,
selected_config_codex_apps_mcp_gateway(config),
)
}
fn codex_apps_mcp_server_config(config: &Config, auth: Option<&CodexAuth>) -> McpServerConfig {
let bearer_token_env_var = codex_apps_mcp_bearer_token_env_var();
let http_headers = if bearer_token_env_var.is_some() {
@@ -89,7 +122,7 @@ fn codex_apps_mcp_server_config(config: &Config, auth: Option<&CodexAuth>) -> Mc
} else {
codex_apps_mcp_http_headers(auth)
};
let url = codex_apps_mcp_url(&config.chatgpt_base_url);
let url = codex_apps_mcp_url(config);
McpServerConfig {
transport: McpServerTransportConfig::StreamableHttp {
@@ -385,4 +418,128 @@ mod tests {
assert_eq!(group_tools_by_server(&tools), expected);
}
#[test]
fn codex_apps_mcp_url_for_default_gateway_keeps_existing_paths() {
assert_eq!(
codex_apps_mcp_url_for_gateway(
"https://chatgpt.com/backend-api",
CodexAppsMcpGateway::LegacyMCPGateway
),
"https://chatgpt.com/backend-api/wham/apps"
);
assert_eq!(
codex_apps_mcp_url_for_gateway(
"https://chat.openai.com",
CodexAppsMcpGateway::LegacyMCPGateway
),
"https://chat.openai.com/backend-api/wham/apps"
);
assert_eq!(
codex_apps_mcp_url_for_gateway(
"http://localhost:8080/api/codex",
CodexAppsMcpGateway::LegacyMCPGateway
),
"http://localhost:8080/api/codex/apps"
);
assert_eq!(
codex_apps_mcp_url_for_gateway(
"http://localhost:8080",
CodexAppsMcpGateway::LegacyMCPGateway
),
"http://localhost:8080/api/codex/apps"
);
}
#[test]
fn codex_apps_mcp_url_for_gateway_uses_openai_connectors_gateway() {
let expected_url = format!("{OPENAI_CONNECTORS_MCP_BASE_URL}{OPENAI_CONNECTORS_MCP_PATH}");
assert_eq!(
codex_apps_mcp_url_for_gateway(
"https://chatgpt.com/backend-api",
CodexAppsMcpGateway::MCPGateway
),
expected_url.as_str()
);
assert_eq!(
codex_apps_mcp_url_for_gateway(
"https://chat.openai.com",
CodexAppsMcpGateway::MCPGateway
),
expected_url.as_str()
);
assert_eq!(
codex_apps_mcp_url_for_gateway(
"http://localhost:8080/api/codex",
CodexAppsMcpGateway::MCPGateway
),
expected_url.as_str()
);
assert_eq!(
codex_apps_mcp_url_for_gateway(
"http://localhost:8080",
CodexAppsMcpGateway::MCPGateway
),
expected_url.as_str()
);
}
#[test]
fn codex_apps_mcp_url_uses_default_gateway_when_feature_is_disabled() {
let mut config = crate::config::test_config();
config.chatgpt_base_url = "https://chatgpt.com".to_string();
assert_eq!(
codex_apps_mcp_url(&config),
"https://chatgpt.com/backend-api/wham/apps"
);
}
#[test]
fn codex_apps_mcp_url_uses_openai_connectors_gateway_when_feature_is_enabled() {
let mut config = crate::config::test_config();
config.chatgpt_base_url = "https://chatgpt.com".to_string();
config.features.enable(Feature::AppsMcpGateway);
assert_eq!(
codex_apps_mcp_url(&config),
format!("{OPENAI_CONNECTORS_MCP_BASE_URL}{OPENAI_CONNECTORS_MCP_PATH}")
);
}
#[test]
fn codex_apps_server_config_switches_gateway_with_flags() {
let mut config = crate::config::test_config();
config.chatgpt_base_url = "https://chatgpt.com".to_string();
let mut servers = with_codex_apps_mcp(HashMap::new(), false, None, &config);
assert!(!servers.contains_key(CODEX_APPS_MCP_SERVER_NAME));
config.features.enable(Feature::Apps);
servers = with_codex_apps_mcp(servers, true, None, &config);
let server = servers
.get(CODEX_APPS_MCP_SERVER_NAME)
.expect("codex apps should be present when apps is enabled");
let url = match &server.transport {
McpServerTransportConfig::StreamableHttp { url, .. } => url,
_ => panic!("expected streamable http transport for codex apps"),
};
assert_eq!(url, "https://chatgpt.com/backend-api/wham/apps");
config.features.enable(Feature::AppsMcpGateway);
servers = with_codex_apps_mcp(servers, true, None, &config);
let server = servers
.get(CODEX_APPS_MCP_SERVER_NAME)
.expect("codex apps should remain present when apps stays enabled");
let url = match &server.transport {
McpServerTransportConfig::StreamableHttp { url, .. } => url,
_ => panic!("expected streamable http transport for codex apps"),
};
let expected_url = format!("{OPENAI_CONNECTORS_MCP_BASE_URL}{OPENAI_CONNECTORS_MCP_PATH}");
assert_eq!(url, &expected_url);
}
}
@@ -432,6 +432,7 @@ mod tests {
interface: None,
dependencies: Some(SkillDependencies { tools }),
policy: None,
permissions: None,
path: PathBuf::from("skill"),
scope: SkillScope::User,
}
+127 -14
View File
@@ -89,6 +89,9 @@ pub const DEFAULT_STARTUP_TIMEOUT: Duration = Duration::from_secs(10);
const DEFAULT_TOOL_TIMEOUT: Duration = Duration::from_secs(60);
const CODEX_APPS_TOOLS_CACHE_TTL: Duration = Duration::from_secs(3600);
const MCP_TOOLS_LIST_DURATION_METRIC: &str = "codex.mcp.tools.list.duration_ms";
const MCP_TOOLS_FETCH_UNCACHED_DURATION_METRIC: &str = "codex.mcp.tools.fetch_uncached.duration_ms";
const MCP_TOOLS_CACHE_WRITE_DURATION_METRIC: &str = "codex.mcp.tools.cache_write.duration_ms";
/// The Responses API requires tool names to match `^[a-zA-Z0-9_-]+$`.
/// MCP server/tool names are user-controlled, so sanitize the fully-qualified
@@ -882,7 +885,7 @@ fn filter_tools(tools: Vec<ToolInfo>, filter: ToolFilter) -> Vec<ToolInfo> {
}
pub(crate) fn filter_codex_apps_mcp_tools_only(
mut mcp_tools: HashMap<String, ToolInfo>,
mcp_tools: &HashMap<String, ToolInfo>,
connectors: &[crate::connectors::AppInfo],
) -> HashMap<String, ToolInfo> {
let allowed: HashSet<&str> = connectors
@@ -890,26 +893,42 @@ pub(crate) fn filter_codex_apps_mcp_tools_only(
.map(|connector| connector.id.as_str())
.collect();
mcp_tools.retain(|_, tool| {
if tool.server_name != CODEX_APPS_MCP_SERVER_NAME {
return false;
}
let Some(connector_id) = tool.connector_id.as_deref() else {
return false;
};
allowed.contains(connector_id)
});
mcp_tools
.iter()
.filter(|(_, tool)| {
if tool.server_name != CODEX_APPS_MCP_SERVER_NAME {
return false;
}
let Some(connector_id) = tool.connector_id.as_deref() else {
return false;
};
allowed.contains(connector_id)
})
.map(|(name, tool)| (name.clone(), tool.clone()))
.collect()
}
pub(crate) fn filter_non_codex_apps_mcp_tools_only(
mcp_tools: &HashMap<String, ToolInfo>,
) -> HashMap<String, ToolInfo> {
mcp_tools
.iter()
.filter(|(_, tool)| tool.server_name != CODEX_APPS_MCP_SERVER_NAME)
.map(|(name, tool)| (name.clone(), tool.clone()))
.collect()
}
pub(crate) fn filter_mcp_tools_by_name(
mut mcp_tools: HashMap<String, ToolInfo>,
mcp_tools: &HashMap<String, ToolInfo>,
selected_tools: &[String],
) -> HashMap<String, ToolInfo> {
let allowed: HashSet<&str> = selected_tools.iter().map(String::as_str).collect();
mcp_tools.retain(|name, _| allowed.contains(name.as_str()));
mcp_tools
.iter()
.filter(|(name, _)| allowed.contains(name.as_str()))
.map(|(name, tool)| (name.clone(), tool.clone()))
.collect()
}
fn normalize_codex_apps_tool_title(
@@ -1098,15 +1117,42 @@ async fn list_tools_for_client(
client: &Arc<RmcpClient>,
timeout: Option<Duration>,
) -> Result<Vec<ToolInfo>> {
let total_start = Instant::now();
if server_name == CODEX_APPS_MCP_SERVER_NAME
&& let Some(cached_tools) = read_cached_codex_apps_tools()
{
emit_duration(
MCP_TOOLS_LIST_DURATION_METRIC,
total_start.elapsed(),
&[("cache", "hit")],
);
return Ok(cached_tools);
}
let fetch_start = Instant::now();
let tools = list_tools_for_client_uncached(server_name, client, timeout).await?;
emit_duration(
MCP_TOOLS_FETCH_UNCACHED_DURATION_METRIC,
fetch_start.elapsed(),
&[],
);
if server_name == CODEX_APPS_MCP_SERVER_NAME {
let cache_write_start = Instant::now();
write_cached_codex_apps_tools(&tools);
emit_duration(
MCP_TOOLS_CACHE_WRITE_DURATION_METRIC,
cache_write_start.elapsed(),
&[],
);
}
if server_name == CODEX_APPS_MCP_SERVER_NAME {
emit_duration(
MCP_TOOLS_LIST_DURATION_METRIC,
total_start.elapsed(),
&[("cache", "miss")],
);
}
Ok(tools)
}
@@ -1123,7 +1169,12 @@ fn read_cached_codex_apps_tools() -> Option<Vec<ToolInfo>> {
return Some(cached.tools.clone());
}
*cache_guard = None;
if cache_guard
.as_ref()
.is_some_and(|cached| now >= cached.expires_at)
{
*cache_guard = None;
}
None
}
@@ -1137,6 +1188,12 @@ fn write_cached_codex_apps_tools(tools: &[ToolInfo]) {
});
}
fn emit_duration(metric: &str, duration: Duration, tags: &[(&str, &str)]) {
if let Some(metrics) = codex_otel::metrics::global() {
let _ = metrics.record_duration(metric, duration, tags);
}
}
async fn list_tools_for_client_uncached(
server_name: &str,
client: &Arc<RmcpClient>,
@@ -1272,6 +1329,21 @@ mod tests {
}
}
fn with_clean_codex_apps_tools_cache<T>(f: impl FnOnce() -> T) -> T {
let previous_cache = {
let mut cache_guard = CODEX_APPS_TOOLS_CACHE
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
cache_guard.take()
};
let result = f();
let mut cache_guard = CODEX_APPS_TOOLS_CACHE
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
*cache_guard = previous_cache;
result
}
#[test]
fn test_qualify_tools_short_non_duplicated_names() {
let tools = vec![
@@ -1424,6 +1496,47 @@ mod tests {
assert_eq!(filtered[0].tool_name, "tool_a");
}
#[test]
fn codex_apps_tools_cache_is_overwritten_by_last_write() {
with_clean_codex_apps_tools_cache(|| {
let tools_gateway_1 = vec![create_test_tool(CODEX_APPS_MCP_SERVER_NAME, "one")];
let tools_gateway_2 = vec![create_test_tool(CODEX_APPS_MCP_SERVER_NAME, "two")];
write_cached_codex_apps_tools(&tools_gateway_1);
let cached_gateway_1 =
read_cached_codex_apps_tools().expect("cache entry exists for first write");
assert_eq!(cached_gateway_1[0].tool_name, "one");
write_cached_codex_apps_tools(&tools_gateway_2);
let cached_gateway_2 =
read_cached_codex_apps_tools().expect("cache entry exists for second write");
assert_eq!(cached_gateway_2[0].tool_name, "two");
});
}
#[test]
fn codex_apps_tools_cache_is_cleared_when_expired() {
with_clean_codex_apps_tools_cache(|| {
let tools = vec![create_test_tool(CODEX_APPS_MCP_SERVER_NAME, "stale_tool")];
write_cached_codex_apps_tools(&tools);
{
let mut cache_guard = CODEX_APPS_TOOLS_CACHE
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
cache_guard.as_mut().expect("cache exists").expires_at =
Instant::now() - Duration::from_secs(1);
}
assert!(read_cached_codex_apps_tools().is_none());
let cache_guard = CODEX_APPS_TOOLS_CACHE
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
assert!(cache_guard.is_none());
});
}
#[test]
fn mcp_init_error_display_prompts_for_github_pat() {
let server_name = "github";
+73
View File
@@ -3,6 +3,8 @@ use std::time::Instant;
use tracing::error;
use crate::analytics_client::AppInvocation;
use crate::analytics_client::build_track_events_context;
use crate::codex::Session;
use crate::codex::TurnContext;
use crate::mcp::CODEX_APPS_MCP_SERVER_NAME;
@@ -102,6 +104,7 @@ pub(crate) async fn handle_mcp_tool_call(
tool_call_end_event.clone(),
)
.await;
maybe_track_codex_app_used(sess.as_ref(), turn_context, &server, &tool_name).await;
result
}
McpToolApprovalDecision::Decline => {
@@ -166,6 +169,7 @@ pub(crate) async fn handle_mcp_tool_call(
});
notify_mcp_tool_call_event(sess.as_ref(), turn_context, tool_call_end_event.clone()).await;
maybe_track_codex_app_used(sess.as_ref(), turn_context, &server, &tool_name).await;
let status = if result.is_ok() { "ok" } else { "error" };
turn_context
@@ -210,6 +214,50 @@ async fn notify_mcp_tool_call_event(sess: &Session, turn_context: &TurnContext,
sess.send_event(turn_context, event).await;
}
struct McpAppUsageMetadata {
connector_id: Option<String>,
app_name: Option<String>,
}
async fn maybe_track_codex_app_used(
sess: &Session,
turn_context: &TurnContext,
server: &str,
tool_name: &str,
) {
if server != CODEX_APPS_MCP_SERVER_NAME {
return;
}
let metadata = lookup_mcp_app_usage_metadata(sess, server, tool_name).await;
let (connector_id, app_name) = metadata
.map(|metadata| (metadata.connector_id, metadata.app_name))
.unwrap_or((None, None));
let invoke_type = if let Some(connector_id) = connector_id.as_deref() {
let mentioned_connector_ids = sess.get_connector_selection().await;
if mentioned_connector_ids.contains(connector_id) {
"explicit"
} else {
"implicit"
}
} else {
"implicit"
};
let tracking = build_track_events_context(
turn_context.model_info.slug.clone(),
sess.conversation_id.to_string(),
turn_context.sub_id.clone(),
);
sess.services.analytics_events_client.track_app_used(
tracking,
AppInvocation {
connector_id,
app_name,
invoke_type: Some(invoke_type.to_string()),
},
);
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum McpToolApprovalDecision {
Accept,
@@ -332,6 +380,31 @@ async fn lookup_mcp_tool_metadata(
})
}
async fn lookup_mcp_app_usage_metadata(
sess: &Session,
server: &str,
tool_name: &str,
) -> Option<McpAppUsageMetadata> {
let tools = sess
.services
.mcp_connection_manager
.read()
.await
.list_all_tools()
.await;
tools.into_values().find_map(|tool_info| {
if tool_info.server_name == server && tool_info.tool_name == tool_name {
Some(McpAppUsageMetadata {
connector_id: tool_info.connector_id,
app_name: tool_info.connector_name,
})
} else {
None
}
})
}
fn build_mcp_tool_approval_question(
question_id: String,
tool_name: &str,
-688
View File
@@ -1,688 +0,0 @@
use crate::codex::Session;
use crate::config::Config;
use crate::config::Constrained;
use crate::memories::memory_root;
use crate::memories::metrics;
use crate::memories::phase_two;
use crate::memories::phase2::spawn_phase2_completion_task;
use crate::memories::prompts::build_consolidation_prompt;
use crate::memories::storage::rebuild_raw_memories_file_from_memories;
use crate::memories::storage::sync_rollout_summaries_from_memories;
use codex_protocol::protocol::AskForApproval;
use codex_protocol::protocol::SandboxPolicy;
use codex_protocol::protocol::SessionSource;
use codex_protocol::protocol::SubAgentSource;
use codex_protocol::user_input::UserInput;
use codex_utils_absolute_path::AbsolutePathBuf;
use std::sync::Arc;
use tracing::debug;
use tracing::info;
use tracing::warn;
//TODO(jif) clean.
fn completion_watermark(
claimed_watermark: i64,
latest_memories: &[codex_state::Stage1Output],
) -> i64 {
latest_memories
.iter()
.map(|memory| memory.source_updated_at.timestamp())
.max()
.unwrap_or(claimed_watermark)
.max(claimed_watermark)
}
pub(in crate::memories) async fn run_global_memory_consolidation(
session: &Arc<Session>,
config: Arc<Config>,
) -> bool {
let otel_manager = &session.services.otel_manager;
let Some(state_db) = session.services.state_db.as_deref() else {
warn!("state db unavailable; skipping global memory consolidation");
otel_manager.counter(
metrics::MEMORY_PHASE_TWO_JOBS,
1,
&[("status", "skipped_state_db_unavailable")],
);
return false;
};
let claim = match state_db
.try_claim_global_phase2_job(session.conversation_id, phase_two::JOB_LEASE_SECONDS)
.await
{
Ok(claim) => claim,
Err(err) => {
warn!("state db try_claim_global_phase2_job failed during memories startup: {err}");
otel_manager.counter(
metrics::MEMORY_PHASE_TWO_JOBS,
1,
&[("status", "failed_claim")],
);
return false;
}
};
let (ownership_token, claimed_watermark) = match claim {
codex_state::Phase2JobClaimOutcome::Claimed {
ownership_token,
input_watermark,
} => {
otel_manager.counter(metrics::MEMORY_PHASE_TWO_JOBS, 1, &[("status", "claimed")]);
(ownership_token, input_watermark)
}
codex_state::Phase2JobClaimOutcome::SkippedNotDirty => {
debug!("memory phase-2 global lock is up-to-date; skipping consolidation");
otel_manager.counter(
metrics::MEMORY_PHASE_TWO_JOBS,
1,
&[("status", "skipped_not_dirty")],
);
return false;
}
codex_state::Phase2JobClaimOutcome::SkippedRunning => {
debug!("memory phase-2 global consolidation already running; skipping");
otel_manager.counter(
metrics::MEMORY_PHASE_TWO_JOBS,
1,
&[("status", "skipped_running")],
);
return false;
}
};
let root = memory_root(&config.codex_home);
let consolidation_config = {
let mut consolidation_config = config.as_ref().clone();
consolidation_config.cwd = root.clone();
consolidation_config.permissions.approval_policy =
Constrained::allow_only(AskForApproval::Never);
let mut writable_roots = Vec::new();
match AbsolutePathBuf::from_absolute_path(consolidation_config.codex_home.clone()) {
Ok(codex_home) => writable_roots.push(codex_home),
Err(err) => warn!(
"memory phase-2 consolidation could not add codex_home writable root {}: {err}",
consolidation_config.codex_home.display()
),
}
let consolidation_sandbox_policy = SandboxPolicy::WorkspaceWrite {
writable_roots,
read_only_access: Default::default(),
network_access: false,
exclude_tmpdir_env_var: false,
exclude_slash_tmp: false,
};
if let Err(err) = consolidation_config
.permissions
.sandbox_policy
.set(consolidation_sandbox_policy)
{
warn!("memory phase-2 consolidation sandbox policy was rejected by constraints: {err}");
otel_manager.counter(
metrics::MEMORY_PHASE_TWO_JOBS,
1,
&[("status", "failed_sandbox_policy")],
);
let _ = state_db
.mark_global_phase2_job_failed(
&ownership_token,
"consolidation sandbox policy was rejected by constraints",
phase_two::JOB_RETRY_DELAY_SECONDS,
)
.await;
return false;
}
consolidation_config
};
let latest_memories = match state_db
.list_stage1_outputs_for_global(phase_two::MAX_RAW_MEMORIES_FOR_GLOBAL)
.await
{
Ok(memories) => memories,
Err(err) => {
warn!("state db list_stage1_outputs_for_global failed during consolidation: {err}");
otel_manager.counter(
metrics::MEMORY_PHASE_TWO_JOBS,
1,
&[("status", "failed_load_stage1_outputs")],
);
let _ = state_db
.mark_global_phase2_job_failed(
&ownership_token,
"failed to read stage-1 outputs before global consolidation",
phase_two::JOB_RETRY_DELAY_SECONDS,
)
.await;
return false;
}
};
if !latest_memories.is_empty() {
otel_manager.counter(
metrics::MEMORY_PHASE_TWO_INPUT,
latest_memories.len() as i64,
&[],
);
}
let completion_watermark = completion_watermark(claimed_watermark, &latest_memories);
if let Err(err) = sync_rollout_summaries_from_memories(&root, &latest_memories).await {
warn!("failed syncing local memory artifacts for global consolidation: {err}");
otel_manager.counter(
metrics::MEMORY_PHASE_TWO_JOBS,
1,
&[("status", "failed_sync_artifacts")],
);
let _ = state_db
.mark_global_phase2_job_failed(
&ownership_token,
"failed syncing local memory artifacts",
phase_two::JOB_RETRY_DELAY_SECONDS,
)
.await;
return false;
}
if let Err(err) = rebuild_raw_memories_file_from_memories(&root, &latest_memories).await {
warn!("failed rebuilding raw memories aggregate for global consolidation: {err}");
otel_manager.counter(
metrics::MEMORY_PHASE_TWO_JOBS,
1,
&[("status", "failed_rebuild_raw_memories")],
);
let _ = state_db
.mark_global_phase2_job_failed(
&ownership_token,
"failed rebuilding raw memories aggregate",
phase_two::JOB_RETRY_DELAY_SECONDS,
)
.await;
return false;
}
if latest_memories.is_empty() {
debug!("memory phase-2 has no stage-1 outputs; finalized local memory artifacts");
let _ = state_db
.mark_global_phase2_job_succeeded(&ownership_token, completion_watermark)
.await;
otel_manager.counter(
metrics::MEMORY_PHASE_TWO_JOBS,
1,
&[("status", "succeeded_no_input")],
);
return false;
}
let prompt = build_consolidation_prompt(&root);
let input = vec![UserInput::Text {
text: prompt,
text_elements: vec![],
}];
let source = SessionSource::SubAgent(SubAgentSource::Other(
phase_two::MEMORY_CONSOLIDATION_SUBAGENT_LABEL.to_string(),
));
match session
.services
.agent_control
.spawn_agent(consolidation_config, input, Some(source))
.await
{
Ok(consolidation_agent_id) => {
info!(
"memory phase-2 global consolidation agent started: agent_id={consolidation_agent_id}"
);
otel_manager.counter(
metrics::MEMORY_PHASE_TWO_JOBS,
1,
&[("status", "agent_spawned")],
);
spawn_phase2_completion_task(
session.as_ref(),
ownership_token,
completion_watermark,
consolidation_agent_id,
);
true
}
Err(err) => {
warn!("failed to spawn global memory consolidation agent: {err}");
otel_manager.counter(
metrics::MEMORY_PHASE_TWO_JOBS,
1,
&[("status", "failed_spawn_agent")],
);
let _ = state_db
.mark_global_phase2_job_failed(
&ownership_token,
"failed to spawn consolidation agent",
phase_two::JOB_RETRY_DELAY_SECONDS,
)
.await;
false
}
}
}
#[cfg(test)]
mod tests {
use super::completion_watermark;
use super::run_global_memory_consolidation;
use crate::CodexAuth;
use crate::ThreadManager;
use crate::agent::control::AgentControl;
use crate::codex::Session;
use crate::codex::make_session_and_context;
use crate::config::Config;
use crate::config::test_config;
use crate::memories::memory_root;
use crate::memories::raw_memories_file;
use crate::memories::rollout_summaries_dir;
use chrono::Utc;
use codex_protocol::ThreadId;
use codex_protocol::protocol::AskForApproval;
use codex_protocol::protocol::Op;
use codex_protocol::protocol::SandboxPolicy;
use codex_protocol::protocol::SessionSource;
use codex_state::Phase2JobClaimOutcome;
use codex_state::Stage1Output;
use codex_state::ThreadMetadataBuilder;
use pretty_assertions::assert_eq;
use std::path::PathBuf;
use std::sync::Arc;
use tempfile::TempDir;
struct DispatchHarness {
_codex_home: TempDir,
config: Arc<Config>,
session: Arc<Session>,
manager: ThreadManager,
state_db: Arc<codex_state::StateRuntime>,
}
impl DispatchHarness {
async fn new() -> Self {
let codex_home = tempfile::tempdir().expect("create temp codex home");
let mut config = test_config();
config.codex_home = codex_home.path().to_path_buf();
config.cwd = config.codex_home.clone();
let config = Arc::new(config);
let state_db = codex_state::StateRuntime::init(
config.codex_home.clone(),
config.model_provider_id.clone(),
None,
)
.await
.expect("initialize state db");
let manager = ThreadManager::with_models_provider_and_home_for_tests(
CodexAuth::from_api_key("dummy"),
config.model_provider.clone(),
config.codex_home.clone(),
);
let (mut session, _turn_context) = make_session_and_context().await;
session.services.state_db = Some(Arc::clone(&state_db));
session.services.agent_control = manager.agent_control();
Self {
_codex_home: codex_home,
config,
session: Arc::new(session),
manager,
state_db,
}
}
async fn seed_stage1_output(&self, source_updated_at: i64) {
let thread_id = ThreadId::new();
let mut metadata_builder = ThreadMetadataBuilder::new(
thread_id,
self.config
.codex_home
.join(format!("rollout-{thread_id}.jsonl")),
Utc::now(),
SessionSource::Cli,
);
metadata_builder.cwd = self.config.cwd.clone();
metadata_builder.model_provider = Some(self.config.model_provider_id.clone());
let metadata = metadata_builder.build(&self.config.model_provider_id);
self.state_db
.upsert_thread(&metadata)
.await
.expect("upsert thread metadata");
let claim = self
.state_db
.try_claim_stage1_job(
thread_id,
self.session.conversation_id,
source_updated_at,
3_600,
64,
)
.await
.expect("claim stage-1 job");
let ownership_token = match claim {
codex_state::Stage1JobClaimOutcome::Claimed { ownership_token } => ownership_token,
other => panic!("unexpected stage-1 claim outcome: {other:?}"),
};
assert!(
self.state_db
.mark_stage1_job_succeeded(
thread_id,
&ownership_token,
source_updated_at,
"raw memory",
"rollout summary",
)
.await
.expect("mark stage-1 success"),
"stage-1 success should enqueue global consolidation"
);
}
async fn shutdown_threads(&self) {
self.manager
.remove_and_close_all_threads()
.await
.expect("shutdown spawned threads");
}
fn user_input_ops_count(&self) -> usize {
self.manager
.captured_ops()
.into_iter()
.filter(|(_, op)| matches!(op, Op::UserInput { .. }))
.count()
}
}
#[test]
fn completion_watermark_never_regresses_below_claimed_input_watermark() {
let stage1_output = Stage1Output {
thread_id: ThreadId::new(),
source_updated_at: chrono::DateTime::<Utc>::from_timestamp(123, 0)
.expect("valid source_updated_at timestamp"),
raw_memory: "raw memory".to_string(),
rollout_summary: "rollout summary".to_string(),
cwd: PathBuf::from("/tmp/workspace"),
generated_at: chrono::DateTime::<Utc>::from_timestamp(124, 0)
.expect("valid generated_at timestamp"),
};
let completion = completion_watermark(1_000, &[stage1_output]);
assert_eq!(completion, 1_000);
}
#[tokio::test]
async fn dispatch_reclaims_stale_global_lock_and_starts_consolidation() {
let harness = DispatchHarness::new().await;
harness.seed_stage1_output(100).await;
let stale_claim = harness
.state_db
.try_claim_global_phase2_job(ThreadId::new(), 0)
.await
.expect("claim stale global lock");
assert!(
matches!(stale_claim, Phase2JobClaimOutcome::Claimed { .. }),
"stale lock precondition should be claimed"
);
let scheduled =
run_global_memory_consolidation(&harness.session, Arc::clone(&harness.config)).await;
assert!(
scheduled,
"dispatch should reclaim stale lock and spawn one agent"
);
let running_claim = harness
.state_db
.try_claim_global_phase2_job(ThreadId::new(), 3_600)
.await
.expect("claim while running");
assert_eq!(running_claim, Phase2JobClaimOutcome::SkippedRunning);
let user_input_ops = harness.user_input_ops_count();
assert_eq!(user_input_ops, 1);
let thread_ids = harness.manager.list_thread_ids().await;
assert_eq!(thread_ids.len(), 1);
let subagent = harness
.manager
.get_thread(thread_ids[0])
.await
.expect("get consolidation thread");
let config_snapshot = subagent.config_snapshot().await;
assert_eq!(config_snapshot.approval_policy, AskForApproval::Never);
assert_eq!(config_snapshot.cwd, memory_root(&harness.config.codex_home));
match config_snapshot.sandbox_policy {
SandboxPolicy::WorkspaceWrite { writable_roots, .. } => {
assert!(
writable_roots
.iter()
.any(|root| root.as_path() == harness.config.codex_home.as_path()),
"consolidation subagent should have codex_home as writable root"
);
}
other => panic!("unexpected sandbox policy: {other:?}"),
}
harness.shutdown_threads().await;
}
#[tokio::test]
async fn dispatch_schedules_only_one_agent_while_lock_is_running() {
let harness = DispatchHarness::new().await;
harness.seed_stage1_output(200).await;
let first_run =
run_global_memory_consolidation(&harness.session, Arc::clone(&harness.config)).await;
let second_run =
run_global_memory_consolidation(&harness.session, Arc::clone(&harness.config)).await;
assert!(first_run, "first dispatch should schedule consolidation");
assert!(
!second_run,
"second dispatch should skip while the global lock is running"
);
let user_input_ops = harness.user_input_ops_count();
assert_eq!(user_input_ops, 1);
harness.shutdown_threads().await;
}
#[tokio::test]
async fn dispatch_with_dirty_job_and_no_stage1_outputs_skips_spawn_and_clears_dirty_flag() {
let harness = DispatchHarness::new().await;
harness
.state_db
.enqueue_global_consolidation(999)
.await
.expect("enqueue global consolidation");
let scheduled =
run_global_memory_consolidation(&harness.session, Arc::clone(&harness.config)).await;
assert!(
!scheduled,
"dispatch should not spawn when no stage-1 outputs are available"
);
assert_eq!(harness.user_input_ops_count(), 0);
let claim = harness
.state_db
.try_claim_global_phase2_job(ThreadId::new(), 3_600)
.await
.expect("claim global job after empty dispatch");
assert_eq!(
claim,
Phase2JobClaimOutcome::SkippedNotDirty,
"empty dispatch should finalize global job as up-to-date"
);
harness.shutdown_threads().await;
}
#[tokio::test]
async fn dispatch_with_empty_stage1_outputs_rebuilds_local_artifacts() {
let harness = DispatchHarness::new().await;
let root = memory_root(&harness.config.codex_home);
let summaries_dir = rollout_summaries_dir(&root);
tokio::fs::create_dir_all(&summaries_dir)
.await
.expect("create rollout summaries dir");
let stale_summary_path = summaries_dir.join(format!("{}.md", ThreadId::new()));
tokio::fs::write(&stale_summary_path, "stale summary\n")
.await
.expect("write stale rollout summary");
let raw_memories_path = raw_memories_file(&root);
tokio::fs::write(&raw_memories_path, "stale raw memories\n")
.await
.expect("write stale raw memories");
let memory_index_path = root.join("MEMORY.md");
tokio::fs::write(&memory_index_path, "stale memory index\n")
.await
.expect("write stale memory index");
let memory_summary_path = root.join("memory_summary.md");
tokio::fs::write(&memory_summary_path, "stale memory summary\n")
.await
.expect("write stale memory summary");
let stale_skill_file = root.join("skills/demo/SKILL.md");
tokio::fs::create_dir_all(
stale_skill_file
.parent()
.expect("skills subdirectory parent should exist"),
)
.await
.expect("create stale skills dir");
tokio::fs::write(&stale_skill_file, "stale skill\n")
.await
.expect("write stale skill");
harness
.state_db
.enqueue_global_consolidation(999)
.await
.expect("enqueue global consolidation");
let scheduled =
run_global_memory_consolidation(&harness.session, Arc::clone(&harness.config)).await;
assert!(
!scheduled,
"dispatch should skip subagent spawn when no stage-1 outputs are available"
);
assert!(
!tokio::fs::try_exists(&stale_summary_path)
.await
.expect("check stale summary existence"),
"empty consolidation should prune stale rollout summary files"
);
let raw_memories = tokio::fs::read_to_string(&raw_memories_path)
.await
.expect("read rebuilt raw memories");
assert_eq!(raw_memories, "# Raw Memories\n\nNo raw memories yet.\n");
assert!(
!tokio::fs::try_exists(&memory_index_path)
.await
.expect("check memory index existence"),
"empty consolidation should remove stale MEMORY.md"
);
assert!(
!tokio::fs::try_exists(&memory_summary_path)
.await
.expect("check memory summary existence"),
"empty consolidation should remove stale memory_summary.md"
);
assert!(
!tokio::fs::try_exists(&stale_skill_file)
.await
.expect("check stale skill existence"),
"empty consolidation should remove stale skills artifacts"
);
assert!(
!tokio::fs::try_exists(root.join("skills"))
.await
.expect("check skills dir existence"),
"empty consolidation should remove stale skills directory"
);
harness.shutdown_threads().await;
}
#[tokio::test]
async fn dispatch_marks_job_for_retry_when_spawn_agent_fails() {
let codex_home = tempfile::tempdir().expect("create temp codex home");
let mut config = test_config();
config.codex_home = codex_home.path().to_path_buf();
config.cwd = config.codex_home.clone();
let config = Arc::new(config);
let state_db = codex_state::StateRuntime::init(
config.codex_home.clone(),
config.model_provider_id.clone(),
None,
)
.await
.expect("initialize state db");
let (mut session, _turn_context) = make_session_and_context().await;
session.services.state_db = Some(Arc::clone(&state_db));
session.services.agent_control = AgentControl::default();
let session = Arc::new(session);
let thread_id = ThreadId::new();
let mut metadata_builder = ThreadMetadataBuilder::new(
thread_id,
config.codex_home.join(format!("rollout-{thread_id}.jsonl")),
Utc::now(),
SessionSource::Cli,
);
metadata_builder.cwd = config.cwd.clone();
metadata_builder.model_provider = Some(config.model_provider_id.clone());
let metadata = metadata_builder.build(&config.model_provider_id);
state_db
.upsert_thread(&metadata)
.await
.expect("upsert thread metadata");
let claim = state_db
.try_claim_stage1_job(thread_id, session.conversation_id, 100, 3_600, 64)
.await
.expect("claim stage-1 job");
let ownership_token = match claim {
codex_state::Stage1JobClaimOutcome::Claimed { ownership_token } => ownership_token,
other => panic!("unexpected stage-1 claim outcome: {other:?}"),
};
assert!(
state_db
.mark_stage1_job_succeeded(
thread_id,
&ownership_token,
100,
"raw memory",
"rollout summary",
)
.await
.expect("mark stage-1 success"),
"stage-1 success should enqueue global consolidation"
);
let scheduled = run_global_memory_consolidation(&session, Arc::clone(&config)).await;
assert!(
!scheduled,
"dispatch should return false when consolidation subagent cannot be spawned"
);
let retry_claim = state_db
.try_claim_global_phase2_job(ThreadId::new(), 3_600)
.await
.expect("claim global job after spawn failure");
assert_eq!(
retry_claim,
Phase2JobClaimOutcome::SkippedNotDirty,
"spawn failures should leave the job in retry backoff instead of running"
);
}
}
+6 -11
View File
@@ -4,7 +4,6 @@
//! - Phase 1: select rollouts, extract stage-1 raw memories, persist stage-1 outputs, and enqueue consolidation.
//! - Phase 2: claim a global consolidation lock, materialize consolidation inputs, and dispatch one consolidation agent.
mod dispatch;
mod phase1;
mod phase2;
pub(crate) mod prompts;
@@ -26,10 +25,10 @@ mod artifacts {
/// Phase 1 (startup extraction).
mod phase_one {
/// Default model used for phase 1.
pub(super) const MODEL: &str = "gpt-5.3-codex-spark";
/// Prompt used for phase 1.
pub(super) const PROMPT: &str = include_str!("../../templates/memories/stage_one_system.md");
/// Maximum number of rollout candidates processed per startup pass.
pub(super) const MAX_ROLLOUTS_PER_STARTUP: usize = 8;
/// Concurrency cap for startup memory extraction and consolidation scheduling.
pub(super) const CONCURRENCY_LIMIT: usize = 8;
/// Fallback stage-1 rollout truncation limit (tokens) when model metadata
@@ -44,10 +43,6 @@ mod phase_one {
/// Keeping this below 100% leaves room for system instructions, prompt
/// framing, and model output.
pub(super) const CONTEXT_WINDOW_PERCENT: i64 = 70;
/// Maximum rollout age considered for phase-1 extraction.
pub(super) const MAX_ROLLOUT_AGE_DAYS: i64 = 30;
/// Minimum rollout idle time required before phase-1 extraction.
pub(super) const MIN_ROLLOUT_IDLE_HOURS: i64 = 12;
/// Lease duration (seconds) for phase-1 job ownership.
pub(super) const JOB_LEASE_SECONDS: i64 = 3_600;
/// Backoff delay (seconds) before retrying a failed stage-1 extraction job.
@@ -58,10 +53,8 @@ mod phase_one {
/// Phase 2 (aka `Consolidation`).
mod phase_two {
/// Subagent source label used to identify consolidation tasks.
pub(super) const MEMORY_CONSOLIDATION_SUBAGENT_LABEL: &str = "memory_consolidation";
/// Maximum number of recent raw memories retained for global consolidation.
pub(super) const MAX_RAW_MEMORIES_FOR_GLOBAL: usize = 1_024;
/// Default model used for phase 2.
pub(super) const MODEL: &str = "gpt-5.3-codex";
/// Lease duration (seconds) for phase-2 consolidation job ownership.
pub(super) const JOB_LEASE_SECONDS: i64 = 3_600;
/// Backoff delay (seconds) before retrying a failed phase-2 consolidation
@@ -76,6 +69,8 @@ mod metrics {
pub(super) const MEMORY_PHASE_ONE_JOBS: &str = "codex.memory.phase1";
/// Number of raw memories produced by phase-1 startup extraction.
pub(super) const MEMORY_PHASE_ONE_OUTPUT: &str = "codex.memory.phase1.output";
/// Histogram for aggregate token usage across one phase-1 startup run.
pub(super) const MEMORY_PHASE_ONE_TOKEN_USAGE: &str = "codex.memory.phase1.token_usage";
/// Number of phase-2 startup jobs grouped by status.
pub(super) const MEMORY_PHASE_TWO_JOBS: &str = "codex.memory.phase2";
/// Number of stage-1 memories included in each phase-2 consolidation step.
+213 -65
View File
@@ -2,6 +2,8 @@ use crate::Prompt;
use crate::RolloutRecorder;
use crate::codex::Session;
use crate::codex::TurnContext;
use crate::config::Config;
use crate::config::types::MemoriesConfig;
use crate::error::CodexErr;
use crate::memories::metrics;
use crate::memories::phase_one;
@@ -17,6 +19,7 @@ use codex_protocol::models::ResponseItem;
use codex_protocol::openai_models::ModelInfo;
use codex_protocol::openai_models::ReasoningEffort as ReasoningEffortConfig;
use codex_protocol::protocol::RolloutItem;
use codex_protocol::protocol::TokenUsage;
use codex_utils_sanitizer::redact_secrets;
use futures::StreamExt;
use serde::Deserialize;
@@ -28,7 +31,7 @@ use tracing::info;
use tracing::warn;
#[derive(Clone, Debug)]
pub(in crate::memories) struct Phase1RequestContext {
pub(in crate::memories) struct RequestContext {
pub(in crate::memories) model_info: ModelInfo,
pub(in crate::memories) otel_manager: OtelManager,
pub(in crate::memories) reasoning_effort: Option<ReasoningEffortConfig>,
@@ -36,18 +39,24 @@ pub(in crate::memories) struct Phase1RequestContext {
pub(in crate::memories) turn_metadata_header: Option<String>,
}
struct JobResult {
outcome: JobOutcome,
token_usage: Option<TokenUsage>,
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum PhaseOneJobOutcome {
enum JobOutcome {
SucceededWithOutput,
SucceededNoOutput,
Failed,
}
struct PhaseOneOutcomeCounts {
struct Stats {
claimed: usize,
succeeded_with_output: usize,
succeeded_no_output: usize,
failed: usize,
total_token_usage: Option<TokenUsage>,
}
/// Phase 1 model output payload.
@@ -60,10 +69,9 @@ struct StageOneOutput {
/// Compact summary line used for routing and indexing.
#[serde(rename = "rollout_summary")]
pub(crate) rollout_summary: String,
/// Optional slug accepted from stage-1 output for forward compatibility.
/// This is currently ignored by downstream storage and naming, which remain thread-id based.
/// Optional slug used to derive rollout summary artifact filenames.
#[serde(default, rename = "rollout_slug")]
pub(crate) _rollout_slug: Option<String>,
pub(crate) rollout_slug: Option<String>,
}
/// Runs memory phase 1 in strict step order:
@@ -71,9 +79,9 @@ struct StageOneOutput {
/// 2) build one stage-1 request context
/// 3) run stage-1 extraction jobs in parallel
/// 4) emit metrics and logs
pub(in crate::memories) async fn run(session: &Arc<Session>) {
pub(in crate::memories) async fn run(session: &Arc<Session>, config: &Config) {
// 1. Claim startup job.
let Some(claimed_candidates) = claim_startup_jobs(session).await else {
let Some(claimed_candidates) = claim_startup_jobs(session, &config.memories).await else {
return;
};
if claimed_candidates.is_empty() {
@@ -86,13 +94,13 @@ pub(in crate::memories) async fn run(session: &Arc<Session>) {
}
// 2. Build request.
let stage_one_context = build_request_context(session).await;
let stage_one_context = build_request_context(session, config).await;
// 3. Run the parallel sampling.
let outcomes = run_jobs(session, claimed_candidates, stage_one_context).await;
// 4. Metrics and logs.
let counts = count_outcomes(outcomes);
let counts = aggregate_stats(outcomes);
emit_metrics(session, &counts);
info!(
"memory stage-1 extraction complete: {} job(s) claimed, {} succeeded ({} with output, {} no output), {} failed",
@@ -110,7 +118,7 @@ pub fn output_schema() -> Value {
"type": "object",
"properties": {
"rollout_summary": { "type": "string" },
"rollout_slug": { "type": "string" },
"rollout_slug": { "type": ["string", "null"] },
"raw_memory": { "type": "string" }
},
"required": ["rollout_summary", "rollout_slug", "raw_memory"],
@@ -118,22 +126,26 @@ pub fn output_schema() -> Value {
})
}
impl Phase1RequestContext {
impl RequestContext {
pub(in crate::memories) fn from_turn_context(
turn_context: &TurnContext,
turn_metadata_header: Option<String>,
model_info: ModelInfo,
) -> Self {
Self {
model_info: turn_context.model_info.clone(),
model_info,
turn_metadata_header,
otel_manager: turn_context.otel_manager.clone(),
reasoning_effort: turn_context.reasoning_effort,
reasoning_summary: turn_context.reasoning_summary,
turn_metadata_header,
}
}
}
async fn claim_startup_jobs(session: &Arc<Session>) -> Option<Vec<codex_state::Stage1JobClaim>> {
async fn claim_startup_jobs(
session: &Arc<Session>,
memories_config: &MemoriesConfig,
) -> Option<Vec<codex_state::Stage1JobClaim>> {
let Some(state_db) = session.services.state_db.as_deref() else {
// This should not happen.
warn!("state db unavailable while claiming phase-1 startup jobs; skipping");
@@ -150,9 +162,9 @@ async fn claim_startup_jobs(session: &Arc<Session>) -> Option<Vec<codex_state::S
session.conversation_id,
codex_state::Stage1StartupClaimParams {
scan_limit: phase_one::THREAD_SCAN_LIMIT,
max_claimed: phase_one::MAX_ROLLOUTS_PER_STARTUP,
max_age_days: phase_one::MAX_ROLLOUT_AGE_DAYS,
min_rollout_idle_hours: phase_one::MIN_ROLLOUT_IDLE_HOURS,
max_claimed: memories_config.max_rollouts_per_startup,
max_age_days: memories_config.max_rollout_age_days,
min_rollout_idle_hours: memories_config.min_rollout_idle_hours,
allowed_sources: allowed_sources.as_slice(),
lease_seconds: phase_one::JOB_LEASE_SECONDS,
},
@@ -172,19 +184,30 @@ async fn claim_startup_jobs(session: &Arc<Session>) -> Option<Vec<codex_state::S
}
}
async fn build_request_context(session: &Arc<Session>) -> Phase1RequestContext {
async fn build_request_context(session: &Arc<Session>, config: &Config) -> RequestContext {
let model_name = config
.memories
.phase_1_model
.clone()
.unwrap_or(phase_one::MODEL.to_string());
let model = session
.services
.models_manager
.get_model_info(&model_name, config)
.await;
let turn_context = session.new_default_turn().await;
Phase1RequestContext::from_turn_context(
RequestContext::from_turn_context(
turn_context.as_ref(),
turn_context.resolve_turn_metadata_header().await,
turn_context.turn_metadata_state.current_header_value(),
model,
)
}
async fn run_jobs(
session: &Arc<Session>,
claimed_candidates: Vec<codex_state::Stage1JobClaim>,
stage_one_context: Phase1RequestContext,
) -> Vec<PhaseOneJobOutcome> {
stage_one_context: RequestContext,
) -> Vec<JobResult> {
futures::stream::iter(claimed_candidates.into_iter())
.map(|claim| {
let session = Arc::clone(session);
@@ -202,10 +225,10 @@ mod job {
pub(in crate::memories) async fn run(
session: &Session,
claim: codex_state::Stage1JobClaim,
stage_one_context: &Phase1RequestContext,
) -> PhaseOneJobOutcome {
stage_one_context: &RequestContext,
) -> JobResult {
let thread = claim.thread;
let stage_one_output = match sample(
let (stage_one_output, token_usage) = match sample(
session,
&thread.rollout_path,
&thread.cwd,
@@ -222,23 +245,33 @@ mod job {
&reason.to_string(),
)
.await;
return PhaseOneJobOutcome::Failed;
return JobResult {
outcome: JobOutcome::Failed,
token_usage: None,
};
}
};
if stage_one_output.raw_memory.is_empty() || stage_one_output.rollout_summary.is_empty() {
return result::no_output(session, thread.id, &claim.ownership_token).await;
return JobResult {
outcome: result::no_output(session, thread.id, &claim.ownership_token).await,
token_usage,
};
}
result::success(
session,
thread.id,
&claim.ownership_token,
thread.updated_at.timestamp(),
&stage_one_output.raw_memory,
&stage_one_output.rollout_summary,
)
.await
JobResult {
outcome: result::success(
session,
thread.id,
&claim.ownership_token,
thread.updated_at.timestamp(),
&stage_one_output.raw_memory,
&stage_one_output.rollout_summary,
stage_one_output.rollout_slug.as_deref(),
)
.await,
token_usage,
}
}
/// Extract the rollout and perform the actual sampling.
@@ -246,8 +279,8 @@ mod job {
session: &Session,
rollout_path: &Path,
rollout_cwd: &Path,
stage_one_context: &Phase1RequestContext,
) -> anyhow::Result<StageOneOutput> {
stage_one_context: &RequestContext,
) -> anyhow::Result<(StageOneOutput, Option<TokenUsage>)> {
let (rollout_items, _, _) = RolloutRecorder::load_rollout_items(rollout_path).await?;
let rollout_contents = serialize_filtered_rollout_response_items(&rollout_items)?;
@@ -290,6 +323,7 @@ mod job {
// TODO(jif) we should have a shared helper somewhere for this.
// Unwrap the stream.
let mut result = String::new();
let mut token_usage = None;
while let Some(message) = stream.next().await.transpose()? {
match message {
ResponseEvent::OutputTextDelta(delta) => result.push_str(&delta),
@@ -301,7 +335,12 @@ mod job {
result.push_str(&text);
}
}
ResponseEvent::Completed { .. } => break,
ResponseEvent::Completed {
token_usage: usage, ..
} => {
token_usage = usage;
break;
}
_ => {}
}
}
@@ -309,8 +348,9 @@ mod job {
let mut output: StageOneOutput = serde_json::from_str(&result)?;
output.raw_memory = redact_secrets(output.raw_memory);
output.rollout_summary = redact_secrets(output.rollout_summary);
output.rollout_slug = output.rollout_slug.map(redact_secrets);
Ok(output)
Ok((output, token_usage))
}
mod result {
@@ -339,9 +379,9 @@ mod job {
session: &Session,
thread_id: codex_protocol::ThreadId,
ownership_token: &str,
) -> PhaseOneJobOutcome {
) -> JobOutcome {
let Some(state_db) = session.services.state_db.as_deref() else {
return PhaseOneJobOutcome::Failed;
return JobOutcome::Failed;
};
if state_db
@@ -349,9 +389,9 @@ mod job {
.await
.unwrap_or(false)
{
PhaseOneJobOutcome::SucceededNoOutput
JobOutcome::SucceededNoOutput
} else {
PhaseOneJobOutcome::Failed
JobOutcome::Failed
}
}
@@ -362,9 +402,10 @@ mod job {
source_updated_at: i64,
raw_memory: &str,
rollout_summary: &str,
) -> PhaseOneJobOutcome {
rollout_slug: Option<&str>,
) -> JobOutcome {
let Some(state_db) = session.services.state_db.as_deref() else {
return PhaseOneJobOutcome::Failed;
return JobOutcome::Failed;
};
if state_db
@@ -374,13 +415,14 @@ mod job {
source_updated_at,
raw_memory,
rollout_summary,
rollout_slug,
)
.await
.unwrap_or(false)
{
PhaseOneJobOutcome::SucceededWithOutput
JobOutcome::SucceededWithOutput
} else {
PhaseOneJobOutcome::Failed
JobOutcome::Failed
}
}
}
@@ -407,29 +449,37 @@ mod job {
}
}
fn count_outcomes(outcomes: Vec<PhaseOneJobOutcome>) -> PhaseOneOutcomeCounts {
let succeeded_with_output = outcomes
.iter()
.filter(|outcome| matches!(outcome, PhaseOneJobOutcome::SucceededWithOutput))
.count();
let succeeded_no_output = outcomes
.iter()
.filter(|outcome| matches!(outcome, PhaseOneJobOutcome::SucceededNoOutput))
.count();
let failed = outcomes
.iter()
.filter(|outcome| matches!(outcome, PhaseOneJobOutcome::Failed))
.count();
fn aggregate_stats(outcomes: Vec<JobResult>) -> Stats {
let claimed = outcomes.len();
let mut succeeded_with_output = 0;
let mut succeeded_no_output = 0;
let mut failed = 0;
let mut total_token_usage = TokenUsage::default();
let mut has_token_usage = false;
PhaseOneOutcomeCounts {
claimed: outcomes.len(),
for outcome in outcomes {
match outcome.outcome {
JobOutcome::SucceededWithOutput => succeeded_with_output += 1,
JobOutcome::SucceededNoOutput => succeeded_no_output += 1,
JobOutcome::Failed => failed += 1,
}
if let Some(token_usage) = outcome.token_usage {
total_token_usage.add_assign(&token_usage);
has_token_usage = true;
}
}
Stats {
claimed,
succeeded_with_output,
succeeded_no_output,
failed,
total_token_usage: has_token_usage.then_some(total_token_usage),
}
}
fn emit_metrics(session: &Session, counts: &PhaseOneOutcomeCounts) {
fn emit_metrics(session: &Session, counts: &Stats) {
if counts.claimed > 0 {
session.services.otel_manager.counter(
metrics::MEMORY_PHASE_ONE_JOBS,
@@ -463,4 +513,102 @@ fn emit_metrics(session: &Session, counts: &PhaseOneOutcomeCounts) {
&[("status", "failed")],
);
}
if let Some(token_usage) = counts.total_token_usage.as_ref() {
session.services.otel_manager.histogram(
metrics::MEMORY_PHASE_ONE_TOKEN_USAGE,
token_usage.total_tokens.max(0),
&[("token_type", "total")],
);
session.services.otel_manager.histogram(
metrics::MEMORY_PHASE_ONE_TOKEN_USAGE,
token_usage.input_tokens.max(0),
&[("token_type", "input")],
);
session.services.otel_manager.histogram(
metrics::MEMORY_PHASE_ONE_TOKEN_USAGE,
token_usage.cached_input(),
&[("token_type", "cached_input")],
);
session.services.otel_manager.histogram(
metrics::MEMORY_PHASE_ONE_TOKEN_USAGE,
token_usage.output_tokens.max(0),
&[("token_type", "output")],
);
session.services.otel_manager.histogram(
metrics::MEMORY_PHASE_ONE_TOKEN_USAGE,
token_usage.reasoning_output_tokens.max(0),
&[("token_type", "reasoning_output")],
);
}
}
#[cfg(test)]
mod tests {
use super::JobOutcome;
use super::JobResult;
use super::aggregate_stats;
use codex_protocol::protocol::TokenUsage;
use pretty_assertions::assert_eq;
#[test]
fn count_outcomes_sums_token_usage_across_all_jobs() {
let counts = aggregate_stats(vec![
JobResult {
outcome: JobOutcome::SucceededWithOutput,
token_usage: Some(TokenUsage {
input_tokens: 10,
cached_input_tokens: 2,
output_tokens: 3,
reasoning_output_tokens: 1,
total_tokens: 13,
}),
},
JobResult {
outcome: JobOutcome::SucceededNoOutput,
token_usage: Some(TokenUsage {
input_tokens: 7,
cached_input_tokens: 1,
output_tokens: 2,
reasoning_output_tokens: 0,
total_tokens: 9,
}),
},
JobResult {
outcome: JobOutcome::Failed,
token_usage: None,
},
]);
assert_eq!(counts.claimed, 3);
assert_eq!(counts.succeeded_with_output, 1);
assert_eq!(counts.succeeded_no_output, 1);
assert_eq!(counts.failed, 1);
assert_eq!(
counts.total_token_usage,
Some(TokenUsage {
input_tokens: 17,
cached_input_tokens: 3,
output_tokens: 5,
reasoning_output_tokens: 1,
total_tokens: 22,
})
);
}
#[test]
fn count_outcomes_keeps_usage_empty_when_no_job_reports_it() {
let counts = aggregate_stats(vec![
JobResult {
outcome: JobOutcome::SucceededWithOutput,
token_usage: None,
},
JobResult {
outcome: JobOutcome::Failed,
token_usage: None,
},
]);
assert_eq!(counts.claimed, 2);
assert_eq!(counts.total_token_usage, None);
}
}
+341 -368
View File
@@ -1,136 +1,360 @@
use crate::agent::AgentStatus;
use crate::agent::status::is_final as is_final_agent_status;
use crate::codex::Session;
use crate::config::Config;
use crate::memories::memory_root;
use crate::memories::metrics;
use crate::memories::phase_two;
use crate::memories::prompts::build_consolidation_prompt;
use crate::memories::storage::rebuild_raw_memories_file_from_memories;
use crate::memories::storage::sync_rollout_summaries_from_memories;
use codex_config::Constrained;
use codex_protocol::ThreadId;
use codex_protocol::protocol::AskForApproval;
use codex_protocol::protocol::SandboxPolicy;
use codex_protocol::protocol::SessionSource;
use codex_protocol::protocol::SubAgentSource;
use codex_protocol::user_input::UserInput;
use codex_state::StateRuntime;
use codex_utils_absolute_path::AbsolutePathBuf;
use std::sync::Arc;
use std::time::Duration;
use tokio::sync::watch;
use tracing::debug;
use tracing::info;
use tracing::warn;
pub(in crate::memories) fn spawn_phase2_completion_task(
session: &Session,
ownership_token: String,
completion_watermark: i64,
consolidation_agent_id: ThreadId,
) {
let state_db = session.services.state_db.clone();
let agent_control = session.services.agent_control.clone();
let otel_manager = session.services.otel_manager.clone();
tokio::spawn(async move {
let Some(state_db) = state_db else {
return;
};
let status_rx = match agent_control.subscribe_status(consolidation_agent_id).await {
Ok(status_rx) => status_rx,
Err(err) => {
warn!(
"failed to subscribe to global memory consolidation agent {consolidation_agent_id}: {err}"
);
otel_manager.counter(
metrics::MEMORY_PHASE_TWO_JOBS,
1,
&[("status", "failed_subscribe_status")],
);
mark_phase2_failed_with_recovery(
state_db.as_ref(),
&ownership_token,
"failed to subscribe to consolidation agent status",
)
.await;
return;
}
};
let final_status = run_phase2_completion_task(
Arc::clone(&state_db),
ownership_token,
completion_watermark,
consolidation_agent_id,
status_rx,
)
.await;
if matches!(final_status, AgentStatus::Shutdown | AgentStatus::NotFound) {
otel_manager.counter(
metrics::MEMORY_PHASE_TWO_JOBS,
1,
&[("status", "failed_agent_unavailable")],
);
return;
}
if is_phase2_success(&final_status) {
otel_manager.counter(
metrics::MEMORY_PHASE_TWO_JOBS,
1,
&[("status", "succeeded")],
);
} else {
otel_manager.counter(metrics::MEMORY_PHASE_TWO_JOBS, 1, &[("status", "failed")]);
}
tokio::spawn(async move {
if let Err(err) = agent_control.shutdown_agent(consolidation_agent_id).await {
warn!(
"failed to auto-close global memory consolidation agent {consolidation_agent_id}: {err}"
);
}
});
});
#[derive(Debug, Clone, Default)]
struct Claim {
token: String,
watermark: i64,
}
async fn run_phase2_completion_task(
state_db: Arc<codex_state::StateRuntime>,
ownership_token: String,
completion_watermark: i64,
consolidation_agent_id: ThreadId,
mut status_rx: watch::Receiver<AgentStatus>,
) -> AgentStatus {
let final_status = {
#[derive(Debug, Clone, Default)]
struct Counters {
input: i64,
}
/// Runs memory phase 2 (aka consolidation) in strict order. The method represents the linear
/// flow of the consolidation phase.
pub(super) async fn run(session: &Arc<Session>, config: Arc<Config>) {
let Some(db) = session.services.state_db.as_deref() else {
// This should not happen.
return;
};
let root = memory_root(&config.codex_home);
let max_raw_memories = config.memories.max_raw_memories_for_global;
// 1. Claim the job.
let claim = match job::claim(session, db).await {
Ok(claim) => claim,
Err(e) => {
session.services.otel_manager.counter(
metrics::MEMORY_PHASE_TWO_JOBS,
1,
&[("status", e)],
);
return;
}
};
// 2. Get the config for the agent
let Some(agent_config) = agent::get_config(config.clone()) else {
// If we can't get the config, we can't consolidate.
tracing::error!("failed to get agent config");
job::failed(session, db, &claim, "failed_sandbox_policy").await;
return;
};
// 3. Query the memories
let raw_memories = match db.list_stage1_outputs_for_global(max_raw_memories).await {
Ok(memories) => memories,
Err(err) => {
tracing::error!("failed to list stage1 outputs from global: {}", err);
job::failed(session, db, &claim, "failed_load_stage1_outputs").await;
return;
}
};
let new_watermark = get_watermark(claim.watermark, &raw_memories);
// 4. Update the file system by syncing the raw memories with the one extracted from DB at
// step 3
// [`rollout_summaries/`]
if let Err(err) =
sync_rollout_summaries_from_memories(&root, &raw_memories, max_raw_memories).await
{
tracing::error!("failed syncing local memory artifacts for global consolidation: {err}");
job::failed(session, db, &claim, "failed_sync_artifacts").await;
return;
}
// [`raw_memories.md`]
if let Err(err) =
rebuild_raw_memories_file_from_memories(&root, &raw_memories, max_raw_memories).await
{
tracing::error!("failed syncing local memory artifacts for global consolidation: {err}");
job::failed(session, db, &claim, "failed_rebuild_raw_memories").await;
return;
}
if raw_memories.is_empty() {
// We check only after sync of the file system.
job::succeed(session, db, &claim, new_watermark, "succeeded_no_input").await;
return;
}
// 5. Spawn the agent
let prompt = agent::get_prompt(config);
let source = SessionSource::SubAgent(SubAgentSource::MemoryConsolidation);
let thread_id = match session
.services
.agent_control
.spawn_agent(agent_config, prompt, Some(source))
.await
{
Ok(thread_id) => thread_id,
Err(err) => {
tracing::error!("failed to spawn global memory consolidation agent: {err}");
job::failed(session, db, &claim, "failed_spawn_agent").await;
return;
}
};
// 6. Spawn the agent handler.
agent::handle(session, claim, new_watermark, thread_id);
// 7. Metrics and logs.
let counters = Counters {
input: raw_memories.len() as i64,
};
emit_metrics(session, counters);
}
mod job {
use super::*;
pub(super) async fn claim(
session: &Arc<Session>,
db: &StateRuntime,
) -> Result<Claim, &'static str> {
let otel_manager = &session.services.otel_manager;
let claim = db
.try_claim_global_phase2_job(session.conversation_id, phase_two::JOB_LEASE_SECONDS)
.await
.map_err(|e| {
tracing::error!("failed to claim job: {}", e);
"failed_claim"
})?;
let (token, watermark) = match claim {
codex_state::Phase2JobClaimOutcome::Claimed {
ownership_token,
input_watermark,
} => {
otel_manager.counter(metrics::MEMORY_PHASE_TWO_JOBS, 1, &[("status", "claimed")]);
(ownership_token, input_watermark)
}
codex_state::Phase2JobClaimOutcome::SkippedNotDirty => return Err("skipped_not_dirty"),
codex_state::Phase2JobClaimOutcome::SkippedRunning => return Err("skipped_running"),
};
Ok(Claim { token, watermark })
}
pub(super) async fn failed(
session: &Arc<Session>,
db: &StateRuntime,
claim: &Claim,
reason: &'static str,
) {
session.services.otel_manager.counter(
metrics::MEMORY_PHASE_TWO_JOBS,
1,
&[("status", reason)],
);
if matches!(
db.mark_global_phase2_job_failed(
&claim.token,
reason,
phase_two::JOB_RETRY_DELAY_SECONDS,
)
.await,
Ok(false)
) {
let _ = db
.mark_global_phase2_job_failed_if_unowned(
&claim.token,
reason,
phase_two::JOB_RETRY_DELAY_SECONDS,
)
.await;
}
}
pub(super) async fn succeed(
session: &Arc<Session>,
db: &StateRuntime,
claim: &Claim,
completion_watermark: i64,
reason: &'static str,
) {
session.services.otel_manager.counter(
metrics::MEMORY_PHASE_TWO_JOBS,
1,
&[("status", reason)],
);
let _ = db
.mark_global_phase2_job_succeeded(&claim.token, completion_watermark)
.await;
}
}
mod agent {
use super::*;
pub(super) fn get_config(config: Arc<Config>) -> Option<Config> {
let root = memory_root(&config.codex_home);
let mut agent_config = config.as_ref().clone();
agent_config.cwd = root;
// Approval policy
agent_config.permissions.approval_policy = Constrained::allow_only(AskForApproval::Never);
// Sandbox policy
let mut writable_roots = Vec::new();
match AbsolutePathBuf::from_absolute_path(agent_config.codex_home.clone()) {
Ok(codex_home) => writable_roots.push(codex_home),
Err(err) => warn!(
"memory phase-2 consolidation could not add codex_home writable root {}: {err}",
agent_config.codex_home.display()
),
}
// The consolidation agent only needs local codex_home write access and no network.
let consolidation_sandbox_policy = SandboxPolicy::WorkspaceWrite {
writable_roots,
read_only_access: Default::default(),
network_access: false,
exclude_tmpdir_env_var: false,
exclude_slash_tmp: false,
};
agent_config
.permissions
.sandbox_policy
.set(consolidation_sandbox_policy)
.ok()?;
agent_config.model = Some(
config
.memories
.phase_2_model
.clone()
.unwrap_or(phase_two::MODEL.to_string()),
);
Some(agent_config)
}
pub(super) fn get_prompt(config: Arc<Config>) -> Vec<UserInput> {
let root = memory_root(&config.codex_home);
let prompt = build_consolidation_prompt(&root);
vec![UserInput::Text {
text: prompt,
text_elements: vec![],
}]
}
/// Handle the agent while it is running.
pub(super) fn handle(
session: &Arc<Session>,
claim: Claim,
new_watermark: i64,
thread_id: ThreadId,
) {
let Some(db) = session.services.state_db.clone() else {
return;
};
let session = session.clone();
tokio::spawn(async move {
let agent_control = session.services.agent_control.clone();
// TODO(jif) we might have a very small race here.
let rx = match agent_control.subscribe_status(thread_id).await {
Ok(rx) => rx,
Err(err) => {
tracing::error!("agent_control.subscribe_status failed: {err:?}");
job::failed(&session, &db, &claim, "failed_subscribe_status").await;
return;
}
};
// Loop the agent until we have the final status.
let final_status = loop_agent(
db.clone(),
claim.token.clone(),
new_watermark,
thread_id,
rx,
)
.await;
if matches!(final_status, AgentStatus::Completed(_)) {
job::succeed(&session, &db, &claim, new_watermark, "succeeded").await;
} else {
job::failed(&session, &db, &claim, "failed_agent").await;
}
// Fire and forget close of the agent.
if !matches!(final_status, AgentStatus::Shutdown | AgentStatus::NotFound) {
tokio::spawn(async move {
if let Err(err) = agent_control.shutdown_agent(thread_id).await {
warn!(
"failed to auto-close global memory consolidation agent {thread_id}: {err}"
);
}
});
} else {
tracing::warn!("The agent was already gone");
}
});
}
async fn loop_agent(
db: Arc<StateRuntime>,
token: String,
_new_watermark: i64,
thread_id: ThreadId,
mut rx: watch::Receiver<AgentStatus>,
) -> AgentStatus {
let mut heartbeat_interval =
tokio::time::interval(Duration::from_secs(phase_two::JOB_HEARTBEAT_SECONDS));
heartbeat_interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
loop {
let status = status_rx.borrow().clone();
let status = rx.borrow().clone();
if is_final_agent_status(&status) {
break status;
}
tokio::select! {
changed = status_rx.changed() => {
if changed.is_err() {
warn!(
"lost status updates for global memory consolidation agent {consolidation_agent_id}"
update = rx.changed() => {
if update.is_err() {
tracing::warn!(
"lost status updates for global memory consolidation agent {thread_id}"
);
break status;
}
}
_ = heartbeat_interval.tick() => {
match state_db
match db
.heartbeat_global_phase2_job(
&ownership_token,
&token,
phase_two::JOB_LEASE_SECONDS,
)
.await
{
Ok(true) => {}
Ok(false) => {
warn!(
"memory phase-2 heartbeat lost global ownership; finalizing as failure"
);
break AgentStatus::Errored(
"lost global phase-2 ownership during heartbeat".to_string(),
);
}
Err(err) => {
warn!(
"state db heartbeat_global_phase2_job failed during memories startup: {err}"
);
break AgentStatus::Errored(format!(
"phase-2 heartbeat update failed: {err}"
));
@@ -139,281 +363,30 @@ async fn run_phase2_completion_task(
}
}
}
};
}
}
let phase2_success = is_phase2_success(&final_status);
info!(
"memory phase-2 global consolidation complete: agent_id={consolidation_agent_id} success={phase2_success} final_status={final_status:?}"
pub(super) fn get_watermark(
claimed_watermark: i64,
latest_memories: &[codex_state::Stage1Output],
) -> i64 {
latest_memories
.iter()
.map(|memory| memory.source_updated_at.timestamp())
.max()
.unwrap_or(claimed_watermark)
.max(claimed_watermark) // todo double check the claimed here.
}
fn emit_metrics(session: &Arc<Session>, counters: Counters) {
let otel = session.services.otel_manager.clone();
if counters.input > 0 {
otel.counter(metrics::MEMORY_PHASE_TWO_INPUT, counters.input, &[]);
}
otel.counter(
metrics::MEMORY_PHASE_TWO_JOBS,
1,
&[("status", "agent_spawned")],
);
if phase2_success {
match state_db
.mark_global_phase2_job_succeeded(&ownership_token, completion_watermark)
.await
{
Ok(true) => {}
Ok(false) => {
debug!(
"memory phase-2 success finalization skipped after global ownership changed"
);
}
Err(err) => {
warn!(
"state db mark_global_phase2_job_succeeded failed during memories startup: {err}"
);
}
}
return final_status;
}
let failure_reason = phase2_failure_reason(&final_status);
mark_phase2_failed_with_recovery(state_db.as_ref(), &ownership_token, &failure_reason).await;
warn!(
"memory phase-2 global consolidation agent finished with non-success status: agent_id={consolidation_agent_id} final_status={final_status:?}"
);
final_status
}
async fn mark_phase2_failed_with_recovery(
state_db: &codex_state::StateRuntime,
ownership_token: &str,
failure_reason: &str,
) {
match state_db
.mark_global_phase2_job_failed(
ownership_token,
failure_reason,
phase_two::JOB_RETRY_DELAY_SECONDS,
)
.await
{
Ok(true) => {}
Ok(false) => match state_db
.mark_global_phase2_job_failed_if_unowned(
ownership_token,
failure_reason,
phase_two::JOB_RETRY_DELAY_SECONDS,
)
.await
{
Ok(true) => {
debug!(
"memory phase-2 failure finalization applied fallback update for unowned running job"
);
}
Ok(false) => {
debug!(
"memory phase-2 failure finalization skipped after global ownership changed"
);
}
Err(err) => {
warn!(
"state db mark_global_phase2_job_failed_if_unowned failed during memories startup: {err}"
);
}
},
Err(err) => {
warn!("state db mark_global_phase2_job_failed failed during memories startup: {err}");
}
}
}
fn is_phase2_success(final_status: &AgentStatus) -> bool {
matches!(final_status, AgentStatus::Completed(_))
}
fn phase2_failure_reason(final_status: &AgentStatus) -> String {
format!("consolidation agent finished with status {final_status:?}")
}
#[cfg(test)]
mod tests {
use super::is_phase2_success;
use super::phase2_failure_reason;
use super::run_phase2_completion_task;
use crate::agent::AgentStatus;
use codex_protocol::ThreadId;
use codex_state::Phase2JobClaimOutcome;
use pretty_assertions::assert_eq;
use std::sync::Arc;
#[test]
fn phase2_success_only_for_completed_status() {
assert!(is_phase2_success(&AgentStatus::Completed(None)));
assert!(!is_phase2_success(&AgentStatus::Running));
assert!(!is_phase2_success(&AgentStatus::Errored(
"oops".to_string()
)));
}
#[test]
fn phase2_failure_reason_includes_status() {
let status = AgentStatus::Errored("boom".to_string());
let reason = phase2_failure_reason(&status);
assert!(reason.contains("consolidation agent finished with status"));
assert!(reason.contains("boom"));
}
#[tokio::test]
async fn phase2_completion_marks_succeeded_for_completed_status() {
let codex_home = tempfile::tempdir().expect("create temp codex home");
let state_db = Arc::new(
codex_state::StateRuntime::init(
codex_home.path().to_path_buf(),
"test-provider".to_string(),
None,
)
.await
.expect("initialize state runtime"),
);
let owner = ThreadId::new();
state_db
.enqueue_global_consolidation(123)
.await
.expect("enqueue global consolidation");
let claim = state_db
.try_claim_global_phase2_job(owner, 3_600)
.await
.expect("claim global phase-2 job");
let ownership_token = match claim {
Phase2JobClaimOutcome::Claimed {
ownership_token, ..
} => ownership_token,
other => panic!("unexpected phase-2 claim outcome: {other:?}"),
};
let (_status_tx, status_rx) = tokio::sync::watch::channel(AgentStatus::Completed(None));
run_phase2_completion_task(
Arc::clone(&state_db),
ownership_token.clone(),
123,
ThreadId::new(),
status_rx,
)
.await;
let up_to_date_claim = state_db
.try_claim_global_phase2_job(ThreadId::new(), 3_600)
.await
.expect("claim up-to-date global job");
assert_eq!(up_to_date_claim, Phase2JobClaimOutcome::SkippedNotDirty);
state_db
.enqueue_global_consolidation(124)
.await
.expect("enqueue advanced consolidation watermark");
let rerun_claim = state_db
.try_claim_global_phase2_job(ThreadId::new(), 3_600)
.await
.expect("claim rerun global job");
assert!(
matches!(rerun_claim, Phase2JobClaimOutcome::Claimed { .. }),
"advanced watermark should be claimable after success finalization"
);
}
#[tokio::test]
async fn phase2_completion_marks_failed_when_status_updates_are_lost() {
let codex_home = tempfile::tempdir().expect("create temp codex home");
let state_db = Arc::new(
codex_state::StateRuntime::init(
codex_home.path().to_path_buf(),
"test-provider".to_string(),
None,
)
.await
.expect("initialize state runtime"),
);
state_db
.enqueue_global_consolidation(456)
.await
.expect("enqueue global consolidation");
let claim = state_db
.try_claim_global_phase2_job(ThreadId::new(), 3_600)
.await
.expect("claim global phase-2 job");
let ownership_token = match claim {
Phase2JobClaimOutcome::Claimed {
ownership_token, ..
} => ownership_token,
other => panic!("unexpected phase-2 claim outcome: {other:?}"),
};
let (status_tx, status_rx) = tokio::sync::watch::channel(AgentStatus::Running);
drop(status_tx);
run_phase2_completion_task(
Arc::clone(&state_db),
ownership_token,
456,
ThreadId::new(),
status_rx,
)
.await;
let claim = state_db
.try_claim_global_phase2_job(ThreadId::new(), 3_600)
.await
.expect("claim after failure finalization");
assert_eq!(
claim,
Phase2JobClaimOutcome::SkippedNotDirty,
"failure finalization should leave global job in retry-backoff, not running ownership"
);
}
#[tokio::test]
async fn phase2_completion_heartbeat_loss_does_not_steal_active_other_owner() {
let codex_home = tempfile::tempdir().expect("create temp codex home");
let state_db = Arc::new(
codex_state::StateRuntime::init(
codex_home.path().to_path_buf(),
"test-provider".to_string(),
None,
)
.await
.expect("initialize state runtime"),
);
state_db
.enqueue_global_consolidation(789)
.await
.expect("enqueue global consolidation");
let claim = state_db
.try_claim_global_phase2_job(ThreadId::new(), 3_600)
.await
.expect("claim global phase-2 job");
let claimed_token = match claim {
Phase2JobClaimOutcome::Claimed {
ownership_token, ..
} => ownership_token,
other => panic!("unexpected phase-2 claim outcome: {other:?}"),
};
let (_status_tx, status_rx) = tokio::sync::watch::channel(AgentStatus::Running);
run_phase2_completion_task(
Arc::clone(&state_db),
"non-owner-token".to_string(),
789,
ThreadId::new(),
status_rx,
)
.await;
let claim = state_db
.try_claim_global_phase2_job(ThreadId::new(), 3_600)
.await
.expect("claim after heartbeat ownership loss");
assert_eq!(
claim,
Phase2JobClaimOutcome::SkippedRunning,
"heartbeat ownership-loss handling should not steal a live owner lease"
);
assert_eq!(
state_db
.mark_global_phase2_job_succeeded(claimed_token.as_str(), 789)
.await
.expect("mark original owner success"),
true,
"the original owner should still be able to finalize"
);
}
}
+3 -2
View File
@@ -2,6 +2,7 @@ use crate::codex::Session;
use crate::config::Config;
use crate::features::Feature;
use crate::memories::phase1;
use crate::memories::phase2;
use codex_protocol::protocol::SessionSource;
use std::sync::Arc;
use tracing::warn;
@@ -34,8 +35,8 @@ pub(crate) fn start_memories_startup_task(
};
// Run phase 1.
phase1::run(&session).await;
phase1::run(&session, &config).await;
// Run phase 2.
crate::memories::dispatch::run_global_memory_consolidation(&session, config).await;
phase2::run(&session, config).await;
});
}
+96 -12
View File
@@ -5,7 +5,6 @@ use std::path::Path;
use tracing::warn;
use crate::memories::ensure_layout;
use crate::memories::phase_two;
use crate::memories::raw_memories_file;
use crate::memories::rollout_summaries_dir;
@@ -15,25 +14,27 @@ use crate::memories::rollout_summaries_dir;
pub(super) async fn rebuild_raw_memories_file_from_memories(
root: &Path,
memories: &[Stage1Output],
max_raw_memories_for_global: usize,
) -> std::io::Result<()> {
ensure_layout(root).await?;
rebuild_raw_memories_file(root, memories).await
rebuild_raw_memories_file(root, memories, max_raw_memories_for_global).await
}
/// Syncs canonical rollout summary files from DB-backed stage-1 output rows.
pub(super) async fn sync_rollout_summaries_from_memories(
root: &Path,
memories: &[Stage1Output],
max_raw_memories_for_global: usize,
) -> std::io::Result<()> {
ensure_layout(root).await?;
let retained = memories
.iter()
.take(phase_two::MAX_RAW_MEMORIES_FOR_GLOBAL)
.take(max_raw_memories_for_global)
.collect::<Vec<_>>();
let keep = retained
.iter()
.map(|memory| memory.thread_id.to_string())
.map(|memory| rollout_summary_file_stem(memory))
.collect::<BTreeSet<_>>();
prune_rollout_summaries(root, &keep).await?;
@@ -62,10 +63,14 @@ pub(super) async fn sync_rollout_summaries_from_memories(
Ok(())
}
async fn rebuild_raw_memories_file(root: &Path, memories: &[Stage1Output]) -> std::io::Result<()> {
async fn rebuild_raw_memories_file(
root: &Path,
memories: &[Stage1Output],
max_raw_memories_for_global: usize,
) -> std::io::Result<()> {
let retained = memories
.iter()
.take(phase_two::MAX_RAW_MEMORIES_FOR_GLOBAL)
.take(max_raw_memories_for_global)
.collect::<Vec<_>>();
let mut body = String::from("# Raw Memories\n\n");
@@ -108,10 +113,10 @@ async fn prune_rollout_summaries(root: &Path, keep: &BTreeSet<String>) -> std::i
let Some(file_name) = path.file_name().and_then(|name| name.to_str()) else {
continue;
};
let Some(thread_id) = extract_thread_id_from_rollout_summary_filename(file_name) else {
let Some(stem) = file_name.strip_suffix(".md") else {
continue;
};
if !keep.contains(thread_id)
if !keep.contains(stem)
&& let Err(err) = tokio::fs::remove_file(&path).await
&& err.kind() != std::io::ErrorKind::NotFound
{
@@ -129,7 +134,8 @@ async fn write_rollout_summary_for_thread(
root: &Path,
memory: &Stage1Output,
) -> std::io::Result<()> {
let path = rollout_summaries_dir(root).join(format!("{}.md", memory.thread_id));
let file_stem = rollout_summary_file_stem(memory);
let path = rollout_summaries_dir(root).join(format!("{file_stem}.md"));
let mut body = String::new();
writeln!(body, "thread_id: {}", memory.thread_id)
@@ -150,7 +156,85 @@ async fn write_rollout_summary_for_thread(
tokio::fs::write(path, body).await
}
fn extract_thread_id_from_rollout_summary_filename(file_name: &str) -> Option<&str> {
let stem = file_name.strip_suffix(".md")?;
if stem.is_empty() { None } else { Some(stem) }
fn rollout_summary_file_stem(memory: &Stage1Output) -> String {
const ROLLOUT_SLUG_MAX_LEN: usize = 20;
let thread_id = memory.thread_id.to_string();
let Some(raw_slug) = memory.rollout_slug.as_deref() else {
return thread_id;
};
let mut slug = String::with_capacity(ROLLOUT_SLUG_MAX_LEN);
for ch in raw_slug.chars() {
if slug.len() >= ROLLOUT_SLUG_MAX_LEN {
break;
}
if ch.is_ascii_alphanumeric() {
slug.push(ch.to_ascii_lowercase());
} else {
slug.push('_');
}
}
while slug.ends_with('_') {
slug.pop();
}
if slug.is_empty() {
thread_id
} else {
format!("{thread_id}-{slug}")
}
}
#[cfg(test)]
mod tests {
use super::rollout_summary_file_stem;
use chrono::TimeZone;
use chrono::Utc;
use codex_protocol::ThreadId;
use codex_state::Stage1Output;
use pretty_assertions::assert_eq;
use std::path::PathBuf;
fn stage1_output_with_slug(rollout_slug: Option<&str>) -> Stage1Output {
Stage1Output {
thread_id: ThreadId::new(),
source_updated_at: Utc.timestamp_opt(123, 0).single().expect("timestamp"),
raw_memory: "raw memory".to_string(),
rollout_summary: "summary".to_string(),
rollout_slug: rollout_slug.map(ToString::to_string),
cwd: PathBuf::from("/tmp/workspace"),
generated_at: Utc.timestamp_opt(124, 0).single().expect("timestamp"),
}
}
#[test]
fn rollout_summary_file_stem_uses_thread_id_when_slug_missing() {
let memory = stage1_output_with_slug(None);
let thread_id = memory.thread_id.to_string();
assert_eq!(rollout_summary_file_stem(&memory), thread_id);
}
#[test]
fn rollout_summary_file_stem_sanitizes_and_truncates_slug() {
let memory =
stage1_output_with_slug(Some("Unsafe Slug/With Spaces & Symbols + EXTRA_LONG_12345"));
let thread_id = memory.thread_id.to_string();
assert_eq!(
rollout_summary_file_stem(&memory),
format!("{thread_id}-unsafe_slug_with_spa")
);
}
#[test]
fn rollout_summary_file_stem_uses_thread_id_when_slug_is_empty() {
let memory = stage1_output_with_slug(Some(""));
let thread_id = memory.thread_id.to_string();
assert_eq!(rollout_summary_file_stem(&memory), thread_id);
}
}

Some files were not shown because too many files have changed in this diff Show More