diff --git a/.bazelrc b/.bazelrc index c58667b46e..de01a87443 100644 --- a/.bazelrc +++ b/.bazelrc @@ -107,8 +107,9 @@ common:ci-bazel --config=ci common:ci-bazel --build_metadata=TAG_workflow=bazel # Keep code-mode integration cases out of ordinary Bazel legs. The # Windows-cross config below re-enables them after generating its Windows V8 -# snapshot on the Windows runner. -common:ci-bazel --test_env=CODEX_BAZEL_TEST_SKIP_FILTERS=suite::code_mode:: +# snapshot on the Windows runner. Also skip the ConPTY Ctrl-C integration test +# until Windows Bazel CI can reliably interrupt foreground processes. +common:ci-bazel --test_env=CODEX_BAZEL_TEST_SKIP_FILTERS=suite::code_mode::,tests::windows_tests::conpty_ctrl_c_interrupts_powershell_foreground_child # Shared config for Bazel-backed Rust linting. build:clippy --aspects=@rules_rust//rust:defs.bzl%rust_clippy_aspect @@ -191,11 +192,12 @@ common:ci-windows-cross --strategy=TestRunner=local common:ci-windows-cross --strategy=V8Mksnapshot=local common:ci-windows-cross --local_test_jobs=4 common:ci-windows-cross --test_env=RUST_TEST_THREADS=1 -# Native Windows CI still covers the PowerShell tests. The cross-built gnullvm -# binaries currently hang in PowerShell AST parser tests when those binaries are -# run on the Windows runner. Keep V8-backed code-mode tests enabled except for -# the hidden dynamic-tool callback test, which currently times out on Windows. -common:ci-windows-cross --test_env=CODEX_BAZEL_TEST_SKIP_FILTERS=powershell,suite::code_mode::code_mode_can_call_hidden_dynamic_tools +# Native Windows CI still covers the PowerShell parser-process tests. The +# cross-built gnullvm binaries currently hang in those tests when run on the +# Windows runner. Keep V8-backed code-mode tests enabled except for the hidden +# dynamic-tool callback test, which currently times out on Windows. This config +# replaces the base skip list, so repeat the exact ConPTY Ctrl-C test exclusion. +common:ci-windows-cross --test_env=CODEX_BAZEL_TEST_SKIP_FILTERS=command_safety::powershell_parser::tests::,suite::code_mode::code_mode_can_call_hidden_dynamic_tools,tests::windows_tests::conpty_ctrl_c_interrupts_powershell_foreground_child common:ci-windows-cross --platforms=//:windows_x86_64_gnullvm common:ci-windows-cross --extra_execution_platforms=//:rbe,//:windows_x86_64_msvc common:ci-windows-cross --extra_toolchains=//:windows_gnullvm_tests_on_msvc_host_toolchain diff --git a/.codex/environments/environment.toml b/.codex/environments/environment.toml index 39c40eb07f..b3c5910fbc 100644 --- a/.codex/environments/environment.toml +++ b/.codex/environments/environment.toml @@ -2,9 +2,6 @@ version = 1 name = "codex" -[setup] -script = "python ./.codex/environments/setup.py" - [[actions]] name = "Run" icon = "run" diff --git a/.codex/environments/setup.py b/.codex/environments/setup.py deleted file mode 100644 index a9d78c9405..0000000000 --- a/.codex/environments/setup.py +++ /dev/null @@ -1,65 +0,0 @@ -#!/usr/bin/env python3 - -"""Set up ignored files that should be shared with Codex worktrees.""" - -import shutil -import subprocess -from functools import cache -from pathlib import Path - - -@cache -def worktree_paths() -> tuple[Path, Path]: - script_dir = Path(__file__).resolve().parent - worktree_root = git_path(script_dir / "../..", "--show-toplevel") - common_git_dir = git_path(worktree_root, "--git-common-dir") - return worktree_root, common_git_dir.parent - - -def git_path(working_directory: Path, argument: str) -> Path: - output = subprocess.check_output( - [ - "git", - "-C", - str(working_directory), - "rev-parse", - "--path-format=absolute", - argument, - ], - text=True, - ) - return Path(output.strip()) - - -def copy_from_main_worktree_to_worktree(repo_relative_path: str) -> None: - relative_path = Path(repo_relative_path) - if relative_path.is_absolute() or ".." in relative_path.parts: - raise ValueError(f"path must be repository-relative: {repo_relative_path}") - - worktree_root, main_worktree = worktree_paths() - source_path = main_worktree / relative_path - destination_path = worktree_root / relative_path - - print(f" source: {source_path}") - print(f" destination: {destination_path}") - - if source_path == destination_path: - print(" result: running in the main worktree; nothing to copy") - elif destination_path.exists(): - print(" result: destination already exists; nothing to copy") - elif not source_path.is_file(): - print(" result: source does not exist; nothing to copy") - else: - destination_path.parent.mkdir(parents=True, exist_ok=True) - shutil.copy2(source_path, destination_path) - print(f" result: copied {repo_relative_path}") - - -def main() -> None: - print("Codex environment setup:") - # See codex-rs/docs/bazel.md for the repository's Bazel workflow. - copy_from_main_worktree_to_worktree("user.bazelrc") - - -if __name__ == "__main__": - main() diff --git a/.codex/skills/code-review/SKILL.md b/.codex/skills/code-review/SKILL.md index eec0787c20..ccd37a9868 100644 --- a/.codex/skills/code-review/SKILL.md +++ b/.codex/skills/code-review/SKILL.md @@ -3,7 +3,7 @@ name: code-review description: Run a final code review on a pull request --- -Use subagents to review code using all code-review-* skills in this repository other than this orchestrator. One subagent per skill. Pass full skill path to subagents. Use xhigh reasoning. +Use subagents to review code using all code-review-* skills other than this orchestrator. One subagent per skill. Pass full skill path to subagents. Use xhigh reasoning. You must return every single issue from every subagent. You can return an unlimited number of findings. Use raw Markdown to report findings. diff --git a/.github/dotslash-zsh-config.json b/.github/dotslash-zsh-config.json index 37285f19e2..2c2b11eec5 100644 --- a/.github/dotslash-zsh-config.json +++ b/.github/dotslash-zsh-config.json @@ -5,21 +5,25 @@ "macos-aarch64": { "name": "codex-zsh-aarch64-apple-darwin.tar.gz", "format": "tar.gz", + "hash": "sha256", "path": "codex-zsh/bin/zsh" }, "macos-x86_64": { "name": "codex-zsh-x86_64-apple-darwin.tar.gz", "format": "tar.gz", + "hash": "sha256", "path": "codex-zsh/bin/zsh" }, "linux-x86_64": { "name": "codex-zsh-x86_64-unknown-linux-musl.tar.gz", "format": "tar.gz", + "hash": "sha256", "path": "codex-zsh/bin/zsh" }, "linux-aarch64": { "name": "codex-zsh-aarch64-unknown-linux-musl.tar.gz", "format": "tar.gz", + "hash": "sha256", "path": "codex-zsh/bin/zsh" } } diff --git a/.github/scripts/build-codex-package-archive.sh b/.github/scripts/build-codex-package-archive.sh index 80da4cf20c..229213cc53 100644 --- a/.github/scripts/build-codex-package-archive.sh +++ b/.github/scripts/build-codex-package-archive.sh @@ -9,6 +9,7 @@ Usage: build-codex-package-archive.sh \ --entrypoint-dir \ --archive-dir \ [--bwrap-bin ] \ + [--zsh-manifest ] \ [--codex-command-runner-bin ] \ [--codex-windows-sandbox-setup-bin ] \ [--target-suffixed-entrypoint] @@ -48,6 +49,10 @@ while [[ $# -gt 0 ]]; do bwrap_bin_provided="true" shift 2 ;; + --zsh-manifest) + resource_args+=(--zsh-manifest "${2:?--zsh-manifest requires a value}") + shift 2 + ;; --codex-command-runner-bin) resource_args+=( --codex-command-runner-bin diff --git a/.github/scripts/check_ci_results.py b/.github/scripts/check_ci_results.py new file mode 100644 index 0000000000..36d1eed3b4 --- /dev/null +++ b/.github/scripts/check_ci_results.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 + +"""Fail a terminal CI job unless every serialized dependency succeeded. + +Parent workflows pass GitHub's `toJSON(needs)` object through the NEEDS +environment variable. Treat skipped and cancelled dependencies as failures too: +for a required fan-in job, only an explicit success is safe to accept. +""" + +import json +import os + + +def main() -> None: + # Keep result policy in one script so blocking-ci and postmerge-ci cannot + # drift in how they interpret dependency conclusions. + needs = json.loads(os.environ["NEEDS"]) + failures = sorted( + (name, dependency["result"]) + for name, dependency in needs.items() + if dependency["result"] != "success" + ) + + if failures: + print("CI dependencies did not succeed:") + for name, result in failures: + print(f"{name}: {result}") + raise SystemExit(1) + + print("All CI dependencies succeeded.") + + +if __name__ == "__main__": + main() diff --git a/.github/scripts/v8_canary_changes.py b/.github/scripts/v8_canary_changes.py index 6d9693c60a..0acc4ec3f6 100644 --- a/.github/scripts/v8_canary_changes.py +++ b/.github/scripts/v8_canary_changes.py @@ -1,12 +1,44 @@ #!/usr/bin/env python3 +"""Decide which V8 canary work is needed for a commit range. + +The workflow deliberately has no trigger-level path filters because it is both +directly triggered for pull requests and called by postmerge-ci. Keeping the +patterns here gives those entrypoints one source of truth; unrelated events +still run metadata but skip the expensive build matrices. +""" + import argparse import subprocess import tomllib +from fnmatch import fnmatchcase from pathlib import Path ROOT = Path(__file__).resolve().parents[2] +# These patterns replace the old pull_request/push path filters. Include parent +# workflow changes because they can alter whether the canary is invoked. +CANARY_PATH_PATTERNS = { + ".bazelrc", + ".github/actions/setup-bazel-ci/**", + ".github/scripts/run_bazel_with_buildbuddy.py", + ".github/scripts/rusty_v8_bazel.py", + ".github/scripts/rusty_v8_module_bazel.py", + ".github/scripts/v8_canary_changes.py", + ".github/workflows/postmerge-ci.yml", + ".github/workflows/rusty-v8-release.yml", + ".github/workflows/v8-canary.yml", + "MODULE.bazel", + "MODULE.bazel.lock", + "codex-rs/Cargo.toml", + "patches/BUILD.bazel", + "patches/llvm_*.patch", + "patches/rules_cc_*.patch", + "patches/v8_*.patch", + "third_party/v8/**", +} +# Windows source builds are a narrower, more expensive subset of the canary. +# A V8 version change also requires them even when no path below changed. WINDOWS_SOURCE_BUILD_PATHS = { ".github/scripts/rusty_v8_bazel.py", ".github/scripts/rusty_v8_module_bazel.py", @@ -16,6 +48,30 @@ WINDOWS_SOURCE_BUILD_PATHS = { } +def matching_canary_paths(changed_files: set[str]) -> set[str]: + """Return changed paths that require the general V8 build matrix.""" + return { + path + for path in changed_files + if any(fnmatchcase(path, pattern) for pattern in CANARY_PATH_PATTERNS) + } + + +def canary_required( + changed_files: set[str], + base_v8_version: str, + head_v8_version: str, + *, + force: bool = False, +) -> bool: + """Return whether the general V8 build matrix should run.""" + return ( + force + or base_v8_version != head_v8_version + or bool(matching_canary_paths(changed_files)) + ) + + def resolved_v8_version(cargo_lock: bytes) -> str: versions = sorted( { @@ -36,6 +92,7 @@ def windows_source_required( *, force: bool = False, ) -> bool: + """Return whether Windows must rebuild rusty_v8 from source.""" return ( force or base_v8_version != head_v8_version @@ -58,6 +115,8 @@ def merge_base(base: str, head: str, *, root: Path = ROOT) -> str: def changed_files(base: str, head: str, *, root: Path = ROOT) -> set[str]: + # Three-dot diff gives PRs merge-base semantics while remaining equivalent + # to before/after for ordinary linear pushes to main. output = git_output( "diff", "--name-only", @@ -79,25 +138,44 @@ def parse_args() -> argparse.Namespace: def main() -> None: args = parse_args() if args.force: - required = True - reason = "manual workflow dispatch" + # workflow_dispatch has no comparison range, and callers use it as a + # manual retry path, so it intentionally runs every variant. + canary = True + canary_reason = "manual workflow dispatch" + windows_source = True + windows_source_reason = "manual workflow dispatch" elif not args.base or not args.head: raise SystemExit("--base and --head are required unless --force is set") else: files = changed_files(args.base, args.head) base_version = v8_version_at_revision(merge_base(args.base, args.head)) head_version = v8_version_at_revision(args.head) - required = windows_source_required(files, base_version, head_version) + + matched_canary_paths = sorted(matching_canary_paths(files)) + canary = canary_required(files, base_version, head_version) + windows_source = windows_source_required(files, base_version, head_version) if base_version != head_version: - reason = f"v8 version changed from {base_version} to {head_version}" + canary_reason = ( + f"v8 version changed from {base_version} to {head_version}" + ) + windows_source_reason = canary_reason else: - matched_paths = sorted(files & WINDOWS_SOURCE_BUILD_PATHS) - reason = ( - ", ".join(matched_paths) if matched_paths else "no relevant changes" + canary_reason = ( + ", ".join(matched_canary_paths) + if matched_canary_paths + else "no relevant changes" + ) + matched_windows_paths = sorted(files & WINDOWS_SOURCE_BUILD_PATHS) + windows_source_reason = ( + ", ".join(matched_windows_paths) + if matched_windows_paths + else "no relevant changes" ) - print(f"windows_source_required={str(required).lower()}") - print(f"windows_source_reason={reason}") + print(f"canary_required={str(canary).lower()}") + print(f"canary_reason={canary_reason}") + print(f"windows_source_required={str(windows_source).lower()}") + print(f"windows_source_reason={windows_source_reason}") if __name__ == "__main__": diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index 7a6db869e0..10dd8670de 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -4,10 +4,7 @@ name: Bazel # https://github.com/cerisier/toolchains_llvm_bootstrapped/blob/main/.github/workflows/ci.yaml on: - pull_request: {} - push: - branches: - - main + workflow_call: workflow_dispatch: concurrency: diff --git a/.github/workflows/blob-size-policy.yml b/.github/workflows/blob-size-policy.yml index ea50e8ee2e..fddc51afe7 100644 --- a/.github/workflows/blob-size-policy.yml +++ b/.github/workflows/blob-size-policy.yml @@ -1,7 +1,7 @@ name: blob-size-policy on: - pull_request: {} + workflow_call: jobs: check: @@ -14,13 +14,24 @@ jobs: fetch-depth: 0 persist-credentials: false - - name: Determine PR comparison range + - name: Determine comparison range id: range shell: bash run: | set -euo pipefail - echo "base=${{ github.event.pull_request.base.sha }}" >> "$GITHUB_OUTPUT" - echo "head=${{ github.event.pull_request.head.sha }}" >> "$GITHUB_OUTPUT" + + # PRs inspect the proposed diff; main pushes inspect only the commit + # range that just landed. Both paths feed the same blob-size checker. + if [[ "${{ github.event_name }}" == "pull_request" ]]; then + base='${{ github.event.pull_request.base.sha }}' + head='${{ github.event.pull_request.head.sha }}' + else + base='${{ github.event.before }}' + head='${{ github.sha }}' + fi + + echo "base=$base" >> "$GITHUB_OUTPUT" + echo "head=$head" >> "$GITHUB_OUTPUT" - name: Check changed blob sizes env: diff --git a/.github/workflows/blocking-ci.yml b/.github/workflows/blocking-ci.yml new file mode 100644 index 0000000000..fe66fdd509 --- /dev/null +++ b/.github/workflows/blocking-ci.yml @@ -0,0 +1,79 @@ +name: blocking-ci + +# This is the single entrypoint for checks that block a PR merge. It also runs +# after pushes to main so the same check family stays grouped in the Actions UI. +on: + pull_request: {} + push: + branches: [main] + +jobs: + # Keep reusable workflow calls alphabetized. The `required` job below is the + # version-controlled list that the main-branch ruleset should require. + bazel: + name: Bazel + uses: ./.github/workflows/bazel.yml + secrets: inherit + + blob-size-policy: + name: Blob size policy + uses: ./.github/workflows/blob-size-policy.yml + secrets: inherit + + cargo-deny: + name: cargo-deny + uses: ./.github/workflows/cargo-deny.yml + secrets: inherit + + codespell: + name: Codespell + uses: ./.github/workflows/codespell.yml + secrets: inherit + + repo-checks: + name: repo-checks + uses: ./.github/workflows/repo-checks.yml + secrets: inherit + + rust-ci: + name: rust-ci + uses: ./.github/workflows/rust-ci.yml + secrets: inherit + + sdk: + name: sdk + uses: ./.github/workflows/sdk.yml + secrets: inherit + + required: + name: CI required + # Without `always()`, GitHub skips this job after a failed dependency and a + # required check can appear successful instead of reporting the failure. + if: ${{ always() }} + needs: + - bazel + - blob-size-policy + - cargo-deny + - codespell + - repo-checks + - rust-ci + - sdk + runs-on: ubuntu-24.04 + steps: + # Keep the helper on the same revision as the caller and child workflows. + # CI workflow uploads are restricted, so this repository does not need a + # separate trusted-base checkout for the terminal policy step. Using the + # PR head also lets the introducing PR exercise a newly added helper. + # + # During the initial rollout, PR branches created before + # check_ci_results.py exists must rebase onto main before this gate can + # run. + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + + - name: Require successful dependencies + env: + NEEDS: ${{ toJSON(needs) }} + run: python3 .github/scripts/check_ci_results.py diff --git a/.github/workflows/cargo-deny.yml b/.github/workflows/cargo-deny.yml index aa37d7cec2..be00e1147d 100644 --- a/.github/workflows/cargo-deny.yml +++ b/.github/workflows/cargo-deny.yml @@ -1,10 +1,7 @@ name: cargo-deny on: - pull_request: - push: - branches: - - main + workflow_call: # Cargo's libgit2 transport has been flaky when fetching git dependencies with # nested submodules. Prefer the system git CLI across every Cargo invocation. diff --git a/.github/workflows/codespell.yml b/.github/workflows/codespell.yml index b642a6dbe1..a1751c8d29 100644 --- a/.github/workflows/codespell.yml +++ b/.github/workflows/codespell.yml @@ -3,10 +3,7 @@ name: Codespell on: - push: - branches: [main] - pull_request: - branches: [main] + workflow_call: permissions: contents: read diff --git a/.github/workflows/issue-labeler.yml b/.github/workflows/issue-labeler.yml index 2c4eb6aa68..0f11d2af5e 100644 --- a/.github/workflows/issue-labeler.yml +++ b/.github/workflows/issue-labeler.yml @@ -78,6 +78,7 @@ jobs: 30. automations - Issues involving scheduled automation tasks or heartbeats. 31. pets - Issues involving pets avatars and animations. 32. agent - Fallback only for core agent loop or agent-related issues that do not fit app-server, connectivity, subagent, session, config, plan, computer-use, browser, memory, imagen, remote, performance, automations, or pets. + 33. aws-bedrock - Issues involving the Amazon Bedrock model provider or Bedrock Mantle. Do not use for unrelated AWS services or generic AWS mentions. Issue number: ${{ github.event.issue.number }} diff --git a/.github/workflows/postmerge-ci.yml b/.github/workflows/postmerge-ci.yml new file mode 100644 index 0000000000..ac2b46d1db --- /dev/null +++ b/.github/workflows/postmerge-ci.yml @@ -0,0 +1,40 @@ +name: postmerge-ci + +# This is the single entrypoint for main-only CI that is intentionally outside +# the merge-blocking suite. It keeps the broader postmerge signal in one run. +on: + push: + branches: [main] + +jobs: + # Keep reusable workflow calls alphabetized. Each child retains its own + # workflow_dispatch trigger so maintainers can rerun flaky suites directly. + rust-ci-full: + name: rust-ci-full + uses: ./.github/workflows/rust-ci-full.yml + secrets: inherit + + v8-canary: + name: v8-canary + uses: ./.github/workflows/v8-canary.yml + secrets: inherit + + results: + name: Postmerge CI results + needs: + - rust-ci-full + - v8-canary + if: ${{ always() }} + runs-on: ubuntu-24.04 + steps: + # Postmerge runs use the pushed main commit, so this helper always comes + # from the same revision that defined the parent workflow. + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + ref: ${{ github.sha }} + persist-credentials: false + + - name: Require successful dependencies + env: + NEEDS: ${{ toJSON(needs) }} + run: python3 .github/scripts/check_ci_results.py diff --git a/.github/workflows/ci.yml b/.github/workflows/repo-checks.yml similarity index 95% rename from .github/workflows/ci.yml rename to .github/workflows/repo-checks.yml index a78ae034ed..3d0504172d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/repo-checks.yml @@ -1,8 +1,7 @@ -name: ci +name: repo-checks on: - pull_request: {} - push: { branches: [main] } + workflow_call: jobs: build-test: @@ -29,6 +28,9 @@ jobs: - name: Test Codex package builder run: python3 -m unittest discover -s scripts/codex_package -p 'test_*.py' + - name: Test standalone installer + run: python3 -m unittest discover -s scripts/install -p 'test_*.py' + - name: Setup pnpm uses: pnpm/action-setup@a8198c4bff370c8506180b035930dea56dbd5288 # v5 with: diff --git a/.github/workflows/rust-ci-full.yml b/.github/workflows/rust-ci-full.yml index 7ad1d4b3ad..cb99c8911e 100644 --- a/.github/workflows/rust-ci-full.yml +++ b/.github/workflows/rust-ci-full.yml @@ -1,8 +1,10 @@ name: rust-ci-full on: + workflow_call: push: branches: - - main + # Main pushes enter through postmerge-ci. Keep this opt-in branch trigger + # for developers who want the full suite before merging. - "**full-ci**" workflow_dispatch: diff --git a/.github/workflows/rust-ci.yml b/.github/workflows/rust-ci.yml index f693994d1a..cf1afdfe2d 100644 --- a/.github/workflows/rust-ci.yml +++ b/.github/workflows/rust-ci.yml @@ -1,6 +1,6 @@ name: rust-ci on: - pull_request: {} + workflow_call: workflow_dispatch: # Cargo's libgit2 transport has been flaky when fetching git dependencies with diff --git a/.github/workflows/rust-release-zsh.yml b/.github/workflows/rust-release-zsh.yml index b55d2e714b..e814ec32e1 100644 --- a/.github/workflows/rust-release-zsh.yml +++ b/.github/workflows/rust-release-zsh.yml @@ -1,15 +1,56 @@ name: rust-release-zsh on: - workflow_call: + push: + tags: + - "codex-zsh-v*.*.*" env: ZSH_COMMIT: 77045ef899e53b9598bebc5a41db93a548a40ca6 ZSH_PATCH: codex-rs/shell-escalation/patches/zsh-exec-wrapper.patch +concurrency: + group: ${{ github.workflow }}::${{ github.ref_name }} + cancel-in-progress: false + jobs: + metadata: + runs-on: ubuntu-latest + outputs: + release_tag: ${{ steps.release_tag.outputs.release_tag }} + + steps: + - name: Validate release tag + id: release_tag + env: + RELEASE_TAG: ${{ github.ref_name }} + shell: bash + run: | + set -euo pipefail + + if [[ ! "${RELEASE_TAG}" =~ ^codex-zsh-v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "Tag ${RELEASE_TAG} does not match codex-zsh-vX.Y.Z." >&2 + exit 1 + fi + + echo "release_tag=${RELEASE_TAG}" >> "${GITHUB_OUTPUT}" + + - name: Ensure release does not exist + env: + GH_TOKEN: ${{ github.token }} + RELEASE_TAG: ${{ steps.release_tag.outputs.release_tag }} + shell: bash + run: | + set -euo pipefail + + if gh release view "${RELEASE_TAG}" --repo "${GITHUB_REPOSITORY}" > /dev/null 2>&1; then + echo "Release ${RELEASE_TAG} already exists; publish changed artifacts under a new tag." >&2 + exit 1 + fi + linux: name: Build zsh (Linux) - ${{ matrix.variant }} - ${{ matrix.target }} + needs: metadata runs-on: ${{ matrix.runner }} timeout-minutes: 30 container: @@ -62,6 +103,7 @@ jobs: darwin: name: Build zsh (macOS) - ${{ matrix.variant }} - ${{ matrix.target }} + needs: metadata runs-on: ${{ matrix.runner }} timeout-minutes: 30 @@ -101,3 +143,40 @@ jobs: with: name: codex-zsh-${{ matrix.target }} path: dist/zsh/${{ matrix.target }}/* + + publish-release: + needs: + - metadata + - linux + - darwin + runs-on: ubuntu-latest + permissions: + contents: write + actions: read + + steps: + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + path: dist + + - name: Create GitHub Release + uses: softprops/action-gh-release@153bb8e04406b158c6c84fc1615b65b24149a1fe # v2.6.1 + with: + tag_name: ${{ needs.metadata.outputs.release_tag }} + name: ${{ needs.metadata.outputs.release_tag }} + files: dist/** + # Keep zsh artifact releases out of Codex's normal "latest release" channel. + prerelease: true + + - name: Publish DotSlash manifest + uses: facebook/dotslash-publish-release@9c9ec027515c34db9282a09a25a9cab5880b2c52 # v2 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + tag: ${{ needs.metadata.outputs.release_tag }} + config: .github/dotslash-zsh-config.json diff --git a/.github/workflows/rust-release.yml b/.github/workflows/rust-release.yml index a2646e3eae..49d24abc18 100644 --- a/.github/workflows/rust-release.yml +++ b/.github/workflows/rust-release.yml @@ -14,6 +14,9 @@ on: tags: - "rust-v*.*.*" +env: + CODEX_ZSH_RELEASE_TAG: codex-zsh-v0.1.0 + concurrency: group: ${{ github.workflow }} cancel-in-progress: true @@ -354,6 +357,15 @@ jobs: cp target/${{ matrix.target }}/release/codex-${{ matrix.target }}.dmg "$dest/codex-${{ matrix.target }}.dmg" fi + - name: Download packaged zsh manifest + if: ${{ runner.os != 'macOS' }} + shell: bash + run: | + set -euo pipefail + curl -fsSL \ + "https://github.com/${GITHUB_REPOSITORY}/releases/download/${CODEX_ZSH_RELEASE_TAG}/codex-zsh" \ + -o "${RUNNER_TEMP}/codex-zsh" + - name: Build Codex package archive if: ${{ runner.os != 'macOS' }} shell: bash @@ -366,7 +378,8 @@ jobs: --target "$TARGET" \ --bundle "$BUNDLE" \ --entrypoint-dir "target/${TARGET}/release" \ - --archive-dir "dist/${TARGET}" + --archive-dir "dist/${TARGET}" \ + --zsh-manifest "${RUNNER_TEMP}/codex-zsh" - name: Build Python runtime wheel if: ${{ matrix.bundle == 'primary' && runner.os != 'macOS' }} @@ -697,6 +710,14 @@ jobs: cp "target/${{ matrix.target }}/release/${binary}" "$dest/${binary}-${{ matrix.target }}" done + - name: Download packaged zsh manifest + shell: bash + run: | + set -euo pipefail + curl -fsSL \ + "https://github.com/${GITHUB_REPOSITORY}/releases/download/${CODEX_ZSH_RELEASE_TAG}/codex-zsh" \ + -o "${RUNNER_TEMP}/codex-zsh" + - name: Build Codex package archive shell: bash env: @@ -708,7 +729,8 @@ jobs: --target "$TARGET" \ --bundle "$BUNDLE" \ --entrypoint-dir "target/${TARGET}/release" \ - --archive-dir "dist/${TARGET}" + --archive-dir "dist/${TARGET}" \ + --zsh-manifest "${RUNNER_TEMP}/codex-zsh" - name: Build Python runtime wheel if: ${{ matrix.bundle == 'primary' }} @@ -1043,11 +1065,6 @@ jobs: with: publish: true - zsh-release-assets: - name: zsh release assets - needs: tag-check - uses: ./.github/workflows/rust-release-zsh.yml - release: needs: - tag-check @@ -1055,7 +1072,6 @@ jobs: - finalize-macos - build-windows - argument-comment-lint-release-assets - - zsh-release-assets if: >- ${{ always() && @@ -1063,8 +1079,7 @@ jobs: needs.build.result == 'success' && needs.finalize-macos.result == 'success' && needs.build-windows.result == 'success' && - needs.argument-comment-lint-release-assets.result == 'success' && - needs.zsh-release-assets.result == 'success' + needs.argument-comment-lint-release-assets.result == 'success' }} name: release runs-on: ubuntu-latest @@ -1113,7 +1128,7 @@ jobs: uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: path: dist - pattern: "{*-symbols,argument-comment-lint-*,codex-zsh-*,python-runtime-wheel-*}" + pattern: "{*-symbols,argument-comment-lint-*,python-runtime-wheel-*}" - name: List run: ls -R dist/ @@ -1236,13 +1251,6 @@ jobs: tag: ${{ github.ref_name }} config: .github/dotslash-config.json - - uses: facebook/dotslash-publish-release@9c9ec027515c34db9282a09a25a9cab5880b2c52 # v2 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - tag: ${{ github.ref_name }} - config: .github/dotslash-zsh-config.json - - uses: facebook/dotslash-publish-release@9c9ec027515c34db9282a09a25a9cab5880b2c52 # v2 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/sdk.yml b/.github/workflows/sdk.yml index dcb9406798..41f8874712 100644 --- a/.github/workflows/sdk.yml +++ b/.github/workflows/sdk.yml @@ -1,9 +1,7 @@ name: sdk on: - push: - branches: [main] - pull_request: {} + workflow_call: jobs: python-sdk: diff --git a/.github/workflows/v8-canary.yml b/.github/workflows/v8-canary.yml index efa06aaad7..53d9585108 100644 --- a/.github/workflows/v8-canary.yml +++ b/.github/workflows/v8-canary.yml @@ -1,44 +1,12 @@ name: v8-canary +# Do not use trigger-level path filters here. This workflow is also called by +# postmerge-ci, and GitHub cannot share a path filter between pull_request and +# workflow_call. v8_canary_changes.py is the single source of truth instead; +# unrelated events run only the cheap metadata job below. on: - pull_request: - paths: - - ".bazelrc" - - ".github/actions/setup-bazel-ci/**" - - ".github/scripts/run_bazel_with_buildbuddy.py" - - ".github/scripts/rusty_v8_bazel.py" - - ".github/scripts/rusty_v8_module_bazel.py" - - ".github/scripts/v8_canary_changes.py" - - ".github/workflows/rusty-v8-release.yml" - - ".github/workflows/v8-canary.yml" - - "MODULE.bazel" - - "MODULE.bazel.lock" - - "codex-rs/Cargo.toml" - - "patches/BUILD.bazel" - - "patches/llvm_*.patch" - - "patches/rules_cc_*.patch" - - "patches/v8_*.patch" - - "third_party/v8/**" - push: - branches: - - main - paths: - - ".bazelrc" - - ".github/actions/setup-bazel-ci/**" - - ".github/scripts/run_bazel_with_buildbuddy.py" - - ".github/scripts/rusty_v8_bazel.py" - - ".github/scripts/rusty_v8_module_bazel.py" - - ".github/scripts/v8_canary_changes.py" - - ".github/workflows/rusty-v8-release.yml" - - ".github/workflows/v8-canary.yml" - - "MODULE.bazel" - - "MODULE.bazel.lock" - - "codex-rs/Cargo.toml" - - "patches/BUILD.bazel" - - "patches/llvm_*.patch" - - "patches/rules_cc_*.patch" - - "patches/v8_*.patch" - - "third_party/v8/**" + workflow_call: + pull_request: {} workflow_dispatch: # Cargo's libgit2 transport has been flaky when fetching git dependencies with @@ -54,6 +22,10 @@ jobs: metadata: runs-on: ubuntu-latest outputs: + # A stale PR head can contain the old detector, which does not emit this + # output. Missing must mean "run" so older branches cannot silently skip + # the expensive V8 coverage while reporting success. + canary_required: ${{ steps.changes.outputs.canary_required || 'true' }} v8_version: ${{ steps.v8_version.outputs.version }} windows_source_required: ${{ steps.changes.outputs.windows_source_required }} @@ -77,7 +49,7 @@ jobs: version="$(python3 .github/scripts/rusty_v8_bazel.py resolved-v8-crate-version)" echo "version=${version}" >> "$GITHUB_OUTPUT" - - name: Detect whether Windows source artifacts need rebuilding + - name: Detect V8 canary changes id: changes env: BASE_SHA: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || github.event.before }} @@ -87,6 +59,8 @@ jobs: run: | set -euo pipefail + # Manual runs have no meaningful before/after range. Force every V8 + # variant so workflow_dispatch remains a reliable retry path. if [[ "${EVENT_NAME}" == "workflow_dispatch" ]]; then output="$(python3 .github/scripts/v8_canary_changes.py --force)" else @@ -104,6 +78,8 @@ jobs: build: name: Build ${{ matrix.variant }} ${{ matrix.target }} needs: metadata + # Metadata always runs; only relevant changes pay for the large matrix. + if: ${{ needs.metadata.outputs.canary_required == 'true' }} runs-on: ${{ matrix.runner }} permissions: contents: read diff --git a/.worktreeinclude b/.worktreeinclude new file mode 100644 index 0000000000..74824b01f9 --- /dev/null +++ b/.worktreeinclude @@ -0,0 +1 @@ +user.bazelrc diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 8ae5bef17c..3356b66c14 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -804,7 +804,7 @@ "critical-section_1.2.0": "{\"dependencies\":[],\"features\":{\"restore-state-bool\":[],\"restore-state-none\":[],\"restore-state-u16\":[],\"restore-state-u32\":[],\"restore-state-u64\":[],\"restore-state-u8\":[],\"restore-state-usize\":[],\"std\":[\"restore-state-bool\"]}}", "crossbeam-channel_0.5.15": "{\"dependencies\":[{\"default_features\":false,\"name\":\"crossbeam-utils\",\"req\":\"^0.8.18\"},{\"kind\":\"dev\",\"name\":\"num_cpus\",\"req\":\"^1.13.0\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"},{\"kind\":\"dev\",\"name\":\"signal-hook\",\"req\":\"^0.3\"}],\"features\":{\"default\":[\"std\"],\"std\":[\"crossbeam-utils/std\"]}}", "crossbeam-deque_0.8.6": "{\"dependencies\":[{\"default_features\":false,\"name\":\"crossbeam-epoch\",\"req\":\"^0.9.17\"},{\"default_features\":false,\"name\":\"crossbeam-utils\",\"req\":\"^0.8.18\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"}],\"features\":{\"default\":[\"std\"],\"std\":[\"crossbeam-epoch/std\",\"crossbeam-utils/std\"]}}", - "crossbeam-epoch_0.9.18": "{\"dependencies\":[{\"default_features\":false,\"name\":\"crossbeam-utils\",\"req\":\"^0.8.18\"},{\"name\":\"loom-crate\",\"optional\":true,\"package\":\"loom\",\"req\":\"^0.7.1\",\"target\":\"cfg(crossbeam_loom)\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"}],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"loom\":[\"loom-crate\",\"crossbeam-utils/loom\"],\"nightly\":[\"crossbeam-utils/nightly\"],\"std\":[\"alloc\",\"crossbeam-utils/std\"]}}", + "crossbeam-epoch_0.9.20": "{\"dependencies\":[{\"default_features\":false,\"name\":\"crossbeam-utils\",\"req\":\"^0.8.18\"},{\"name\":\"loom-crate\",\"optional\":true,\"package\":\"loom\",\"req\":\"^0.7.1\",\"target\":\"cfg(crossbeam_loom)\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"}],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"loom\":[\"loom-crate\",\"crossbeam-utils/loom\"],\"nightly\":[\"crossbeam-utils/nightly\"],\"std\":[\"alloc\",\"crossbeam-utils/std\"]}}", "crossbeam-queue_0.3.12": "{\"dependencies\":[{\"default_features\":false,\"name\":\"crossbeam-utils\",\"req\":\"^0.8.18\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"}],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"nightly\":[\"crossbeam-utils/nightly\"],\"std\":[\"alloc\",\"crossbeam-utils/std\"]}}", "crossbeam-utils_0.8.21": "{\"dependencies\":[{\"name\":\"loom\",\"optional\":true,\"req\":\"^0.7.1\",\"target\":\"cfg(crossbeam_loom)\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"}],\"features\":{\"default\":[\"std\"],\"nightly\":[],\"std\":[]}}", "crossterm_winapi_0.9.1": "{\"dependencies\":[{\"features\":[\"winbase\",\"consoleapi\",\"processenv\",\"handleapi\",\"synchapi\",\"impl-default\"],\"name\":\"winapi\",\"req\":\"^0.3.8\",\"target\":\"cfg(windows)\"}],\"features\":{}}", @@ -1343,7 +1343,7 @@ "pkcs8_0.10.2": "{\"dependencies\":[{\"features\":[\"oid\"],\"name\":\"der\",\"req\":\"^0.7\"},{\"kind\":\"dev\",\"name\":\"hex-literal\",\"req\":\"^0.3\"},{\"name\":\"pkcs5\",\"optional\":true,\"req\":\"^0.7\"},{\"default_features\":false,\"name\":\"rand_core\",\"optional\":true,\"req\":\"^0.6\"},{\"name\":\"spki\",\"req\":\"^0.7.1\"},{\"default_features\":false,\"name\":\"subtle\",\"optional\":true,\"req\":\"^2\"},{\"kind\":\"dev\",\"name\":\"tempfile\",\"req\":\"^3\"}],\"features\":{\"3des\":[\"encryption\",\"pkcs5/3des\"],\"alloc\":[\"der/alloc\",\"der/zeroize\",\"spki/alloc\"],\"des-insecure\":[\"encryption\",\"pkcs5/des-insecure\"],\"encryption\":[\"alloc\",\"pkcs5/alloc\",\"pkcs5/pbes2\",\"rand_core\"],\"getrandom\":[\"rand_core/getrandom\"],\"pem\":[\"alloc\",\"der/pem\",\"spki/pem\"],\"sha1-insecure\":[\"encryption\",\"pkcs5/sha1-insecure\"],\"std\":[\"alloc\",\"der/std\",\"spki/std\"]}}", "pkg-config_0.3.32": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"lazy_static\",\"req\":\"^1\"}],\"features\":{}}", "plain_0.2.3": "{\"dependencies\":[],\"features\":{}}", - "plist_1.8.0": "{\"dependencies\":[{\"name\":\"base64\",\"req\":\"^0.22.0\"},{\"name\":\"indexmap\",\"req\":\"^2.1.0\"},{\"name\":\"quick_xml\",\"package\":\"quick-xml\",\"req\":\"^0.38.0\"},{\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.2\"},{\"kind\":\"dev\",\"name\":\"serde_derive\",\"req\":\"^1.0.2\"},{\"kind\":\"dev\",\"name\":\"serde_yaml\",\"req\":\"^0.8.21\"},{\"features\":[\"parsing\",\"formatting\"],\"name\":\"time\",\"req\":\"^0.3.30\"}],\"features\":{\"default\":[\"serde\"],\"enable_unstable_features_that_may_break_with_minor_version_bumps\":[]}}", + "plist_1.9.0": "{\"dependencies\":[{\"name\":\"base64\",\"req\":\"^0.22.0\"},{\"name\":\"indexmap\",\"req\":\"^2.1.0\"},{\"name\":\"quick_xml\",\"package\":\"quick-xml\",\"req\":\"^0.39.2\"},{\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.2\"},{\"kind\":\"dev\",\"name\":\"serde_derive\",\"req\":\"^1.0.2\"},{\"kind\":\"dev\",\"name\":\"serde_yaml\",\"req\":\"^0.8.21\"},{\"features\":[\"parsing\",\"formatting\"],\"name\":\"time\",\"req\":\"^0.3.47\"}],\"features\":{\"default\":[\"serde\"],\"enable_unstable_features_that_may_break_with_minor_version_bumps\":[]}}", "png_0.18.0": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"approx\",\"req\":\"^0.5.1\"},{\"name\":\"bitflags\",\"req\":\"^2.0\"},{\"kind\":\"dev\",\"name\":\"byteorder\",\"req\":\"^1.5.0\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"clap\",\"req\":\"^4.0\"},{\"name\":\"crc32fast\",\"req\":\"^1.2.0\"},{\"default_features\":false,\"features\":[\"cargo_bench_support\"],\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.7.0\"},{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.7.0\",\"target\":\"cfg(not(target_arch = \\\"wasm32\\\"))\"},{\"name\":\"fdeflate\",\"req\":\"^0.3.3\"},{\"name\":\"flate2\",\"req\":\"^1.0.35\"},{\"kind\":\"dev\",\"name\":\"glob\",\"req\":\"^0.3\"},{\"features\":[\"simd\"],\"name\":\"miniz_oxide\",\"req\":\"^0.8\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.9.2\"}],\"features\":{\"benchmarks\":[],\"unstable\":[\"crc32fast/nightly\"],\"zlib-rs\":[\"flate2/zlib-rs\"]}}", "polling_3.11.0": "{\"dependencies\":[{\"name\":\"cfg-if\",\"req\":\"^1\"},{\"name\":\"concurrent-queue\",\"req\":\"^2.2.0\",\"target\":\"cfg(windows)\"},{\"kind\":\"dev\",\"name\":\"easy-parallel\",\"req\":\"^3.1.0\"},{\"kind\":\"dev\",\"name\":\"fastrand\",\"req\":\"^2.0.0\"},{\"name\":\"hermit-abi\",\"req\":\"^0.5.0\",\"target\":\"cfg(target_os = \\\"hermit\\\")\"},{\"kind\":\"dev\",\"name\":\"libc\",\"req\":\"^0.2\",\"target\":\"cfg(unix)\"},{\"name\":\"pin-project-lite\",\"req\":\"^0.2.9\",\"target\":\"cfg(windows)\"},{\"default_features\":false,\"features\":[\"event\",\"fs\",\"pipe\",\"process\",\"std\",\"time\"],\"name\":\"rustix\",\"req\":\"^1.0.5\",\"target\":\"cfg(any(unix, target_os = \\\"fuchsia\\\", target_os = \\\"vxworks\\\"))\"},{\"kind\":\"dev\",\"name\":\"signal-hook\",\"req\":\"^0.3.17\",\"target\":\"cfg(all(unix, not(target_os=\\\"vita\\\")))\"},{\"kind\":\"dev\",\"name\":\"socket2\",\"req\":\"^0.6.0\"},{\"default_features\":false,\"name\":\"tracing\",\"optional\":true,\"req\":\"^0.1.37\"},{\"features\":[\"Wdk_Foundation\",\"Wdk_Storage_FileSystem\",\"Win32_Foundation\",\"Win32_Networking_WinSock\",\"Win32_Security\",\"Win32_Storage_FileSystem\",\"Win32_System_IO\",\"Win32_System_LibraryLoader\",\"Win32_System_Threading\",\"Win32_System_WindowsProgramming\"],\"name\":\"windows-sys\",\"req\":\"^0.61\",\"target\":\"cfg(windows)\"}],\"features\":{}}", "poly1305_0.8.0": "{\"dependencies\":[{\"name\":\"cpufeatures\",\"req\":\"^0.2\",\"target\":\"cfg(any(target_arch = \\\"x86_64\\\", target_arch = \\\"x86\\\"))\"},{\"kind\":\"dev\",\"name\":\"hex-literal\",\"req\":\"^0.3\"},{\"name\":\"opaque-debug\",\"req\":\"^0.3\"},{\"default_features\":false,\"name\":\"universal-hash\",\"req\":\"^0.5\"},{\"default_features\":false,\"name\":\"zeroize\",\"optional\":true,\"req\":\"^1\"}],\"features\":{\"std\":[\"universal-hash/std\"]}}", @@ -1390,7 +1390,8 @@ "pulldown-cmark_0.10.3": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"bincode\",\"req\":\"^1.3.1\"},{\"name\":\"bitflags\",\"req\":\"^2\"},{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.5\"},{\"name\":\"getopts\",\"optional\":true,\"req\":\"^0.2\"},{\"kind\":\"dev\",\"name\":\"lazy_static\",\"req\":\"^1.4\"},{\"name\":\"memchr\",\"req\":\"^2.5\"},{\"name\":\"pulldown-cmark-escape\",\"optional\":true,\"req\":\"^0.10.0\"},{\"kind\":\"dev\",\"name\":\"regex\",\"req\":\"^1.6\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0.61\"},{\"name\":\"unicase\",\"req\":\"^2.6\"}],\"features\":{\"default\":[\"getopts\",\"html\"],\"gen-tests\":[],\"html\":[\"pulldown-cmark-escape\"],\"simd\":[\"pulldown-cmark-escape?/simd\"]}}", "pxfm_0.1.27": "{\"dependencies\":[{\"name\":\"num-traits\",\"req\":\"^0.2.3\"}],\"features\":{}}", "quick-error_2.0.1": "{\"dependencies\":[],\"features\":{}}", - "quick-xml_0.38.4": "{\"dependencies\":[{\"features\":[\"derive\"],\"name\":\"arbitrary\",\"optional\":true,\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\">=0.4, <0.8\"},{\"name\":\"document-features\",\"optional\":true,\"req\":\"^0.2\"},{\"name\":\"encoding_rs\",\"optional\":true,\"req\":\"^0.8\"},{\"name\":\"memchr\",\"req\":\"^2.1\"},{\"kind\":\"dev\",\"name\":\"pretty_assertions\",\"req\":\"^1.4\"},{\"kind\":\"dev\",\"name\":\"regex\",\"req\":\"^1\"},{\"name\":\"serde\",\"optional\":true,\"req\":\">=1.0.139\"},{\"kind\":\"dev\",\"name\":\"serde-value\",\"req\":\"^0.7\"},{\"kind\":\"dev\",\"name\":\"serde_derive\",\"req\":\"^1.0.206\"},{\"default_features\":false,\"features\":[\"io-util\"],\"name\":\"tokio\",\"optional\":true,\"req\":\"^1.10\"},{\"default_features\":false,\"features\":[\"macros\",\"rt\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.21\"},{\"kind\":\"dev\",\"name\":\"tokio-test\",\"req\":\"^0.4\"}],\"features\":{\"async-tokio\":[\"tokio\"],\"default\":[],\"encoding\":[\"encoding_rs\"],\"escape-html\":[],\"overlapped-lists\":[],\"serde-types\":[\"serde/derive\"],\"serialize\":[\"serde\"]}}", + "quick-xml_0.39.4": "{\"dependencies\":[{\"features\":[\"derive\"],\"name\":\"arbitrary\",\"optional\":true,\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\">=0.4, <0.9\"},{\"name\":\"document-features\",\"optional\":true,\"req\":\"^0.2\"},{\"name\":\"encoding_rs\",\"optional\":true,\"req\":\"^0.8\"},{\"name\":\"memchr\",\"req\":\"^2.1\"},{\"kind\":\"dev\",\"name\":\"pretty_assertions\",\"req\":\"^1.4\"},{\"kind\":\"dev\",\"name\":\"regex\",\"req\":\"^1\"},{\"name\":\"serde\",\"optional\":true,\"req\":\">=1.0.180\"},{\"kind\":\"dev\",\"name\":\"serde-value\",\"req\":\"^0.7\"},{\"kind\":\"dev\",\"name\":\"serde_derive\",\"req\":\"^1.0.206\"},{\"default_features\":false,\"features\":[\"io-util\"],\"name\":\"tokio\",\"optional\":true,\"req\":\"^1.10\"},{\"default_features\":false,\"features\":[\"macros\",\"rt\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.21\"},{\"kind\":\"dev\",\"name\":\"tokio-test\",\"req\":\"^0.4\"}],\"features\":{\"async-tokio\":[\"tokio\"],\"default\":[],\"encoding\":[\"encoding_rs\"],\"escape-html\":[],\"overlapped-lists\":[],\"serde-types\":[\"serde/derive\"],\"serialize\":[\"serde\"]}}", + "quick-xml_0.41.0": "{\"dependencies\":[{\"features\":[\"derive\"],\"name\":\"arbitrary\",\"optional\":true,\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\">=0.4, <0.9\"},{\"name\":\"document-features\",\"optional\":true,\"req\":\"^0.2\"},{\"name\":\"encoding_rs\",\"optional\":true,\"req\":\"^0.8\"},{\"name\":\"memchr\",\"req\":\"^2.1\"},{\"kind\":\"dev\",\"name\":\"pretty_assertions\",\"req\":\"^1.4\"},{\"kind\":\"dev\",\"name\":\"regex\",\"req\":\"^1\"},{\"name\":\"serde\",\"optional\":true,\"req\":\">=1.0.180\"},{\"kind\":\"dev\",\"name\":\"serde-value\",\"req\":\"^0.7\"},{\"kind\":\"dev\",\"name\":\"serde_derive\",\"req\":\"^1.0.206\"},{\"default_features\":false,\"features\":[\"io-util\"],\"name\":\"tokio\",\"optional\":true,\"req\":\"^1.10\"},{\"default_features\":false,\"features\":[\"macros\",\"rt\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.21\"},{\"kind\":\"dev\",\"name\":\"tokio-test\",\"req\":\"^0.4\"}],\"features\":{\"async-tokio\":[\"tokio\"],\"default\":[],\"encoding\":[\"encoding_rs\"],\"escape-html\":[],\"overlapped-lists\":[],\"serde-types\":[\"serde/derive\"],\"serialize\":[\"serde\"]}}", "quickcheck_1.1.0": "{\"dependencies\":[{\"default_features\":false,\"name\":\"env_logger\",\"optional\":true,\"req\":\"^0.11\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4\"},{\"default_features\":false,\"features\":[\"sys_rng\"],\"name\":\"rand\",\"req\":\"^0.10\"}],\"features\":{\"default\":[\"regex\",\"use_logging\"],\"regex\":[\"env_logger/regex\"],\"use_logging\":[\"log\",\"env_logger\"]}}", "quinn-proto_0.11.14": "{\"dependencies\":[{\"features\":[\"derive\"],\"name\":\"arbitrary\",\"optional\":true,\"req\":\"^1.0.1\"},{\"kind\":\"dev\",\"name\":\"assert_matches\",\"req\":\"^1.1\"},{\"default_features\":false,\"name\":\"aws-lc-rs\",\"optional\":true,\"req\":\"^1.9\"},{\"name\":\"bytes\",\"req\":\"^1\"},{\"name\":\"fastbloom\",\"optional\":true,\"req\":\"^0.14\"},{\"default_features\":false,\"features\":[\"wasm_js\"],\"name\":\"getrandom\",\"req\":\"^0.3\",\"target\":\"cfg(all(target_family = \\\"wasm\\\", target_os = \\\"unknown\\\"))\"},{\"kind\":\"dev\",\"name\":\"hex-literal\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"lazy_static\",\"req\":\"^1\"},{\"name\":\"lru-slab\",\"req\":\"^0.1.2\"},{\"name\":\"qlog\",\"optional\":true,\"req\":\"^0.15.2\"},{\"name\":\"rand\",\"req\":\"^0.9\"},{\"kind\":\"dev\",\"name\":\"rand_pcg\",\"req\":\"^0.9\"},{\"kind\":\"dev\",\"name\":\"rcgen\",\"req\":\"^0.14\"},{\"features\":[\"wasm32_unknown_unknown_js\"],\"name\":\"ring\",\"req\":\"^0.17\",\"target\":\"cfg(all(target_family = \\\"wasm\\\", target_os = \\\"unknown\\\"))\"},{\"name\":\"ring\",\"optional\":true,\"req\":\"^0.17\"},{\"name\":\"rustc-hash\",\"req\":\"^2\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"rustls\",\"optional\":true,\"req\":\"^0.23.5\"},{\"features\":[\"web\"],\"name\":\"rustls-pki-types\",\"req\":\"^1.7\",\"target\":\"cfg(all(target_family = \\\"wasm\\\", target_os = \\\"unknown\\\"))\"},{\"name\":\"rustls-platform-verifier\",\"optional\":true,\"req\":\"^0.6\"},{\"name\":\"slab\",\"req\":\"^0.4.6\"},{\"name\":\"thiserror\",\"req\":\"^2.0.3\"},{\"features\":[\"alloc\",\"alloc\"],\"name\":\"tinyvec\",\"req\":\"^1.1\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"tracing\",\"req\":\"^0.1.10\"},{\"default_features\":false,\"features\":[\"env-filter\",\"fmt\",\"ansi\",\"time\",\"local-time\"],\"kind\":\"dev\",\"name\":\"tracing-subscriber\",\"req\":\"^0.3.0\"},{\"kind\":\"dev\",\"name\":\"wasm-bindgen-test\",\"req\":\"^0.3.45\"},{\"name\":\"web-time\",\"req\":\"^1\",\"target\":\"cfg(all(target_family = \\\"wasm\\\", target_os = \\\"unknown\\\"))\"}],\"features\":{\"__rustls-post-quantum-test\":[],\"aws-lc-rs\":[\"dep:aws-lc-rs\",\"aws-lc-rs?/aws-lc-sys\",\"aws-lc-rs?/prebuilt-nasm\"],\"aws-lc-rs-fips\":[\"aws-lc-rs\",\"aws-lc-rs?/fips\"],\"bloom\":[\"dep:fastbloom\"],\"default\":[\"rustls-ring\",\"log\",\"bloom\"],\"log\":[\"tracing/log\"],\"platform-verifier\":[\"dep:rustls-platform-verifier\"],\"qlog\":[\"dep:qlog\"],\"ring\":[\"dep:ring\"],\"rustls\":[\"rustls-ring\"],\"rustls-aws-lc-rs\":[\"dep:rustls\",\"rustls?/aws-lc-rs\",\"aws-lc-rs\"],\"rustls-aws-lc-rs-fips\":[\"rustls-aws-lc-rs\",\"aws-lc-rs-fips\"],\"rustls-log\":[\"rustls?/logging\"],\"rustls-ring\":[\"dep:rustls\",\"rustls?/ring\",\"ring\"]}}", "quinn-udp_0.5.14": "{\"dependencies\":[{\"kind\":\"build\",\"name\":\"cfg_aliases\",\"req\":\"^0.2\"},{\"default_features\":false,\"features\":[\"async_tokio\"],\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.7\"},{\"name\":\"libc\",\"req\":\"^0.2.158\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4\"},{\"name\":\"once_cell\",\"req\":\"^1.19\",\"target\":\"cfg(windows)\"},{\"name\":\"socket2\",\"req\":\">=0.5, <0.7\",\"target\":\"cfg(not(all(target_family = \\\"wasm\\\", target_os = \\\"unknown\\\")))\"},{\"features\":[\"sync\",\"rt\",\"rt-multi-thread\",\"net\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.28.1\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"tracing\",\"optional\":true,\"req\":\"^0.1.10\"},{\"features\":[\"Win32_Foundation\",\"Win32_System_IO\",\"Win32_Networking_WinSock\"],\"name\":\"windows-sys\",\"req\":\">=0.52, <=0.60\",\"target\":\"cfg(windows)\"}],\"features\":{\"default\":[\"tracing\",\"log\"],\"direct-log\":[\"dep:log\"],\"fast-apple-datapath\":[],\"log\":[\"tracing/log\"]}}", @@ -1749,7 +1750,7 @@ "wayland-client_0.31.12": "{\"dependencies\":[{\"name\":\"bitflags\",\"req\":\"^2\"},{\"kind\":\"dev\",\"name\":\"futures-channel\",\"req\":\"^0.3.16\"},{\"kind\":\"dev\",\"name\":\"futures-util\",\"req\":\"^0.3\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4\"},{\"features\":[\"event\"],\"name\":\"rustix\",\"req\":\"^1.0.2\"},{\"kind\":\"dev\",\"name\":\"tempfile\",\"req\":\"^3.2\"},{\"name\":\"wayland-backend\",\"req\":\"^0.3.12\"},{\"name\":\"wayland-scanner\",\"req\":\"^0.31.8\"}],\"features\":{}}", "wayland-protocols-wlr_0.3.10": "{\"dependencies\":[{\"name\":\"bitflags\",\"req\":\"^2\"},{\"name\":\"wayland-backend\",\"req\":\"^0.3.12\"},{\"name\":\"wayland-client\",\"optional\":true,\"req\":\"^0.31.12\"},{\"name\":\"wayland-protocols\",\"req\":\"^0.32.10\"},{\"name\":\"wayland-scanner\",\"req\":\"^0.31.8\"},{\"name\":\"wayland-server\",\"optional\":true,\"req\":\"^0.31.11\"}],\"features\":{\"client\":[\"wayland-client\",\"wayland-protocols/client\"],\"server\":[\"wayland-server\",\"wayland-protocols/server\"]}}", "wayland-protocols_0.32.10": "{\"dependencies\":[{\"name\":\"bitflags\",\"req\":\"^2\"},{\"name\":\"wayland-backend\",\"req\":\"^0.3.12\"},{\"name\":\"wayland-client\",\"optional\":true,\"req\":\"^0.31.12\"},{\"name\":\"wayland-scanner\",\"req\":\"^0.31.8\"},{\"name\":\"wayland-server\",\"optional\":true,\"req\":\"^0.31.11\"}],\"features\":{\"client\":[\"wayland-client\"],\"server\":[\"wayland-server\"],\"staging\":[],\"unstable\":[]}}", - "wayland-scanner_0.31.8": "{\"dependencies\":[{\"name\":\"proc-macro2\",\"req\":\"^1.0.11\"},{\"name\":\"quick-xml\",\"req\":\"^0.38.3\"},{\"name\":\"quote\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"similar\",\"req\":\"^2\"}],\"features\":{}}", + "wayland-scanner_0.31.10": "{\"dependencies\":[{\"name\":\"proc-macro2\",\"req\":\"^1.0.11\"},{\"name\":\"quick-xml\",\"req\":\"^0.39\"},{\"name\":\"quote\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"similar\",\"req\":\"^2\"}],\"features\":{}}", "wayland-sys_0.31.8": "{\"dependencies\":[{\"name\":\"dlib\",\"optional\":true,\"req\":\"^0.5.1\"},{\"name\":\"libc\",\"optional\":true,\"req\":\"^0.2\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4\"},{\"name\":\"memoffset\",\"optional\":true,\"req\":\"^0.9\"},{\"name\":\"once_cell\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"build\",\"name\":\"pkg-config\",\"req\":\"^0.3.7\"}],\"features\":{\"client\":[\"dep:dlib\",\"dep:log\"],\"cursor\":[\"client\"],\"dlopen\":[\"once_cell\"],\"egl\":[\"client\"],\"server\":[\"libc\",\"memoffset\",\"dep:dlib\",\"dep:log\"]}}", "web-sys_0.3.85": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3\",\"target\":\"cfg(target_arch = \\\"wasm32\\\")\"},{\"default_features\":false,\"name\":\"js-sys\",\"req\":\"=0.3.85\"},{\"default_features\":false,\"name\":\"wasm-bindgen\",\"req\":\"=0.2.108\"}],\"features\":{\"AbortController\":[],\"AbortSignal\":[\"EventTarget\"],\"AddEventListenerOptions\":[],\"AesCbcParams\":[],\"AesCtrParams\":[],\"AesDerivedKeyParams\":[],\"AesGcmParams\":[],\"AesKeyAlgorithm\":[],\"AesKeyGenParams\":[],\"Algorithm\":[],\"AlignSetting\":[],\"AllowedBluetoothDevice\":[],\"AllowedUsbDevice\":[],\"AlphaOption\":[],\"AnalyserNode\":[\"AudioNode\",\"EventTarget\"],\"AnalyserOptions\":[],\"AngleInstancedArrays\":[],\"Animation\":[\"EventTarget\"],\"AnimationEffect\":[],\"AnimationEvent\":[\"Event\"],\"AnimationEventInit\":[],\"AnimationPlayState\":[],\"AnimationPlaybackEvent\":[\"Event\"],\"AnimationPlaybackEventInit\":[],\"AnimationPropertyDetails\":[],\"AnimationPropertyValueDetails\":[],\"AnimationTimeline\":[],\"AssignedNodesOptions\":[],\"AttestationConveyancePreference\":[],\"Attr\":[\"EventTarget\",\"Node\"],\"AttributeNameValue\":[],\"AudioBuffer\":[],\"AudioBufferOptions\":[],\"AudioBufferSourceNode\":[\"AudioNode\",\"AudioScheduledSourceNode\",\"EventTarget\"],\"AudioBufferSourceOptions\":[],\"AudioConfiguration\":[],\"AudioContext\":[\"BaseAudioContext\",\"EventTarget\"],\"AudioContextLatencyCategory\":[],\"AudioContextOptions\":[],\"AudioContextState\":[],\"AudioData\":[],\"AudioDataCopyToOptions\":[],\"AudioDataInit\":[],\"AudioDecoder\":[],\"AudioDecoderConfig\":[],\"AudioDecoderInit\":[],\"AudioDecoderSupport\":[],\"AudioDestinationNode\":[\"AudioNode\",\"EventTarget\"],\"AudioEncoder\":[],\"AudioEncoderConfig\":[],\"AudioEncoderInit\":[],\"AudioEncoderSupport\":[],\"AudioListener\":[],\"AudioNode\":[\"EventTarget\"],\"AudioNodeOptions\":[],\"AudioParam\":[],\"AudioParamMap\":[],\"AudioProcessingEvent\":[\"Event\"],\"AudioSampleFormat\":[],\"AudioScheduledSourceNode\":[\"AudioNode\",\"EventTarget\"],\"AudioSinkInfo\":[],\"AudioSinkOptions\":[],\"AudioSinkType\":[],\"AudioStreamTrack\":[\"EventTarget\",\"MediaStreamTrack\"],\"AudioTrack\":[],\"AudioTrackList\":[\"EventTarget\"],\"AudioWorklet\":[\"Worklet\"],\"AudioWorkletGlobalScope\":[\"WorkletGlobalScope\"],\"AudioWorkletNode\":[\"AudioNode\",\"EventTarget\"],\"AudioWorkletNodeOptions\":[],\"AudioWorkletProcessor\":[],\"AuthenticationExtensionsClientInputs\":[],\"AuthenticationExtensionsClientInputsJson\":[],\"AuthenticationExtensionsClientOutputs\":[],\"AuthenticationExtensionsClientOutputsJson\":[],\"AuthenticationExtensionsDevicePublicKeyInputs\":[],\"AuthenticationExtensionsDevicePublicKeyOutputs\":[],\"AuthenticationExtensionsLargeBlobInputs\":[],\"AuthenticationExtensionsLargeBlobOutputs\":[],\"AuthenticationExtensionsPrfInputs\":[],\"AuthenticationExtensionsPrfOutputs\":[],\"AuthenticationExtensionsPrfValues\":[],\"AuthenticationResponseJson\":[],\"AuthenticatorAssertionResponse\":[\"AuthenticatorResponse\"],\"AuthenticatorAssertionResponseJson\":[],\"AuthenticatorAttachment\":[],\"AuthenticatorAttestationResponse\":[\"AuthenticatorResponse\"],\"AuthenticatorAttestationResponseJson\":[],\"AuthenticatorResponse\":[],\"AuthenticatorSelectionCriteria\":[],\"AuthenticatorTransport\":[],\"AutoKeyword\":[],\"AutocompleteInfo\":[],\"BarProp\":[],\"BaseAudioContext\":[\"EventTarget\"],\"BaseComputedKeyframe\":[],\"BaseKeyframe\":[],\"BasePropertyIndexedKeyframe\":[],\"BasicCardRequest\":[],\"BasicCardResponse\":[],\"BasicCardType\":[],\"BatteryManager\":[\"EventTarget\"],\"BeforeUnloadEvent\":[\"Event\"],\"BinaryType\":[],\"BiquadFilterNode\":[\"AudioNode\",\"EventTarget\"],\"BiquadFilterOptions\":[],\"BiquadFilterType\":[],\"Blob\":[],\"BlobEvent\":[\"Event\"],\"BlobEventInit\":[],\"BlobPropertyBag\":[],\"BlockParsingOptions\":[],\"Bluetooth\":[\"EventTarget\"],\"BluetoothAdvertisingEvent\":[\"Event\"],\"BluetoothAdvertisingEventInit\":[],\"BluetoothCharacteristicProperties\":[],\"BluetoothDataFilterInit\":[],\"BluetoothDevice\":[\"EventTarget\"],\"BluetoothLeScanFilterInit\":[],\"BluetoothManufacturerDataMap\":[],\"BluetoothPermissionDescriptor\":[],\"BluetoothPermissionResult\":[\"EventTarget\",\"PermissionStatus\"],\"BluetoothPermissionStorage\":[],\"BluetoothRemoteGattCharacteristic\":[\"EventTarget\"],\"BluetoothRemoteGattDescriptor\":[],\"BluetoothRemoteGattServer\":[],\"BluetoothRemoteGattService\":[\"EventTarget\"],\"BluetoothServiceDataMap\":[],\"BluetoothUuid\":[],\"BoxQuadOptions\":[],\"BroadcastChannel\":[\"EventTarget\"],\"BrowserElementDownloadOptions\":[],\"BrowserElementExecuteScriptOptions\":[],\"BrowserFeedWriter\":[],\"BrowserFindCaseSensitivity\":[],\"BrowserFindDirection\":[],\"ByteLengthQueuingStrategy\":[],\"Cache\":[],\"CacheBatchOperation\":[],\"CacheQueryOptions\":[],\"CacheStorage\":[],\"CacheStorageNamespace\":[],\"CanvasCaptureMediaStream\":[\"EventTarget\",\"MediaStream\"],\"CanvasCaptureMediaStreamTrack\":[\"EventTarget\",\"MediaStreamTrack\"],\"CanvasGradient\":[],\"CanvasPattern\":[],\"CanvasRenderingContext2d\":[],\"CanvasWindingRule\":[],\"CaretChangedReason\":[],\"CaretPosition\":[],\"CaretStateChangedEventInit\":[],\"CdataSection\":[\"CharacterData\",\"EventTarget\",\"Node\",\"Text\"],\"ChannelCountMode\":[],\"ChannelInterpretation\":[],\"ChannelMergerNode\":[\"AudioNode\",\"EventTarget\"],\"ChannelMergerOptions\":[],\"ChannelSplitterNode\":[\"AudioNode\",\"EventTarget\"],\"ChannelSplitterOptions\":[],\"CharacterData\":[\"EventTarget\",\"Node\"],\"CheckerboardReason\":[],\"CheckerboardReport\":[],\"CheckerboardReportService\":[],\"ChromeFilePropertyBag\":[],\"ChromeWorker\":[\"EventTarget\",\"Worker\"],\"Client\":[],\"ClientQueryOptions\":[],\"ClientRectsAndTexts\":[],\"ClientType\":[],\"Clients\":[],\"Clipboard\":[\"EventTarget\"],\"ClipboardEvent\":[\"Event\"],\"ClipboardEventInit\":[],\"ClipboardItem\":[],\"ClipboardItemOptions\":[],\"ClipboardPermissionDescriptor\":[],\"ClipboardUnsanitizedFormats\":[],\"CloseEvent\":[\"Event\"],\"CloseEventInit\":[],\"CodecState\":[],\"CollectedClientData\":[],\"ColorSpaceConversion\":[],\"Comment\":[\"CharacterData\",\"EventTarget\",\"Node\"],\"CompositeOperation\":[],\"CompositionEvent\":[\"Event\",\"UiEvent\"],\"CompositionEventInit\":[],\"CompressionFormat\":[],\"CompressionStream\":[],\"ComputedEffectTiming\":[],\"ConnStatusDict\":[],\"ConnectionType\":[],\"ConsoleCounter\":[],\"ConsoleCounterError\":[],\"ConsoleEvent\":[],\"ConsoleInstance\":[],\"ConsoleInstanceOptions\":[],\"ConsoleLevel\":[],\"ConsoleLogLevel\":[],\"ConsoleProfileEvent\":[],\"ConsoleStackEntry\":[],\"ConsoleTimerError\":[],\"ConsoleTimerLogOrEnd\":[],\"ConsoleTimerStart\":[],\"ConstantSourceNode\":[\"AudioNode\",\"AudioScheduledSourceNode\",\"EventTarget\"],\"ConstantSourceOptions\":[],\"ConstrainBooleanParameters\":[],\"ConstrainDomStringParameters\":[],\"ConstrainDoubleRange\":[],\"ConstrainLongRange\":[],\"ContextAttributes2d\":[],\"ConvertCoordinateOptions\":[],\"ConvolverNode\":[\"AudioNode\",\"EventTarget\"],\"ConvolverOptions\":[],\"CookieChangeEvent\":[\"Event\"],\"CookieChangeEventInit\":[],\"CookieInit\":[],\"CookieListItem\":[],\"CookieSameSite\":[],\"CookieStore\":[\"EventTarget\"],\"CookieStoreDeleteOptions\":[],\"CookieStoreGetOptions\":[],\"CookieStoreManager\":[],\"Coordinates\":[],\"CountQueuingStrategy\":[],\"Credential\":[],\"CredentialCreationOptions\":[],\"CredentialPropertiesOutput\":[],\"CredentialRequestOptions\":[],\"CredentialsContainer\":[],\"Crypto\":[],\"CryptoKey\":[],\"CryptoKeyPair\":[],\"CssAnimation\":[\"Animation\",\"EventTarget\"],\"CssBoxType\":[],\"CssConditionRule\":[\"CssGroupingRule\",\"CssRule\"],\"CssCounterStyleRule\":[\"CssRule\"],\"CssFontFaceRule\":[\"CssRule\"],\"CssFontFeatureValuesRule\":[\"CssRule\"],\"CssGroupingRule\":[\"CssRule\"],\"CssImportRule\":[\"CssRule\"],\"CssKeyframeRule\":[\"CssRule\"],\"CssKeyframesRule\":[\"CssRule\"],\"CssMediaRule\":[\"CssConditionRule\",\"CssGroupingRule\",\"CssRule\"],\"CssNamespaceRule\":[\"CssRule\"],\"CssPageRule\":[\"CssRule\"],\"CssPseudoElement\":[],\"CssRule\":[],\"CssRuleList\":[],\"CssStyleDeclaration\":[],\"CssStyleRule\":[\"CssRule\"],\"CssStyleSheet\":[\"StyleSheet\"],\"CssStyleSheetParsingMode\":[],\"CssSupportsRule\":[\"CssConditionRule\",\"CssGroupingRule\",\"CssRule\"],\"CssTransition\":[\"Animation\",\"EventTarget\"],\"CustomElementRegistry\":[],\"CustomEvent\":[\"Event\"],\"CustomEventInit\":[],\"DataTransfer\":[],\"DataTransferItem\":[],\"DataTransferItemList\":[],\"DateTimeValue\":[],\"DecoderDoctorNotification\":[],\"DecoderDoctorNotificationType\":[],\"DecompressionStream\":[],\"DedicatedWorkerGlobalScope\":[\"EventTarget\",\"WorkerGlobalScope\"],\"DelayNode\":[\"AudioNode\",\"EventTarget\"],\"DelayOptions\":[],\"DeviceAcceleration\":[],\"DeviceAccelerationInit\":[],\"DeviceLightEvent\":[\"Event\"],\"DeviceLightEventInit\":[],\"DeviceMotionEvent\":[\"Event\"],\"DeviceMotionEventInit\":[],\"DeviceOrientationEvent\":[\"Event\"],\"DeviceOrientationEventInit\":[],\"DeviceProximityEvent\":[\"Event\"],\"DeviceProximityEventInit\":[],\"DeviceRotationRate\":[],\"DeviceRotationRateInit\":[],\"DhKeyDeriveParams\":[],\"DirectionSetting\":[],\"Directory\":[],\"DirectoryPickerOptions\":[],\"DisplayMediaStreamConstraints\":[],\"DisplayNameOptions\":[],\"DisplayNameResult\":[],\"DistanceModelType\":[],\"DnsCacheDict\":[],\"DnsCacheEntry\":[],\"DnsLookupDict\":[],\"Document\":[\"EventTarget\",\"Node\"],\"DocumentFragment\":[\"EventTarget\",\"Node\"],\"DocumentTimeline\":[\"AnimationTimeline\"],\"DocumentTimelineOptions\":[],\"DocumentType\":[\"EventTarget\",\"Node\"],\"DomError\":[],\"DomException\":[],\"DomImplementation\":[],\"DomMatrix\":[\"DomMatrixReadOnly\"],\"DomMatrix2dInit\":[],\"DomMatrixInit\":[],\"DomMatrixReadOnly\":[],\"DomParser\":[],\"DomPoint\":[\"DomPointReadOnly\"],\"DomPointInit\":[],\"DomPointReadOnly\":[],\"DomQuad\":[],\"DomQuadInit\":[],\"DomQuadJson\":[],\"DomRect\":[\"DomRectReadOnly\"],\"DomRectInit\":[],\"DomRectList\":[],\"DomRectReadOnly\":[],\"DomRequest\":[\"EventTarget\"],\"DomRequestReadyState\":[],\"DomStringList\":[],\"DomStringMap\":[],\"DomTokenList\":[],\"DomWindowResizeEventDetail\":[],\"DoubleRange\":[],\"DragEvent\":[\"Event\",\"MouseEvent\",\"UiEvent\"],\"DragEventInit\":[],\"DynamicsCompressorNode\":[\"AudioNode\",\"EventTarget\"],\"DynamicsCompressorOptions\":[],\"EcKeyAlgorithm\":[],\"EcKeyGenParams\":[],\"EcKeyImportParams\":[],\"EcdhKeyDeriveParams\":[],\"EcdsaParams\":[],\"EffectTiming\":[],\"Element\":[\"EventTarget\",\"Node\"],\"ElementCreationOptions\":[],\"ElementDefinitionOptions\":[],\"EncodedAudioChunk\":[],\"EncodedAudioChunkInit\":[],\"EncodedAudioChunkMetadata\":[],\"EncodedAudioChunkType\":[],\"EncodedVideoChunk\":[],\"EncodedVideoChunkInit\":[],\"EncodedVideoChunkMetadata\":[],\"EncodedVideoChunkType\":[],\"EndingTypes\":[],\"ErrorCallback\":[],\"ErrorEvent\":[\"Event\"],\"ErrorEventInit\":[],\"Event\":[],\"EventInit\":[],\"EventListener\":[],\"EventListenerOptions\":[],\"EventModifierInit\":[],\"EventSource\":[\"EventTarget\"],\"EventSourceInit\":[],\"EventTarget\":[],\"Exception\":[],\"ExtBlendMinmax\":[],\"ExtColorBufferFloat\":[],\"ExtColorBufferHalfFloat\":[],\"ExtDisjointTimerQuery\":[],\"ExtFragDepth\":[],\"ExtSRgb\":[],\"ExtShaderTextureLod\":[],\"ExtTextureFilterAnisotropic\":[],\"ExtTextureNorm16\":[],\"ExtendableCookieChangeEvent\":[\"Event\",\"ExtendableEvent\"],\"ExtendableCookieChangeEventInit\":[],\"ExtendableEvent\":[\"Event\"],\"ExtendableEventInit\":[],\"ExtendableMessageEvent\":[\"Event\",\"ExtendableEvent\"],\"ExtendableMessageEventInit\":[],\"External\":[],\"FakePluginMimeEntry\":[],\"FakePluginTagInit\":[],\"FetchEvent\":[\"Event\",\"ExtendableEvent\"],\"FetchEventInit\":[],\"FetchObserver\":[\"EventTarget\"],\"FetchReadableStreamReadDataArray\":[],\"FetchReadableStreamReadDataDone\":[],\"FetchState\":[],\"File\":[\"Blob\"],\"FileCallback\":[],\"FileList\":[],\"FilePickerAcceptType\":[],\"FilePickerOptions\":[],\"FilePropertyBag\":[],\"FileReader\":[\"EventTarget\"],\"FileReaderSync\":[],\"FileSystem\":[],\"FileSystemCreateWritableOptions\":[],\"FileSystemDirectoryEntry\":[\"FileSystemEntry\"],\"FileSystemDirectoryHandle\":[\"FileSystemHandle\"],\"FileSystemDirectoryReader\":[],\"FileSystemEntriesCallback\":[],\"FileSystemEntry\":[],\"FileSystemEntryCallback\":[],\"FileSystemFileEntry\":[\"FileSystemEntry\"],\"FileSystemFileHandle\":[\"FileSystemHandle\"],\"FileSystemFlags\":[],\"FileSystemGetDirectoryOptions\":[],\"FileSystemGetFileOptions\":[],\"FileSystemHandle\":[],\"FileSystemHandleKind\":[],\"FileSystemHandlePermissionDescriptor\":[],\"FileSystemPermissionDescriptor\":[],\"FileSystemPermissionMode\":[],\"FileSystemReadWriteOptions\":[],\"FileSystemRemoveOptions\":[],\"FileSystemSyncAccessHandle\":[],\"FileSystemWritableFileStream\":[\"WritableStream\"],\"FillMode\":[],\"FlashClassification\":[],\"FlowControlType\":[],\"FocusEvent\":[\"Event\",\"UiEvent\"],\"FocusEventInit\":[],\"FocusOptions\":[],\"FontData\":[],\"FontFace\":[],\"FontFaceDescriptors\":[],\"FontFaceLoadStatus\":[],\"FontFaceSet\":[\"EventTarget\"],\"FontFaceSetIterator\":[],\"FontFaceSetIteratorResult\":[],\"FontFaceSetLoadEvent\":[\"Event\"],\"FontFaceSetLoadEventInit\":[],\"FontFaceSetLoadStatus\":[],\"FormData\":[],\"FrameType\":[],\"FuzzingFunctions\":[],\"GainNode\":[\"AudioNode\",\"EventTarget\"],\"GainOptions\":[],\"Gamepad\":[],\"GamepadButton\":[],\"GamepadEffectParameters\":[],\"GamepadEvent\":[\"Event\"],\"GamepadEventInit\":[],\"GamepadHand\":[],\"GamepadHapticActuator\":[],\"GamepadHapticActuatorType\":[],\"GamepadHapticEffectType\":[],\"GamepadHapticsResult\":[],\"GamepadMappingType\":[],\"GamepadPose\":[],\"GamepadTouch\":[],\"Geolocation\":[],\"GestureEvent\":[\"Event\",\"UiEvent\"],\"GetAnimationsOptions\":[],\"GetRootNodeOptions\":[],\"GetUserMediaRequest\":[],\"Gpu\":[],\"GpuAdapter\":[],\"GpuAdapterInfo\":[],\"GpuAddressMode\":[],\"GpuAutoLayoutMode\":[],\"GpuBindGroup\":[],\"GpuBindGroupDescriptor\":[],\"GpuBindGroupEntry\":[],\"GpuBindGroupLayout\":[],\"GpuBindGroupLayoutDescriptor\":[],\"GpuBindGroupLayoutEntry\":[],\"GpuBlendComponent\":[],\"GpuBlendFactor\":[],\"GpuBlendOperation\":[],\"GpuBlendState\":[],\"GpuBuffer\":[],\"GpuBufferBinding\":[],\"GpuBufferBindingLayout\":[],\"GpuBufferBindingType\":[],\"GpuBufferDescriptor\":[],\"GpuBufferMapState\":[],\"GpuCanvasAlphaMode\":[],\"GpuCanvasConfiguration\":[],\"GpuCanvasContext\":[],\"GpuCanvasToneMapping\":[],\"GpuCanvasToneMappingMode\":[],\"GpuColorDict\":[],\"GpuColorTargetState\":[],\"GpuCommandBuffer\":[],\"GpuCommandBufferDescriptor\":[],\"GpuCommandEncoder\":[],\"GpuCommandEncoderDescriptor\":[],\"GpuCompareFunction\":[],\"GpuCompilationInfo\":[],\"GpuCompilationMessage\":[],\"GpuCompilationMessageType\":[],\"GpuComputePassDescriptor\":[],\"GpuComputePassEncoder\":[],\"GpuComputePassTimestampWrites\":[],\"GpuComputePipeline\":[],\"GpuComputePipelineDescriptor\":[],\"GpuCopyExternalImageDestInfo\":[],\"GpuCopyExternalImageSourceInfo\":[],\"GpuCullMode\":[],\"GpuDepthStencilState\":[],\"GpuDevice\":[\"EventTarget\"],\"GpuDeviceDescriptor\":[],\"GpuDeviceLostInfo\":[],\"GpuDeviceLostReason\":[],\"GpuError\":[],\"GpuErrorFilter\":[],\"GpuExtent3dDict\":[],\"GpuExternalTexture\":[],\"GpuExternalTextureBindingLayout\":[],\"GpuExternalTextureDescriptor\":[],\"GpuFeatureName\":[],\"GpuFilterMode\":[],\"GpuFragmentState\":[],\"GpuFrontFace\":[],\"GpuIndexFormat\":[],\"GpuInternalError\":[\"GpuError\"],\"GpuLoadOp\":[],\"GpuMipmapFilterMode\":[],\"GpuMultisampleState\":[],\"GpuObjectDescriptorBase\":[],\"GpuOrigin2dDict\":[],\"GpuOrigin3dDict\":[],\"GpuOutOfMemoryError\":[\"GpuError\"],\"GpuPipelineDescriptorBase\":[],\"GpuPipelineError\":[\"DomException\"],\"GpuPipelineErrorInit\":[],\"GpuPipelineErrorReason\":[],\"GpuPipelineLayout\":[],\"GpuPipelineLayoutDescriptor\":[],\"GpuPowerPreference\":[],\"GpuPrimitiveState\":[],\"GpuPrimitiveTopology\":[],\"GpuProgrammableStage\":[],\"GpuQuerySet\":[],\"GpuQuerySetDescriptor\":[],\"GpuQueryType\":[],\"GpuQueue\":[],\"GpuQueueDescriptor\":[],\"GpuRenderBundle\":[],\"GpuRenderBundleDescriptor\":[],\"GpuRenderBundleEncoder\":[],\"GpuRenderBundleEncoderDescriptor\":[],\"GpuRenderPassColorAttachment\":[],\"GpuRenderPassDepthStencilAttachment\":[],\"GpuRenderPassDescriptor\":[],\"GpuRenderPassEncoder\":[],\"GpuRenderPassLayout\":[],\"GpuRenderPassTimestampWrites\":[],\"GpuRenderPipeline\":[],\"GpuRenderPipelineDescriptor\":[],\"GpuRequestAdapterOptions\":[],\"GpuSampler\":[],\"GpuSamplerBindingLayout\":[],\"GpuSamplerBindingType\":[],\"GpuSamplerDescriptor\":[],\"GpuShaderModule\":[],\"GpuShaderModuleCompilationHint\":[],\"GpuShaderModuleDescriptor\":[],\"GpuStencilFaceState\":[],\"GpuStencilOperation\":[],\"GpuStorageTextureAccess\":[],\"GpuStorageTextureBindingLayout\":[],\"GpuStoreOp\":[],\"GpuSupportedFeatures\":[],\"GpuSupportedLimits\":[],\"GpuTexelCopyBufferInfo\":[],\"GpuTexelCopyBufferLayout\":[],\"GpuTexelCopyTextureInfo\":[],\"GpuTexture\":[],\"GpuTextureAspect\":[],\"GpuTextureBindingLayout\":[],\"GpuTextureDescriptor\":[],\"GpuTextureDimension\":[],\"GpuTextureFormat\":[],\"GpuTextureSampleType\":[],\"GpuTextureView\":[],\"GpuTextureViewDescriptor\":[],\"GpuTextureViewDimension\":[],\"GpuUncapturedErrorEvent\":[\"Event\"],\"GpuUncapturedErrorEventInit\":[],\"GpuValidationError\":[\"GpuError\"],\"GpuVertexAttribute\":[],\"GpuVertexBufferLayout\":[],\"GpuVertexFormat\":[],\"GpuVertexState\":[],\"GpuVertexStepMode\":[],\"GroupedHistoryEventInit\":[],\"HalfOpenInfoDict\":[],\"HardwareAcceleration\":[],\"HashChangeEvent\":[\"Event\"],\"HashChangeEventInit\":[],\"Headers\":[],\"HeadersGuardEnum\":[],\"Hid\":[\"EventTarget\"],\"HidCollectionInfo\":[],\"HidConnectionEvent\":[\"Event\"],\"HidConnectionEventInit\":[],\"HidDevice\":[\"EventTarget\"],\"HidDeviceFilter\":[],\"HidDeviceRequestOptions\":[],\"HidInputReportEvent\":[\"Event\"],\"HidInputReportEventInit\":[],\"HidReportInfo\":[],\"HidReportItem\":[],\"HidUnitSystem\":[],\"HiddenPluginEventInit\":[],\"History\":[],\"HitRegionOptions\":[],\"HkdfParams\":[],\"HmacDerivedKeyParams\":[],\"HmacImportParams\":[],\"HmacKeyAlgorithm\":[],\"HmacKeyGenParams\":[],\"HtmlAllCollection\":[],\"HtmlAnchorElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlAreaElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlAudioElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"HtmlMediaElement\",\"Node\"],\"HtmlBaseElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlBodyElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlBrElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlButtonElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlCanvasElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlCollection\":[],\"HtmlDListElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlDataElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlDataListElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlDetailsElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlDialogElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlDirectoryElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlDivElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlDocument\":[\"Document\",\"EventTarget\",\"Node\"],\"HtmlElement\":[\"Element\",\"EventTarget\",\"Node\"],\"HtmlEmbedElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlFieldSetElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlFontElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlFormControlsCollection\":[\"HtmlCollection\"],\"HtmlFormElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlFrameElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlFrameSetElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlHeadElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlHeadingElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlHrElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlHtmlElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlIFrameElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlImageElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlInputElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlLabelElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlLegendElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlLiElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlLinkElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlMapElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlMediaElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlMenuElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlMenuItemElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlMetaElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlMeterElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlModElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlOListElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlObjectElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlOptGroupElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlOptionElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlOptionsCollection\":[\"HtmlCollection\"],\"HtmlOutputElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlParagraphElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlParamElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlPictureElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlPreElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlProgressElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlQuoteElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlScriptElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlSelectElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlSlotElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlSourceElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlSpanElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlStyleElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlTableCaptionElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlTableCellElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlTableColElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlTableElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlTableRowElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlTableSectionElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlTemplateElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlTextAreaElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlTimeElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlTitleElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlTrackElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlUListElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlUnknownElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlVideoElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"HtmlMediaElement\",\"Node\"],\"HttpConnDict\":[],\"HttpConnInfo\":[],\"HttpConnectionElement\":[],\"IdbCursor\":[],\"IdbCursorDirection\":[],\"IdbCursorWithValue\":[\"IdbCursor\"],\"IdbDatabase\":[\"EventTarget\"],\"IdbFactory\":[],\"IdbFileHandle\":[\"EventTarget\"],\"IdbFileMetadataParameters\":[],\"IdbFileRequest\":[\"DomRequest\",\"EventTarget\"],\"IdbIndex\":[],\"IdbIndexParameters\":[],\"IdbKeyRange\":[],\"IdbLocaleAwareKeyRange\":[\"IdbKeyRange\"],\"IdbMutableFile\":[\"EventTarget\"],\"IdbObjectStore\":[],\"IdbObjectStoreParameters\":[],\"IdbOpenDbOptions\":[],\"IdbOpenDbRequest\":[\"EventTarget\",\"IdbRequest\"],\"IdbRequest\":[\"EventTarget\"],\"IdbRequestReadyState\":[],\"IdbTransaction\":[\"EventTarget\"],\"IdbTransactionDurability\":[],\"IdbTransactionMode\":[],\"IdbTransactionOptions\":[],\"IdbVersionChangeEvent\":[\"Event\"],\"IdbVersionChangeEventInit\":[],\"IdleDeadline\":[],\"IdleRequestOptions\":[],\"IirFilterNode\":[\"AudioNode\",\"EventTarget\"],\"IirFilterOptions\":[],\"ImageBitmap\":[],\"ImageBitmapOptions\":[],\"ImageBitmapRenderingContext\":[],\"ImageCapture\":[],\"ImageCaptureError\":[],\"ImageCaptureErrorEvent\":[\"Event\"],\"ImageCaptureErrorEventInit\":[],\"ImageData\":[],\"ImageDecodeOptions\":[],\"ImageDecodeResult\":[],\"ImageDecoder\":[],\"ImageDecoderInit\":[],\"ImageEncodeOptions\":[],\"ImageOrientation\":[],\"ImageTrack\":[\"EventTarget\"],\"ImageTrackList\":[],\"InputDeviceInfo\":[\"MediaDeviceInfo\"],\"InputEvent\":[\"Event\",\"UiEvent\"],\"InputEventInit\":[],\"IntersectionObserver\":[],\"IntersectionObserverEntry\":[],\"IntersectionObserverEntryInit\":[],\"IntersectionObserverInit\":[],\"IntlUtils\":[],\"IsInputPendingOptions\":[],\"IterableKeyAndValueResult\":[],\"IterableKeyOrValueResult\":[],\"IterationCompositeOperation\":[],\"JsonWebKey\":[],\"KeyAlgorithm\":[],\"KeyEvent\":[],\"KeyFrameRequestEvent\":[\"Event\"],\"KeyIdsInitData\":[],\"KeyboardEvent\":[\"Event\",\"UiEvent\"],\"KeyboardEventInit\":[],\"KeyframeAnimationOptions\":[],\"KeyframeEffect\":[\"AnimationEffect\"],\"KeyframeEffectOptions\":[],\"L10nElement\":[],\"L10nValue\":[],\"LargeBlobSupport\":[],\"LatencyMode\":[],\"LifecycleCallbacks\":[],\"LineAlignSetting\":[],\"ListBoxObject\":[],\"LocalMediaStream\":[\"EventTarget\",\"MediaStream\"],\"LocaleInfo\":[],\"Location\":[],\"Lock\":[],\"LockInfo\":[],\"LockManager\":[],\"LockManagerSnapshot\":[],\"LockMode\":[],\"LockOptions\":[],\"MathMlElement\":[\"Element\",\"EventTarget\",\"Node\"],\"MediaCapabilities\":[],\"MediaCapabilitiesInfo\":[],\"MediaConfiguration\":[],\"MediaDecodingConfiguration\":[],\"MediaDecodingType\":[],\"MediaDeviceInfo\":[],\"MediaDeviceKind\":[],\"MediaDevices\":[\"EventTarget\"],\"MediaElementAudioSourceNode\":[\"AudioNode\",\"EventTarget\"],\"MediaElementAudioSourceOptions\":[],\"MediaEncodingConfiguration\":[],\"MediaEncodingType\":[],\"MediaEncryptedEvent\":[\"Event\"],\"MediaError\":[],\"MediaImage\":[],\"MediaKeyError\":[\"Event\"],\"MediaKeyMessageEvent\":[\"Event\"],\"MediaKeyMessageEventInit\":[],\"MediaKeyMessageType\":[],\"MediaKeyNeededEventInit\":[],\"MediaKeySession\":[\"EventTarget\"],\"MediaKeySessionType\":[],\"MediaKeyStatus\":[],\"MediaKeyStatusMap\":[],\"MediaKeySystemAccess\":[],\"MediaKeySystemConfiguration\":[],\"MediaKeySystemMediaCapability\":[],\"MediaKeySystemStatus\":[],\"MediaKeys\":[],\"MediaKeysPolicy\":[],\"MediaKeysRequirement\":[],\"MediaList\":[],\"MediaMetadata\":[],\"MediaMetadataInit\":[],\"MediaPositionState\":[],\"MediaQueryList\":[\"EventTarget\"],\"MediaQueryListEvent\":[\"Event\"],\"MediaQueryListEventInit\":[],\"MediaRecorder\":[\"EventTarget\"],\"MediaRecorderErrorEvent\":[\"Event\"],\"MediaRecorderErrorEventInit\":[],\"MediaRecorderOptions\":[],\"MediaSession\":[],\"MediaSessionAction\":[],\"MediaSessionActionDetails\":[],\"MediaSessionPlaybackState\":[],\"MediaSource\":[\"EventTarget\"],\"MediaSourceEndOfStreamError\":[],\"MediaSourceEnum\":[],\"MediaSourceReadyState\":[],\"MediaStream\":[\"EventTarget\"],\"MediaStreamAudioDestinationNode\":[\"AudioNode\",\"EventTarget\"],\"MediaStreamAudioSourceNode\":[\"AudioNode\",\"EventTarget\"],\"MediaStreamAudioSourceOptions\":[],\"MediaStreamConstraints\":[],\"MediaStreamError\":[],\"MediaStreamEvent\":[\"Event\"],\"MediaStreamEventInit\":[],\"MediaStreamTrack\":[\"EventTarget\"],\"MediaStreamTrackEvent\":[\"Event\"],\"MediaStreamTrackEventInit\":[],\"MediaStreamTrackGenerator\":[\"EventTarget\",\"MediaStreamTrack\"],\"MediaStreamTrackGeneratorInit\":[],\"MediaStreamTrackProcessor\":[],\"MediaStreamTrackProcessorInit\":[],\"MediaStreamTrackState\":[],\"MediaTrackCapabilities\":[],\"MediaTrackConstraintSet\":[],\"MediaTrackConstraints\":[],\"MediaTrackSettings\":[],\"MediaTrackSupportedConstraints\":[],\"MemoryAttribution\":[],\"MemoryAttributionContainer\":[],\"MemoryBreakdownEntry\":[],\"MemoryMeasurement\":[],\"MessageChannel\":[],\"MessageEvent\":[\"Event\"],\"MessageEventInit\":[],\"MessagePort\":[\"EventTarget\"],\"MidiAccess\":[\"EventTarget\"],\"MidiConnectionEvent\":[\"Event\"],\"MidiConnectionEventInit\":[],\"MidiInput\":[\"EventTarget\",\"MidiPort\"],\"MidiInputMap\":[],\"MidiMessageEvent\":[\"Event\"],\"MidiMessageEventInit\":[],\"MidiOptions\":[],\"MidiOutput\":[\"EventTarget\",\"MidiPort\"],\"MidiOutputMap\":[],\"MidiPort\":[\"EventTarget\"],\"MidiPortConnectionState\":[],\"MidiPortDeviceState\":[],\"MidiPortType\":[],\"MimeType\":[],\"MimeTypeArray\":[],\"MouseEvent\":[\"Event\",\"UiEvent\"],\"MouseEventInit\":[],\"MouseScrollEvent\":[\"Event\",\"MouseEvent\",\"UiEvent\"],\"MozDebug\":[],\"MutationEvent\":[\"Event\"],\"MutationObserver\":[],\"MutationObserverInit\":[],\"MutationObservingInfo\":[],\"MutationRecord\":[],\"NamedNodeMap\":[],\"NativeOsFileReadOptions\":[],\"NativeOsFileWriteAtomicOptions\":[],\"NavigationType\":[],\"Navigator\":[],\"NavigatorAutomationInformation\":[],\"NavigatorUaBrandVersion\":[],\"NavigatorUaData\":[],\"NetworkCommandOptions\":[],\"NetworkInformation\":[\"EventTarget\"],\"NetworkResultOptions\":[],\"Node\":[\"EventTarget\"],\"NodeFilter\":[],\"NodeIterator\":[],\"NodeList\":[],\"Notification\":[\"EventTarget\"],\"NotificationAction\":[],\"NotificationDirection\":[],\"NotificationEvent\":[\"Event\",\"ExtendableEvent\"],\"NotificationEventInit\":[],\"NotificationOptions\":[],\"NotificationPermission\":[],\"ObserverCallback\":[],\"OesElementIndexUint\":[],\"OesStandardDerivatives\":[],\"OesTextureFloat\":[],\"OesTextureFloatLinear\":[],\"OesTextureHalfFloat\":[],\"OesTextureHalfFloatLinear\":[],\"OesVertexArrayObject\":[],\"OfflineAudioCompletionEvent\":[\"Event\"],\"OfflineAudioCompletionEventInit\":[],\"OfflineAudioContext\":[\"BaseAudioContext\",\"EventTarget\"],\"OfflineAudioContextOptions\":[],\"OfflineResourceList\":[\"EventTarget\"],\"OffscreenCanvas\":[\"EventTarget\"],\"OffscreenCanvasRenderingContext2d\":[],\"OpenFilePickerOptions\":[],\"OpenWindowEventDetail\":[],\"OptionalEffectTiming\":[],\"OrientationLockType\":[],\"OrientationType\":[],\"OscillatorNode\":[\"AudioNode\",\"AudioScheduledSourceNode\",\"EventTarget\"],\"OscillatorOptions\":[],\"OscillatorType\":[],\"OverSampleType\":[],\"OvrMultiview2\":[],\"PageTransitionEvent\":[\"Event\"],\"PageTransitionEventInit\":[],\"PaintRequest\":[],\"PaintRequestList\":[],\"PaintWorkletGlobalScope\":[\"WorkletGlobalScope\"],\"PannerNode\":[\"AudioNode\",\"EventTarget\"],\"PannerOptions\":[],\"PanningModelType\":[],\"ParityType\":[],\"Path2d\":[],\"PaymentAddress\":[],\"PaymentComplete\":[],\"PaymentMethodChangeEvent\":[\"Event\",\"PaymentRequestUpdateEvent\"],\"PaymentMethodChangeEventInit\":[],\"PaymentRequestUpdateEvent\":[\"Event\"],\"PaymentRequestUpdateEventInit\":[],\"PaymentResponse\":[],\"Pbkdf2Params\":[],\"PcImplIceConnectionState\":[],\"PcImplIceGatheringState\":[],\"PcImplSignalingState\":[],\"PcObserverStateType\":[],\"Performance\":[\"EventTarget\"],\"PerformanceEntry\":[],\"PerformanceEntryEventInit\":[],\"PerformanceEntryFilterOptions\":[],\"PerformanceMark\":[\"PerformanceEntry\"],\"PerformanceMeasure\":[\"PerformanceEntry\"],\"PerformanceNavigation\":[],\"PerformanceNavigationTiming\":[\"PerformanceEntry\",\"PerformanceResourceTiming\"],\"PerformanceObserver\":[],\"PerformanceObserverEntryList\":[],\"PerformanceObserverInit\":[],\"PerformanceResourceTiming\":[\"PerformanceEntry\"],\"PerformanceServerTiming\":[],\"PerformanceTiming\":[],\"PeriodicWave\":[],\"PeriodicWaveConstraints\":[],\"PeriodicWaveOptions\":[],\"PermissionDescriptor\":[],\"PermissionName\":[],\"PermissionState\":[],\"PermissionStatus\":[\"EventTarget\"],\"Permissions\":[],\"PictureInPictureEvent\":[\"Event\"],\"PictureInPictureEventInit\":[],\"PictureInPictureWindow\":[\"EventTarget\"],\"PlaneLayout\":[],\"PlaybackDirection\":[],\"Plugin\":[],\"PluginArray\":[],\"PluginCrashedEventInit\":[],\"PointerEvent\":[\"Event\",\"MouseEvent\",\"UiEvent\"],\"PointerEventInit\":[],\"PopStateEvent\":[\"Event\"],\"PopStateEventInit\":[],\"PopupBlockedEvent\":[\"Event\"],\"PopupBlockedEventInit\":[],\"Position\":[],\"PositionAlignSetting\":[],\"PositionError\":[],\"PositionOptions\":[],\"PremultiplyAlpha\":[],\"Presentation\":[],\"PresentationAvailability\":[\"EventTarget\"],\"PresentationConnection\":[\"EventTarget\"],\"PresentationConnectionAvailableEvent\":[\"Event\"],\"PresentationConnectionAvailableEventInit\":[],\"PresentationConnectionBinaryType\":[],\"PresentationConnectionCloseEvent\":[\"Event\"],\"PresentationConnectionCloseEventInit\":[],\"PresentationConnectionClosedReason\":[],\"PresentationConnectionList\":[\"EventTarget\"],\"PresentationConnectionState\":[],\"PresentationReceiver\":[],\"PresentationRequest\":[\"EventTarget\"],\"PresentationStyle\":[],\"ProcessingInstruction\":[\"CharacterData\",\"EventTarget\",\"Node\"],\"ProfileTimelineLayerRect\":[],\"ProfileTimelineMarker\":[],\"ProfileTimelineMessagePortOperationType\":[],\"ProfileTimelineStackFrame\":[],\"ProfileTimelineWorkerOperationType\":[],\"ProgressEvent\":[\"Event\"],\"ProgressEventInit\":[],\"PromiseNativeHandler\":[],\"PromiseRejectionEvent\":[\"Event\"],\"PromiseRejectionEventInit\":[],\"PublicKeyCredential\":[\"Credential\"],\"PublicKeyCredentialCreationOptions\":[],\"PublicKeyCredentialCreationOptionsJson\":[],\"PublicKeyCredentialDescriptor\":[],\"PublicKeyCredentialDescriptorJson\":[],\"PublicKeyCredentialEntity\":[],\"PublicKeyCredentialHints\":[],\"PublicKeyCredentialParameters\":[],\"PublicKeyCredentialRequestOptions\":[],\"PublicKeyCredentialRequestOptionsJson\":[],\"PublicKeyCredentialRpEntity\":[],\"PublicKeyCredentialType\":[],\"PublicKeyCredentialUserEntity\":[],\"PublicKeyCredentialUserEntityJson\":[],\"PushEncryptionKeyName\":[],\"PushEvent\":[\"Event\",\"ExtendableEvent\"],\"PushEventInit\":[],\"PushManager\":[],\"PushMessageData\":[],\"PushPermissionState\":[],\"PushSubscription\":[],\"PushSubscriptionInit\":[],\"PushSubscriptionJson\":[],\"PushSubscriptionKeys\":[],\"PushSubscriptionOptions\":[],\"PushSubscriptionOptionsInit\":[],\"QueryOptions\":[],\"QueuingStrategy\":[],\"QueuingStrategyInit\":[],\"RadioNodeList\":[\"NodeList\"],\"Range\":[],\"RcwnPerfStats\":[],\"RcwnStatus\":[],\"ReadableByteStreamController\":[],\"ReadableStream\":[],\"ReadableStreamByobReader\":[],\"ReadableStreamByobRequest\":[],\"ReadableStreamDefaultController\":[],\"ReadableStreamDefaultReader\":[],\"ReadableStreamGetReaderOptions\":[],\"ReadableStreamIteratorOptions\":[],\"ReadableStreamReadResult\":[],\"ReadableStreamReaderMode\":[],\"ReadableStreamType\":[],\"ReadableWritablePair\":[],\"RecordingState\":[],\"ReferrerPolicy\":[],\"RegisterRequest\":[],\"RegisterResponse\":[],\"RegisteredKey\":[],\"RegistrationOptions\":[],\"RegistrationResponseJson\":[],\"Request\":[],\"RequestCache\":[],\"RequestCredentials\":[],\"RequestDestination\":[],\"RequestDeviceOptions\":[],\"RequestInit\":[],\"RequestMediaKeySystemAccessNotification\":[],\"RequestMode\":[],\"RequestRedirect\":[],\"ResidentKeyRequirement\":[],\"ResizeObserver\":[],\"ResizeObserverBoxOptions\":[],\"ResizeObserverEntry\":[],\"ResizeObserverOptions\":[],\"ResizeObserverSize\":[],\"ResizeQuality\":[],\"Response\":[],\"ResponseInit\":[],\"ResponseType\":[],\"RsaHashedImportParams\":[],\"RsaOaepParams\":[],\"RsaOtherPrimesInfo\":[],\"RsaPssParams\":[],\"RtcAnswerOptions\":[],\"RtcBundlePolicy\":[],\"RtcCertificate\":[],\"RtcCertificateExpiration\":[],\"RtcCodecStats\":[],\"RtcConfiguration\":[],\"RtcDataChannel\":[\"EventTarget\"],\"RtcDataChannelEvent\":[\"Event\"],\"RtcDataChannelEventInit\":[],\"RtcDataChannelInit\":[],\"RtcDataChannelState\":[],\"RtcDataChannelType\":[],\"RtcDegradationPreference\":[],\"RtcEncodedAudioFrame\":[],\"RtcEncodedAudioFrameMetadata\":[],\"RtcEncodedAudioFrameOptions\":[],\"RtcEncodedVideoFrame\":[],\"RtcEncodedVideoFrameMetadata\":[],\"RtcEncodedVideoFrameOptions\":[],\"RtcEncodedVideoFrameType\":[],\"RtcFecParameters\":[],\"RtcIceCandidate\":[],\"RtcIceCandidateInit\":[],\"RtcIceCandidatePairStats\":[],\"RtcIceCandidateStats\":[],\"RtcIceComponentStats\":[],\"RtcIceConnectionState\":[],\"RtcIceCredentialType\":[],\"RtcIceGatheringState\":[],\"RtcIceServer\":[],\"RtcIceTransportPolicy\":[],\"RtcIdentityAssertion\":[],\"RtcIdentityAssertionResult\":[],\"RtcIdentityProvider\":[],\"RtcIdentityProviderDetails\":[],\"RtcIdentityProviderOptions\":[],\"RtcIdentityProviderRegistrar\":[],\"RtcIdentityValidationResult\":[],\"RtcInboundRtpStreamStats\":[],\"RtcMediaStreamStats\":[],\"RtcMediaStreamTrackStats\":[],\"RtcOfferAnswerOptions\":[],\"RtcOfferOptions\":[],\"RtcOutboundRtpStreamStats\":[],\"RtcPeerConnection\":[\"EventTarget\"],\"RtcPeerConnectionIceErrorEvent\":[\"Event\"],\"RtcPeerConnectionIceEvent\":[\"Event\"],\"RtcPeerConnectionIceEventInit\":[],\"RtcPeerConnectionState\":[],\"RtcPriorityType\":[],\"RtcRtcpParameters\":[],\"RtcRtpCapabilities\":[],\"RtcRtpCodecCapability\":[],\"RtcRtpCodecParameters\":[],\"RtcRtpContributingSource\":[],\"RtcRtpEncodingParameters\":[],\"RtcRtpHeaderExtensionCapability\":[],\"RtcRtpHeaderExtensionParameters\":[],\"RtcRtpParameters\":[],\"RtcRtpReceiver\":[],\"RtcRtpScriptTransform\":[],\"RtcRtpScriptTransformer\":[\"EventTarget\"],\"RtcRtpSender\":[],\"RtcRtpSourceEntry\":[],\"RtcRtpSourceEntryType\":[],\"RtcRtpSynchronizationSource\":[],\"RtcRtpTransceiver\":[],\"RtcRtpTransceiverDirection\":[],\"RtcRtpTransceiverInit\":[],\"RtcRtxParameters\":[],\"RtcSdpType\":[],\"RtcSessionDescription\":[],\"RtcSessionDescriptionInit\":[],\"RtcSignalingState\":[],\"RtcStats\":[],\"RtcStatsIceCandidatePairState\":[],\"RtcStatsIceCandidateType\":[],\"RtcStatsReport\":[],\"RtcStatsReportInternal\":[],\"RtcStatsType\":[],\"RtcTrackEvent\":[\"Event\"],\"RtcTrackEventInit\":[],\"RtcTransformEvent\":[\"Event\"],\"RtcTransportStats\":[],\"RtcdtmfSender\":[\"EventTarget\"],\"RtcdtmfToneChangeEvent\":[\"Event\"],\"RtcdtmfToneChangeEventInit\":[],\"RtcrtpContributingSourceStats\":[],\"RtcrtpStreamStats\":[],\"SFrameTransform\":[\"EventTarget\"],\"SFrameTransformErrorEvent\":[\"Event\"],\"SFrameTransformErrorEventInit\":[],\"SFrameTransformErrorEventType\":[],\"SFrameTransformOptions\":[],\"SFrameTransformRole\":[],\"SaveFilePickerOptions\":[],\"Scheduler\":[],\"SchedulerPostTaskOptions\":[],\"Scheduling\":[],\"Screen\":[\"EventTarget\"],\"ScreenColorGamut\":[],\"ScreenDetailed\":[\"EventTarget\",\"Screen\"],\"ScreenDetails\":[\"EventTarget\"],\"ScreenLuminance\":[],\"ScreenOrientation\":[\"EventTarget\"],\"ScriptProcessorNode\":[\"AudioNode\",\"EventTarget\"],\"ScrollAreaEvent\":[\"Event\",\"UiEvent\"],\"ScrollBehavior\":[],\"ScrollBoxObject\":[],\"ScrollIntoViewContainer\":[],\"ScrollIntoViewOptions\":[],\"ScrollLogicalPosition\":[],\"ScrollOptions\":[],\"ScrollRestoration\":[],\"ScrollSetting\":[],\"ScrollState\":[],\"ScrollToOptions\":[],\"ScrollViewChangeEventInit\":[],\"SecurityPolicyViolationEvent\":[\"Event\"],\"SecurityPolicyViolationEventDisposition\":[],\"SecurityPolicyViolationEventInit\":[],\"Selection\":[],\"SelectionMode\":[],\"Serial\":[\"EventTarget\"],\"SerialInputSignals\":[],\"SerialOptions\":[],\"SerialOutputSignals\":[],\"SerialPort\":[\"EventTarget\"],\"SerialPortFilter\":[],\"SerialPortInfo\":[],\"SerialPortRequestOptions\":[],\"ServerSocketOptions\":[],\"ServiceWorker\":[\"EventTarget\"],\"ServiceWorkerContainer\":[\"EventTarget\"],\"ServiceWorkerGlobalScope\":[\"EventTarget\",\"WorkerGlobalScope\"],\"ServiceWorkerRegistration\":[\"EventTarget\"],\"ServiceWorkerState\":[],\"ServiceWorkerUpdateViaCache\":[],\"ShadowRoot\":[\"DocumentFragment\",\"EventTarget\",\"Node\"],\"ShadowRootInit\":[],\"ShadowRootMode\":[],\"ShareData\":[],\"SharedWorker\":[\"EventTarget\"],\"SharedWorkerGlobalScope\":[\"EventTarget\",\"WorkerGlobalScope\"],\"SignResponse\":[],\"SocketElement\":[],\"SocketOptions\":[],\"SocketReadyState\":[],\"SocketsDict\":[],\"SourceBuffer\":[\"EventTarget\"],\"SourceBufferAppendMode\":[],\"SourceBufferList\":[\"EventTarget\"],\"SpeechGrammar\":[],\"SpeechGrammarList\":[],\"SpeechRecognition\":[\"EventTarget\"],\"SpeechRecognitionAlternative\":[],\"SpeechRecognitionError\":[\"Event\"],\"SpeechRecognitionErrorCode\":[],\"SpeechRecognitionErrorInit\":[],\"SpeechRecognitionEvent\":[\"Event\"],\"SpeechRecognitionEventInit\":[],\"SpeechRecognitionResult\":[],\"SpeechRecognitionResultList\":[],\"SpeechSynthesis\":[\"EventTarget\"],\"SpeechSynthesisErrorCode\":[],\"SpeechSynthesisErrorEvent\":[\"Event\",\"SpeechSynthesisEvent\"],\"SpeechSynthesisErrorEventInit\":[],\"SpeechSynthesisEvent\":[\"Event\"],\"SpeechSynthesisEventInit\":[],\"SpeechSynthesisUtterance\":[\"EventTarget\"],\"SpeechSynthesisVoice\":[],\"StereoPannerNode\":[\"AudioNode\",\"EventTarget\"],\"StereoPannerOptions\":[],\"Storage\":[],\"StorageEstimate\":[],\"StorageEvent\":[\"Event\"],\"StorageEventInit\":[],\"StorageManager\":[],\"StorageType\":[],\"StreamPipeOptions\":[],\"StyleRuleChangeEventInit\":[],\"StyleSheet\":[],\"StyleSheetApplicableStateChangeEventInit\":[],\"StyleSheetChangeEventInit\":[],\"StyleSheetList\":[],\"SubmitEvent\":[\"Event\"],\"SubmitEventInit\":[],\"SubtleCrypto\":[],\"SupportedType\":[],\"SvcOutputMetadata\":[],\"SvgAngle\":[],\"SvgAnimateElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgAnimationElement\",\"SvgElement\"],\"SvgAnimateMotionElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgAnimationElement\",\"SvgElement\"],\"SvgAnimateTransformElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgAnimationElement\",\"SvgElement\"],\"SvgAnimatedAngle\":[],\"SvgAnimatedBoolean\":[],\"SvgAnimatedEnumeration\":[],\"SvgAnimatedInteger\":[],\"SvgAnimatedLength\":[],\"SvgAnimatedLengthList\":[],\"SvgAnimatedNumber\":[],\"SvgAnimatedNumberList\":[],\"SvgAnimatedPreserveAspectRatio\":[],\"SvgAnimatedRect\":[],\"SvgAnimatedString\":[],\"SvgAnimatedTransformList\":[],\"SvgAnimationElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgBoundingBoxOptions\":[],\"SvgCircleElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\",\"SvgGeometryElement\",\"SvgGraphicsElement\"],\"SvgClipPathElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgComponentTransferFunctionElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgDefsElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\",\"SvgGraphicsElement\"],\"SvgDescElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgElement\":[\"Element\",\"EventTarget\",\"Node\"],\"SvgEllipseElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\",\"SvgGeometryElement\",\"SvgGraphicsElement\"],\"SvgFilterElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgForeignObjectElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\",\"SvgGraphicsElement\"],\"SvgGeometryElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\",\"SvgGraphicsElement\"],\"SvgGradientElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgGraphicsElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgImageElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\",\"SvgGraphicsElement\"],\"SvgLength\":[],\"SvgLengthList\":[],\"SvgLineElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\",\"SvgGeometryElement\",\"SvgGraphicsElement\"],\"SvgLinearGradientElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\",\"SvgGradientElement\"],\"SvgMarkerElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgMaskElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgMatrix\":[],\"SvgMetadataElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgNumber\":[],\"SvgNumberList\":[],\"SvgPathElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\",\"SvgGeometryElement\",\"SvgGraphicsElement\"],\"SvgPathSeg\":[],\"SvgPathSegArcAbs\":[\"SvgPathSeg\"],\"SvgPathSegArcRel\":[\"SvgPathSeg\"],\"SvgPathSegClosePath\":[\"SvgPathSeg\"],\"SvgPathSegCurvetoCubicAbs\":[\"SvgPathSeg\"],\"SvgPathSegCurvetoCubicRel\":[\"SvgPathSeg\"],\"SvgPathSegCurvetoCubicSmoothAbs\":[\"SvgPathSeg\"],\"SvgPathSegCurvetoCubicSmoothRel\":[\"SvgPathSeg\"],\"SvgPathSegCurvetoQuadraticAbs\":[\"SvgPathSeg\"],\"SvgPathSegCurvetoQuadraticRel\":[\"SvgPathSeg\"],\"SvgPathSegCurvetoQuadraticSmoothAbs\":[\"SvgPathSeg\"],\"SvgPathSegCurvetoQuadraticSmoothRel\":[\"SvgPathSeg\"],\"SvgPathSegLinetoAbs\":[\"SvgPathSeg\"],\"SvgPathSegLinetoHorizontalAbs\":[\"SvgPathSeg\"],\"SvgPathSegLinetoHorizontalRel\":[\"SvgPathSeg\"],\"SvgPathSegLinetoRel\":[\"SvgPathSeg\"],\"SvgPathSegLinetoVerticalAbs\":[\"SvgPathSeg\"],\"SvgPathSegLinetoVerticalRel\":[\"SvgPathSeg\"],\"SvgPathSegList\":[],\"SvgPathSegMovetoAbs\":[\"SvgPathSeg\"],\"SvgPathSegMovetoRel\":[\"SvgPathSeg\"],\"SvgPatternElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgPoint\":[],\"SvgPointList\":[],\"SvgPolygonElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\",\"SvgGeometryElement\",\"SvgGraphicsElement\"],\"SvgPolylineElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\",\"SvgGeometryElement\",\"SvgGraphicsElement\"],\"SvgPreserveAspectRatio\":[],\"SvgRadialGradientElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\",\"SvgGradientElement\"],\"SvgRect\":[],\"SvgRectElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\",\"SvgGeometryElement\",\"SvgGraphicsElement\"],\"SvgScriptElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgSetElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgAnimationElement\",\"SvgElement\"],\"SvgStopElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgStringList\":[],\"SvgStyleElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgSwitchElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\",\"SvgGraphicsElement\"],\"SvgSymbolElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgTextContentElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\",\"SvgGraphicsElement\"],\"SvgTextElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\",\"SvgGraphicsElement\",\"SvgTextContentElement\",\"SvgTextPositioningElement\"],\"SvgTextPathElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\",\"SvgGraphicsElement\",\"SvgTextContentElement\"],\"SvgTextPositioningElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\",\"SvgGraphicsElement\",\"SvgTextContentElement\"],\"SvgTitleElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgTransform\":[],\"SvgTransformList\":[],\"SvgUnitTypes\":[],\"SvgUseElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\",\"SvgGraphicsElement\"],\"SvgViewElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgZoomAndPan\":[],\"SvgaElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\",\"SvgGraphicsElement\"],\"SvgfeBlendElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgfeColorMatrixElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgfeComponentTransferElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgfeCompositeElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgfeConvolveMatrixElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgfeDiffuseLightingElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgfeDisplacementMapElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgfeDistantLightElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgfeDropShadowElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgfeFloodElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgfeFuncAElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgComponentTransferFunctionElement\",\"SvgElement\"],\"SvgfeFuncBElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgComponentTransferFunctionElement\",\"SvgElement\"],\"SvgfeFuncGElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgComponentTransferFunctionElement\",\"SvgElement\"],\"SvgfeFuncRElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgComponentTransferFunctionElement\",\"SvgElement\"],\"SvgfeGaussianBlurElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgfeImageElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgfeMergeElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgfeMergeNodeElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgfeMorphologyElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgfeOffsetElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgfePointLightElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgfeSpecularLightingElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgfeSpotLightElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgfeTileElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgfeTurbulenceElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvggElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\",\"SvgGraphicsElement\"],\"SvgmPathElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgsvgElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\",\"SvgGraphicsElement\"],\"SvgtSpanElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\",\"SvgGraphicsElement\",\"SvgTextContentElement\",\"SvgTextPositioningElement\"],\"TaskController\":[\"AbortController\"],\"TaskControllerInit\":[],\"TaskPriority\":[],\"TaskPriorityChangeEvent\":[\"Event\"],\"TaskPriorityChangeEventInit\":[],\"TaskSignal\":[\"AbortSignal\",\"EventTarget\"],\"TaskSignalAnyInit\":[],\"TcpReadyState\":[],\"TcpServerSocket\":[\"EventTarget\"],\"TcpServerSocketEvent\":[\"Event\"],\"TcpServerSocketEventInit\":[],\"TcpSocket\":[\"EventTarget\"],\"TcpSocketBinaryType\":[],\"TcpSocketErrorEvent\":[\"Event\"],\"TcpSocketErrorEventInit\":[],\"TcpSocketEvent\":[\"Event\"],\"TcpSocketEventInit\":[],\"Text\":[\"CharacterData\",\"EventTarget\",\"Node\"],\"TextDecodeOptions\":[],\"TextDecoder\":[],\"TextDecoderOptions\":[],\"TextEncoder\":[],\"TextMetrics\":[],\"TextTrack\":[\"EventTarget\"],\"TextTrackCue\":[\"EventTarget\"],\"TextTrackCueList\":[],\"TextTrackKind\":[],\"TextTrackList\":[\"EventTarget\"],\"TextTrackMode\":[],\"TimeEvent\":[\"Event\"],\"TimeRanges\":[],\"ToggleEvent\":[\"Event\"],\"ToggleEventInit\":[],\"TokenBinding\":[],\"TokenBindingStatus\":[],\"Touch\":[],\"TouchEvent\":[\"Event\",\"UiEvent\"],\"TouchEventInit\":[],\"TouchInit\":[],\"TouchList\":[],\"TrackEvent\":[\"Event\"],\"TrackEventInit\":[],\"TransformStream\":[],\"TransformStreamDefaultController\":[],\"Transformer\":[],\"TransitionEvent\":[\"Event\"],\"TransitionEventInit\":[],\"Transport\":[],\"TreeBoxObject\":[],\"TreeCellInfo\":[],\"TreeView\":[],\"TreeWalker\":[],\"U2f\":[],\"U2fClientData\":[],\"ULongRange\":[],\"UaDataValues\":[],\"UaLowEntropyJson\":[],\"UdpMessageEventInit\":[],\"UdpOptions\":[],\"UiEvent\":[\"Event\"],\"UiEventInit\":[],\"UnderlyingSink\":[],\"UnderlyingSource\":[],\"Url\":[],\"UrlSearchParams\":[],\"Usb\":[\"EventTarget\"],\"UsbAlternateInterface\":[],\"UsbConfiguration\":[],\"UsbConnectionEvent\":[\"Event\"],\"UsbConnectionEventInit\":[],\"UsbControlTransferParameters\":[],\"UsbDevice\":[],\"UsbDeviceFilter\":[],\"UsbDeviceRequestOptions\":[],\"UsbDirection\":[],\"UsbEndpoint\":[],\"UsbEndpointType\":[],\"UsbInTransferResult\":[],\"UsbInterface\":[],\"UsbIsochronousInTransferPacket\":[],\"UsbIsochronousInTransferResult\":[],\"UsbIsochronousOutTransferPacket\":[],\"UsbIsochronousOutTransferResult\":[],\"UsbOutTransferResult\":[],\"UsbPermissionDescriptor\":[],\"UsbPermissionResult\":[\"EventTarget\",\"PermissionStatus\"],\"UsbPermissionStorage\":[],\"UsbRecipient\":[],\"UsbRequestType\":[],\"UsbTransferStatus\":[],\"UserActivation\":[],\"UserProximityEvent\":[\"Event\"],\"UserProximityEventInit\":[],\"UserVerificationRequirement\":[],\"ValidityState\":[],\"ValueEvent\":[\"Event\"],\"ValueEventInit\":[],\"VideoColorPrimaries\":[],\"VideoColorSpace\":[],\"VideoColorSpaceInit\":[],\"VideoConfiguration\":[],\"VideoDecoder\":[],\"VideoDecoderConfig\":[],\"VideoDecoderInit\":[],\"VideoDecoderSupport\":[],\"VideoEncoder\":[],\"VideoEncoderConfig\":[],\"VideoEncoderEncodeOptions\":[],\"VideoEncoderInit\":[],\"VideoEncoderSupport\":[],\"VideoFacingModeEnum\":[],\"VideoFrame\":[],\"VideoFrameBufferInit\":[],\"VideoFrameCopyToOptions\":[],\"VideoFrameInit\":[],\"VideoMatrixCoefficients\":[],\"VideoPixelFormat\":[],\"VideoPlaybackQuality\":[],\"VideoStreamTrack\":[\"EventTarget\",\"MediaStreamTrack\"],\"VideoTrack\":[],\"VideoTrackList\":[\"EventTarget\"],\"VideoTransferCharacteristics\":[],\"ViewTransition\":[],\"VisibilityState\":[],\"VisualViewport\":[\"EventTarget\"],\"VoidCallback\":[],\"VrDisplay\":[\"EventTarget\"],\"VrDisplayCapabilities\":[],\"VrEye\":[],\"VrEyeParameters\":[],\"VrFieldOfView\":[],\"VrFrameData\":[],\"VrLayer\":[],\"VrMockController\":[],\"VrMockDisplay\":[],\"VrPose\":[],\"VrServiceTest\":[],\"VrStageParameters\":[],\"VrSubmitFrameResult\":[],\"VttCue\":[\"EventTarget\",\"TextTrackCue\"],\"VttRegion\":[],\"WakeLock\":[],\"WakeLockSentinel\":[\"EventTarget\"],\"WakeLockType\":[],\"WatchAdvertisementsOptions\":[],\"WaveShaperNode\":[\"AudioNode\",\"EventTarget\"],\"WaveShaperOptions\":[],\"WebGl2RenderingContext\":[],\"WebGlActiveInfo\":[],\"WebGlBuffer\":[],\"WebGlContextAttributes\":[],\"WebGlContextEvent\":[\"Event\"],\"WebGlContextEventInit\":[],\"WebGlFramebuffer\":[],\"WebGlPowerPreference\":[],\"WebGlProgram\":[],\"WebGlQuery\":[],\"WebGlRenderbuffer\":[],\"WebGlRenderingContext\":[],\"WebGlSampler\":[],\"WebGlShader\":[],\"WebGlShaderPrecisionFormat\":[],\"WebGlSync\":[],\"WebGlTexture\":[],\"WebGlTransformFeedback\":[],\"WebGlUniformLocation\":[],\"WebGlVertexArrayObject\":[],\"WebKitCssMatrix\":[\"DomMatrix\",\"DomMatrixReadOnly\"],\"WebSocket\":[\"EventTarget\"],\"WebSocketDict\":[],\"WebSocketElement\":[],\"WebTransport\":[],\"WebTransportBidirectionalStream\":[],\"WebTransportCloseInfo\":[],\"WebTransportCongestionControl\":[],\"WebTransportDatagramDuplexStream\":[],\"WebTransportDatagramStats\":[],\"WebTransportError\":[\"DomException\"],\"WebTransportErrorOptions\":[],\"WebTransportErrorSource\":[],\"WebTransportHash\":[],\"WebTransportOptions\":[],\"WebTransportReceiveStream\":[\"ReadableStream\"],\"WebTransportReceiveStreamStats\":[],\"WebTransportReliabilityMode\":[],\"WebTransportSendStream\":[\"WritableStream\"],\"WebTransportSendStreamOptions\":[],\"WebTransportSendStreamStats\":[],\"WebTransportStats\":[],\"WebglColorBufferFloat\":[],\"WebglCompressedTextureAstc\":[],\"WebglCompressedTextureAtc\":[],\"WebglCompressedTextureEtc\":[],\"WebglCompressedTextureEtc1\":[],\"WebglCompressedTexturePvrtc\":[],\"WebglCompressedTextureS3tc\":[],\"WebglCompressedTextureS3tcSrgb\":[],\"WebglDebugRendererInfo\":[],\"WebglDebugShaders\":[],\"WebglDepthTexture\":[],\"WebglDrawBuffers\":[],\"WebglLoseContext\":[],\"WebglMultiDraw\":[],\"WellKnownDirectory\":[],\"WgslLanguageFeatures\":[],\"WheelEvent\":[\"Event\",\"MouseEvent\",\"UiEvent\"],\"WheelEventInit\":[],\"WidevineCdmManifest\":[],\"Window\":[\"EventTarget\"],\"WindowClient\":[\"Client\"],\"Worker\":[\"EventTarget\"],\"WorkerDebuggerGlobalScope\":[\"EventTarget\"],\"WorkerGlobalScope\":[\"EventTarget\"],\"WorkerLocation\":[],\"WorkerNavigator\":[],\"WorkerOptions\":[],\"WorkerType\":[],\"Worklet\":[],\"WorkletGlobalScope\":[],\"WorkletOptions\":[],\"WritableStream\":[],\"WritableStreamDefaultController\":[],\"WritableStreamDefaultWriter\":[],\"WriteCommandType\":[],\"WriteParams\":[],\"XPathExpression\":[],\"XPathNsResolver\":[],\"XPathResult\":[],\"XmlDocument\":[\"Document\",\"EventTarget\",\"Node\"],\"XmlHttpRequest\":[\"EventTarget\",\"XmlHttpRequestEventTarget\"],\"XmlHttpRequestEventTarget\":[\"EventTarget\"],\"XmlHttpRequestResponseType\":[],\"XmlHttpRequestUpload\":[\"EventTarget\",\"XmlHttpRequestEventTarget\"],\"XmlSerializer\":[],\"XrBoundedReferenceSpace\":[\"EventTarget\",\"XrReferenceSpace\",\"XrSpace\"],\"XrEye\":[],\"XrFrame\":[],\"XrHand\":[],\"XrHandJoint\":[],\"XrHandedness\":[],\"XrInputSource\":[],\"XrInputSourceArray\":[],\"XrInputSourceEvent\":[\"Event\"],\"XrInputSourceEventInit\":[],\"XrInputSourcesChangeEvent\":[\"Event\"],\"XrInputSourcesChangeEventInit\":[],\"XrJointPose\":[\"XrPose\"],\"XrJointSpace\":[\"EventTarget\",\"XrSpace\"],\"XrLayer\":[\"EventTarget\"],\"XrPermissionDescriptor\":[],\"XrPermissionStatus\":[\"EventTarget\",\"PermissionStatus\"],\"XrPose\":[],\"XrReferenceSpace\":[\"EventTarget\",\"XrSpace\"],\"XrReferenceSpaceEvent\":[\"Event\"],\"XrReferenceSpaceEventInit\":[],\"XrReferenceSpaceType\":[],\"XrRenderState\":[],\"XrRenderStateInit\":[],\"XrRigidTransform\":[],\"XrSession\":[\"EventTarget\"],\"XrSessionEvent\":[\"Event\"],\"XrSessionEventInit\":[],\"XrSessionInit\":[],\"XrSessionMode\":[],\"XrSessionSupportedPermissionDescriptor\":[],\"XrSpace\":[\"EventTarget\"],\"XrSystem\":[\"EventTarget\"],\"XrTargetRayMode\":[],\"XrView\":[],\"XrViewerPose\":[\"XrPose\"],\"XrViewport\":[],\"XrVisibilityState\":[],\"XrWebGlLayer\":[\"EventTarget\",\"XrLayer\"],\"XrWebGlLayerInit\":[],\"XsltProcessor\":[],\"console\":[],\"css\":[],\"default\":[\"std\"],\"gpu_buffer_usage\":[],\"gpu_color_write\":[],\"gpu_map_mode\":[],\"gpu_shader_stage\":[],\"gpu_texture_usage\":[],\"std\":[\"wasm-bindgen/std\",\"js-sys/std\"]}}", "web-time_1.1.0": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"alloc\"],\"kind\":\"dev\",\"name\":\"futures-channel\",\"req\":\"^0.3\",\"target\":\"cfg(all(target_family = \\\"wasm\\\", target_feature = \\\"atomics\\\"))\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"futures-util\",\"req\":\"^0.3\",\"target\":\"cfg(all(target_family = \\\"wasm\\\", target_feature = \\\"atomics\\\"))\"},{\"features\":[\"js\"],\"kind\":\"dev\",\"name\":\"getrandom\",\"req\":\"^0.2\",\"target\":\"cfg(target_family = \\\"wasm\\\")\"},{\"name\":\"js-sys\",\"req\":\"^0.3.20\",\"target\":\"cfg(all(target_family = \\\"wasm\\\", target_os = \\\"unknown\\\"))\"},{\"features\":[\"macro\"],\"kind\":\"dev\",\"name\":\"pollster\",\"req\":\"^0.3\",\"target\":\"cfg(not(target_family = \\\"wasm\\\"))\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\",\"target\":\"cfg(target_family = \\\"wasm\\\")\"},{\"name\":\"serde\",\"optional\":true,\"req\":\"^1\",\"target\":\"cfg(all(target_family = \\\"wasm\\\", target_os = \\\"unknown\\\"))\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1\",\"target\":\"cfg(target_family = \\\"wasm\\\")\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"wasm-bindgen\",\"req\":\"^0.2.70\",\"target\":\"cfg(all(target_family = \\\"wasm\\\", target_os = \\\"unknown\\\"))\"},{\"kind\":\"dev\",\"name\":\"wasm-bindgen-futures\",\"req\":\"^0.4\",\"target\":\"cfg(target_family = \\\"wasm\\\")\"},{\"kind\":\"dev\",\"name\":\"wasm-bindgen-test\",\"req\":\"^0.3\",\"target\":\"cfg(target_family = \\\"wasm\\\")\"},{\"features\":[\"WorkerGlobalScope\"],\"kind\":\"dev\",\"name\":\"web-sys\",\"req\":\"^0.3\",\"target\":\"cfg(all(target_family = \\\"wasm\\\", target_feature = \\\"atomics\\\"))\"},{\"features\":[\"CssStyleDeclaration\",\"Document\",\"Element\",\"HtmlTableElement\",\"HtmlTableRowElement\",\"Performance\",\"Window\"],\"kind\":\"dev\",\"name\":\"web-sys\",\"req\":\"^0.3\",\"target\":\"cfg(target_family = \\\"wasm\\\")\"}],\"features\":{\"serde\":[\"dep:serde\"]}}", diff --git a/README.md b/README.md index 77c8d2199c..bb2a9d18c3 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ curl -fsSL https://chatgpt.com/codex/install.sh | sh Run the following on Windows to install Codex CLI: -``` +```shell powershell -ExecutionPolicy ByPass -c "irm https://chatgpt.com/codex/install.ps1 | iex" ``` diff --git a/cliff.toml b/cliff.toml deleted file mode 100644 index f31e1bd89c..0000000000 --- a/cliff.toml +++ /dev/null @@ -1,46 +0,0 @@ -# https://git-cliff.org/docs/configuration - -[changelog] -header = """ -# Changelog - -You can install any of these versions: `npm install -g @openai/codex@` -""" - -body = """ -{% if version -%} -## [{{ version | trim_start_matches(pat="v") }}] - {{ timestamp | date(format="%Y-%m-%d") }} -{%- else %} -## [unreleased] -{% endif %} - -{%- for group, commits in commits | group_by(attribute="group") %} -### {{ group | striptags | trim }} - -{% for commit in commits %}- {% if commit.scope %}*({{ commit.scope }})* {% endif %}{% if commit.breaking %}[**breaking**] {% endif %}{{ commit.message | upper_first }} -{% endfor %} - -{%- endfor -%} -""" - -footer = """ - -""" - -trim = true -postprocessors = [] - -[git] -conventional_commits = true - -commit_parsers = [ - { message = "^feat", group = "🚀 Features" }, - { message = "^fix", group = "🪲 Bug Fixes" }, - { message = "^bump", group = "🛳️ Release" }, - # Fallback – skip anything that didn't match the above rules. - { message = ".*", group = "💼 Other" }, -] - -filter_unconventional = false -sort_commits = "oldest" -topo_order = false \ No newline at end of file diff --git a/codex-rs/.cargo/audit.toml b/codex-rs/.cargo/audit.toml index c1d8ff87ee..56feb5edb3 100644 --- a/codex-rs/.cargo/audit.toml +++ b/codex-rs/.cargo/audit.toml @@ -1,5 +1,5 @@ [advisories] -# Reviewed 2026-06-07. Keep this list in sync with ../deny.toml. +# Reviewed 2026-07-02. Keep this list in sync with ../deny.toml. ignore = [ "RUSTSEC-2024-0388", # derivative 2.2.0 via starlark/starlark_syntax; upstream crate is unmaintained "RUSTSEC-2025-0057", # fxhash 0.2.1 via starlark_map/bm25; upstream crate is unmaintained @@ -10,4 +10,6 @@ ignore = [ "RUSTSEC-2026-0118", # hickory-proto via rama-dns/rama-tcp; remove when rama updates to hickory 0.26.1 or hickory-net "RUSTSEC-2026-0119", # hickory-proto via rama-dns/rama-tcp; remove when rama updates to hickory 0.26.1 or hickory-net "RUSTSEC-2026-0173", # proc-macro-error2 via i18n-embed-fl/age/codex-secrets; remove when local secrets storage migrates off age or age drops i18n-embed-fl + "RUSTSEC-2026-0194", # quick-xml via plist/syntect and wayland-scanner; trusted inputs only; remove when rust-plist#191 and wayland-rs#938 are released + "RUSTSEC-2026-0195", # quick-xml via plist/syntect and wayland-scanner; trusted inputs only; remove when rust-plist#191 and wayland-rs#938 are released ] diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index 2c50f268f0..c0ff4a585a 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -2173,9 +2173,11 @@ dependencies = [ "futures", "gethostname", "hmac 0.12.1", + "httpdate", "jsonwebtoken", "owo-colors", "pretty_assertions", + "rand 0.9.3", "serde", "serde_json", "sha2 0.10.9", @@ -2314,6 +2316,7 @@ name = "codex-cli" version = "0.0.0" dependencies = [ "anyhow", + "app_test_support", "assert_cmd", "assert_matches", "clap", @@ -2503,6 +2506,7 @@ dependencies = [ "codex-code-mode-protocol", "codex-protocol", "deno_core_icudata", + "futures", "pretty_assertions", "serde_json", "tokio", @@ -2514,6 +2518,18 @@ dependencies = [ [[package]] name = "codex-code-mode-host" version = "0.0.0" +dependencies = [ + "anyhow", + "codex-code-mode", + "codex-code-mode-protocol", + "codex-protocol", + "codex-utils-cargo-bin", + "pretty_assertions", + "serde_json", + "tempfile", + "tokio", + "tokio-util", +] [[package]] name = "codex-code-mode-protocol" @@ -2557,6 +2573,7 @@ dependencies = [ "multimap", "pretty_assertions", "prost 0.14.3", + "regex-lite", "schemars 0.8.22", "serde", "serde_ignored", @@ -2919,7 +2936,9 @@ dependencies = [ "opentelemetry_sdk", "pretty_assertions", "prost 0.14.3", + "rcgen", "reqwest 0.12.28", + "rustls", "serde", "serde_json", "serial_test", @@ -3009,6 +3028,7 @@ dependencies = [ "codex-tools", "codex-utils-absolute-path", "pretty_assertions", + "serde_json", "tokio", ] @@ -3058,6 +3078,7 @@ dependencies = [ "anyhow", "codex-login", "codex-protocol", + "mime_guess", "pretty_assertions", "sentry", "tracing", @@ -3244,6 +3265,7 @@ dependencies = [ "clap", "codex-core", "codex-install-context", + "codex-network-proxy", "codex-process-hardening", "codex-protocol", "codex-sandboxing", @@ -3356,6 +3378,7 @@ name = "codex-mcp-extension" version = "0.0.0" dependencies = [ "codex-config", + "codex-connectors-extension", "codex-core", "codex-core-plugins", "codex-exec-server", @@ -3692,7 +3715,7 @@ dependencies = [ "icu_provider", "landlock", "pretty_assertions", - "quick-xml", + "quick-xml 0.41.0", "reqwest 0.12.28", "schemars 0.8.22", "seccompiler", @@ -3757,7 +3780,6 @@ dependencies = [ "base64 0.22.1", "bytes", "codex-api", - "codex-client", "codex-config", "codex-exec-server", "codex-keyring-store", @@ -4801,9 +4823,9 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.18" +version = "0.9.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" dependencies = [ "crossbeam-utils", ] @@ -10252,13 +10274,13 @@ checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" [[package]] name = "plist" -version = "1.8.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "740ebea15c5d1428f910cd1a5f52cebf8d25006245ed8ade92702f4943d91e07" +checksum = "092791278e026273c1b65bbdcfbba3a300f2994c896bd01ab01da613c29c46f1" dependencies = [ "base64 0.22.1", "indexmap 2.14.0", - "quick-xml", + "quick-xml 0.39.4", "serde", "time", ] @@ -10691,9 +10713,18 @@ checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" [[package]] name = "quick-xml" -version = "0.38.4" +version = "0.39.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b66c2058c55a409d601666cffe35f04333cf1013010882cec174a7467cd4e21c" +checksum = "cdcc8dd4e2f670d309a5f0e83fe36dfdc05af317008fea29144da1a2ac858e5e" +dependencies = [ + "memchr", +] + +[[package]] +name = "quick-xml" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1" dependencies = [ "memchr", "serde", @@ -14654,12 +14685,12 @@ dependencies = [ [[package]] name = "wayland-scanner" -version = "0.31.8" +version = "0.31.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5423e94b6a63e68e439803a3e153a9252d5ead12fd853334e2ad33997e3889e3" +checksum = "9c324a910fd86ebdc364a3e61ec1f11737d3b1d6c273c0239ee8ff4bc0d24b4a" dependencies = [ "proc-macro2", - "quick-xml", + "quick-xml 0.39.4", "quote", ] diff --git a/codex-rs/Cargo.toml b/codex-rs/Cargo.toml index ef17cb9ce5..2f73c216a9 100644 --- a/codex-rs/Cargo.toml +++ b/codex-rs/Cargo.toml @@ -166,6 +166,7 @@ codex-code-mode-protocol = { path = "code-mode-protocol" } codex-home = { path = "codex-home" } codex-config = { path = "config" } codex-connectors = { path = "connectors" } +codex-connectors-extension = { path = "ext/connectors" } codex-context-fragments = { path = "context-fragments" } codex-core = { path = "core" } codex-core-api = { path = "core-api" } @@ -317,6 +318,7 @@ glob = "0.3" globset = "0.4" hmac = "0.12.1" http = "1.3.1" +httpdate = "1.0.3" iana-time-zone = "0.1.64" icu_decimal = "2.1" icu_locale_core = "2.1" @@ -360,7 +362,7 @@ portable-pty = "0.9.0" predicates = "3" pretty_assertions = "1.4.1" pulldown-cmark = "0.10" -quick-xml = "0.38.4" +quick-xml = "0.41.0" rand = "0.9" ratatui = "0.29.0" ratatui-macros = "0.6.0" diff --git a/codex-rs/analytics/src/analytics_client_tests.rs b/codex-rs/analytics/src/analytics_client_tests.rs index c183ad925a..0f759fd938 100644 --- a/codex-rs/analytics/src/analytics_client_tests.rs +++ b/codex-rs/analytics/src/analytics_client_tests.rs @@ -198,6 +198,7 @@ fn sample_thread_with_metadata( parent_thread_id, preview: "first prompt".to_string(), ephemeral, + history_mode: Default::default(), model_provider: "openai".to_string(), created_at: 1, updated_at: 2, @@ -558,6 +559,7 @@ async fn ingest_rejected_turn_steer( response: Box::new(sample_thread_resume_response( "thread-2", /*ephemeral*/ false, "gpt-5", )), + thread_originator: None, }, out, ) @@ -631,6 +633,7 @@ async fn ingest_turn_prerequisites( response: Box::new(sample_thread_start_response( "thread-2", /*ephemeral*/ false, "gpt-5", )), + thread_originator: None, }, out, ) @@ -654,6 +657,7 @@ async fn ingest_turn_prerequisites( connection_id: 7, request_id: RequestId::Integer(3), response: Box::new(sample_turn_start_response("turn-2")), + thread_originator: None, }, out, ) @@ -720,6 +724,7 @@ async fn ingest_review_prerequisites( response: Box::new(sample_thread_start_response( "thread-1", /*ephemeral*/ false, "gpt-5", )), + thread_originator: None, }, events, ) @@ -1652,6 +1657,7 @@ async fn initialize_caches_client_and_thread_lifecycle_publishes_once_initialize /*ephemeral*/ false, "gpt-5", )), + thread_originator: None, }, &mut events, ) @@ -1697,6 +1703,7 @@ async fn initialize_caches_client_and_thread_lifecycle_publishes_once_initialize response: Box::new(sample_thread_resume_response( "thread-1", /*ephemeral*/ true, "gpt-5", )), + thread_originator: None, }, &mut events, ) @@ -1741,6 +1748,157 @@ async fn initialize_caches_client_and_thread_lifecycle_publishes_once_initialize ); } +#[tokio::test] +async fn thread_originator_overrides_shared_connection_across_thread_events() { + let mut reducer = AnalyticsReducer::default(); + let mut events = Vec::new(); + + reducer + .ingest(sample_initialize_fact(/*connection_id*/ 7), &mut events) + .await; + for (request_id, thread_id, thread_originator) in [ + (1, "thread-work", Some(TEST_PRODUCT_CLIENT_ID.to_string())), + (2, "thread-default", None), + ] { + reducer + .ingest( + AnalyticsFact::ClientResponse { + connection_id: 7, + request_id: RequestId::Integer(request_id), + response: Box::new(sample_thread_start_response( + thread_id, /*ephemeral*/ false, "gpt-5", + )), + thread_originator, + }, + &mut events, + ) + .await; + } + + let initialized = serde_json::to_value(&events).expect("serialize thread events"); + assert_eq!( + initialized + .as_array() + .expect("thread events") + .iter() + .map(|event| { + json!({ + "thread_id": event["event_params"]["thread_id"], + "app_server_client": event["event_params"]["app_server_client"], + }) + }) + .collect::>(), + vec![ + json!({ + "thread_id": "thread-work", + "app_server_client": { + "product_client_id": TEST_PRODUCT_CLIENT_ID, + "client_name": "codex-tui", + "client_version": "1.0.0", + "rpc_transport": "websocket", + "experimental_api_enabled": false, + }, + }), + json!({ + "thread_id": "thread-default", + "app_server_client": { + "product_client_id": DEFAULT_ORIGINATOR, + "client_name": "codex-tui", + "client_version": "1.0.0", + "rpc_transport": "websocket", + "experimental_api_enabled": false, + }, + }), + ] + ); + + events.clear(); + reducer + .ingest( + AnalyticsFact::ClientRequest { + connection_id: 7, + request_id: RequestId::Integer(3), + request: Box::new(sample_turn_start_request( + "thread-work", + /*request_id*/ 3, + )), + }, + &mut events, + ) + .await; + reducer + .ingest( + AnalyticsFact::ClientResponse { + connection_id: 7, + request_id: RequestId::Integer(3), + response: Box::new(sample_turn_start_response("turn-1")), + thread_originator: None, + }, + &mut events, + ) + .await; + ingest_completed_command_execution_item(&mut reducer, &mut events, "thread-work", "item-work") + .await; + ingest_complete_child_turn(&mut reducer, &mut events, "thread-work", "turn-1").await; + reducer + .ingest( + AnalyticsFact::Custom(CustomAnalyticsFact::Compaction(Box::new( + CodexCompactionEvent { + thread_id: "thread-work".to_string(), + turn_id: "turn-compact".to_string(), + trigger: CompactionTrigger::Manual, + reason: CompactionReason::UserRequested, + implementation: CompactionImplementation::Responses, + phase: CompactionPhase::StandaloneTurn, + strategy: CompactionStrategy::Memento, + status: CompactionStatus::Completed, + codex_error_kind: None, + codex_error_http_status_code: None, + active_context_tokens_before: 131_000, + active_context_tokens_after: 64_000, + retained_image_count: None, + compaction_summary_tokens: None, + cached_input_tokens: None, + started_at: 100, + completed_at: 101, + duration_ms: Some(1200), + }, + ))), + &mut events, + ) + .await; + + let lifecycle = serde_json::to_value(&events).expect("serialize lifecycle events"); + assert_eq!( + lifecycle + .as_array() + .expect("lifecycle events") + .iter() + .map(|event| { + json!({ + "event_type": event["event_type"], + "product_client_id": + event["event_params"]["app_server_client"]["product_client_id"], + }) + }) + .collect::>(), + vec![ + json!({ + "event_type": "codex_command_execution_event", + "product_client_id": TEST_PRODUCT_CLIENT_ID, + }), + json!({ + "event_type": "codex_turn_event", + "product_client_id": TEST_PRODUCT_CLIENT_ID, + }), + json!({ + "event_type": "codex_compaction_event", + "product_client_id": TEST_PRODUCT_CLIENT_ID, + }), + ] + ); +} + #[tokio::test] async fn unrelated_client_requests_are_ignored_by_reducer() { let mut reducer = AnalyticsReducer::default(); @@ -1767,6 +1925,7 @@ async fn unrelated_client_requests_are_ignored_by_reducer() { connection_id: 7, request_id: RequestId::Integer(3), response: Box::new(sample_turn_start_response("turn-2")), + thread_originator: None, }, &mut events, ) @@ -1792,6 +1951,7 @@ async fn unrelated_client_responses_are_ignored_by_reducer() { response: Box::new(ClientResponsePayload::ThreadArchive( ThreadArchiveResponse {}, )), + thread_originator: None, }, &mut events, ) @@ -1851,6 +2011,7 @@ async fn compaction_event_ingests_custom_fact() { Some(AppServerThreadSource::Subagent), Some(parent_thread_id.to_string()), )), + thread_originator: None, }, &mut events, ) @@ -1971,6 +2132,7 @@ async fn guardian_review_event_ingests_custom_fact_with_optional_target_item() { /*ephemeral*/ false, "gpt-5", )), + thread_originator: None, }, &mut events, ) @@ -2499,6 +2661,7 @@ async fn item_review_summaries_do_not_cross_threads_with_reused_item_ids() { response: Box::new(sample_thread_start_response( "thread-2", /*ephemeral*/ false, "gpt-5", )), + thread_originator: None, }, &mut events, ) @@ -2749,7 +2912,7 @@ async fn subagent_thread_started_publishes_without_initialize() { } #[tokio::test] -async fn subagent_events_use_inherited_connection_unless_turn_connection_is_explicit() { +async fn subagent_events_keep_thread_originator_with_explicit_turn_connection() { let mut reducer = AnalyticsReducer::default(); let mut events = Vec::new(); let parent_thread_id = @@ -2786,6 +2949,7 @@ async fn subagent_events_use_inherited_connection_unless_turn_connection_is_expl /*ephemeral*/ false, "gpt-5", )), + thread_originator: None, }, &mut events, ) @@ -2908,6 +3072,7 @@ async fn subagent_events_use_inherited_connection_unless_turn_connection_is_expl connection_id: 8, request_id: RequestId::Integer(3), response: Box::new(sample_turn_start_response("turn-explicit")), + thread_originator: None, }, &mut events, ) @@ -2918,7 +3083,11 @@ async fn subagent_events_use_inherited_connection_unless_turn_connection_is_expl }; assert_eq!( event.event_params.app_server_client.product_client_id, - DEFAULT_ORIGINATOR + "parent-client" + ); + assert_eq!( + event.event_params.app_server_client.client_name.as_deref(), + Some("codex-tui") ); } @@ -3868,6 +4037,7 @@ async fn accepted_turn_steer_emits_expected_event() { connection_id: 7, request_id: RequestId::Integer(4), response: Box::new(sample_turn_steer_response("turn-2")), + thread_originator: None, }, &mut out, ) @@ -4039,6 +4209,7 @@ async fn turn_start_error_response_discards_pending_start_request() { connection_id: 7, request_id: RequestId::Integer(3), response: Box::new(sample_turn_start_response("turn-2")), + thread_originator: None, }, &mut out, ) @@ -4367,6 +4538,7 @@ async fn accepted_steers_increment_turn_steer_count() { connection_id: 7, request_id: RequestId::Integer(4), response: Box::new(sample_turn_steer_response("turn-2")), + thread_originator: None, }, &mut out, ) @@ -4414,6 +4586,7 @@ async fn accepted_steers_increment_turn_steer_count() { connection_id: 7, request_id: RequestId::Integer(6), response: Box::new(sample_turn_steer_response("turn-2")), + thread_originator: None, }, &mut out, ) diff --git a/codex-rs/analytics/src/client.rs b/codex-rs/analytics/src/client.rs index 5996734f8e..f957a665a0 100644 --- a/codex-rs/analytics/src/client.rs +++ b/codex-rs/analytics/src/client.rs @@ -435,6 +435,31 @@ impl AnalyticsEventsClient { connection_id: u64, request_id: RequestId, response: ClientResponsePayload, + ) { + self.track_response_inner( + connection_id, + request_id, + response, + /*thread_originator*/ None, + ); + } + + pub fn track_response_with_thread_originator( + &self, + connection_id: u64, + request_id: RequestId, + response: ClientResponsePayload, + thread_originator: String, + ) { + self.track_response_inner(connection_id, request_id, response, Some(thread_originator)); + } + + fn track_response_inner( + &self, + connection_id: u64, + request_id: RequestId, + response: ClientResponsePayload, + thread_originator: Option, ) { if !matches!( response, @@ -450,6 +475,7 @@ impl AnalyticsEventsClient { connection_id, request_id, response: Box::new(response), + thread_originator, }); } diff --git a/codex-rs/analytics/src/client_tests.rs b/codex-rs/analytics/src/client_tests.rs index bbc5b5f0b6..ac50d6ebc7 100644 --- a/codex-rs/analytics/src/client_tests.rs +++ b/codex-rs/analytics/src/client_tests.rs @@ -283,6 +283,7 @@ fn sample_thread(thread_id: &str) -> Thread { parent_thread_id: None, preview: "first prompt".to_string(), ephemeral: false, + history_mode: Default::default(), model_provider: "openai".to_string(), created_at: 1, updated_at: 2, diff --git a/codex-rs/analytics/src/facts.rs b/codex-rs/analytics/src/facts.rs index 89b8708648..3346a6e19a 100644 --- a/codex-rs/analytics/src/facts.rs +++ b/codex-rs/analytics/src/facts.rs @@ -464,6 +464,7 @@ pub(crate) enum AnalyticsFact { connection_id: u64, request_id: RequestId, response: Box, + thread_originator: Option, }, ErrorResponse { connection_id: u64, diff --git a/codex-rs/analytics/src/reducer.rs b/codex-rs/analytics/src/reducer.rs index 522a307688..29f21bac52 100644 --- a/codex-rs/analytics/src/reducer.rs +++ b/codex-rs/analytics/src/reducer.rs @@ -165,6 +165,20 @@ struct ConnectionState { struct ThreadAnalyticsState { connection_id: Option, metadata: Option, + originator: Option, +} + +impl ThreadAnalyticsState { + fn app_server_client( + &self, + connection_state: &ConnectionState, + ) -> CodexAppServerClientMetadata { + let mut app_server_client = connection_state.app_server_client.clone(); + if let Some(originator) = self.originator.as_ref() { + app_server_client.product_client_id.clone_from(originator); + } + app_server_client + } } #[derive(Clone, Copy)] @@ -434,9 +448,11 @@ impl AnalyticsReducer { connection_id, request_id, response, + thread_originator, } => { if let Some(response) = response.into_client_response(request_id) { - self.ingest_response(connection_id, response, out).await; + self.ingest_response(connection_id, response, thread_originator, out) + .await; } } AnalyticsFact::ErrorResponse { @@ -575,6 +591,9 @@ impl AnalyticsReducer { .and_then(|parent_thread_id| self.threads.get(parent_thread_id)) .and_then(|thread| thread.connection_id); let thread_state = self.threads.entry(input.thread_id.clone()).or_default(); + thread_state + .originator + .get_or_insert_with(|| input.product_client_id.clone()); thread_state .metadata .get_or_insert_with(|| ThreadMetadataState { @@ -597,7 +616,7 @@ impl AnalyticsReducer { input: GuardianReviewEventParams, out: &mut Vec, ) { - let Some((connection_state, thread_metadata)) = + let Some((connection_state, thread_state, thread_metadata)) = self.thread_context_or_warn(AnalyticsDropSite::guardian(&input)) else { return; @@ -607,7 +626,7 @@ impl AnalyticsReducer { event_type: "codex_guardian_review", event_params: GuardianReviewEventPayload { session_id: thread_metadata.session_id.clone(), - app_server_client: connection_state.app_server_client.clone(), + app_server_client: thread_state.app_server_client(connection_state), runtime: connection_state.runtime.clone(), guardian_review: input, }, @@ -874,6 +893,7 @@ impl AnalyticsReducer { &mut self, connection_id: u64, response: ClientResponse, + thread_originator: Option, out: &mut Vec, ) { match response { @@ -883,6 +903,7 @@ impl AnalyticsReducer { response.thread, response.model, ThreadInitializationMode::New, + thread_originator, out, ); } @@ -892,6 +913,7 @@ impl AnalyticsReducer { response.thread, response.model, ThreadInitializationMode::Resumed, + thread_originator, out, ); } @@ -901,6 +923,7 @@ impl AnalyticsReducer { response.thread, response.model, ThreadInitializationMode::Forked, + thread_originator, out, ); } @@ -1248,7 +1271,7 @@ impl AnalyticsReducer { else { return; }; - let Some((connection_state, thread_metadata)) = self + let Some((connection_state, thread_state, thread_metadata)) = self .thread_context_or_warn(AnalyticsDropSite::tool_item(¬ification, item_id)) else { return; @@ -1260,6 +1283,7 @@ impl AnalyticsReducer { started_at_ms, completed_at_ms, connection_state, + thread_state, thread_metadata, review_summary: self.item_review_summaries.get(&key), }) { @@ -1316,6 +1340,7 @@ impl AnalyticsReducer { thread: codex_app_server_protocol::Thread, model: String, initialization_mode: ThreadInitializationMode, + thread_originator: Option, out: &mut Vec, ) { let session_source: SessionSource = thread.source.into(); @@ -1333,20 +1358,20 @@ impl AnalyticsReducer { parent_thread_id, initialization_mode, ); - self.threads.insert( - thread_id.clone(), - ThreadAnalyticsState { - connection_id: Some(connection_id), - metadata: Some(thread_metadata.clone()), - }, - ); + let thread_state = self.threads.entry(thread_id.clone()).or_default(); + if let Some(originator) = thread_originator { + thread_state.originator = Some(originator); + } + thread_state.connection_id = Some(connection_id); + thread_state.metadata = Some(thread_metadata.clone()); + let app_server_client = thread_state.app_server_client(connection_state); out.push(TrackEventRequest::ThreadInitialized( ThreadInitializedEvent { event_type: "codex_thread_initialized", event_params: ThreadInitializedEventParams { thread_id, session_id, - app_server_client: connection_state.app_server_client.clone(), + app_server_client, runtime: connection_state.runtime.clone(), model, ephemeral: thread.ephemeral, @@ -1362,7 +1387,7 @@ impl AnalyticsReducer { } fn ingest_compaction(&mut self, input: CodexCompactionEvent, out: &mut Vec) { - let Some((connection_state, thread_metadata)) = + let Some((connection_state, thread_state, thread_metadata)) = self.thread_context_or_warn(AnalyticsDropSite::compaction(&input)) else { return; @@ -1373,7 +1398,7 @@ impl AnalyticsReducer { event_params: codex_compaction_event_params( input, thread_metadata.session_id.clone(), - connection_state.app_server_client.clone(), + thread_state.app_server_client(connection_state), connection_state.runtime.clone(), thread_metadata.thread_source.clone(), thread_metadata.subagent_source.clone(), @@ -1384,7 +1409,7 @@ impl AnalyticsReducer { } fn ingest_goal(&mut self, input: CodexGoalEvent, out: &mut Vec) { - let Some((connection_state, thread_metadata)) = + let Some((connection_state, thread_state, thread_metadata)) = self.thread_context_or_warn(AnalyticsDropSite::goal(&input)) else { return; @@ -1394,7 +1419,7 @@ impl AnalyticsReducer { event_params: codex_goal_event_params( input, thread_metadata.session_id.clone(), - connection_state.app_server_client.clone(), + thread_state.app_server_client(connection_state), connection_state.runtime.clone(), thread_metadata.thread_source.clone(), thread_metadata.subagent_source.clone(), @@ -1483,11 +1508,11 @@ impl AnalyticsReducer { return; }; let drop_site = AnalyticsDropSite::turn_steer(&pending_request.thread_id); - let Some(thread_metadata) = self - .threads - .get(drop_site.thread_id) - .and_then(|thread| thread.metadata.as_ref()) - else { + let Some(thread_state) = self.threads.get(drop_site.thread_id) else { + warn_missing_analytics_context(&drop_site, MissingAnalyticsContext::ThreadMetadata); + return; + }; + let Some(thread_metadata) = thread_state.metadata.as_ref() else { warn_missing_analytics_context(&drop_site, MissingAnalyticsContext::ThreadMetadata); return; }; @@ -1498,7 +1523,7 @@ impl AnalyticsReducer { session_id: thread_metadata.session_id.clone(), expected_turn_id: Some(pending_request.expected_turn_id), accepted_turn_id, - app_server_client: connection_state.app_server_client.clone(), + app_server_client: thread_state.app_server_client(connection_state), runtime: connection_state.runtime.clone(), thread_source: thread_metadata.thread_source.clone(), subagent_source: thread_metadata.subagent_source.clone(), @@ -1529,7 +1554,7 @@ impl AnalyticsReducer { &pending_review, ); } - let Some((connection_state, thread_metadata)) = + let Some((connection_state, thread_state, thread_metadata)) = self.thread_context_or_warn(AnalyticsDropSite::review(&pending_review)) else { return; @@ -1541,7 +1566,7 @@ impl AnalyticsReducer { turn_id: pending_review.turn_id, item_id: pending_review.item_id, review_id: pending_review.review_id, - app_server_client: connection_state.app_server_client.clone(), + app_server_client: thread_state.app_server_client(connection_state), runtime: connection_state.runtime.clone(), thread_source: thread_metadata.thread_source.clone(), subagent_source: thread_metadata.subagent_source.clone(), @@ -1610,18 +1635,18 @@ impl AnalyticsReducer { ); return; }; - let Some(thread_metadata) = self - .threads - .get(drop_site.thread_id) - .and_then(|thread| thread.metadata.as_ref()) - else { + let Some(thread_state) = self.threads.get(drop_site.thread_id) else { + warn_missing_analytics_context(&drop_site, MissingAnalyticsContext::ThreadMetadata); + return; + }; + let Some(thread_metadata) = thread_state.metadata.as_ref() else { warn_missing_analytics_context(&drop_site, MissingAnalyticsContext::ThreadMetadata); return; }; let turn_event = TrackEventRequest::TurnEvent(Box::new(CodexTurnEventRequest { event_type: "codex_turn_event", event_params: codex_turn_event_params( - connection_state.app_server_client.clone(), + thread_state.app_server_client(connection_state), connection_state.runtime.clone(), turn_id.to_string(), turn_state, @@ -1663,17 +1688,18 @@ impl AnalyticsReducer { fn thread_context_or_warn( &self, drop_site: AnalyticsDropSite<'_>, - ) -> Option<(&ConnectionState, &ThreadMetadataState)> { + ) -> Option<( + &ConnectionState, + &ThreadAnalyticsState, + &ThreadMetadataState, + )> { let connection_state = self.thread_connection_or_warn(drop_site)?; - let Some(thread_metadata) = self - .threads - .get(drop_site.thread_id) - .and_then(|thread| thread.metadata.as_ref()) - else { + let thread_state = self.threads.get(drop_site.thread_id)?; + let Some(thread_metadata) = thread_state.metadata.as_ref() else { warn_missing_analytics_context(&drop_site, MissingAnalyticsContext::ThreadMetadata); return None; }; - Some((connection_state, thread_metadata)) + Some((connection_state, thread_state, thread_metadata)) } } @@ -1743,6 +1769,7 @@ struct ToolItemEventInput<'a> { started_at_ms: u64, completed_at_ms: u64, connection_state: &'a ConnectionState, + thread_state: &'a ThreadAnalyticsState, thread_metadata: &'a ThreadMetadataState, review_summary: Option<&'a ItemReviewSummary>, } @@ -1755,6 +1782,7 @@ fn tool_item_event(input: ToolItemEventInput<'_>) -> Option { started_at_ms, completed_at_ms, connection_state, + thread_state, thread_metadata, review_summary, } = input; @@ -1784,6 +1812,7 @@ fn tool_item_event(input: ToolItemEventInput<'_>) -> Option { started_at_ms, completed_at_ms, connection_state, + thread_state, thread_metadata, review_summary, }, @@ -1825,6 +1854,7 @@ fn tool_item_event(input: ToolItemEventInput<'_>) -> Option { started_at_ms, completed_at_ms, connection_state, + thread_state, thread_metadata, review_summary, }, @@ -1866,6 +1896,7 @@ fn tool_item_event(input: ToolItemEventInput<'_>) -> Option { started_at_ms, completed_at_ms, connection_state, + thread_state, thread_metadata, review_summary, }, @@ -1910,6 +1941,7 @@ fn tool_item_event(input: ToolItemEventInput<'_>) -> Option { started_at_ms, completed_at_ms, connection_state, + thread_state, thread_metadata, review_summary, }, @@ -1954,6 +1986,7 @@ fn tool_item_event(input: ToolItemEventInput<'_>) -> Option { started_at_ms, completed_at_ms, connection_state, + thread_state, thread_metadata, review_summary, }, @@ -2009,6 +2042,7 @@ fn tool_item_event(input: ToolItemEventInput<'_>) -> Option { started_at_ms, completed_at_ms, connection_state, + thread_state, thread_metadata, review_summary, }, @@ -2045,6 +2079,7 @@ fn tool_item_event(input: ToolItemEventInput<'_>) -> Option { started_at_ms, completed_at_ms, connection_state, + thread_state, thread_metadata, review_summary, }, @@ -2100,6 +2135,7 @@ struct ToolItemContext<'a> { started_at_ms: u64, completed_at_ms: u64, connection_state: &'a ConnectionState, + thread_state: &'a ThreadAnalyticsState, thread_metadata: &'a ThreadMetadataState, review_summary: Option<&'a ItemReviewSummary>, } @@ -2118,7 +2154,9 @@ fn tool_item_base( thread_id: thread_id.to_string(), turn_id: turn_id.to_string(), item_id, - app_server_client: context.connection_state.app_server_client.clone(), + app_server_client: context + .thread_state + .app_server_client(context.connection_state), runtime: context.connection_state.runtime.clone(), thread_source: thread_metadata.thread_source.clone(), subagent_source: thread_metadata.subagent_source.clone(), diff --git a/codex-rs/app-server-client/src/lib.rs b/codex-rs/app-server-client/src/lib.rs index b7a958eff1..4eec8a2348 100644 --- a/codex-rs/app-server-client/src/lib.rs +++ b/codex-rs/app-server-client/src/lib.rs @@ -78,9 +78,6 @@ pub use crate::remote::RemoteAppServerEndpoint; /// module exists so clients can remove a direct `codex-core` dependency /// while legacy startup/config paths are migrated to RPCs. pub mod legacy_core { - pub use codex_core::check_execpolicy_for_warnings; - pub use codex_core::format_exec_policy_error_with_source; - pub mod config { pub use codex_core::config::*; diff --git a/codex-rs/app-server-protocol/schema/json/ClientRequest.json b/codex-rs/app-server-protocol/schema/json/ClientRequest.json index 68c242504f..2f39f821b6 100644 --- a/codex-rs/app-server-protocol/schema/json/ClientRequest.json +++ b/codex-rs/app-server-protocol/schema/json/ClientRequest.json @@ -556,6 +556,13 @@ }, "ConsumeAccountRateLimitResetCreditParams": { "properties": { + "creditId": { + "description": "Opaque reset-credit identifier to redeem. When omitted, the backend selects the next available credit.", + "type": [ + "string", + "null" + ] + }, "idempotencyKey": { "description": "Identifies one logical reset attempt. A UUID is recommended; reuse the same value when retrying that attempt.", "type": "string" @@ -1619,6 +1626,12 @@ "null" ] }, + "threadId": { + "type": [ + "string", + "null" + ] + }, "timeoutSecs": { "format": "int64", "type": [ @@ -1780,13 +1793,29 @@ "type": "object" }, "MultiAgentMode": { - "description": "Controls whether the model receives multi-agent delegation instructions and, when it does, whether it should only spawn sub-agents after an explicit user request or may delegate proactively when doing so would help. `none` leaves the multi-agent tools available without injecting delegation instructions.", - "enum": [ - "none", - "explicitRequestOnly", - "proactive" - ], - "type": "string" + "description": "Controls the effective multi-agent delegation instructions for a turn. `custom` means the configured mode hint defines the policy instead of a built-in policy.", + "oneOf": [ + { + "enum": [ + "explicitRequestOnly", + "proactive" + ], + "type": "string" + }, + { + "additionalProperties": false, + "properties": { + "custom": { + "type": "string" + } + }, + "required": [ + "custom" + ], + "title": "CustomMultiAgentMode", + "type": "object" + } + ] }, "NetworkAccess": { "enum": [ @@ -2677,6 +2706,12 @@ "name": { "type": "string" }, + "namespace": { + "type": [ + "string", + "null" + ] + }, "status": { "type": [ "string", @@ -3625,6 +3660,13 @@ "ephemeral": { "type": "boolean" }, + "lastTurnId": { + "description": "Optional last turn id to fork through, inclusive.\n\nWhen specified, turns after `last_turn_id` are omitted from the fork. The referenced turn cannot be in progress.", + "type": [ + "string", + "null" + ] + }, "model": { "description": "Configuration overrides for the forked thread, if any.", "type": [ @@ -3741,6 +3783,13 @@ ], "type": "string" }, + "ThreadHistoryMode": { + "enum": [ + "legacy", + "paginated" + ], + "type": "string" + }, "ThreadInjectItemsParams": { "properties": { "items": { @@ -4168,6 +4217,7 @@ "type": "object" }, "ThreadRollbackParams": { + "description": "DEPRECATED: `thread/rollback` will be removed soon.", "properties": { "numTurns": { "description": "The number of turns to drop from the end of the thread. Must be >= 1.\n\nThis only modifies the thread's history and does not revert local file changes that have been made by the agent. Clients are responsible for reverting these changes.", diff --git a/codex-rs/app-server-protocol/schema/json/ServerNotification.json b/codex-rs/app-server-protocol/schema/json/ServerNotification.json index a3d07c24a0..e7a533d76b 100644 --- a/codex-rs/app-server-protocol/schema/json/ServerNotification.json +++ b/codex-rs/app-server-protocol/schema/json/ServerNotification.json @@ -2370,6 +2370,12 @@ }, "success": { "type": "boolean" + }, + "threadId": { + "type": [ + "string", + "null" + ] } }, "required": [ @@ -2378,6 +2384,12 @@ ], "type": "object" }, + "McpServerStartupFailureReason": { + "enum": [ + "reauthenticationRequired" + ], + "type": "string" + }, "McpServerStartupState": { "enum": [ "starting", @@ -2395,6 +2407,16 @@ "null" ] }, + "failureReason": { + "anyOf": [ + { + "$ref": "#/definitions/McpServerStartupFailureReason" + }, + { + "type": "null" + } + ] + }, "name": { "type": "string" }, @@ -2416,6 +2438,18 @@ }, "McpToolCallAppContext": { "properties": { + "actionName": { + "type": [ + "string", + "null" + ] + }, + "appName": { + "type": [ + "string", + "null" + ] + }, "connectorId": { "type": "string" }, @@ -2430,6 +2464,12 @@ "string", "null" ] + }, + "templateId": { + "type": [ + "string", + "null" + ] } }, "required": [ @@ -2673,13 +2713,29 @@ "type": "object" }, "MultiAgentMode": { - "description": "Controls whether the model receives multi-agent delegation instructions and, when it does, whether it should only spawn sub-agents after an explicit user request or may delegate proactively when doing so would help. `none` leaves the multi-agent tools available without injecting delegation instructions.", - "enum": [ - "none", - "explicitRequestOnly", - "proactive" - ], - "type": "string" + "description": "Controls the effective multi-agent delegation instructions for a turn. `custom` means the configured mode hint defines the policy instead of a built-in policy.", + "oneOf": [ + { + "enum": [ + "explicitRequestOnly", + "proactive" + ], + "type": "string" + }, + { + "additionalProperties": false, + "properties": { + "custom": { + "type": "string" + } + }, + "required": [ + "custom" + ], + "title": "CustomMultiAgentMode", + "type": "object" + } + ] }, "NetworkAccess": { "enum": [ @@ -3877,6 +3933,13 @@ ], "type": "object" }, + "ThreadHistoryMode": { + "enum": [ + "legacy", + "paginated" + ], + "type": "string" + }, "ThreadId": { "type": "string" }, diff --git a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json index 4d417ffb2f..0eedef903a 100644 --- a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json +++ b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json @@ -8414,6 +8414,16 @@ "object", "null" ] + }, + "models": { + "anyOf": [ + { + "$ref": "#/definitions/v2/ModelsRequirements" + }, + { + "type": "null" + } + ] } }, "type": "object" @@ -8678,6 +8688,13 @@ "ConsumeAccountRateLimitResetCreditParams": { "$schema": "http://json-schema.org/draft-07/schema#", "properties": { + "creditId": { + "description": "Opaque reset-credit identifier to redeem. When omitted, the backend selects the next available credit.", + "type": [ + "string", + "null" + ] + }, "idempotencyKey": { "description": "Identifies one logical reset attempt. A UUID is recommended; reuse the same value when retrying that attempt.", "type": "string" @@ -12026,6 +12043,12 @@ }, "success": { "type": "boolean" + }, + "threadId": { + "type": [ + "string", + "null" + ] } }, "required": [ @@ -12050,6 +12073,12 @@ "null" ] }, + "threadId": { + "type": [ + "string", + "null" + ] + }, "timeoutSecs": { "format": "int64", "type": [ @@ -12082,6 +12111,12 @@ "title": "McpServerRefreshResponse", "type": "object" }, + "McpServerStartupFailureReason": { + "enum": [ + "reauthenticationRequired" + ], + "type": "string" + }, "McpServerStartupState": { "enum": [ "starting", @@ -12153,6 +12188,16 @@ "null" ] }, + "failureReason": { + "anyOf": [ + { + "$ref": "#/definitions/v2/McpServerStartupFailureReason" + }, + { + "type": "null" + } + ] + }, "name": { "type": "string" }, @@ -12220,6 +12265,18 @@ }, "McpToolCallAppContext": { "properties": { + "actionName": { + "type": [ + "string", + "null" + ] + }, + "appName": { + "type": [ + "string", + "null" + ] + }, "connectorId": { "type": "string" }, @@ -12234,6 +12291,12 @@ "string", "null" ] + }, + "templateId": { + "type": [ + "string", + "null" + ] } }, "required": [ @@ -12788,14 +12851,45 @@ "title": "ModelVerificationNotification", "type": "object" }, + "ModelsRequirements": { + "properties": { + "newThread": { + "anyOf": [ + { + "$ref": "#/definitions/v2/NewThreadModelDefaults" + }, + { + "type": "null" + } + ] + } + }, + "type": "object" + }, "MultiAgentMode": { - "description": "Controls whether the model receives multi-agent delegation instructions and, when it does, whether it should only spawn sub-agents after an explicit user request or may delegate proactively when doing so would help. `none` leaves the multi-agent tools available without injecting delegation instructions.", - "enum": [ - "none", - "explicitRequestOnly", - "proactive" - ], - "type": "string" + "description": "Controls the effective multi-agent delegation instructions for a turn. `custom` means the configured mode hint defines the policy instead of a built-in policy.", + "oneOf": [ + { + "enum": [ + "explicitRequestOnly", + "proactive" + ], + "type": "string" + }, + { + "additionalProperties": false, + "properties": { + "custom": { + "type": "string" + } + }, + "required": [ + "custom" + ], + "title": "CustomMultiAgentMode", + "type": "object" + } + ] }, "NetworkAccess": { "enum": [ @@ -12935,6 +13029,33 @@ ], "type": "string" }, + "NewThreadModelDefaults": { + "properties": { + "model": { + "type": [ + "string", + "null" + ] + }, + "modelReasoningEffort": { + "anyOf": [ + { + "$ref": "#/definitions/v2/ReasoningEffort" + }, + { + "type": "null" + } + ] + }, + "serviceTier": { + "type": [ + "string", + "null" + ] + } + }, + "type": "object" + }, "NonSteerableTurnKind": { "enum": [ "review", @@ -14109,6 +14230,40 @@ "title": "GitPluginSource", "type": "object" }, + { + "properties": { + "package": { + "type": "string" + }, + "registry": { + "description": "Optional HTTPS registry URL. Authentication stays in the user's npm config.", + "type": [ + "string", + "null" + ] + }, + "type": { + "enum": [ + "npm" + ], + "title": "NpmPluginSourceType", + "type": "string" + }, + "version": { + "description": "Optional npm version or version range.", + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "package", + "type" + ], + "title": "NpmPluginSource", + "type": "object" + }, { "description": "The plugin is available in the remote catalog. Download metadata is kept server-side and is not exposed through the app-server API.", "properties": { @@ -14202,6 +14357,14 @@ }, "source": { "$ref": "#/definitions/v2/PluginSource" + }, + "version": { + "default": null, + "description": "Version advertised by the remote marketplace backend when available.", + "type": [ + "string", + "null" + ] } }, "required": [ @@ -14377,11 +14540,78 @@ ], "type": "string" }, + "RateLimitResetCredit": { + "properties": { + "description": { + "description": "Backend-provided display description for this credit, or `null` when unavailable.", + "type": [ + "string", + "null" + ] + }, + "expiresAt": { + "description": "Unix timestamp in seconds when the credit expires, or `null` if it does not expire.", + "format": "int64", + "type": [ + "integer", + "null" + ] + }, + "grantedAt": { + "description": "Unix timestamp in seconds when the credit was granted.", + "format": "int64", + "type": "integer" + }, + "id": { + "description": "Opaque backend identifier for this reset credit.", + "type": "string" + }, + "resetType": { + "$ref": "#/definitions/v2/RateLimitResetType" + }, + "status": { + "$ref": "#/definitions/v2/RateLimitResetCreditStatus" + }, + "title": { + "description": "Backend-provided display title for this credit, or `null` when unavailable.", + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "grantedAt", + "id", + "resetType", + "status" + ], + "type": "object" + }, + "RateLimitResetCreditStatus": { + "enum": [ + "available", + "redeeming", + "redeemed", + "unknown" + ], + "type": "string" + }, "RateLimitResetCreditsSummary": { "properties": { "availableCount": { "format": "int64", "type": "integer" + }, + "credits": { + "description": "Detail rows for available reset credits, when the backend provides them.\n\n`null` means only `availableCount` is known, while an empty array means details were fetched and no available credits were returned. The backend may cap this list, so its length can be less than `availableCount`.", + "items": { + "$ref": "#/definitions/v2/RateLimitResetCredit" + }, + "type": [ + "array", + "null" + ] } }, "required": [ @@ -14389,6 +14619,13 @@ ], "type": "object" }, + "RateLimitResetType": { + "enum": [ + "codexRateLimits", + "unknown" + ], + "type": "string" + }, "RateLimitSnapshot": { "properties": { "credits": { @@ -15378,6 +15615,12 @@ "name": { "type": "string" }, + "namespace": { + "type": [ + "string", + "null" + ] + }, "status": { "type": [ "string", @@ -17147,6 +17390,13 @@ "ephemeral": { "type": "boolean" }, + "lastTurnId": { + "description": "Optional last turn id to fork through, inclusive.\n\nWhen specified, turns after `last_turn_id` are omitted from the fork. The referenced turn cannot be in progress.", + "type": [ + "string", + "null" + ] + }, "model": { "description": "Configuration overrides for the forked thread, if any.", "type": [ @@ -17466,6 +17716,13 @@ "title": "ThreadGoalUpdatedNotification", "type": "object" }, + "ThreadHistoryMode": { + "enum": [ + "legacy", + "paginated" + ], + "type": "string" + }, "ThreadId": { "type": "string" }, @@ -18985,6 +19242,7 @@ }, "ThreadRollbackParams": { "$schema": "http://json-schema.org/draft-07/schema#", + "description": "DEPRECATED: `thread/rollback` will be removed soon.", "properties": { "numTurns": { "description": "The number of turns to drop from the end of the thread. Must be >= 1.\n\nThis only modifies the thread's history and does not revert local file changes that have been made by the agent. Clients are responsible for reverting these changes.", diff --git a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json index 0df923a0f4..b428f25258 100644 --- a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json +++ b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json @@ -4654,6 +4654,16 @@ "object", "null" ] + }, + "models": { + "anyOf": [ + { + "$ref": "#/definitions/ModelsRequirements" + }, + { + "type": "null" + } + ] } }, "type": "object" @@ -4918,6 +4928,13 @@ "ConsumeAccountRateLimitResetCreditParams": { "$schema": "http://json-schema.org/draft-07/schema#", "properties": { + "creditId": { + "description": "Opaque reset-credit identifier to redeem. When omitted, the backend selects the next available credit.", + "type": [ + "string", + "null" + ] + }, "idempotencyKey": { "description": "Identifies one logical reset attempt. A UUID is recommended; reuse the same value when retrying that attempt.", "type": "string" @@ -8430,6 +8447,12 @@ }, "success": { "type": "boolean" + }, + "threadId": { + "type": [ + "string", + "null" + ] } }, "required": [ @@ -8454,6 +8477,12 @@ "null" ] }, + "threadId": { + "type": [ + "string", + "null" + ] + }, "timeoutSecs": { "format": "int64", "type": [ @@ -8486,6 +8515,12 @@ "title": "McpServerRefreshResponse", "type": "object" }, + "McpServerStartupFailureReason": { + "enum": [ + "reauthenticationRequired" + ], + "type": "string" + }, "McpServerStartupState": { "enum": [ "starting", @@ -8557,6 +8592,16 @@ "null" ] }, + "failureReason": { + "anyOf": [ + { + "$ref": "#/definitions/McpServerStartupFailureReason" + }, + { + "type": "null" + } + ] + }, "name": { "type": "string" }, @@ -8624,6 +8669,18 @@ }, "McpToolCallAppContext": { "properties": { + "actionName": { + "type": [ + "string", + "null" + ] + }, + "appName": { + "type": [ + "string", + "null" + ] + }, "connectorId": { "type": "string" }, @@ -8638,6 +8695,12 @@ "string", "null" ] + }, + "templateId": { + "type": [ + "string", + "null" + ] } }, "required": [ @@ -9192,14 +9255,45 @@ "title": "ModelVerificationNotification", "type": "object" }, + "ModelsRequirements": { + "properties": { + "newThread": { + "anyOf": [ + { + "$ref": "#/definitions/NewThreadModelDefaults" + }, + { + "type": "null" + } + ] + } + }, + "type": "object" + }, "MultiAgentMode": { - "description": "Controls whether the model receives multi-agent delegation instructions and, when it does, whether it should only spawn sub-agents after an explicit user request or may delegate proactively when doing so would help. `none` leaves the multi-agent tools available without injecting delegation instructions.", - "enum": [ - "none", - "explicitRequestOnly", - "proactive" - ], - "type": "string" + "description": "Controls the effective multi-agent delegation instructions for a turn. `custom` means the configured mode hint defines the policy instead of a built-in policy.", + "oneOf": [ + { + "enum": [ + "explicitRequestOnly", + "proactive" + ], + "type": "string" + }, + { + "additionalProperties": false, + "properties": { + "custom": { + "type": "string" + } + }, + "required": [ + "custom" + ], + "title": "CustomMultiAgentMode", + "type": "object" + } + ] }, "NetworkAccess": { "enum": [ @@ -9339,6 +9433,33 @@ ], "type": "string" }, + "NewThreadModelDefaults": { + "properties": { + "model": { + "type": [ + "string", + "null" + ] + }, + "modelReasoningEffort": { + "anyOf": [ + { + "$ref": "#/definitions/ReasoningEffort" + }, + { + "type": "null" + } + ] + }, + "serviceTier": { + "type": [ + "string", + "null" + ] + } + }, + "type": "object" + }, "NonSteerableTurnKind": { "enum": [ "review", @@ -10513,6 +10634,40 @@ "title": "GitPluginSource", "type": "object" }, + { + "properties": { + "package": { + "type": "string" + }, + "registry": { + "description": "Optional HTTPS registry URL. Authentication stays in the user's npm config.", + "type": [ + "string", + "null" + ] + }, + "type": { + "enum": [ + "npm" + ], + "title": "NpmPluginSourceType", + "type": "string" + }, + "version": { + "description": "Optional npm version or version range.", + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "package", + "type" + ], + "title": "NpmPluginSource", + "type": "object" + }, { "description": "The plugin is available in the remote catalog. Download metadata is kept server-side and is not exposed through the app-server API.", "properties": { @@ -10606,6 +10761,14 @@ }, "source": { "$ref": "#/definitions/PluginSource" + }, + "version": { + "default": null, + "description": "Version advertised by the remote marketplace backend when available.", + "type": [ + "string", + "null" + ] } }, "required": [ @@ -10781,11 +10944,78 @@ ], "type": "string" }, + "RateLimitResetCredit": { + "properties": { + "description": { + "description": "Backend-provided display description for this credit, or `null` when unavailable.", + "type": [ + "string", + "null" + ] + }, + "expiresAt": { + "description": "Unix timestamp in seconds when the credit expires, or `null` if it does not expire.", + "format": "int64", + "type": [ + "integer", + "null" + ] + }, + "grantedAt": { + "description": "Unix timestamp in seconds when the credit was granted.", + "format": "int64", + "type": "integer" + }, + "id": { + "description": "Opaque backend identifier for this reset credit.", + "type": "string" + }, + "resetType": { + "$ref": "#/definitions/RateLimitResetType" + }, + "status": { + "$ref": "#/definitions/RateLimitResetCreditStatus" + }, + "title": { + "description": "Backend-provided display title for this credit, or `null` when unavailable.", + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "grantedAt", + "id", + "resetType", + "status" + ], + "type": "object" + }, + "RateLimitResetCreditStatus": { + "enum": [ + "available", + "redeeming", + "redeemed", + "unknown" + ], + "type": "string" + }, "RateLimitResetCreditsSummary": { "properties": { "availableCount": { "format": "int64", "type": "integer" + }, + "credits": { + "description": "Detail rows for available reset credits, when the backend provides them.\n\n`null` means only `availableCount` is known, while an empty array means details were fetched and no available credits were returned. The backend may cap this list, so its length can be less than `availableCount`.", + "items": { + "$ref": "#/definitions/RateLimitResetCredit" + }, + "type": [ + "array", + "null" + ] } }, "required": [ @@ -10793,6 +11023,13 @@ ], "type": "object" }, + "RateLimitResetType": { + "enum": [ + "codexRateLimits", + "unknown" + ], + "type": "string" + }, "RateLimitSnapshot": { "properties": { "credits": { @@ -11782,6 +12019,12 @@ "name": { "type": "string" }, + "namespace": { + "type": [ + "string", + "null" + ] + }, "status": { "type": [ "string", @@ -14926,6 +15169,13 @@ "ephemeral": { "type": "boolean" }, + "lastTurnId": { + "description": "Optional last turn id to fork through, inclusive.\n\nWhen specified, turns after `last_turn_id` are omitted from the fork. The referenced turn cannot be in progress.", + "type": [ + "string", + "null" + ] + }, "model": { "description": "Configuration overrides for the forked thread, if any.", "type": [ @@ -15245,6 +15495,13 @@ "title": "ThreadGoalUpdatedNotification", "type": "object" }, + "ThreadHistoryMode": { + "enum": [ + "legacy", + "paginated" + ], + "type": "string" + }, "ThreadId": { "type": "string" }, @@ -16764,6 +17021,7 @@ }, "ThreadRollbackParams": { "$schema": "http://json-schema.org/draft-07/schema#", + "description": "DEPRECATED: `thread/rollback` will be removed soon.", "properties": { "numTurns": { "description": "The number of turns to drop from the end of the thread. Must be >= 1.\n\nThis only modifies the thread's history and does not revert local file changes that have been made by the agent. Clients are responsible for reverting these changes.", diff --git a/codex-rs/app-server-protocol/schema/json/v2/ConfigRequirementsReadResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ConfigRequirementsReadResponse.json index 3fe6a89787..4e35f7aead 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ConfigRequirementsReadResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ConfigRequirementsReadResponse.json @@ -169,6 +169,16 @@ "object", "null" ] + }, + "models": { + "anyOf": [ + { + "$ref": "#/definitions/ModelsRequirements" + }, + { + "type": "null" + } + ] } }, "type": "object" @@ -362,6 +372,21 @@ ], "type": "object" }, + "ModelsRequirements": { + "properties": { + "newThread": { + "anyOf": [ + { + "$ref": "#/definitions/NewThreadModelDefaults" + }, + { + "type": "null" + } + ] + } + }, + "type": "object" + }, "NetworkDomainPermission": { "enum": [ "allow", @@ -484,6 +509,38 @@ ], "type": "string" }, + "NewThreadModelDefaults": { + "properties": { + "model": { + "type": [ + "string", + "null" + ] + }, + "modelReasoningEffort": { + "anyOf": [ + { + "$ref": "#/definitions/ReasoningEffort" + }, + { + "type": "null" + } + ] + }, + "serviceTier": { + "type": [ + "string", + "null" + ] + } + }, + "type": "object" + }, + "ReasoningEffort": { + "description": "A non-empty reasoning effort value advertised by the model.", + "minLength": 1, + "type": "string" + }, "ResidencyRequirement": { "enum": [ "us" diff --git a/codex-rs/app-server-protocol/schema/json/v2/ConsumeAccountRateLimitResetCreditParams.json b/codex-rs/app-server-protocol/schema/json/v2/ConsumeAccountRateLimitResetCreditParams.json index c9bf023a63..3d9d2c1e98 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ConsumeAccountRateLimitResetCreditParams.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ConsumeAccountRateLimitResetCreditParams.json @@ -1,6 +1,13 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "properties": { + "creditId": { + "description": "Opaque reset-credit identifier to redeem. When omitted, the backend selects the next available credit.", + "type": [ + "string", + "null" + ] + }, "idempotencyKey": { "description": "Identifies one logical reset attempt. A UUID is recommended; reuse the same value when retrying that attempt.", "type": "string" diff --git a/codex-rs/app-server-protocol/schema/json/v2/GetAccountRateLimitsResponse.json b/codex-rs/app-server-protocol/schema/json/v2/GetAccountRateLimitsResponse.json index 11b9712510..a2883a1293 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/GetAccountRateLimitsResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/GetAccountRateLimitsResponse.json @@ -49,11 +49,78 @@ ], "type": "string" }, + "RateLimitResetCredit": { + "properties": { + "description": { + "description": "Backend-provided display description for this credit, or `null` when unavailable.", + "type": [ + "string", + "null" + ] + }, + "expiresAt": { + "description": "Unix timestamp in seconds when the credit expires, or `null` if it does not expire.", + "format": "int64", + "type": [ + "integer", + "null" + ] + }, + "grantedAt": { + "description": "Unix timestamp in seconds when the credit was granted.", + "format": "int64", + "type": "integer" + }, + "id": { + "description": "Opaque backend identifier for this reset credit.", + "type": "string" + }, + "resetType": { + "$ref": "#/definitions/RateLimitResetType" + }, + "status": { + "$ref": "#/definitions/RateLimitResetCreditStatus" + }, + "title": { + "description": "Backend-provided display title for this credit, or `null` when unavailable.", + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "grantedAt", + "id", + "resetType", + "status" + ], + "type": "object" + }, + "RateLimitResetCreditStatus": { + "enum": [ + "available", + "redeeming", + "redeemed", + "unknown" + ], + "type": "string" + }, "RateLimitResetCreditsSummary": { "properties": { "availableCount": { "format": "int64", "type": "integer" + }, + "credits": { + "description": "Detail rows for available reset credits, when the backend provides them.\n\n`null` means only `availableCount` is known, while an empty array means details were fetched and no available credits were returned. The backend may cap this list, so its length can be less than `availableCount`.", + "items": { + "$ref": "#/definitions/RateLimitResetCredit" + }, + "type": [ + "array", + "null" + ] } }, "required": [ @@ -61,6 +128,13 @@ ], "type": "object" }, + "RateLimitResetType": { + "enum": [ + "codexRateLimits", + "unknown" + ], + "type": "string" + }, "RateLimitSnapshot": { "properties": { "credits": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ItemCompletedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/ItemCompletedNotification.json index 8f4b533d8f..dcf6312c7d 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ItemCompletedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ItemCompletedNotification.json @@ -299,6 +299,18 @@ }, "McpToolCallAppContext": { "properties": { + "actionName": { + "type": [ + "string", + "null" + ] + }, + "appName": { + "type": [ + "string", + "null" + ] + }, "connectorId": { "type": "string" }, @@ -313,6 +325,12 @@ "string", "null" ] + }, + "templateId": { + "type": [ + "string", + "null" + ] } }, "required": [ diff --git a/codex-rs/app-server-protocol/schema/json/v2/ItemStartedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/ItemStartedNotification.json index 4c930afde1..1933225ade 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ItemStartedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ItemStartedNotification.json @@ -299,6 +299,18 @@ }, "McpToolCallAppContext": { "properties": { + "actionName": { + "type": [ + "string", + "null" + ] + }, + "appName": { + "type": [ + "string", + "null" + ] + }, "connectorId": { "type": "string" }, @@ -313,6 +325,12 @@ "string", "null" ] + }, + "templateId": { + "type": [ + "string", + "null" + ] } }, "required": [ diff --git a/codex-rs/app-server-protocol/schema/json/v2/McpServerOauthLoginCompletedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/McpServerOauthLoginCompletedNotification.json index 35efd2baf2..6204ff672b 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/McpServerOauthLoginCompletedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/McpServerOauthLoginCompletedNotification.json @@ -12,6 +12,12 @@ }, "success": { "type": "boolean" + }, + "threadId": { + "type": [ + "string", + "null" + ] } }, "required": [ diff --git a/codex-rs/app-server-protocol/schema/json/v2/McpServerOauthLoginParams.json b/codex-rs/app-server-protocol/schema/json/v2/McpServerOauthLoginParams.json index 4370f444b9..de66dcefd6 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/McpServerOauthLoginParams.json +++ b/codex-rs/app-server-protocol/schema/json/v2/McpServerOauthLoginParams.json @@ -13,6 +13,12 @@ "null" ] }, + "threadId": { + "type": [ + "string", + "null" + ] + }, "timeoutSecs": { "format": "int64", "type": [ diff --git a/codex-rs/app-server-protocol/schema/json/v2/McpServerStatusUpdatedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/McpServerStatusUpdatedNotification.json index 87dab90cff..8efb36bdad 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/McpServerStatusUpdatedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/McpServerStatusUpdatedNotification.json @@ -1,6 +1,12 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "definitions": { + "McpServerStartupFailureReason": { + "enum": [ + "reauthenticationRequired" + ], + "type": "string" + }, "McpServerStartupState": { "enum": [ "starting", @@ -18,6 +24,16 @@ "null" ] }, + "failureReason": { + "anyOf": [ + { + "$ref": "#/definitions/McpServerStartupFailureReason" + }, + { + "type": "null" + } + ] + }, "name": { "type": "string" }, diff --git a/codex-rs/app-server-protocol/schema/json/v2/PluginInstalledResponse.json b/codex-rs/app-server-protocol/schema/json/v2/PluginInstalledResponse.json index ce4ae5aa0b..55ad95be2e 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/PluginInstalledResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/PluginInstalledResponse.json @@ -413,6 +413,40 @@ "title": "GitPluginSource", "type": "object" }, + { + "properties": { + "package": { + "type": "string" + }, + "registry": { + "description": "Optional HTTPS registry URL. Authentication stays in the user's npm config.", + "type": [ + "string", + "null" + ] + }, + "type": { + "enum": [ + "npm" + ], + "title": "NpmPluginSourceType", + "type": "string" + }, + "version": { + "description": "Optional npm version or version range.", + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "package", + "type" + ], + "title": "NpmPluginSource", + "type": "object" + }, { "description": "The plugin is available in the remote catalog. Download metadata is kept server-side and is not exposed through the app-server API.", "properties": { @@ -506,6 +540,14 @@ }, "source": { "$ref": "#/definitions/PluginSource" + }, + "version": { + "default": null, + "description": "Version advertised by the remote marketplace backend when available.", + "type": [ + "string", + "null" + ] } }, "required": [ diff --git a/codex-rs/app-server-protocol/schema/json/v2/PluginListResponse.json b/codex-rs/app-server-protocol/schema/json/v2/PluginListResponse.json index 1002ad1a8d..25d12e50dd 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/PluginListResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/PluginListResponse.json @@ -413,6 +413,40 @@ "title": "GitPluginSource", "type": "object" }, + { + "properties": { + "package": { + "type": "string" + }, + "registry": { + "description": "Optional HTTPS registry URL. Authentication stays in the user's npm config.", + "type": [ + "string", + "null" + ] + }, + "type": { + "enum": [ + "npm" + ], + "title": "NpmPluginSourceType", + "type": "string" + }, + "version": { + "description": "Optional npm version or version range.", + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "package", + "type" + ], + "title": "NpmPluginSource", + "type": "object" + }, { "description": "The plugin is available in the remote catalog. Download metadata is kept server-side and is not exposed through the app-server API.", "properties": { @@ -506,6 +540,14 @@ }, "source": { "$ref": "#/definitions/PluginSource" + }, + "version": { + "default": null, + "description": "Version advertised by the remote marketplace backend when available.", + "type": [ + "string", + "null" + ] } }, "required": [ diff --git a/codex-rs/app-server-protocol/schema/json/v2/PluginReadResponse.json b/codex-rs/app-server-protocol/schema/json/v2/PluginReadResponse.json index a2d2d442c5..b4b917602a 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/PluginReadResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/PluginReadResponse.json @@ -553,6 +553,40 @@ "title": "GitPluginSource", "type": "object" }, + { + "properties": { + "package": { + "type": "string" + }, + "registry": { + "description": "Optional HTTPS registry URL. Authentication stays in the user's npm config.", + "type": [ + "string", + "null" + ] + }, + "type": { + "enum": [ + "npm" + ], + "title": "NpmPluginSourceType", + "type": "string" + }, + "version": { + "description": "Optional npm version or version range.", + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "package", + "type" + ], + "title": "NpmPluginSource", + "type": "object" + }, { "description": "The plugin is available in the remote catalog. Download metadata is kept server-side and is not exposed through the app-server API.", "properties": { @@ -646,6 +680,14 @@ }, "source": { "$ref": "#/definitions/PluginSource" + }, + "version": { + "default": null, + "description": "Version advertised by the remote marketplace backend when available.", + "type": [ + "string", + "null" + ] } }, "required": [ diff --git a/codex-rs/app-server-protocol/schema/json/v2/PluginShareListResponse.json b/codex-rs/app-server-protocol/schema/json/v2/PluginShareListResponse.json index be97e20b65..751a6c5e7f 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/PluginShareListResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/PluginShareListResponse.json @@ -369,6 +369,40 @@ "title": "GitPluginSource", "type": "object" }, + { + "properties": { + "package": { + "type": "string" + }, + "registry": { + "description": "Optional HTTPS registry URL. Authentication stays in the user's npm config.", + "type": [ + "string", + "null" + ] + }, + "type": { + "enum": [ + "npm" + ], + "title": "NpmPluginSourceType", + "type": "string" + }, + "version": { + "description": "Optional npm version or version range.", + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "package", + "type" + ], + "title": "NpmPluginSource", + "type": "object" + }, { "description": "The plugin is available in the remote catalog. Download metadata is kept server-side and is not exposed through the app-server API.", "properties": { @@ -462,6 +496,14 @@ }, "source": { "$ref": "#/definitions/PluginSource" + }, + "version": { + "default": null, + "description": "Version advertised by the remote marketplace backend when available.", + "type": [ + "string", + "null" + ] } }, "required": [ diff --git a/codex-rs/app-server-protocol/schema/json/v2/RawResponseItemCompletedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/RawResponseItemCompletedNotification.json index 01ac63c268..dad669b139 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/RawResponseItemCompletedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/RawResponseItemCompletedNotification.json @@ -751,6 +751,12 @@ "name": { "type": "string" }, + "namespace": { + "type": [ + "string", + "null" + ] + }, "status": { "type": [ "string", diff --git a/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json index 16da9f205a..6851ddb487 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json @@ -437,6 +437,18 @@ }, "McpToolCallAppContext": { "properties": { + "actionName": { + "type": [ + "string", + "null" + ] + }, + "appName": { + "type": [ + "string", + "null" + ] + }, "connectorId": { "type": "string" }, @@ -451,6 +463,12 @@ "string", "null" ] + }, + "templateId": { + "type": [ + "string", + "null" + ] } }, "required": [ diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadForkParams.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadForkParams.json index ad0174cfe0..76278b106b 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadForkParams.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadForkParams.json @@ -126,6 +126,13 @@ "ephemeral": { "type": "boolean" }, + "lastTurnId": { + "description": "Optional last turn id to fork through, inclusive.\n\nWhen specified, turns after `last_turn_id` are omitted from the fork. The referenced turn cannot be in progress.", + "type": [ + "string", + "null" + ] + }, "model": { "description": "Configuration overrides for the forked thread, if any.", "type": [ diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json index 37a78533c2..59a14cc070 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json @@ -541,6 +541,18 @@ }, "McpToolCallAppContext": { "properties": { + "actionName": { + "type": [ + "string", + "null" + ] + }, + "appName": { + "type": [ + "string", + "null" + ] + }, "connectorId": { "type": "string" }, @@ -555,6 +567,12 @@ "string", "null" ] + }, + "templateId": { + "type": [ + "string", + "null" + ] } }, "required": [ @@ -663,13 +681,29 @@ ] }, "MultiAgentMode": { - "description": "Controls whether the model receives multi-agent delegation instructions and, when it does, whether it should only spawn sub-agents after an explicit user request or may delegate proactively when doing so would help. `none` leaves the multi-agent tools available without injecting delegation instructions.", - "enum": [ - "none", - "explicitRequestOnly", - "proactive" - ], - "type": "string" + "description": "Controls the effective multi-agent delegation instructions for a turn. `custom` means the configured mode hint defines the policy instead of a built-in policy.", + "oneOf": [ + { + "enum": [ + "explicitRequestOnly", + "proactive" + ], + "type": "string" + }, + { + "additionalProperties": false, + "properties": { + "custom": { + "type": "string" + } + }, + "required": [ + "custom" + ], + "title": "CustomMultiAgentMode", + "type": "object" + } + ] }, "NetworkAccess": { "enum": [ @@ -1170,6 +1204,13 @@ "description": "Extra app-server data for a thread.", "type": "object" }, + "ThreadHistoryMode": { + "enum": [ + "legacy", + "paginated" + ], + "type": "string" + }, "ThreadId": { "type": "string" }, diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json index 67726c13a6..e5cbc2e606 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json @@ -463,6 +463,18 @@ }, "McpToolCallAppContext": { "properties": { + "actionName": { + "type": [ + "string", + "null" + ] + }, + "appName": { + "type": [ + "string", + "null" + ] + }, "connectorId": { "type": "string" }, @@ -477,6 +489,12 @@ "string", "null" ] + }, + "templateId": { + "type": [ + "string", + "null" + ] } }, "required": [ @@ -977,6 +995,13 @@ "description": "Extra app-server data for a thread.", "type": "object" }, + "ThreadHistoryMode": { + "enum": [ + "legacy", + "paginated" + ], + "type": "string" + }, "ThreadId": { "type": "string" }, diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json index cdd74a735a..cdd1d47a69 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json @@ -463,6 +463,18 @@ }, "McpToolCallAppContext": { "properties": { + "actionName": { + "type": [ + "string", + "null" + ] + }, + "appName": { + "type": [ + "string", + "null" + ] + }, "connectorId": { "type": "string" }, @@ -477,6 +489,12 @@ "string", "null" ] + }, + "templateId": { + "type": [ + "string", + "null" + ] } }, "required": [ @@ -977,6 +995,13 @@ "description": "Extra app-server data for a thread.", "type": "object" }, + "ThreadHistoryMode": { + "enum": [ + "legacy", + "paginated" + ], + "type": "string" + }, "ThreadId": { "type": "string" }, diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json index c3e388d1fd..613dd9648e 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json @@ -463,6 +463,18 @@ }, "McpToolCallAppContext": { "properties": { + "actionName": { + "type": [ + "string", + "null" + ] + }, + "appName": { + "type": [ + "string", + "null" + ] + }, "connectorId": { "type": "string" }, @@ -477,6 +489,12 @@ "string", "null" ] + }, + "templateId": { + "type": [ + "string", + "null" + ] } }, "required": [ @@ -977,6 +995,13 @@ "description": "Extra app-server data for a thread.", "type": "object" }, + "ThreadHistoryMode": { + "enum": [ + "legacy", + "paginated" + ], + "type": "string" + }, "ThreadId": { "type": "string" }, diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeParams.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeParams.json index af078af68e..0cd7d9ae7b 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeParams.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeParams.json @@ -821,6 +821,12 @@ "name": { "type": "string" }, + "namespace": { + "type": [ + "string", + "null" + ] + }, "status": { "type": [ "string", diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json index ad44bf406b..80fbb183f8 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json @@ -541,6 +541,18 @@ }, "McpToolCallAppContext": { "properties": { + "actionName": { + "type": [ + "string", + "null" + ] + }, + "appName": { + "type": [ + "string", + "null" + ] + }, "connectorId": { "type": "string" }, @@ -555,6 +567,12 @@ "string", "null" ] + }, + "templateId": { + "type": [ + "string", + "null" + ] } }, "required": [ @@ -663,13 +681,29 @@ ] }, "MultiAgentMode": { - "description": "Controls whether the model receives multi-agent delegation instructions and, when it does, whether it should only spawn sub-agents after an explicit user request or may delegate proactively when doing so would help. `none` leaves the multi-agent tools available without injecting delegation instructions.", - "enum": [ - "none", - "explicitRequestOnly", - "proactive" - ], - "type": "string" + "description": "Controls the effective multi-agent delegation instructions for a turn. `custom` means the configured mode hint defines the policy instead of a built-in policy.", + "oneOf": [ + { + "enum": [ + "explicitRequestOnly", + "proactive" + ], + "type": "string" + }, + { + "additionalProperties": false, + "properties": { + "custom": { + "type": "string" + } + }, + "required": [ + "custom" + ], + "title": "CustomMultiAgentMode", + "type": "object" + } + ] }, "NetworkAccess": { "enum": [ @@ -1170,6 +1204,13 @@ "description": "Extra app-server data for a thread.", "type": "object" }, + "ThreadHistoryMode": { + "enum": [ + "legacy", + "paginated" + ], + "type": "string" + }, "ThreadId": { "type": "string" }, diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadRollbackParams.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadRollbackParams.json index cb3ba0db39..aa52fbd598 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadRollbackParams.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadRollbackParams.json @@ -1,5 +1,6 @@ { "$schema": "http://json-schema.org/draft-07/schema#", + "description": "DEPRECATED: `thread/rollback` will be removed soon.", "properties": { "numTurns": { "description": "The number of turns to drop from the end of the thread. Must be >= 1.\n\nThis only modifies the thread's history and does not revert local file changes that have been made by the agent. Clients are responsible for reverting these changes.", diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadRollbackResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadRollbackResponse.json index 5ae94d8d11..793f11c226 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadRollbackResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadRollbackResponse.json @@ -463,6 +463,18 @@ }, "McpToolCallAppContext": { "properties": { + "actionName": { + "type": [ + "string", + "null" + ] + }, + "appName": { + "type": [ + "string", + "null" + ] + }, "connectorId": { "type": "string" }, @@ -477,6 +489,12 @@ "string", "null" ] + }, + "templateId": { + "type": [ + "string", + "null" + ] } }, "required": [ @@ -977,6 +995,13 @@ "description": "Extra app-server data for a thread.", "type": "object" }, + "ThreadHistoryMode": { + "enum": [ + "legacy", + "paginated" + ], + "type": "string" + }, "ThreadId": { "type": "string" }, diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadSettingsUpdatedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadSettingsUpdatedNotification.json index 8d528246b5..f3296fa376 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadSettingsUpdatedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadSettingsUpdatedNotification.json @@ -108,13 +108,29 @@ "type": "string" }, "MultiAgentMode": { - "description": "Controls whether the model receives multi-agent delegation instructions and, when it does, whether it should only spawn sub-agents after an explicit user request or may delegate proactively when doing so would help. `none` leaves the multi-agent tools available without injecting delegation instructions.", - "enum": [ - "none", - "explicitRequestOnly", - "proactive" - ], - "type": "string" + "description": "Controls the effective multi-agent delegation instructions for a turn. `custom` means the configured mode hint defines the policy instead of a built-in policy.", + "oneOf": [ + { + "enum": [ + "explicitRequestOnly", + "proactive" + ], + "type": "string" + }, + { + "additionalProperties": false, + "properties": { + "custom": { + "type": "string" + } + }, + "required": [ + "custom" + ], + "title": "CustomMultiAgentMode", + "type": "object" + } + ] }, "NetworkAccess": { "enum": [ diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartParams.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartParams.json index 50c0a9cf7f..3819f1f761 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartParams.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartParams.json @@ -195,13 +195,29 @@ "type": "string" }, "MultiAgentMode": { - "description": "Controls whether the model receives multi-agent delegation instructions and, when it does, whether it should only spawn sub-agents after an explicit user request or may delegate proactively when doing so would help. `none` leaves the multi-agent tools available without injecting delegation instructions.", - "enum": [ - "none", - "explicitRequestOnly", - "proactive" - ], - "type": "string" + "description": "Controls the effective multi-agent delegation instructions for a turn. `custom` means the configured mode hint defines the policy instead of a built-in policy.", + "oneOf": [ + { + "enum": [ + "explicitRequestOnly", + "proactive" + ], + "type": "string" + }, + { + "additionalProperties": false, + "properties": { + "custom": { + "type": "string" + } + }, + "required": [ + "custom" + ], + "title": "CustomMultiAgentMode", + "type": "object" + } + ] }, "Personality": { "enum": [ @@ -241,6 +257,13 @@ ], "type": "object" }, + "ThreadHistoryMode": { + "enum": [ + "legacy", + "paginated" + ], + "type": "string" + }, "ThreadSource": { "type": "string" }, diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json index df70c08c43..3ef1577038 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json @@ -541,6 +541,18 @@ }, "McpToolCallAppContext": { "properties": { + "actionName": { + "type": [ + "string", + "null" + ] + }, + "appName": { + "type": [ + "string", + "null" + ] + }, "connectorId": { "type": "string" }, @@ -555,6 +567,12 @@ "string", "null" ] + }, + "templateId": { + "type": [ + "string", + "null" + ] } }, "required": [ @@ -663,13 +681,29 @@ ] }, "MultiAgentMode": { - "description": "Controls whether the model receives multi-agent delegation instructions and, when it does, whether it should only spawn sub-agents after an explicit user request or may delegate proactively when doing so would help. `none` leaves the multi-agent tools available without injecting delegation instructions.", - "enum": [ - "none", - "explicitRequestOnly", - "proactive" - ], - "type": "string" + "description": "Controls the effective multi-agent delegation instructions for a turn. `custom` means the configured mode hint defines the policy instead of a built-in policy.", + "oneOf": [ + { + "enum": [ + "explicitRequestOnly", + "proactive" + ], + "type": "string" + }, + { + "additionalProperties": false, + "properties": { + "custom": { + "type": "string" + } + }, + "required": [ + "custom" + ], + "title": "CustomMultiAgentMode", + "type": "object" + } + ] }, "NetworkAccess": { "enum": [ @@ -1170,6 +1204,13 @@ "description": "Extra app-server data for a thread.", "type": "object" }, + "ThreadHistoryMode": { + "enum": [ + "legacy", + "paginated" + ], + "type": "string" + }, "ThreadId": { "type": "string" }, diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json index 157f6193b4..a56d59c01d 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json @@ -463,6 +463,18 @@ }, "McpToolCallAppContext": { "properties": { + "actionName": { + "type": [ + "string", + "null" + ] + }, + "appName": { + "type": [ + "string", + "null" + ] + }, "connectorId": { "type": "string" }, @@ -477,6 +489,12 @@ "string", "null" ] + }, + "templateId": { + "type": [ + "string", + "null" + ] } }, "required": [ @@ -977,6 +995,13 @@ "description": "Extra app-server data for a thread.", "type": "object" }, + "ThreadHistoryMode": { + "enum": [ + "legacy", + "paginated" + ], + "type": "string" + }, "ThreadId": { "type": "string" }, diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json index 265578dcdd..6cc9b44cba 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json @@ -463,6 +463,18 @@ }, "McpToolCallAppContext": { "properties": { + "actionName": { + "type": [ + "string", + "null" + ] + }, + "appName": { + "type": [ + "string", + "null" + ] + }, "connectorId": { "type": "string" }, @@ -477,6 +489,12 @@ "string", "null" ] + }, + "templateId": { + "type": [ + "string", + "null" + ] } }, "required": [ @@ -977,6 +995,13 @@ "description": "Extra app-server data for a thread.", "type": "object" }, + "ThreadHistoryMode": { + "enum": [ + "legacy", + "paginated" + ], + "type": "string" + }, "ThreadId": { "type": "string" }, diff --git a/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json index ae1b07cfc9..6ffb343595 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json @@ -437,6 +437,18 @@ }, "McpToolCallAppContext": { "properties": { + "actionName": { + "type": [ + "string", + "null" + ] + }, + "appName": { + "type": [ + "string", + "null" + ] + }, "connectorId": { "type": "string" }, @@ -451,6 +463,12 @@ "string", "null" ] + }, + "templateId": { + "type": [ + "string", + "null" + ] } }, "required": [ diff --git a/codex-rs/app-server-protocol/schema/json/v2/TurnStartParams.json b/codex-rs/app-server-protocol/schema/json/v2/TurnStartParams.json index 36b4d2a15d..cc4869e1d1 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/TurnStartParams.json +++ b/codex-rs/app-server-protocol/schema/json/v2/TurnStartParams.json @@ -141,13 +141,29 @@ "type": "string" }, "MultiAgentMode": { - "description": "Controls whether the model receives multi-agent delegation instructions and, when it does, whether it should only spawn sub-agents after an explicit user request or may delegate proactively when doing so would help. `none` leaves the multi-agent tools available without injecting delegation instructions.", - "enum": [ - "none", - "explicitRequestOnly", - "proactive" - ], - "type": "string" + "description": "Controls the effective multi-agent delegation instructions for a turn. `custom` means the configured mode hint defines the policy instead of a built-in policy.", + "oneOf": [ + { + "enum": [ + "explicitRequestOnly", + "proactive" + ], + "type": "string" + }, + { + "additionalProperties": false, + "properties": { + "custom": { + "type": "string" + } + }, + "required": [ + "custom" + ], + "title": "CustomMultiAgentMode", + "type": "object" + } + ] }, "NetworkAccess": { "enum": [ diff --git a/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json b/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json index 5853612dd9..805a5099a2 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json @@ -437,6 +437,18 @@ }, "McpToolCallAppContext": { "properties": { + "actionName": { + "type": [ + "string", + "null" + ] + }, + "appName": { + "type": [ + "string", + "null" + ] + }, "connectorId": { "type": "string" }, @@ -451,6 +463,12 @@ "string", "null" ] + }, + "templateId": { + "type": [ + "string", + "null" + ] } }, "required": [ diff --git a/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json index 7db58289ee..1f3f7c7036 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json @@ -437,6 +437,18 @@ }, "McpToolCallAppContext": { "properties": { + "actionName": { + "type": [ + "string", + "null" + ] + }, + "appName": { + "type": [ + "string", + "null" + ] + }, "connectorId": { "type": "string" }, @@ -451,6 +463,12 @@ "string", "null" ] + }, + "templateId": { + "type": [ + "string", + "null" + ] } }, "required": [ diff --git a/codex-rs/app-server-protocol/schema/typescript/MultiAgentMode.ts b/codex-rs/app-server-protocol/schema/typescript/MultiAgentMode.ts index b59be94f8e..7784a6f5ca 100644 --- a/codex-rs/app-server-protocol/schema/typescript/MultiAgentMode.ts +++ b/codex-rs/app-server-protocol/schema/typescript/MultiAgentMode.ts @@ -3,9 +3,7 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. /** - * Controls whether the model receives multi-agent delegation instructions and, - * when it does, whether it should only spawn sub-agents after an explicit user - * request or may delegate proactively when doing so would help. `none` leaves - * the multi-agent tools available without injecting delegation instructions. + * Controls the effective multi-agent delegation instructions for a turn. `custom` means the + * configured mode hint defines the policy instead of a built-in policy. */ -export type MultiAgentMode = "none" | "explicitRequestOnly" | "proactive"; +export type MultiAgentMode = { "custom": string } | "explicitRequestOnly" | "proactive"; diff --git a/codex-rs/app-server-protocol/schema/typescript/ResponseItem.ts b/codex-rs/app-server-protocol/schema/typescript/ResponseItem.ts index 62547d3e40..769b1b08fc 100644 --- a/codex-rs/app-server-protocol/schema/typescript/ResponseItem.ts +++ b/codex-rs/app-server-protocol/schema/typescript/ResponseItem.ts @@ -20,4 +20,4 @@ id?: string, /** * Set when using the Responses API. */ -call_id: string | null, status: LocalShellStatus, action: LocalShellAction, internal_chat_message_metadata_passthrough?: InternalChatMessageMetadataPassthrough, } | { "type": "function_call", id?: string, name: string, namespace?: string, arguments: string, call_id: string, internal_chat_message_metadata_passthrough?: InternalChatMessageMetadataPassthrough, } | { "type": "tool_search_call", id?: string, call_id: string | null, status?: string, execution: string, arguments: unknown, internal_chat_message_metadata_passthrough?: InternalChatMessageMetadataPassthrough, } | { "type": "function_call_output", id?: string, call_id: string, output: FunctionCallOutputBody, internal_chat_message_metadata_passthrough?: InternalChatMessageMetadataPassthrough, } | { "type": "custom_tool_call", id?: string, status?: string, call_id: string, name: string, input: string, internal_chat_message_metadata_passthrough?: InternalChatMessageMetadataPassthrough, } | { "type": "custom_tool_call_output", id?: string, call_id: string, name?: string, output: FunctionCallOutputBody, internal_chat_message_metadata_passthrough?: InternalChatMessageMetadataPassthrough, } | { "type": "tool_search_output", id?: string, call_id: string | null, status: string, execution: string, tools: unknown[], internal_chat_message_metadata_passthrough?: InternalChatMessageMetadataPassthrough, } | { "type": "web_search_call", id?: string, status?: string, action?: WebSearchAction, internal_chat_message_metadata_passthrough?: InternalChatMessageMetadataPassthrough, } | { "type": "image_generation_call", id?: string, status: string, revised_prompt?: string, result: string, internal_chat_message_metadata_passthrough?: InternalChatMessageMetadataPassthrough, } | { "type": "compaction", id?: string, encrypted_content: string, internal_chat_message_metadata_passthrough?: InternalChatMessageMetadataPassthrough, } | { "type": "compaction_trigger", } | { "type": "context_compaction", id?: string, encrypted_content?: string, internal_chat_message_metadata_passthrough?: InternalChatMessageMetadataPassthrough, } | { "type": "other" }; +call_id: string | null, status: LocalShellStatus, action: LocalShellAction, internal_chat_message_metadata_passthrough?: InternalChatMessageMetadataPassthrough, } | { "type": "function_call", id?: string, name: string, namespace?: string, arguments: string, call_id: string, internal_chat_message_metadata_passthrough?: InternalChatMessageMetadataPassthrough, } | { "type": "tool_search_call", id?: string, call_id: string | null, status?: string, execution: string, arguments: unknown, internal_chat_message_metadata_passthrough?: InternalChatMessageMetadataPassthrough, } | { "type": "function_call_output", id?: string, call_id: string, output: FunctionCallOutputBody, internal_chat_message_metadata_passthrough?: InternalChatMessageMetadataPassthrough, } | { "type": "custom_tool_call", id?: string, status?: string, call_id: string, name: string, namespace?: string, input: string, internal_chat_message_metadata_passthrough?: InternalChatMessageMetadataPassthrough, } | { "type": "custom_tool_call_output", id?: string, call_id: string, name?: string, output: FunctionCallOutputBody, internal_chat_message_metadata_passthrough?: InternalChatMessageMetadataPassthrough, } | { "type": "tool_search_output", id?: string, call_id: string | null, status: string, execution: string, tools: unknown[], internal_chat_message_metadata_passthrough?: InternalChatMessageMetadataPassthrough, } | { "type": "web_search_call", id?: string, status?: string, action?: WebSearchAction, internal_chat_message_metadata_passthrough?: InternalChatMessageMetadataPassthrough, } | { "type": "image_generation_call", id?: string, status: string, revised_prompt?: string, result: string, internal_chat_message_metadata_passthrough?: InternalChatMessageMetadataPassthrough, } | { "type": "compaction", id?: string, encrypted_content: string, internal_chat_message_metadata_passthrough?: InternalChatMessageMetadataPassthrough, } | { "type": "compaction_trigger", } | { "type": "context_compaction", id?: string, encrypted_content?: string, internal_chat_message_metadata_passthrough?: InternalChatMessageMetadataPassthrough, } | { "type": "other" }; diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/ConfigRequirements.ts b/codex-rs/app-server-protocol/schema/typescript/v2/ConfigRequirements.ts index 3eb9a65aa6..e2fd573a32 100644 --- a/codex-rs/app-server-protocol/schema/typescript/v2/ConfigRequirements.ts +++ b/codex-rs/app-server-protocol/schema/typescript/v2/ConfigRequirements.ts @@ -4,8 +4,9 @@ import type { WebSearchMode } from "../WebSearchMode"; import type { AskForApproval } from "./AskForApproval"; import type { ComputerUseRequirements } from "./ComputerUseRequirements"; +import type { ModelsRequirements } from "./ModelsRequirements"; import type { ResidencyRequirement } from "./ResidencyRequirement"; import type { SandboxMode } from "./SandboxMode"; import type { WindowsSandboxSetupMode } from "./WindowsSandboxSetupMode"; -export type ConfigRequirements = {allowedApprovalPolicies: Array | null, allowedSandboxModes: Array | null, allowedWindowsSandboxImplementations: Array | null, allowedPermissionProfiles: { [key in string]?: boolean } | null, defaultPermissions: string | null, allowedWebSearchModes: Array | null, allowManagedHooksOnly: boolean | null, allowAppshots: boolean | null, allowRemoteControl: boolean | null, computerUse: ComputerUseRequirements | null, featureRequirements: { [key in string]?: boolean } | null, enforceResidency: ResidencyRequirement | null}; +export type ConfigRequirements = {allowedApprovalPolicies: Array | null, allowedSandboxModes: Array | null, allowedWindowsSandboxImplementations: Array | null, allowedPermissionProfiles: { [key in string]?: boolean } | null, defaultPermissions: string | null, allowedWebSearchModes: Array | null, allowManagedHooksOnly: boolean | null, allowAppshots: boolean | null, allowRemoteControl: boolean | null, computerUse: ComputerUseRequirements | null, featureRequirements: { [key in string]?: boolean } | null, enforceResidency: ResidencyRequirement | null, models: ModelsRequirements | null}; diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/ConsumeAccountRateLimitResetCreditParams.ts b/codex-rs/app-server-protocol/schema/typescript/v2/ConsumeAccountRateLimitResetCreditParams.ts index c3cef64f36..f1c5bf351e 100644 --- a/codex-rs/app-server-protocol/schema/typescript/v2/ConsumeAccountRateLimitResetCreditParams.ts +++ b/codex-rs/app-server-protocol/schema/typescript/v2/ConsumeAccountRateLimitResetCreditParams.ts @@ -7,4 +7,9 @@ export type ConsumeAccountRateLimitResetCreditParams = { * Identifies one logical reset attempt. A UUID is recommended; reuse the same value when * retrying that attempt. */ -idempotencyKey: string, }; +idempotencyKey: string, +/** + * Opaque reset-credit identifier to redeem. When omitted, the backend selects the next + * available credit. + */ +creditId?: string | null, }; diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/McpServerOauthLoginCompletedNotification.ts b/codex-rs/app-server-protocol/schema/typescript/v2/McpServerOauthLoginCompletedNotification.ts index 592860ae39..cfa6603094 100644 --- a/codex-rs/app-server-protocol/schema/typescript/v2/McpServerOauthLoginCompletedNotification.ts +++ b/codex-rs/app-server-protocol/schema/typescript/v2/McpServerOauthLoginCompletedNotification.ts @@ -2,4 +2,4 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. -export type McpServerOauthLoginCompletedNotification = { name: string, success: boolean, error?: string, }; +export type McpServerOauthLoginCompletedNotification = { name: string, threadId: string | null, success: boolean, error?: string, }; diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/McpServerOauthLoginParams.ts b/codex-rs/app-server-protocol/schema/typescript/v2/McpServerOauthLoginParams.ts index a61c304609..ff088b8d9f 100644 --- a/codex-rs/app-server-protocol/schema/typescript/v2/McpServerOauthLoginParams.ts +++ b/codex-rs/app-server-protocol/schema/typescript/v2/McpServerOauthLoginParams.ts @@ -2,4 +2,4 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. -export type McpServerOauthLoginParams = { name: string, scopes?: Array | null, timeoutSecs?: bigint | null, }; +export type McpServerOauthLoginParams = { name: string, threadId?: string | null, scopes?: Array | null, timeoutSecs?: bigint | null, }; diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/McpServerStartupFailureReason.ts b/codex-rs/app-server-protocol/schema/typescript/v2/McpServerStartupFailureReason.ts new file mode 100644 index 0000000000..0373e5444b --- /dev/null +++ b/codex-rs/app-server-protocol/schema/typescript/v2/McpServerStartupFailureReason.ts @@ -0,0 +1,5 @@ +// GENERATED CODE! DO NOT MODIFY BY HAND! + +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +export type McpServerStartupFailureReason = "reauthenticationRequired"; diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/McpServerStatusUpdatedNotification.ts b/codex-rs/app-server-protocol/schema/typescript/v2/McpServerStatusUpdatedNotification.ts index b5c0a9058c..fd192f22c6 100644 --- a/codex-rs/app-server-protocol/schema/typescript/v2/McpServerStatusUpdatedNotification.ts +++ b/codex-rs/app-server-protocol/schema/typescript/v2/McpServerStatusUpdatedNotification.ts @@ -1,6 +1,7 @@ // GENERATED CODE! DO NOT MODIFY BY HAND! // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { McpServerStartupFailureReason } from "./McpServerStartupFailureReason"; import type { McpServerStartupState } from "./McpServerStartupState"; -export type McpServerStatusUpdatedNotification = { threadId: string | null, name: string, status: McpServerStartupState, error: string | null, }; +export type McpServerStatusUpdatedNotification = { threadId: string | null, name: string, status: McpServerStartupState, error: string | null, failureReason: McpServerStartupFailureReason | null, }; diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/McpToolCallAppContext.ts b/codex-rs/app-server-protocol/schema/typescript/v2/McpToolCallAppContext.ts index 0a8343428d..e4bc5a11d9 100644 --- a/codex-rs/app-server-protocol/schema/typescript/v2/McpToolCallAppContext.ts +++ b/codex-rs/app-server-protocol/schema/typescript/v2/McpToolCallAppContext.ts @@ -2,4 +2,4 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. -export type McpToolCallAppContext = { connectorId: string, linkId: string | null, resourceUri: string | null, }; +export type McpToolCallAppContext = { connectorId: string, linkId: string | null, resourceUri: string | null, appName: string | null, templateId: string | null, actionName: string | null, }; diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/ModelsRequirements.ts b/codex-rs/app-server-protocol/schema/typescript/v2/ModelsRequirements.ts new file mode 100644 index 0000000000..9041fff834 --- /dev/null +++ b/codex-rs/app-server-protocol/schema/typescript/v2/ModelsRequirements.ts @@ -0,0 +1,6 @@ +// GENERATED CODE! DO NOT MODIFY BY HAND! + +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { NewThreadModelDefaults } from "./NewThreadModelDefaults"; + +export type ModelsRequirements = { newThread: NewThreadModelDefaults | null, }; diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/NewThreadModelDefaults.ts b/codex-rs/app-server-protocol/schema/typescript/v2/NewThreadModelDefaults.ts new file mode 100644 index 0000000000..fed8b25eaa --- /dev/null +++ b/codex-rs/app-server-protocol/schema/typescript/v2/NewThreadModelDefaults.ts @@ -0,0 +1,6 @@ +// GENERATED CODE! DO NOT MODIFY BY HAND! + +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { ReasoningEffort } from "../ReasoningEffort"; + +export type NewThreadModelDefaults = { model: string | null, modelReasoningEffort: ReasoningEffort | null, serviceTier: string | null, }; diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/PluginSource.ts b/codex-rs/app-server-protocol/schema/typescript/v2/PluginSource.ts index f6e867195d..c7ba3bcf1c 100644 --- a/codex-rs/app-server-protocol/schema/typescript/v2/PluginSource.ts +++ b/codex-rs/app-server-protocol/schema/typescript/v2/PluginSource.ts @@ -3,4 +3,12 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. import type { AbsolutePathBuf } from "../AbsolutePathBuf"; -export type PluginSource = { "type": "local", path: AbsolutePathBuf, } | { "type": "git", url: string, path: string | null, refName: string | null, sha: string | null, } | { "type": "remote" }; +export type PluginSource = { "type": "local", path: AbsolutePathBuf, } | { "type": "git", url: string, path: string | null, refName: string | null, sha: string | null, } | { "type": "npm", package: string, +/** + * Optional npm version or version range. + */ +version: string | null, +/** + * Optional HTTPS registry URL. Authentication stays in the user's npm config. + */ +registry: string | null, } | { "type": "remote" }; diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/PluginSummary.ts b/codex-rs/app-server-protocol/schema/typescript/v2/PluginSummary.ts index 268349cb9b..d12f55b3a3 100644 --- a/codex-rs/app-server-protocol/schema/typescript/v2/PluginSummary.ts +++ b/codex-rs/app-server-protocol/schema/typescript/v2/PluginSummary.ts @@ -13,6 +13,10 @@ export type PluginSummary = { id: string, * Backend remote plugin identifier when available. */ remotePluginId: string | null, +/** + * Version advertised by the remote marketplace backend when available. + */ +version: string | null, /** * Version of the locally materialized plugin package when available. */ diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/RateLimitResetCredit.ts b/codex-rs/app-server-protocol/schema/typescript/v2/RateLimitResetCredit.ts new file mode 100644 index 0000000000..514c1558a2 --- /dev/null +++ b/codex-rs/app-server-protocol/schema/typescript/v2/RateLimitResetCredit.ts @@ -0,0 +1,27 @@ +// GENERATED CODE! DO NOT MODIFY BY HAND! + +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { RateLimitResetCreditStatus } from "./RateLimitResetCreditStatus"; +import type { RateLimitResetType } from "./RateLimitResetType"; + +export type RateLimitResetCredit = { +/** + * Opaque backend identifier for this reset credit. + */ +id: string, resetType: RateLimitResetType, status: RateLimitResetCreditStatus, +/** + * Unix timestamp in seconds when the credit was granted. + */ +grantedAt: number, +/** + * Unix timestamp in seconds when the credit expires, or `null` if it does not expire. + */ +expiresAt: number | null, +/** + * Backend-provided display title for this credit, or `null` when unavailable. + */ +title: string | null, +/** + * Backend-provided display description for this credit, or `null` when unavailable. + */ +description: string | null, }; diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/RateLimitResetCreditStatus.ts b/codex-rs/app-server-protocol/schema/typescript/v2/RateLimitResetCreditStatus.ts new file mode 100644 index 0000000000..fa15861b3a --- /dev/null +++ b/codex-rs/app-server-protocol/schema/typescript/v2/RateLimitResetCreditStatus.ts @@ -0,0 +1,5 @@ +// GENERATED CODE! DO NOT MODIFY BY HAND! + +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +export type RateLimitResetCreditStatus = "available" | "redeeming" | "redeemed" | "unknown"; diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/RateLimitResetCreditsSummary.ts b/codex-rs/app-server-protocol/schema/typescript/v2/RateLimitResetCreditsSummary.ts index e42dd8a703..46a8eee2e7 100644 --- a/codex-rs/app-server-protocol/schema/typescript/v2/RateLimitResetCreditsSummary.ts +++ b/codex-rs/app-server-protocol/schema/typescript/v2/RateLimitResetCreditsSummary.ts @@ -1,5 +1,14 @@ // GENERATED CODE! DO NOT MODIFY BY HAND! // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { RateLimitResetCredit } from "./RateLimitResetCredit"; -export type RateLimitResetCreditsSummary = { availableCount: bigint, }; +export type RateLimitResetCreditsSummary = { availableCount: bigint, +/** + * Detail rows for available reset credits, when the backend provides them. + * + * `null` means only `availableCount` is known, while an empty array means details were fetched + * and no available credits were returned. The backend may cap this list, so its length can be + * less than `availableCount`. + */ +credits: Array | null, }; diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/RateLimitResetType.ts b/codex-rs/app-server-protocol/schema/typescript/v2/RateLimitResetType.ts new file mode 100644 index 0000000000..718145bf47 --- /dev/null +++ b/codex-rs/app-server-protocol/schema/typescript/v2/RateLimitResetType.ts @@ -0,0 +1,5 @@ +// GENERATED CODE! DO NOT MODIFY BY HAND! + +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +export type RateLimitResetType = "codexRateLimits" | "unknown"; diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/ThreadForkParams.ts b/codex-rs/app-server-protocol/schema/typescript/v2/ThreadForkParams.ts index c5109b2c7d..3ace4d4417 100644 --- a/codex-rs/app-server-protocol/schema/typescript/v2/ThreadForkParams.ts +++ b/codex-rs/app-server-protocol/schema/typescript/v2/ThreadForkParams.ts @@ -18,6 +18,12 @@ import type { ThreadSource } from "./ThreadSource"; * Prefer using thread_id whenever possible. */ export type ThreadForkParams = {threadId: string, /** + * Optional last turn id to fork through, inclusive. + * + * When specified, turns after `last_turn_id` are omitted from the fork. + * The referenced turn cannot be in progress. + */ +lastTurnId?: string | null, /** * Configuration overrides for the forked thread, if any. */ model?: string | null, modelProvider?: string | null, serviceTier?: string | null | null, cwd?: string | null, approvalPolicy?: AskForApproval | null, /** diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/ThreadHistoryMode.ts b/codex-rs/app-server-protocol/schema/typescript/v2/ThreadHistoryMode.ts new file mode 100644 index 0000000000..db0f2d824e --- /dev/null +++ b/codex-rs/app-server-protocol/schema/typescript/v2/ThreadHistoryMode.ts @@ -0,0 +1,5 @@ +// GENERATED CODE! DO NOT MODIFY BY HAND! + +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +export type ThreadHistoryMode = "legacy" | "paginated"; diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/ThreadRollbackParams.ts b/codex-rs/app-server-protocol/schema/typescript/v2/ThreadRollbackParams.ts index 1c938e3bfd..af416d1872 100644 --- a/codex-rs/app-server-protocol/schema/typescript/v2/ThreadRollbackParams.ts +++ b/codex-rs/app-server-protocol/schema/typescript/v2/ThreadRollbackParams.ts @@ -2,6 +2,9 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +/** + * DEPRECATED: `thread/rollback` will be removed soon. + */ export type ThreadRollbackParams = { threadId: string, /** * The number of turns to drop from the end of the thread. Must be >= 1. diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/index.ts b/codex-rs/app-server-protocol/schema/typescript/v2/index.ts index ff7df9e9ec..c335f144ec 100644 --- a/codex-rs/app-server-protocol/schema/typescript/v2/index.ts +++ b/codex-rs/app-server-protocol/schema/typescript/v2/index.ts @@ -238,6 +238,7 @@ export type { McpServerOauthLoginCompletedNotification } from "./McpServerOauthL export type { McpServerOauthLoginParams } from "./McpServerOauthLoginParams"; export type { McpServerOauthLoginResponse } from "./McpServerOauthLoginResponse"; export type { McpServerRefreshResponse } from "./McpServerRefreshResponse"; +export type { McpServerStartupFailureReason } from "./McpServerStartupFailureReason"; export type { McpServerStartupState } from "./McpServerStartupState"; export type { McpServerStatus } from "./McpServerStatus"; export type { McpServerStatusDetail } from "./McpServerStatusDetail"; @@ -266,6 +267,7 @@ export type { ModelServiceTier } from "./ModelServiceTier"; export type { ModelUpgradeInfo } from "./ModelUpgradeInfo"; export type { ModelVerification } from "./ModelVerification"; export type { ModelVerificationNotification } from "./ModelVerificationNotification"; +export type { ModelsRequirements } from "./ModelsRequirements"; export type { NetworkAccess } from "./NetworkAccess"; export type { NetworkApprovalContext } from "./NetworkApprovalContext"; export type { NetworkApprovalProtocol } from "./NetworkApprovalProtocol"; @@ -274,6 +276,7 @@ export type { NetworkPolicyAmendment } from "./NetworkPolicyAmendment"; export type { NetworkPolicyRuleAction } from "./NetworkPolicyRuleAction"; export type { NetworkRequirements } from "./NetworkRequirements"; export type { NetworkUnixSocketPermission } from "./NetworkUnixSocketPermission"; +export type { NewThreadModelDefaults } from "./NewThreadModelDefaults"; export type { NonSteerableTurnKind } from "./NonSteerableTurnKind"; export type { OverriddenMetadata } from "./OverriddenMetadata"; export type { PatchApplyStatus } from "./PatchApplyStatus"; @@ -332,7 +335,10 @@ export type { ProcessOutputDeltaNotification } from "./ProcessOutputDeltaNotific export type { ProcessOutputStream } from "./ProcessOutputStream"; export type { ProcessTerminalSize } from "./ProcessTerminalSize"; export type { RateLimitReachedType } from "./RateLimitReachedType"; +export type { RateLimitResetCredit } from "./RateLimitResetCredit"; +export type { RateLimitResetCreditStatus } from "./RateLimitResetCreditStatus"; export type { RateLimitResetCreditsSummary } from "./RateLimitResetCreditsSummary"; +export type { RateLimitResetType } from "./RateLimitResetType"; export type { RateLimitSnapshot } from "./RateLimitSnapshot"; export type { RateLimitWindow } from "./RateLimitWindow"; export type { RawResponseItemCompletedNotification } from "./RawResponseItemCompletedNotification"; @@ -409,6 +415,7 @@ export type { ThreadGoalSetParams } from "./ThreadGoalSetParams"; export type { ThreadGoalSetResponse } from "./ThreadGoalSetResponse"; export type { ThreadGoalStatus } from "./ThreadGoalStatus"; export type { ThreadGoalUpdatedNotification } from "./ThreadGoalUpdatedNotification"; +export type { ThreadHistoryMode } from "./ThreadHistoryMode"; export type { ThreadInjectItemsParams } from "./ThreadInjectItemsParams"; export type { ThreadInjectItemsResponse } from "./ThreadInjectItemsResponse"; export type { ThreadItem } from "./ThreadItem"; diff --git a/codex-rs/app-server-protocol/src/export.rs b/codex-rs/app-server-protocol/src/export.rs index 91eca690f1..9f27dff7fa 100644 --- a/codex-rs/app-server-protocol/src/export.rs +++ b/codex-rs/app-server-protocol/src/export.rs @@ -43,6 +43,8 @@ pub(crate) const GENERATED_TS_HEADER: &str = "// GENERATED CODE! DO NOT MODIFY B const IGNORED_DEFINITIONS: &[&str] = &["Option<()>"]; const JSON_V1_ALLOWLIST: &[&str] = &["InitializeParams", "InitializeResponse"]; const EXPERIMENTAL_CLIENT_METHOD_DEPENDENCY_TYPES: &[&str] = &[ + "EnvironmentShellInfo", + "PathUri", "RemoteControlClient", "RemoteControlClientsListOrder", "ThreadBackgroundTerminal", diff --git a/codex-rs/app-server-protocol/src/protocol/common.rs b/codex-rs/app-server-protocol/src/protocol/common.rs index 51a37fbaa9..c4b94d8b3f 100644 --- a/codex-rs/app-server-protocol/src/protocol/common.rs +++ b/codex-rs/app-server-protocol/src/protocol/common.rs @@ -943,6 +943,13 @@ client_request_definitions! { serialization: global("environment"), response: v2::EnvironmentAddResponse, }, + #[experimental("environment/info")] + /// Reads information from a configured execution environment. + EnvironmentInfo => "environment/info" { + params: v2::EnvironmentInfoParams, + serialization: global_shared_read("environment"), + response: v2::EnvironmentInfoResponse, + }, McpServerOauthLogin => "mcpServer/oauth/login" { params: v2::McpServerOauthLoginParams, @@ -1913,6 +1920,7 @@ mod tests { request_id: request_id(), params: v2::McpServerOauthLoginParams { name: "server-a".to_string(), + thread_id: None, scopes: None, timeout_secs: None, }, @@ -2579,6 +2587,7 @@ mod tests { parent_thread_id: None, preview: "first prompt".to_string(), ephemeral: true, + history_mode: Default::default(), model_provider: "openai".to_string(), created_at: 1, updated_at: 2, @@ -2629,6 +2638,7 @@ mod tests { "parentThreadId": null, "preview": "first prompt", "ephemeral": true, + "historyMode": "legacy", "modelProvider": "openai", "createdAt": 1, "updatedAt": 2, @@ -3185,6 +3195,7 @@ mod tests { request_id: RequestId::Integer(9), params: v2::ThreadRealtimeStartParams { client_managed_handoffs: Some(true), + flush_transcript_tail_on_session_end: Some(true), codex_responses_as_items: None, codex_response_item_prefix: None, codex_response_handoff_prefix: Some("silent context".to_string()), @@ -3206,6 +3217,7 @@ mod tests { "params": { "threadId": "thr_123", "clientManagedHandoffs": true, + "flushTranscriptTailOnSessionEnd": true, "codexResponsesAsItems": null, "codexResponseItemPrefix": null, "codexResponseHandoffPrefix": "silent context", @@ -3230,6 +3242,7 @@ mod tests { request_id: RequestId::Integer(9), params: v2::ThreadRealtimeStartParams { client_managed_handoffs: None, + flush_transcript_tail_on_session_end: None, codex_responses_as_items: None, codex_response_item_prefix: None, codex_response_handoff_prefix: None, @@ -3251,6 +3264,7 @@ mod tests { "params": { "threadId": "thr_123", "clientManagedHandoffs": null, + "flushTranscriptTailOnSessionEnd": null, "codexResponsesAsItems": null, "codexResponseItemPrefix": null, "codexResponseHandoffPrefix": null, @@ -3270,6 +3284,7 @@ mod tests { request_id: RequestId::Integer(9), params: v2::ThreadRealtimeStartParams { client_managed_handoffs: None, + flush_transcript_tail_on_session_end: None, codex_responses_as_items: None, codex_response_item_prefix: None, codex_response_handoff_prefix: None, @@ -3291,6 +3306,7 @@ mod tests { "params": { "threadId": "thr_123", "clientManagedHandoffs": null, + "flushTranscriptTailOnSessionEnd": null, "codexResponsesAsItems": null, "codexResponseItemPrefix": null, "codexResponseHandoffPrefix": null, @@ -3508,6 +3524,7 @@ mod tests { request_id: RequestId::Integer(1), params: v2::ThreadRealtimeStartParams { client_managed_handoffs: None, + flush_transcript_tail_on_session_end: None, codex_responses_as_items: None, codex_response_item_prefix: None, codex_response_handoff_prefix: None, diff --git a/codex-rs/app-server-protocol/src/protocol/item_builders.rs b/codex-rs/app-server-protocol/src/protocol/item_builders.rs index 66019980a3..c041fe9621 100644 --- a/codex-rs/app-server-protocol/src/protocol/item_builders.rs +++ b/codex-rs/app-server-protocol/src/protocol/item_builders.rs @@ -128,7 +128,10 @@ pub fn build_command_execution_end_item(payload: &ExecCommandEndEvent) -> Thread } } -fn command_actions_for_path_uri(parsed_cmd: &[ParsedCommand], cwd: &PathUri) -> Vec { +pub(crate) fn command_actions_for_path_uri( + parsed_cmd: &[ParsedCommand], + cwd: &PathUri, +) -> Vec { // TODO(anp): Carry PathUri into CommandAction so foreign Read actions retain resolved paths. // Until then, omit those actions rather than project a foreign cwd onto the host. let native_cwd = if cwd.infer_path_convention() == Some(PathConvention::native()) { diff --git a/codex-rs/app-server-protocol/src/protocol/thread_history.rs b/codex-rs/app-server-protocol/src/protocol/thread_history.rs index 98702267ab..2d9c6b6a1a 100644 --- a/codex-rs/app-server-protocol/src/protocol/thread_history.rs +++ b/codex-rs/app-server-protocol/src/protocol/thread_history.rs @@ -574,52 +574,25 @@ impl ThreadHistoryBuilder { } fn handle_item_started(&mut self, payload: &ItemStartedEvent) { - match &payload.item { - codex_protocol::items::TurnItem::Plan(plan) => { - if plan.text.is_empty() { - return; - } - self.upsert_item_in_turn_id( - &payload.turn_id, - ThreadItem::from(payload.item.clone()), - ); - } - codex_protocol::items::TurnItem::Sleep(_) => { - self.upsert_item_in_turn_id( - &payload.turn_id, - ThreadItem::from(payload.item.clone()), - ); - } - codex_protocol::items::TurnItem::UserMessage(_) - | codex_protocol::items::TurnItem::HookPrompt(_) - | codex_protocol::items::TurnItem::AgentMessage(_) - | codex_protocol::items::TurnItem::Reasoning(_) - | codex_protocol::items::TurnItem::WebSearch(_) - | codex_protocol::items::TurnItem::ImageView(_) - | codex_protocol::items::TurnItem::ImageGeneration(_) - | codex_protocol::items::TurnItem::FileChange(_) - | codex_protocol::items::TurnItem::McpToolCall(_) - | codex_protocol::items::TurnItem::ContextCompaction(_) => {} - } + self.handle_materialized_item_lifecycle(&payload.turn_id, &payload.item); } fn handle_item_completed(&mut self, payload: &ItemCompletedEvent) { - match &payload.item { - codex_protocol::items::TurnItem::Plan(plan) => { - if plan.text.is_empty() { - return; - } - self.upsert_item_in_turn_id( - &payload.turn_id, - ThreadItem::from(payload.item.clone()), - ); - } - codex_protocol::items::TurnItem::Sleep(_) => { - self.upsert_item_in_turn_id( - &payload.turn_id, - ThreadItem::from(payload.item.clone()), - ); - } + self.handle_materialized_item_lifecycle(&payload.turn_id, &payload.item); + } + + fn handle_materialized_item_lifecycle( + &mut self, + turn_id: &str, + item: &codex_protocol::items::TurnItem, + ) { + let should_upsert = match item { + codex_protocol::items::TurnItem::Plan(plan) => !plan.text.is_empty(), + codex_protocol::items::TurnItem::Sleep(_) + | codex_protocol::items::TurnItem::CommandExecution(_) + | codex_protocol::items::TurnItem::DynamicToolCall(_) + | codex_protocol::items::TurnItem::CollabAgentToolCall(_) + | codex_protocol::items::TurnItem::SubAgentActivity(_) => true, codex_protocol::items::TurnItem::UserMessage(_) | codex_protocol::items::TurnItem::HookPrompt(_) | codex_protocol::items::TurnItem::AgentMessage(_) @@ -629,7 +602,11 @@ impl ThreadHistoryBuilder { | codex_protocol::items::TurnItem::ImageGeneration(_) | codex_protocol::items::TurnItem::FileChange(_) | codex_protocol::items::TurnItem::McpToolCall(_) - | codex_protocol::items::TurnItem::ContextCompaction(_) => {} + | codex_protocol::items::TurnItem::ContextCompaction(_) => false, + }; + + if should_upsert { + self.upsert_item_in_turn_id(turn_id, ThreadItem::from(item.clone())); } } @@ -775,6 +752,9 @@ impl ThreadHistoryBuilder { connector_id, link_id: payload.link_id.clone(), resource_uri: payload.mcp_app_resource_uri.clone(), + app_name: payload.app_name.clone(), + template_id: payload.template_id.clone(), + action_name: payload.action_name.clone(), }), mcp_app_resource_uri: payload.mcp_app_resource_uri.clone(), plugin_id: payload.plugin_id.clone(), @@ -825,6 +805,9 @@ impl ThreadHistoryBuilder { connector_id, link_id: payload.link_id.clone(), resource_uri: payload.mcp_app_resource_uri.clone(), + app_name: payload.app_name.clone(), + template_id: payload.template_id.clone(), + action_name: payload.action_name.clone(), }), mcp_app_resource_uri: payload.mcp_app_resource_uri.clone(), plugin_id: payload.plugin_id.clone(), @@ -1567,6 +1550,8 @@ mod tests { use crate::protocol::v2::CommandExecutionSource; use codex_protocol::ThreadId; use codex_protocol::dynamic_tools::DynamicToolCallOutputContentItem as CoreDynamicToolCallOutputContentItem; + use codex_protocol::items::CommandExecutionItem as CoreCommandExecutionItem; + use codex_protocol::items::CommandExecutionStatus as CoreCommandExecutionStatus; use codex_protocol::items::HookPromptFragment as CoreHookPromptFragment; use codex_protocol::items::SleepItem as CoreSleepItem; use codex_protocol::items::TurnItem as CoreTurnItem; @@ -1861,6 +1846,77 @@ mod tests { ); } + #[test] + fn rebuilds_command_execution_item_from_persisted_completion() { + let turn_id = "turn-1"; + let thread_id = ThreadId::new(); + let command_item = CoreTurnItem::CommandExecution(CoreCommandExecutionItem { + id: "exec-1".to_string(), + process_id: Some("pid-1".to_string()), + command: vec!["echo".to_string(), "hello world".to_string()], + cwd: test_path_buf("/tmp").abs().into(), + parsed_cmd: vec![ParsedCommand::Unknown { + cmd: "echo hello world".to_string(), + }], + source: ExecCommandSource::Agent, + interaction_input: None, + status: CoreCommandExecutionStatus::Completed, + stdout: Some("hello world\n".to_string()), + stderr: Some(String::new()), + aggregated_output: Some("hello world\n".to_string()), + exit_code: Some(0), + duration: Some(Duration::from_millis(12)), + formatted_output: Some("hello world\n".to_string()), + }); + let events = vec![ + EventMsg::TurnStarted(TurnStartedEvent { + turn_id: turn_id.to_string(), + trace_id: None, + started_at: None, + model_context_window: None, + collaboration_mode_kind: Default::default(), + }), + EventMsg::ItemCompleted(ItemCompletedEvent { + thread_id, + turn_id: turn_id.to_string(), + item: command_item, + completed_at_ms: 1_000, + }), + EventMsg::TurnComplete(TurnCompleteEvent { + turn_id: turn_id.to_string(), + last_agent_message: None, + completed_at: None, + duration_ms: None, + time_to_first_token_ms: None, + }), + ]; + + let items = events + .into_iter() + .map(RolloutItem::EventMsg) + .collect::>(); + let turns = build_turns_from_rollout_items(&items); + + assert_eq!(turns.len(), 1); + assert_eq!( + turns[0].items, + vec![ThreadItem::CommandExecution { + id: "exec-1".to_string(), + command: "echo 'hello world'".to_string(), + cwd: test_path_buf("/tmp").abs().into(), + process_id: Some("pid-1".to_string()), + source: CommandExecutionSource::Agent, + status: CommandExecutionStatus::Completed, + command_actions: vec![CommandAction::Unknown { + command: "echo hello world".to_string(), + }], + aggregated_output: Some("hello world\n".to_string()), + exit_code: Some(0), + duration_ms: Some(12), + }] + ); + } + #[test] fn preserves_user_message_client_id_from_legacy_event() { let turn_id = "turn-1"; @@ -2418,6 +2474,9 @@ mod tests { connector_id: None, mcp_app_resource_uri: None, link_id: None, + app_name: None, + template_id: None, + action_name: None, plugin_id: None, duration: Duration::from_millis(8), result: Err("boom".into()), @@ -2499,6 +2558,9 @@ mod tests { connector_id: Some("calendar".into()), mcp_app_resource_uri: Some("ui://widget/lookup.html".into()), link_id: Some("link_calendar".into()), + app_name: Some("Calendar".into()), + template_id: Some("calendar_template".into()), + action_name: Some("lookup".into()), plugin_id: Some("sample@test".into()), duration: Duration::from_millis(8), result: Ok(CallToolResult { @@ -2533,6 +2595,9 @@ mod tests { connector_id: "calendar".into(), link_id: Some("link_calendar".into()), resource_uri: Some("ui://widget/lookup.html".into()), + app_name: Some("Calendar".into()), + template_id: Some("calendar_template".into()), + action_name: Some("lookup".into()), }), mcp_app_resource_uri: Some("ui://widget/lookup.html".into()), plugin_id: Some("sample@test".into()), diff --git a/codex-rs/app-server-protocol/src/protocol/v2/account.rs b/codex-rs/app-server-protocol/src/protocol/v2/account.rs index 635d001bba..788c8f7aaf 100644 --- a/codex-rs/app-server-protocol/src/protocol/v2/account.rs +++ b/codex-rs/app-server-protocol/src/protocol/v2/account.rs @@ -284,6 +284,52 @@ pub struct GetAccountRateLimitsResponse { #[ts(export_to = "v2/")] pub struct RateLimitResetCreditsSummary { pub available_count: i64, + /// Detail rows for available reset credits, when the backend provides them. + /// + /// `null` means only `availableCount` is known, while an empty array means details were fetched + /// and no available credits were returned. The backend may cap this list, so its length can be + /// less than `availableCount`. + pub credits: Option>, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct RateLimitResetCredit { + /// Opaque backend identifier for this reset credit. + pub id: String, + pub reset_type: RateLimitResetType, + pub status: RateLimitResetCreditStatus, + /// Unix timestamp in seconds when the credit was granted. + #[ts(type = "number")] + pub granted_at: i64, + /// Unix timestamp in seconds when the credit expires, or `null` if it does not expire. + #[ts(type = "number | null")] + pub expires_at: Option, + /// Backend-provided display title for this credit, or `null` when unavailable. + pub title: Option, + /// Backend-provided display description for this credit, or `null` when unavailable. + pub description: Option, +} + +#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/", rename_all = "camelCase")] +pub enum RateLimitResetType { + CodexRateLimits, + #[serde(other)] + Unknown, +} + +#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/", rename_all = "camelCase")] +pub enum RateLimitResetCreditStatus { + Available, + Redeeming, + Redeemed, + #[serde(other)] + Unknown, } #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, JsonSchema, TS)] @@ -293,6 +339,10 @@ pub struct ConsumeAccountRateLimitResetCreditParams { /// Identifies one logical reset attempt. A UUID is recommended; reuse the same value when /// retrying that attempt. pub idempotency_key: String, + /// Opaque reset-credit identifier to redeem. When omitted, the backend selects the next + /// available credit. + #[ts(optional = nullable)] + pub credit_id: Option, } #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, JsonSchema, TS)] diff --git a/codex-rs/app-server-protocol/src/protocol/v2/config.rs b/codex-rs/app-server-protocol/src/protocol/v2/config.rs index 4c9e355b47..f615b25d44 100644 --- a/codex-rs/app-server-protocol/src/protocol/v2/config.rs +++ b/codex-rs/app-server-protocol/src/protocol/v2/config.rs @@ -390,6 +390,23 @@ pub struct ConfigRequirements { pub enforce_residency: Option, #[experimental("configRequirements/read.network")] pub network: Option, + pub models: Option, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct ModelsRequirements { + pub new_thread: Option, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct NewThreadModelDefaults { + pub model: Option, + pub model_reasoning_effort: Option, + pub service_tier: Option, } #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, JsonSchema, TS)] diff --git a/codex-rs/app-server-protocol/src/protocol/v2/environment.rs b/codex-rs/app-server-protocol/src/protocol/v2/environment.rs index ccffd5813b..320e2af894 100644 --- a/codex-rs/app-server-protocol/src/protocol/v2/environment.rs +++ b/codex-rs/app-server-protocol/src/protocol/v2/environment.rs @@ -1,3 +1,4 @@ +use codex_utils_path_uri::PathUri; use schemars::JsonSchema; use serde::Deserialize; use serde::Serialize; @@ -19,3 +20,29 @@ pub struct EnvironmentAddParams { #[serde(rename_all = "camelCase")] #[ts(export_to = "v2/")] pub struct EnvironmentAddResponse {} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct EnvironmentInfoParams { + pub environment_id: String, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct EnvironmentInfoResponse { + pub shell: EnvironmentShellInfo, + /// Default working directory reported by the environment, as a canonical file URI. + pub cwd: Option, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct EnvironmentShellInfo { + /// Stable shell name, for example `zsh`, `bash`, `powershell`, `sh`, or `cmd`. + pub name: String, + /// Target-native shell executable path or command name. + pub path: String, +} diff --git a/codex-rs/app-server-protocol/src/protocol/v2/item.rs b/codex-rs/app-server-protocol/src/protocol/v2/item.rs index e2a0d45129..cd04752449 100644 --- a/codex-rs/app-server-protocol/src/protocol/v2/item.rs +++ b/codex-rs/app-server-protocol/src/protocol/v2/item.rs @@ -8,12 +8,17 @@ use super::NetworkPolicyAmendment; use super::RequestPermissionProfile; use super::UserInput; use super::shared::v2_enum_from_core; +use crate::protocol::item_builders::command_actions_for_path_uri; use crate::protocol::item_builders::convert_patch_changes; use codex_experimental_api_macros::ExperimentalApi; use codex_protocol::approvals::GuardianAssessmentAction as CoreGuardianAssessmentAction; use codex_protocol::approvals::GuardianAssessmentDecisionSource as CoreGuardianAssessmentDecisionSource; use codex_protocol::approvals::GuardianCommandSource as CoreGuardianCommandSource; use codex_protocol::items::AgentMessageContent as CoreAgentMessageContent; +use codex_protocol::items::CollabAgentTool as CoreCollabAgentTool; +use codex_protocol::items::CollabAgentToolCallStatus as CoreCollabAgentToolCallStatus; +use codex_protocol::items::CommandExecutionStatus as CoreCommandExecutionStatus; +use codex_protocol::items::DynamicToolCallStatus as CoreDynamicToolCallStatus; use codex_protocol::items::McpToolCallStatus as CoreMcpToolCallStatus; use codex_protocol::items::TurnItem as CoreTurnItem; use codex_protocol::memory_citation::MemoryCitation as CoreMemoryCitation; @@ -30,6 +35,7 @@ use codex_protocol::protocol::GuardianUserAuthorization as CoreGuardianUserAutho use codex_protocol::protocol::PatchApplyStatus as CorePatchApplyStatus; use codex_protocol::protocol::ReviewDecision as CoreReviewDecision; use codex_protocol::protocol::SubAgentActivityKind as CoreSubAgentActivityKind; +use codex_shell_command::parse_command::shlex_join; use codex_utils_absolute_path::AbsolutePathBuf; use codex_utils_path_uri::LegacyAppPathString; use schemars::JsonSchema; @@ -396,6 +402,9 @@ pub struct McpToolCallAppContext { pub connector_id: String, pub link_id: Option, pub resource_uri: Option, + pub app_name: Option, + pub template_id: Option, + pub action_name: Option, } #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, JsonSchema, TS)] @@ -851,6 +860,64 @@ impl From for ThreadItem { summary: reasoning.summary_text, content: reasoning.raw_content, }, + CoreTurnItem::CommandExecution(command) => ThreadItem::CommandExecution { + id: command.id, + command: shlex_join(&command.command), + cwd: command.cwd.clone().into(), + process_id: command.process_id, + source: command.source.into(), + status: command.status.into(), + command_actions: command_actions_for_path_uri(&command.parsed_cmd, &command.cwd), + aggregated_output: command + .aggregated_output + .filter(|output| !output.is_empty()), + exit_code: command.exit_code, + duration_ms: command + .duration + .and_then(|duration| i64::try_from(duration.as_millis()).ok()), + }, + CoreTurnItem::DynamicToolCall(call) => ThreadItem::DynamicToolCall { + id: call.id, + namespace: call.namespace, + tool: call.tool, + arguments: call.arguments, + status: call.status.into(), + content_items: call.content_items.map(|items| { + items + .into_iter() + .map(DynamicToolCallOutputContentItem::from) + .collect() + }), + success: call.success, + duration_ms: call + .duration + .and_then(|duration| i64::try_from(duration.as_millis()).ok()), + }, + CoreTurnItem::CollabAgentToolCall(call) => ThreadItem::CollabAgentToolCall { + id: call.id, + tool: call.tool.into(), + status: call.status.into(), + sender_thread_id: call.sender_thread_id.to_string(), + receiver_thread_ids: call + .receiver_thread_ids + .into_iter() + .map(String::from) + .collect(), + prompt: call.prompt, + model: call.model, + reasoning_effort: call.reasoning_effort, + agents_states: call + .agents_states + .into_iter() + .map(|(thread_id, status)| (thread_id.to_string(), status.into())) + .collect(), + }, + CoreTurnItem::SubAgentActivity(activity) => ThreadItem::SubAgentActivity { + id: activity.id, + kind: activity.kind.into(), + agent_thread_id: activity.agent_thread_id.to_string(), + agent_path: String::from(activity.agent_path), + }, CoreTurnItem::WebSearch(search) => ThreadItem::WebSearch { id: search.id, query: search.query, @@ -895,6 +962,9 @@ impl From for ThreadItem { connector_id, link_id: mcp.link_id, resource_uri: mcp.mcp_app_resource_uri.clone(), + app_name: mcp.app_name, + template_id: mcp.template_id, + action_name: mcp.action_name, }), mcp_app_resource_uri: mcp.mcp_app_resource_uri, plugin_id: mcp.plugin_id, @@ -935,6 +1005,17 @@ impl From for CommandExecutionStatus { } } +impl From for CommandExecutionStatus { + fn from(value: CoreCommandExecutionStatus) -> Self { + match value { + CoreCommandExecutionStatus::InProgress => Self::InProgress, + CoreCommandExecutionStatus::Completed => Self::Completed, + CoreCommandExecutionStatus::Failed => Self::Failed, + CoreCommandExecutionStatus::Declined => Self::Declined, + } + } +} + impl From<&CoreExecCommandStatus> for CommandExecutionStatus { fn from(value: &CoreExecCommandStatus) -> Self { match value { @@ -1022,6 +1103,16 @@ impl From for McpToolCallStatus { } } +impl From for DynamicToolCallStatus { + fn from(value: CoreDynamicToolCallStatus) -> Self { + match value { + CoreDynamicToolCallStatus::InProgress => Self::InProgress, + CoreDynamicToolCallStatus::Completed => Self::Completed, + CoreDynamicToolCallStatus::Failed => Self::Failed, + } + } +} + #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] #[serde(rename_all = "camelCase")] #[ts(export_to = "v2/")] @@ -1049,6 +1140,28 @@ pub enum CollabAgentToolCallStatus { Failed, } +impl From for CollabAgentTool { + fn from(value: CoreCollabAgentTool) -> Self { + match value { + CoreCollabAgentTool::SpawnAgent => Self::SpawnAgent, + CoreCollabAgentTool::SendInput => Self::SendInput, + CoreCollabAgentTool::ResumeAgent => Self::ResumeAgent, + CoreCollabAgentTool::Wait => Self::Wait, + CoreCollabAgentTool::CloseAgent => Self::CloseAgent, + } + } +} + +impl From for CollabAgentToolCallStatus { + fn from(value: CoreCollabAgentToolCallStatus) -> Self { + match value { + CoreCollabAgentToolCallStatus::InProgress => Self::InProgress, + CoreCollabAgentToolCallStatus::Completed => Self::Completed, + CoreCollabAgentToolCallStatus::Failed => Self::Failed, + } + } +} + #[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, JsonSchema, TS)] #[serde(rename_all = "camelCase")] #[ts(export_to = "v2/")] @@ -1456,6 +1569,21 @@ pub enum DynamicToolCallOutputContentItem { InputImage { image_url: String }, } +impl From + for DynamicToolCallOutputContentItem +{ + fn from(item: codex_protocol::dynamic_tools::DynamicToolCallOutputContentItem) -> Self { + match item { + codex_protocol::dynamic_tools::DynamicToolCallOutputContentItem::InputText { text } => { + Self::InputText { text } + } + codex_protocol::dynamic_tools::DynamicToolCallOutputContentItem::InputImage { + image_url, + } => Self::InputImage { image_url }, + } + } +} + impl From for codex_protocol::dynamic_tools::DynamicToolCallOutputContentItem { diff --git a/codex-rs/app-server-protocol/src/protocol/v2/mcp.rs b/codex-rs/app-server-protocol/src/protocol/v2/mcp.rs index cdfeb524f5..d24da0c879 100644 --- a/codex-rs/app-server-protocol/src/protocol/v2/mcp.rs +++ b/codex-rs/app-server-protocol/src/protocol/v2/mcp.rs @@ -23,6 +23,12 @@ v2_enum_from_core!( } ); +v2_enum_from_core!( + pub enum McpServerStartupFailureReason from codex_protocol::protocol::McpStartupFailureReason { + ReauthenticationRequired + } +); + #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] #[serde(rename_all = "camelCase")] #[ts(export_to = "v2/")] @@ -186,6 +192,8 @@ pub struct McpServerRefreshResponse {} #[ts(export_to = "v2/")] pub struct McpServerOauthLoginParams { pub name: String, + #[ts(optional = nullable)] + pub thread_id: Option, #[serde(default, skip_serializing_if = "Option::is_none")] #[ts(optional = nullable)] pub scopes: Option>, @@ -215,6 +223,7 @@ pub struct McpToolCallProgressNotification { #[ts(export_to = "v2/")] pub struct McpServerOauthLoginCompletedNotification { pub name: String, + pub thread_id: Option, pub success: bool, #[serde(default, skip_serializing_if = "Option::is_none")] #[ts(optional)] @@ -239,6 +248,7 @@ pub struct McpServerStatusUpdatedNotification { pub name: String, pub status: McpServerStartupState, pub error: Option, + pub failure_reason: Option, } #[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, JsonSchema, TS)] diff --git a/codex-rs/app-server-protocol/src/protocol/v2/plugin.rs b/codex-rs/app-server-protocol/src/protocol/v2/plugin.rs index dbe0b3d55d..19a4ccd889 100644 --- a/codex-rs/app-server-protocol/src/protocol/v2/plugin.rs +++ b/codex-rs/app-server-protocol/src/protocol/v2/plugin.rs @@ -600,6 +600,9 @@ pub struct PluginSummary { pub id: String, /// Backend remote plugin identifier when available. pub remote_plugin_id: Option, + /// Version advertised by the remote marketplace backend when available. + #[serde(default)] + pub version: Option, /// Version of the locally materialized plugin package when available. #[serde(default)] pub local_version: Option, @@ -744,6 +747,15 @@ pub enum PluginSource { ref_name: Option, sha: Option, }, + #[serde(rename_all = "camelCase")] + #[ts(rename_all = "camelCase")] + Npm { + package: String, + /// Optional npm version or version range. + version: Option, + /// Optional HTTPS registry URL. Authentication stays in the user's npm config. + registry: Option, + }, /// The plugin is available in the remote catalog. Download metadata is /// kept server-side and is not exposed through the app-server API. Remote, diff --git a/codex-rs/app-server-protocol/src/protocol/v2/realtime.rs b/codex-rs/app-server-protocol/src/protocol/v2/realtime.rs index 4b8b2056ea..b78f390bec 100644 --- a/codex-rs/app-server-protocol/src/protocol/v2/realtime.rs +++ b/codex-rs/app-server-protocol/src/protocol/v2/realtime.rs @@ -70,6 +70,10 @@ pub struct ThreadRealtimeStartParams { /// them automatically. Defaults to false. #[ts(optional = nullable)] pub client_managed_handoffs: Option, + /// Routes any transcript tail remaining at session end through Codex. Defaults to false. + /// TODO: Remove this rollout knob once transcript-tail flushing is always enabled. + #[ts(optional = nullable)] + pub flush_transcript_tail_on_session_end: Option, /// Sends automatic Codex responses as realtime conversation items instead of handoff appends. #[ts(optional = nullable)] pub codex_responses_as_items: Option, diff --git a/codex-rs/app-server-protocol/src/protocol/v2/tests.rs b/codex-rs/app-server-protocol/src/protocol/v2/tests.rs index 7c78fee4c8..77d8a1439b 100644 --- a/codex-rs/app-server-protocol/src/protocol/v2/tests.rs +++ b/codex-rs/app-server-protocol/src/protocol/v2/tests.rs @@ -1,13 +1,22 @@ use super::*; +use crate::ServerNotification; use codex_protocol::approvals::ElicitationRequest as CoreElicitationRequest; use codex_protocol::config_types::MultiAgentMode; use codex_protocol::items::AgentMessageContent; use codex_protocol::items::AgentMessageItem; +use codex_protocol::items::CollabAgentTool as CoreCollabAgentTool; +use codex_protocol::items::CollabAgentToolCallItem; +use codex_protocol::items::CollabAgentToolCallStatus as CoreCollabAgentToolCallStatus; +use codex_protocol::items::CommandExecutionItem; +use codex_protocol::items::CommandExecutionStatus as CoreCommandExecutionStatus; +use codex_protocol::items::DynamicToolCallItem; +use codex_protocol::items::DynamicToolCallStatus as CoreDynamicToolCallStatus; use codex_protocol::items::FileChangeItem; use codex_protocol::items::ImageViewItem; use codex_protocol::items::McpToolCallItem; use codex_protocol::items::McpToolCallStatus as CoreMcpToolCallStatus; use codex_protocol::items::ReasoningItem; +use codex_protocol::items::SubAgentActivityItem; use codex_protocol::items::TurnItem; use codex_protocol::items::UserMessageItem; use codex_protocol::items::WebSearchItem; @@ -29,8 +38,10 @@ use codex_protocol::permissions::FileSystemSpecialPath as CoreFileSystemSpecialP use codex_protocol::protocol::AgentStatus as CoreAgentStatus; use codex_protocol::protocol::AskForApproval as CoreAskForApproval; use codex_protocol::protocol::ConversationTextRole; +use codex_protocol::protocol::ExecCommandSource as CoreExecCommandSource; use codex_protocol::protocol::GranularApprovalConfig as CoreGranularApprovalConfig; use codex_protocol::protocol::NetworkAccess as CoreNetworkAccess; +use codex_protocol::protocol::SubAgentActivityKind as CoreSubAgentActivityKind; use codex_protocol::request_permissions::RequestPermissionProfile as CoreRequestPermissionProfile; use codex_protocol::user_input::UserInput as CoreUserInput; use codex_utils_absolute_path::AbsolutePathBuf; @@ -174,6 +185,7 @@ fn thread_resume_response_round_trips_initial_turns_page() { parent_thread_id: None, preview: String::new(), ephemeral: false, + history_mode: Default::default(), model_provider: "openai".to_string(), created_at: 1, updated_at: 1, @@ -837,6 +849,30 @@ fn thread_path_params_deserialize_empty_path_as_none() { ); } +#[test] +fn thread_fork_last_turn_id_round_trips() { + let params: ThreadForkParams = serde_json::from_value(json!({ + "threadId": "thread-1", + "lastTurnId": "turn-2", + })) + .expect("thread/fork params deserialize"); + + assert_eq!(params.last_turn_id, Some("turn-2".to_string())); + let serialized = serde_json::to_value(params).expect("thread/fork params serialize"); + assert_eq!(serialized["lastTurnId"], json!("turn-2")); + + let omitted = serde_json::to_value(ThreadForkParams { + thread_id: "thread-1".to_string(), + ..Default::default() + }) + .expect("thread/fork params without last turn id serialize"); + assert_eq!( + omitted["lastTurnId"], + serde_json::Value::Null, + "optional lastTurnId should serialize as null when omitted" + ); +} + #[test] fn fs_get_metadata_response_round_trips_minimal_fields() { let response = FsGetMetadataResponse { @@ -1751,6 +1787,7 @@ fn config_requirements_granular_allowed_approval_policy_is_marked_experimental() hooks: None, enforce_residency: None, network: None, + models: None, }); assert_eq!(reason, Some("askForApproval.granular")); @@ -2155,6 +2192,7 @@ fn mcp_server_status_updated_accepts_missing_thread_id() { name: "optional_broken".to_string(), status: McpServerStartupState::Failed, error: Some("handshake failed".to_string()), + failure_reason: None, }; assert_eq!(notification, expected); assert_eq!( @@ -2164,6 +2202,33 @@ fn mcp_server_status_updated_accepts_missing_thread_id() { "name": "optional_broken", "status": "failed", "error": "handshake failed", + "failureReason": null, + }) + ); +} + +#[test] +fn mcp_server_status_updated_serializes_failure_reason() { + let notification = + ServerNotification::McpServerStatusUpdated(McpServerStatusUpdatedNotification { + thread_id: Some("thread-1".to_string()), + name: "expired-oauth".to_string(), + status: McpServerStartupState::Failed, + error: Some("OAuth credentials expired".to_string()), + failure_reason: Some(McpServerStartupFailureReason::ReauthenticationRequired), + }); + + assert_eq!( + serde_json::to_value(notification).expect("notification should serialize"), + json!({ + "method": "mcpServer/startupStatus/updated", + "params": { + "threadId": "thread-1", + "name": "expired-oauth", + "status": "failed", + "error": "OAuth credentials expired", + "failureReason": "reauthenticationRequired", + }, }) ); } @@ -2577,6 +2642,134 @@ fn core_turn_item_into_thread_item_converts_supported_variants() { } ); + let command_item = TurnItem::CommandExecution(CommandExecutionItem { + id: "exec-1".to_string(), + process_id: Some("pid-1".to_string()), + command: vec!["echo".to_string(), "done".to_string()], + cwd: PathUri::from_abs_path(&test_path_buf("/tmp").abs()), + parsed_cmd: vec![codex_protocol::parse_command::ParsedCommand::Unknown { + cmd: "echo done".to_string(), + }], + source: CoreExecCommandSource::Agent, + interaction_input: None, + status: CoreCommandExecutionStatus::Completed, + stdout: Some("done\n".to_string()), + stderr: Some(String::new()), + aggregated_output: Some("done\n".to_string()), + exit_code: Some(0), + duration: Some(Duration::from_millis(5)), + formatted_output: Some("done\n".to_string()), + }); + + assert_eq!( + ThreadItem::from(command_item), + ThreadItem::CommandExecution { + id: "exec-1".to_string(), + command: "echo done".to_string(), + cwd: LegacyAppPathString::from_abs_path(&test_path_buf("/tmp").abs()), + process_id: Some("pid-1".to_string()), + source: CommandExecutionSource::Agent, + status: CommandExecutionStatus::Completed, + command_actions: vec![CommandAction::Unknown { + command: "echo done".to_string(), + }], + aggregated_output: Some("done\n".to_string()), + exit_code: Some(0), + duration_ms: Some(5), + } + ); + + let dynamic_tool_call_item = TurnItem::DynamicToolCall(DynamicToolCallItem { + id: "dynamic-1".to_string(), + namespace: Some("apps".to_string()), + tool: "lookup".to_string(), + arguments: json!({"id": "123"}), + status: CoreDynamicToolCallStatus::Completed, + content_items: Some(vec![ + codex_protocol::dynamic_tools::DynamicToolCallOutputContentItem::InputText { + text: "ok".to_string(), + }, + ]), + success: Some(true), + error: None, + duration: Some(Duration::from_millis(5)), + }); + + assert_eq!( + ThreadItem::from(dynamic_tool_call_item), + ThreadItem::DynamicToolCall { + id: "dynamic-1".to_string(), + namespace: Some("apps".to_string()), + tool: "lookup".to_string(), + arguments: json!({"id": "123"}), + status: DynamicToolCallStatus::Completed, + content_items: Some(vec![DynamicToolCallOutputContentItem::InputText { + text: "ok".to_string(), + }]), + success: Some(true), + duration_ms: Some(5), + } + ); + + let sender_thread_id = codex_protocol::ThreadId::default(); + let receiver_thread_id = codex_protocol::ThreadId::default(); + let collab_item = TurnItem::CollabAgentToolCall(CollabAgentToolCallItem { + id: "collab-1".to_string(), + tool: CoreCollabAgentTool::SendInput, + status: CoreCollabAgentToolCallStatus::Completed, + sender_thread_id, + receiver_thread_ids: vec![receiver_thread_id], + receiver_agents: Vec::new(), + prompt: Some("continue".to_string()), + model: None, + reasoning_effort: None, + agents_states: [(receiver_thread_id, CoreAgentStatus::Completed(None))] + .into_iter() + .collect(), + }); + + assert_eq!( + ThreadItem::from(collab_item), + ThreadItem::CollabAgentToolCall { + id: "collab-1".to_string(), + tool: CollabAgentTool::SendInput, + status: CollabAgentToolCallStatus::Completed, + sender_thread_id: sender_thread_id.to_string(), + receiver_thread_ids: vec![receiver_thread_id.to_string()], + prompt: Some("continue".to_string()), + model: None, + reasoning_effort: None, + agents_states: [( + receiver_thread_id.to_string(), + CollabAgentState { + status: CollabAgentStatus::Completed, + message: None, + }, + )] + .into_iter() + .collect(), + } + ); + + let sub_agent_activity_item = TurnItem::SubAgentActivity(SubAgentActivityItem { + id: "activity-1".to_string(), + kind: CoreSubAgentActivityKind::Interrupted, + agent_thread_id: receiver_thread_id, + agent_path: codex_protocol::AgentPath::root() + .join("worker") + .expect("worker path"), + }); + + assert_eq!( + ThreadItem::from(sub_agent_activity_item), + ThreadItem::SubAgentActivity { + id: "activity-1".to_string(), + kind: SubAgentActivityKind::Interrupted, + agent_thread_id: receiver_thread_id.to_string(), + agent_path: "/root/worker".to_string(), + } + ); + let search_item = TurnItem::WebSearch(WebSearchItem { id: "search-1".to_string(), query: "docs".to_string(), @@ -2648,6 +2841,9 @@ fn core_turn_item_into_thread_item_converts_supported_variants() { connector_id: Some("calendar".to_string()), mcp_app_resource_uri: Some("app://connector".to_string()), link_id: Some("link_calendar".to_string()), + app_name: Some("Calendar".to_string()), + template_id: Some("calendar_template".to_string()), + action_name: Some("create_event".to_string()), plugin_id: Some("sample@test".to_string()), status: CoreMcpToolCallStatus::InProgress, result: None, @@ -2667,6 +2863,9 @@ fn core_turn_item_into_thread_item_converts_supported_variants() { connector_id: "calendar".to_string(), link_id: Some("link_calendar".to_string()), resource_uri: Some("app://connector".to_string()), + app_name: Some("Calendar".to_string()), + template_id: Some("calendar_template".to_string()), + action_name: Some("create_event".to_string()), }), mcp_app_resource_uri: Some("app://connector".to_string()), plugin_id: Some("sample@test".to_string()), @@ -2684,6 +2883,9 @@ fn core_turn_item_into_thread_item_converts_supported_variants() { connector_id: None, mcp_app_resource_uri: None, link_id: None, + app_name: None, + template_id: None, + action_name: None, plugin_id: None, status: CoreMcpToolCallStatus::Completed, result: Some(CallToolResult { @@ -2730,6 +2932,9 @@ fn mcp_tool_call_app_context_serializes_connector_id() { connector_id: "calendar".to_string(), link_id: Some("link_calendar".to_string()), resource_uri: Some("app://connector".to_string()), + app_name: Some("Calendar".to_string()), + template_id: Some("calendar_template".to_string()), + action_name: Some("create_event".to_string()), }), mcp_app_resource_uri: Some("app://connector".to_string()), plugin_id: None, @@ -2751,6 +2956,9 @@ fn mcp_tool_call_app_context_serializes_connector_id() { "connectorId": "calendar", "linkId": "link_calendar", "resourceUri": "app://connector", + "appName": "Calendar", + "templateId": "calendar_template", + "actionName": "create_event", }, "mcpAppResourceUri": "app://connector", "pluginId": null, @@ -2761,6 +2969,29 @@ fn mcp_tool_call_app_context_serializes_connector_id() { ); } +#[test] +fn mcp_tool_call_app_context_serializes_missing_mixed_version_fields_as_null() { + assert_eq!( + serde_json::to_value(McpToolCallAppContext { + connector_id: "calendar".to_string(), + link_id: None, + resource_uri: None, + app_name: None, + template_id: None, + action_name: None, + }) + .expect("MCP tool call app context should serialize"), + json!({ + "connectorId": "calendar", + "linkId": null, + "resourceUri": null, + "appName": null, + "templateId": null, + "actionName": null, + }) + ); +} + #[test] fn user_input_into_core_preserves_image_detail() { assert_eq!( @@ -2834,7 +3065,7 @@ fn skills_extra_roots_set_params_rejects_relative_roots() { } #[test] -fn plugin_source_serializes_local_git_and_remote_variants() { +fn plugin_source_serializes_local_git_npm_and_remote_variants() { let local_path = if cfg!(windows) { r"C:\plugins\linear" } else { @@ -2868,6 +3099,21 @@ fn plugin_source_serializes_local_git_and_remote_variants() { }), ); + assert_eq!( + serde_json::to_value(PluginSource::Npm { + package: "@acme/plugin".to_string(), + version: Some("^1.2.0".to_string()), + registry: Some("https://npm.example.com".to_string()), + }) + .unwrap(), + json!({ + "type": "npm", + "package": "@acme/plugin", + "version": "^1.2.0", + "registry": "https://npm.example.com", + }), + ); + assert_eq!( serde_json::to_value(PluginSource::Remote).unwrap(), json!({ @@ -3396,6 +3642,7 @@ fn plugin_share_list_response_serializes_share_items() { remote_plugin_id: Some( "plugins~Plugin_00000000000000000000000000000000".to_string(), ), + version: None, local_version: None, name: "gmail".to_string(), share_context: None, @@ -3417,6 +3664,7 @@ fn plugin_share_list_response_serializes_share_items() { "plugin": { "id": "gmail@openai-curated-remote", "remotePluginId": "plugins~Plugin_00000000000000000000000000000000", + "version": null, "localVersion": null, "name": "gmail", "shareContext": null, diff --git a/codex-rs/app-server-protocol/src/protocol/v2/thread.rs b/codex-rs/app-server-protocol/src/protocol/v2/thread.rs index c330c7eefd..367a4a64c8 100644 --- a/codex-rs/app-server-protocol/src/protocol/v2/thread.rs +++ b/codex-rs/app-server-protocol/src/protocol/v2/thread.rs @@ -4,6 +4,7 @@ use super::AskForApproval; use super::SandboxMode; use super::SandboxPolicy; use super::Thread; +use super::ThreadHistoryMode; use super::ThreadItem; use super::ThreadSource; use super::Turn; @@ -57,6 +58,11 @@ pub struct ThreadStartParams { pub model: Option, #[ts(optional = nullable)] pub model_provider: Option, + /// Allow a provider with an authoritative static model catalog to replace an unavailable + /// requested model with its default. + #[experimental("thread/start.allowProviderModelFallback")] + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub allow_provider_model_fallback: bool, #[serde( default, deserialize_with = "crate::protocol::serde_helpers::deserialize_double_option", @@ -100,6 +106,10 @@ pub struct ThreadStartParams { pub multi_agent_mode: Option, #[ts(optional = nullable)] pub ephemeral: Option, + /// Persisted thread history contract to use for this new thread. + #[experimental("thread/start.historyMode")] + #[ts(optional = nullable)] + pub history_mode: Option, #[ts(optional = nullable)] pub session_start_source: Option, /// Optional client-supplied analytics source classification for this thread. @@ -484,6 +494,13 @@ impl From for TurnsPage { pub struct ThreadForkParams { pub thread_id: String, + /// Optional last turn id to fork through, inclusive. + /// + /// When specified, turns after `last_turn_id` are omitted from the fork. + /// The referenced turn cannot be in progress. + #[ts(optional = nullable)] + pub last_turn_id: Option, + /// [UNSTABLE] Specify the rollout path to fork from. /// If specified, the thread_id param will be ignored. #[experimental("thread/fork.path")] @@ -668,7 +685,7 @@ pub struct ThreadIncrementElicitationParams { #[ts(export_to = "v2/")] pub struct ThreadIncrementElicitationResponse { /// Current out-of-band elicitation count after the increment. - pub count: u64, + pub count: i64, /// Whether timeout accounting is paused after applying the increment. pub paused: bool, } @@ -688,7 +705,7 @@ pub struct ThreadDecrementElicitationParams { #[ts(export_to = "v2/")] pub struct ThreadDecrementElicitationResponse { /// Current out-of-band elicitation count after the decrement. - pub count: u64, + pub count: i64, /// Whether timeout accounting remains paused after applying the decrement. pub paused: bool, } @@ -1025,6 +1042,7 @@ pub struct ThreadBackgroundTerminalsTerminateResponse { #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] #[serde(rename_all = "camelCase")] #[ts(export_to = "v2/")] +/// DEPRECATED: `thread/rollback` will be removed soon. pub struct ThreadRollbackParams { pub thread_id: String, /// The number of turns to drop from the end of the thread. Must be >= 1. diff --git a/codex-rs/app-server-protocol/src/protocol/v2/thread_data.rs b/codex-rs/app-server-protocol/src/protocol/v2/thread_data.rs index 3338513df7..2393efb25c 100644 --- a/codex-rs/app-server-protocol/src/protocol/v2/thread_data.rs +++ b/codex-rs/app-server-protocol/src/protocol/v2/thread_data.rs @@ -5,6 +5,7 @@ use super::TurnStatus; use codex_experimental_api_macros::ExperimentalApi; use codex_protocol::protocol::SessionSource as CoreSessionSource; use codex_protocol::protocol::SubAgentSource as CoreSubAgentSource; +use codex_protocol::protocol::ThreadHistoryMode as CoreThreadHistoryMode; use codex_protocol::protocol::ThreadSource as CoreThreadSource; use codex_utils_absolute_path::AbsolutePathBuf; use schemars::JsonSchema; @@ -64,6 +65,33 @@ impl From for CoreSessionSource { } } +#[derive(Default, Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, JsonSchema, TS)] +#[serde(rename_all = "lowercase")] +#[ts(rename_all = "lowercase", export_to = "v2/")] +pub enum ThreadHistoryMode { + #[default] + Legacy, + Paginated, +} + +impl From for ThreadHistoryMode { + fn from(value: CoreThreadHistoryMode) -> Self { + match value { + CoreThreadHistoryMode::Legacy => Self::Legacy, + CoreThreadHistoryMode::Paginated => Self::Paginated, + } + } +} + +impl From for CoreThreadHistoryMode { + fn from(value: ThreadHistoryMode) -> Self { + match value { + ThreadHistoryMode::Legacy => Self::Legacy, + ThreadHistoryMode::Paginated => Self::Paginated, + } + } +} + #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, TS)] #[serde(try_from = "String", into = "String")] #[ts(type = "string")] @@ -155,6 +183,10 @@ pub struct Thread { pub preview: String, /// Whether the thread is ephemeral and should not be materialized on disk. pub ephemeral: bool, + /// Persisted thread history contract selected when this thread was created. + #[experimental("thread.historyMode")] + #[serde(default)] + pub history_mode: ThreadHistoryMode, /// Model provider used for this thread (for example, 'openai'). pub model_provider: String, /// Unix timestamp (in seconds) when the thread was created. diff --git a/codex-rs/app-server-test-client/src/plugin_analytics_mutation_smoke.rs b/codex-rs/app-server-test-client/src/plugin_analytics_mutation_smoke.rs index 70aa13ab9c..c9f9653155 100644 --- a/codex-rs/app-server-test-client/src/plugin_analytics_mutation_smoke.rs +++ b/codex-rs/app-server-test-client/src/plugin_analytics_mutation_smoke.rs @@ -124,7 +124,6 @@ pub(super) fn run_cleanup( overrides.extend([ "analytics.enabled=false".to_string(), "features.plugins=true".to_string(), - "features.remote_plugin=true".to_string(), ]); let mut client = CodexClient::spawn_stdio(codex_bin, &overrides)?; client.initialize()?; @@ -199,7 +198,6 @@ fn spawn_client( overrides.extend([ "analytics.enabled=true".to_string(), "features.plugins=true".to_string(), - "features.remote_plugin=true".to_string(), ]); let environment = vec![( OsString::from(ANALYTICS_CAPTURE_ENV_VAR), diff --git a/codex-rs/app-server-test-client/src/plugin_analytics_smoke.rs b/codex-rs/app-server-test-client/src/plugin_analytics_smoke.rs index 65704eeb65..aaf54dd2da 100644 --- a/codex-rs/app-server-test-client/src/plugin_analytics_smoke.rs +++ b/codex-rs/app-server-test-client/src/plugin_analytics_smoke.rs @@ -263,7 +263,6 @@ fn smoke_config_overrides(responses_base_url: &str) -> Result> { Ok(vec![ "analytics.enabled=true".to_string(), "features.plugins=true".to_string(), - "features.remote_plugin=true".to_string(), format!("model={}", quoted(MOCK_MODEL_SLUG)?), format!("model_provider={}", quoted(MOCK_PROVIDER_ID)?), format!( diff --git a/codex-rs/app-server-transport/Cargo.toml b/codex-rs/app-server-transport/Cargo.toml index d4c0a83739..af1f0e9603 100644 --- a/codex-rs/app-server-transport/Cargo.toml +++ b/codex-rs/app-server-transport/Cargo.toml @@ -35,8 +35,10 @@ constant_time_eq = { workspace = true } futures = { workspace = true } gethostname = { workspace = true } hmac = { workspace = true } +httpdate = { workspace = true } jsonwebtoken = { workspace = true } owo-colors = { workspace = true, features = ["supports-colors"] } +rand = { workspace = true } serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } sha2 = { workspace = true } diff --git a/codex-rs/app-server-transport/src/transport/remote_control/enroll.rs b/codex-rs/app-server-transport/src/transport/remote_control/enroll.rs index e327503a93..be64072476 100644 --- a/codex-rs/app-server-transport/src/transport/remote_control/enroll.rs +++ b/codex-rs/app-server-transport/src/transport/remote_control/enroll.rs @@ -1,8 +1,4 @@ -use super::auth::RemoteControlConnectionAuth; use super::pairing_unavailable_error; -use super::protocol::EnrollRemoteServerRequest; -use super::protocol::EnrollRemoteServerResponse; -use super::protocol::RefreshRemoteServerRequest; use super::protocol::RemoteControlPairingStatusRequest; use super::protocol::RemoteControlPairingStatusResponse as BackendRemoteControlPairingStatusResponse; use super::protocol::RemoteControlTarget; @@ -14,8 +10,6 @@ use codex_app_server_protocol::RemoteControlPairingStatusResponse; use codex_login::default_client::build_reqwest_client; use codex_state::RemoteControlEnrollmentRecord; use codex_state::StateRuntime; -use serde::Serialize; -use serde::de::DeserializeOwned; use std::io; use std::io::ErrorKind; use time::OffsetDateTime; @@ -23,15 +17,13 @@ use time::format_description::well_known::Rfc3339; use tracing::info; use tracing::warn; -const REMOTE_CONTROL_ENROLL_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30); const REMOTE_CONTROL_PAIRING_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30); const REMOTE_CONTROL_RESPONSE_BODY_MAX_BYTES: usize = 4096; -const REMOTE_CONTROL_SERVER_TOKEN_REFRESH_SKEW_SECS: i64 = 30; +const REMOTE_CONTROL_SERVER_TOKEN_REFRESH_SKEW_SECS: i64 = 5 * 60; const REQUEST_ID_HEADER: &str = "x-request-id"; const OAI_REQUEST_ID_HEADER: &str = "x-oai-request-id"; const CF_RAY_HEADER: &str = "cf-ray"; -pub(super) const REMOTE_CONTROL_INSTALLATION_ID_HEADER: &str = "x-codex-installation-id"; #[derive(Debug, Clone, PartialEq, Eq)] pub(super) struct RemoteControlEnrollment { @@ -42,6 +34,14 @@ pub(super) struct RemoteControlEnrollment { pub(super) server_name: String, pub(super) remote_control_token: Option, pub(super) expires_at: Option, + pub(super) next_refresh_at: Option, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum RemoteControlServerTokenRefreshRequirement { + Required, + Proactive, + NotNeeded, } impl RemoteControlEnrollment { @@ -49,7 +49,9 @@ impl RemoteControlEnrollment { &self, request: StartRemoteControlPairingRequest, ) -> io::Result { - if self.should_refresh_server_token() { + if self.server_token_refresh_requirement() + == RemoteControlServerTokenRefreshRequirement::Required + { return Err(pairing_unavailable_error()); } let remote_control_token = self @@ -142,7 +144,9 @@ impl RemoteControlEnrollment { &self, request: RemoteControlPairingStatusRequest, ) -> io::Result { - if self.should_refresh_server_token() { + if self.server_token_refresh_requirement() + == RemoteControlServerTokenRefreshRequirement::Required + { return Err(pairing_unavailable_error()); } let remote_control_token = self @@ -201,13 +205,35 @@ impl RemoteControlEnrollment { }) } + pub(super) fn server_token_refresh_requirement( + &self, + ) -> RemoteControlServerTokenRefreshRequirement { + self.server_token_refresh_requirement_at(OffsetDateTime::now_utc()) + } + pub(super) fn should_refresh_server_token(&self) -> bool { - self.remote_control_token.is_none() - || self.expires_at.is_none_or(|expires_at| { - expires_at.unix_timestamp() - <= OffsetDateTime::now_utc().unix_timestamp() - + REMOTE_CONTROL_SERVER_TOKEN_REFRESH_SKEW_SECS - }) + self.server_token_refresh_requirement() + != RemoteControlServerTokenRefreshRequirement::NotNeeded + } + + pub(super) fn server_token_refresh_requirement_at( + &self, + now: OffsetDateTime, + ) -> RemoteControlServerTokenRefreshRequirement { + let Some(expires_at) = self.remote_control_token.as_ref().and(self.expires_at) else { + return RemoteControlServerTokenRefreshRequirement::Required; + }; + if expires_at <= now { + return RemoteControlServerTokenRefreshRequirement::Required; + } + if expires_at > now + time::Duration::seconds(REMOTE_CONTROL_SERVER_TOKEN_REFRESH_SKEW_SECS) + || self + .next_refresh_at + .is_some_and(|next_refresh_at| next_refresh_at > now) + { + return RemoteControlServerTokenRefreshRequirement::NotNeeded; + } + RemoteControlServerTokenRefreshRequirement::Proactive } pub(super) fn clear_server_token(&mut self) { @@ -267,6 +293,7 @@ pub(super) async fn load_persisted_remote_control_enrollment( server_name: enrollment.server_name, remote_control_token: None, expires_at: None, + next_refresh_at: None, })) } None => { @@ -398,164 +425,12 @@ pub(crate) fn format_headers(headers: &HeaderMap) -> String { format!("request-id: {request_id_str}, cf-ray: {cf_ray_str}") } -pub(super) async fn enroll_remote_control_server( - remote_control_target: &RemoteControlTarget, - auth: &RemoteControlConnectionAuth, - installation_id: &str, - server_name: &str, -) -> io::Result { - let enroll_url = &remote_control_target.enroll_url; - let request = EnrollRemoteServerRequest { - name: server_name.to_string(), - os: std::env::consts::OS, - arch: std::env::consts::ARCH, - app_server_version: env!("CARGO_PKG_VERSION"), - installation_id: installation_id.to_string(), - }; - let enrollment_response = send_remote_control_server_request::<_, EnrollRemoteServerResponse>( - enroll_url, - auth, - installation_id, - &request, - "enroll", - "server enrollment", - ) - .await?; - let mut enrollment = RemoteControlEnrollment { - remote_control_target: remote_control_target.clone(), - account_id: auth.account_id.clone(), - environment_id: enrollment_response.environment_id, - server_id: enrollment_response.server_id, - server_name: server_name.to_string(), - remote_control_token: None, - expires_at: None, - }; - update_remote_control_server_token( - &mut enrollment, - enroll_url, - enrollment_response.remote_control_token, - enrollment_response.expires_at, - )?; - Ok(enrollment) -} - -pub(super) async fn refresh_remote_control_server( - auth: &RemoteControlConnectionAuth, - installation_id: &str, - enrollment: &mut RemoteControlEnrollment, -) -> io::Result<()> { - let refresh_url = enrollment.remote_control_target.refresh_url.clone(); - let request = RefreshRemoteServerRequest { - server_id: enrollment.server_id.clone(), - installation_id: installation_id.to_string(), - }; - let refreshed = send_remote_control_server_request::<_, EnrollRemoteServerResponse>( - &refresh_url, - auth, - installation_id, - &request, - "refresh", - "server refresh", - ) - .await?; - if refreshed.server_id != enrollment.server_id - || refreshed.environment_id != enrollment.environment_id - { - return Err(io::Error::other(format!( - "remote control server refresh returned mismatched enrollment: expected server_id={}, environment_id={}; got server_id={}, environment_id={}", - enrollment.server_id, - enrollment.environment_id, - refreshed.server_id, - refreshed.environment_id - ))); - } - - update_remote_control_server_token( - enrollment, - &refresh_url, - refreshed.remote_control_token, - refreshed.expires_at, - ) -} - -async fn send_remote_control_server_request( - url: &str, - auth: &RemoteControlConnectionAuth, - installation_id: &str, - request: &Request, - action: &str, - response_kind: &str, -) -> io::Result -where - Request: Serialize, - Response: DeserializeOwned, -{ - let client = build_reqwest_client(); - let auth_headers = auth.request_headers()?; - let response = client - .post(url) - .timeout(REMOTE_CONTROL_ENROLL_TIMEOUT) - .headers(auth_headers) - .header(REMOTE_CONTROL_INSTALLATION_ID_HEADER, installation_id) - .json(request) - .send() - .await - .map_err(|err| { - io::Error::other(format!( - "failed to {action} remote control server at `{url}`: {err}" - )) - })?; - let headers = response.headers().clone(); - let status = response.status(); - let body = response.bytes().await.map_err(|err| { - io::Error::other(format!( - "failed to read remote control {response_kind} response from `{url}`: {err}" - )) - })?; - let body_preview = preview_remote_control_response_body(&body); - if !status.is_success() { - let headers_str = format_headers(&headers); - let error_kind = match status.as_u16() { - 401 | 403 => ErrorKind::PermissionDenied, - 404 => ErrorKind::NotFound, - _ => ErrorKind::Other, - }; - return Err(io::Error::new( - error_kind, - format!( - "remote control {response_kind} failed at `{url}`: HTTP {status}, {headers_str}, body: {body_preview}" - ), - )); - } - - serde_json::from_slice::(&body).map_err(|err| { - let headers_str = format_headers(&headers); - io::Error::other(format!( - "failed to parse remote control {response_kind} response from `{url}`: HTTP {status}, {headers_str}, body: {body_preview}, decode error: {err}" - )) - }) -} - -fn update_remote_control_server_token( - enrollment: &mut RemoteControlEnrollment, - url: &str, - token: String, - expires_at: String, -) -> io::Result<()> { - let expires_at = OffsetDateTime::parse(&expires_at, &Rfc3339).map_err(|err| { - io::Error::other(format!( - "failed to parse remote control server token expiry from `{url}`: {err}" - )) - })?; - enrollment.remote_control_token = Some(token); - enrollment.expires_at = Some(expires_at); - Ok(()) -} - #[cfg(test)] mod tests { use super::*; + use crate::transport::remote_control::auth::RemoteControlConnectionAuth; use crate::transport::remote_control::protocol::normalize_remote_control_url; + use crate::transport::remote_control::server_api::enroll_remote_control_server; use codex_state::StateRuntime; use pretty_assertions::assert_eq; use serde_json::json; @@ -575,28 +450,6 @@ mod tests { .expect("state runtime should initialize") } - #[test] - fn remote_control_enrollment_refreshes_server_token_before_expiry() { - let expires_soon = RemoteControlEnrollment { - remote_control_target: normalize_remote_control_url("http://localhost/backend-api/") - .expect("target should normalize"), - account_id: "account-a".to_string(), - environment_id: "env_first".to_string(), - server_id: "srv_e_first".to_string(), - server_name: "first-server".to_string(), - remote_control_token: Some("expires-soon".to_string()), - expires_at: Some(OffsetDateTime::now_utc() + time::Duration::seconds(29)), - }; - let expires_later = RemoteControlEnrollment { - expires_at: Some(OffsetDateTime::now_utc() + time::Duration::seconds(31)), - remote_control_token: Some("expires-later".to_string()), - ..expires_soon.clone() - }; - - assert!(expires_soon.should_refresh_server_token()); - assert!(!expires_later.should_refresh_server_token()); - } - #[test] fn preview_remote_control_response_body_redacts_server_token() { assert_eq!( @@ -630,6 +483,7 @@ mod tests { server_name: "first-server".to_string(), remote_control_token: None, expires_at: None, + next_refresh_at: None, }; let second_enrollment = RemoteControlEnrollment { remote_control_target: second_target.clone(), @@ -639,6 +493,7 @@ mod tests { server_name: "second-server".to_string(), remote_control_token: None, expires_at: None, + next_refresh_at: None, }; update_persisted_remote_control_enrollment( @@ -714,6 +569,7 @@ mod tests { server_name: "first-server".to_string(), remote_control_token: None, expires_at: None, + next_refresh_at: None, }; let second_enrollment = RemoteControlEnrollment { remote_control_target: second_target.clone(), @@ -723,6 +579,7 @@ mod tests { server_name: "second-server".to_string(), remote_control_token: None, expires_at: None, + next_refresh_at: None, }; update_persisted_remote_control_enrollment( diff --git a/codex-rs/app-server-transport/src/transport/remote_control/mod.rs b/codex-rs/app-server-transport/src/transport/remote_control/mod.rs index d0e934d90b..7422c92c42 100644 --- a/codex-rs/app-server-transport/src/transport/remote_control/mod.rs +++ b/codex-rs/app-server-transport/src/transport/remote_control/mod.rs @@ -5,6 +5,7 @@ mod desired_state; mod enroll; mod protocol; mod segment; +mod server_api; mod websocket; use self::auth::load_remote_control_auth; @@ -12,10 +13,10 @@ use self::auth::recover_remote_control_auth; use self::desired_state::RemoteControlDesiredState; use self::desired_state::acquire_persistence_lock; use self::enroll::RemoteControlEnrollment; -use self::enroll::enroll_remote_control_server; use self::enroll::load_persisted_remote_control_enrollment; -use self::enroll::refresh_remote_control_server; use self::enroll::update_persisted_remote_control_enrollment; +use self::server_api::enroll_remote_control_server; +use self::server_api::refresh_remote_control_server; use crate::transport::remote_control::websocket::RemoteControlChannels; use crate::transport::remote_control::websocket::RemoteControlStatusPublisher; use crate::transport::remote_control::websocket::RemoteControlWebsocket; @@ -825,27 +826,39 @@ async fn refresh_pairing_enrollment( installation_id: &str, enrollment: &mut RemoteControlEnrollment, ) -> io::Result<()> { - if let Err(err) = refresh_remote_control_server(auth, installation_id, enrollment).await { - if err.kind() != io::ErrorKind::PermissionDenied { - return Err(err); - } + let mut refresh_result = refresh_remote_control_server(auth, installation_id, enrollment).await; + if refresh_result + .as_ref() + .is_err_and(|err| err.kind() == io::ErrorKind::PermissionDenied) + { let mut auth_recovery = auth_manager.unauthorized_recovery(); let mut auth_change_rx = auth_manager.auth_change_receiver(); - if !recover_remote_control_auth(&mut auth_recovery, &mut auth_change_rx).await { - return Err(err); + if recover_remote_control_auth(&mut auth_recovery, &mut auth_change_rx).await { + match load_remote_control_auth(auth_manager).await { + Ok(recovered_auth) if recovered_auth.account_id == enrollment.account_id => { + *auth = recovered_auth; + refresh_result = + refresh_remote_control_server(auth, installation_id, enrollment).await; + } + Ok(_) | Err(_) => { + enrollment.clear_server_token(); + refresh_result = Err(pairing_unavailable_error()); + } + } + } else { + enrollment.clear_server_token(); } - *auth = load_remote_control_auth(auth_manager) - .await - .map_err(|_| pairing_unavailable_error())?; - if auth.account_id != enrollment.account_id { - return Err(pairing_unavailable_error()); - } - refresh_remote_control_server(auth, installation_id, enrollment).await? } - if replace_current_enrollment(current_enrollment, enrollment) { - Ok(()) - } else { + if refresh_result + .as_ref() + .is_err_and(|err| err.kind() == io::ErrorKind::PermissionDenied) + { + enrollment.clear_server_token(); + } + if !replace_current_enrollment(current_enrollment, enrollment) { Err(pairing_unavailable_error()) + } else { + refresh_result } } diff --git a/codex-rs/app-server-transport/src/transport/remote_control/server_api.rs b/codex-rs/app-server-transport/src/transport/remote_control/server_api.rs new file mode 100644 index 0000000000..def38907c7 --- /dev/null +++ b/codex-rs/app-server-transport/src/transport/remote_control/server_api.rs @@ -0,0 +1,339 @@ +use super::auth::RemoteControlConnectionAuth; +use super::enroll::RemoteControlEnrollment; +use super::enroll::RemoteControlServerTokenRefreshRequirement; +use super::enroll::format_headers; +use super::enroll::preview_remote_control_response_body; +use super::protocol::EnrollRemoteServerRequest; +use super::protocol::EnrollRemoteServerResponse; +use super::protocol::RefreshRemoteServerRequest; +use super::protocol::RemoteControlTarget; +use axum::http::HeaderMap; +use axum::http::StatusCode; +use codex_login::default_client::build_reqwest_client; +use rand::Rng; +use serde::Serialize; +use serde::de::DeserializeOwned; +use std::fmt; +use std::io; +use std::io::ErrorKind; +use std::time::Duration; +use time::OffsetDateTime; +use time::format_description::well_known::Rfc3339; +use tracing::warn; + +const REMOTE_CONTROL_ENROLL_TIMEOUT: Duration = Duration::from_secs(30); +const REMOTE_CONTROL_SERVER_TOKEN_REFRESH_BACKOFF_MIN_SECS: u64 = 24; +const REMOTE_CONTROL_SERVER_TOKEN_REFRESH_BACKOFF_MAX_SECS: u64 = 36; + +pub(super) const REMOTE_CONTROL_INSTALLATION_ID_HEADER: &str = "x-codex-installation-id"; + +#[derive(Debug)] +struct RemoteControlServerRequestError { + message: String, + status: Option, + retry_at: Option, +} + +impl RemoteControlServerRequestError { + fn io_error( + message: String, + status: Option, + retry_at: Option, + timed_out: bool, + ) -> io::Error { + let kind = match status { + Some(StatusCode::UNAUTHORIZED | StatusCode::FORBIDDEN) => ErrorKind::PermissionDenied, + Some(StatusCode::NOT_FOUND) => ErrorKind::NotFound, + Some(status) if timed_out && !status.is_client_error() => ErrorKind::TimedOut, + None if timed_out => ErrorKind::TimedOut, + Some(_) | None => ErrorKind::Other, + }; + io::Error::new( + kind, + Self { + message, + status, + retry_at, + }, + ) + } + + fn is_transient(&self, kind: ErrorKind) -> bool { + kind == ErrorKind::TimedOut + || self.status.is_none() + || self.status.is_some_and(|status| { + status == StatusCode::TOO_MANY_REQUESTS || status.is_server_error() + }) + } +} + +impl fmt::Display for RemoteControlServerRequestError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(&self.message) + } +} + +impl std::error::Error for RemoteControlServerRequestError {} + +pub(super) async fn enroll_remote_control_server( + remote_control_target: &RemoteControlTarget, + auth: &RemoteControlConnectionAuth, + installation_id: &str, + server_name: &str, +) -> io::Result { + let enroll_url = &remote_control_target.enroll_url; + let request = EnrollRemoteServerRequest { + name: server_name.to_string(), + os: std::env::consts::OS, + arch: std::env::consts::ARCH, + app_server_version: env!("CARGO_PKG_VERSION"), + installation_id: installation_id.to_string(), + }; + let enrollment_response = send_remote_control_server_request::<_, EnrollRemoteServerResponse>( + enroll_url, + auth, + installation_id, + &request, + "enroll", + "server enrollment", + REMOTE_CONTROL_ENROLL_TIMEOUT, + ) + .await?; + let mut enrollment = RemoteControlEnrollment { + remote_control_target: remote_control_target.clone(), + account_id: auth.account_id.clone(), + environment_id: enrollment_response.environment_id, + server_id: enrollment_response.server_id, + server_name: server_name.to_string(), + remote_control_token: None, + expires_at: None, + next_refresh_at: None, + }; + update_remote_control_server_token( + &mut enrollment, + enroll_url, + enrollment_response.remote_control_token, + enrollment_response.expires_at, + )?; + Ok(enrollment) +} + +pub(super) async fn refresh_remote_control_server( + auth: &RemoteControlConnectionAuth, + installation_id: &str, + enrollment: &mut RemoteControlEnrollment, +) -> io::Result<()> { + let now = OffsetDateTime::now_utc(); + let refresh_requirement = enrollment.server_token_refresh_requirement_at(now); + if refresh_requirement == RemoteControlServerTokenRefreshRequirement::NotNeeded { + return Ok(()); + } + if refresh_requirement == RemoteControlServerTokenRefreshRequirement::Required + && let Some(next_refresh_at) = enrollment.next_refresh_at + && next_refresh_at > now + { + return Err(io::Error::new( + ErrorKind::WouldBlock, + format!("remote control server token refresh deferred until {next_refresh_at}"), + )); + } + let refresh_url = enrollment.remote_control_target.refresh_url.clone(); + let request = RefreshRemoteServerRequest { + server_id: enrollment.server_id.clone(), + installation_id: installation_id.to_string(), + }; + let refreshed = match send_remote_control_server_request::<_, EnrollRemoteServerResponse>( + &refresh_url, + auth, + installation_id, + &request, + "refresh", + "server refresh", + REMOTE_CONTROL_ENROLL_TIMEOUT, + ) + .await + { + Ok(refreshed) => refreshed, + Err(err) => { + let Some(refresh_error) = remote_control_server_request_error(&err) else { + return Err(err); + }; + if !refresh_error.is_transient(err.kind()) { + return Err(err); + } + let now = OffsetDateTime::now_utc(); + let refresh_is_required = enrollment.server_token_refresh_requirement_at(now) + == RemoteControlServerTokenRefreshRequirement::Required; + let (refresh_delay, next_refresh_at) = refresh_deferral(refresh_error.retry_at, now); + enrollment.next_refresh_at = Some(next_refresh_at); + if refresh_is_required { + warn!( + refresh_url, + server_id = %enrollment.server_id, + environment_id = %enrollment.environment_id, + error = %err, + ?refresh_delay, + %next_refresh_at, + "required remote control server token refresh failed; deferring next attempt" + ); + return Err(err); + } + warn!( + refresh_url, + server_id = %enrollment.server_id, + environment_id = %enrollment.environment_id, + error = %err, + ?refresh_delay, + %next_refresh_at, + "proactive remote control server token refresh failed; continuing with valid token" + ); + return Ok(()); + } + }; + if refreshed.server_id != enrollment.server_id + || refreshed.environment_id != enrollment.environment_id + { + return Err(io::Error::other(format!( + "remote control server refresh returned mismatched enrollment: expected server_id={}, environment_id={}; got server_id={}, environment_id={}", + enrollment.server_id, + enrollment.environment_id, + refreshed.server_id, + refreshed.environment_id + ))); + } + + update_remote_control_server_token( + enrollment, + &refresh_url, + refreshed.remote_control_token, + refreshed.expires_at, + ) +} + +async fn send_remote_control_server_request( + url: &str, + auth: &RemoteControlConnectionAuth, + installation_id: &str, + request: &Request, + action: &str, + response_kind: &str, + timeout: Duration, +) -> io::Result +where + Request: Serialize, + Response: DeserializeOwned, +{ + let client = build_reqwest_client(); + let auth_headers = auth.request_headers()?; + let response = client + .post(url) + .timeout(timeout) + .headers(auth_headers) + .header(REMOTE_CONTROL_INSTALLATION_ID_HEADER, installation_id) + .json(request) + .send() + .await + .map_err(|err| { + let timed_out = err.is_timeout(); + RemoteControlServerRequestError::io_error( + format!("failed to {action} remote control server at `{url}`: {err}"), + /*status*/ None, + /*retry_at*/ None, + timed_out, + ) + })?; + let headers = response.headers().clone(); + let status = response.status(); + let retry_at = parse_retry_after(&headers, OffsetDateTime::now_utc()); + let body = response.bytes().await.map_err(|err| { + let timed_out = err.is_timeout(); + RemoteControlServerRequestError::io_error( + format!("failed to read remote control {response_kind} response from `{url}`: {err}"), + Some(status), + retry_at, + timed_out, + ) + })?; + let body_preview = preview_remote_control_response_body(&body); + if !status.is_success() { + let headers_str = format_headers(&headers); + return Err(RemoteControlServerRequestError::io_error( + format!( + "remote control {response_kind} failed at `{url}`: HTTP {status}, {headers_str}, body: {body_preview}" + ), + Some(status), + retry_at, + /*timed_out*/ false, + )); + } + + serde_json::from_slice::(&body).map_err(|err| { + let headers_str = format_headers(&headers); + io::Error::other(format!( + "failed to parse remote control {response_kind} response from `{url}`: HTTP {status}, {headers_str}, body: {body_preview}, decode error: {err}" + )) + }) +} + +fn update_remote_control_server_token( + enrollment: &mut RemoteControlEnrollment, + url: &str, + token: String, + expires_at: String, +) -> io::Result<()> { + let expires_at = OffsetDateTime::parse(&expires_at, &Rfc3339).map_err(|err| { + io::Error::other(format!( + "failed to parse remote control server token expiry from `{url}`: {err}" + )) + })?; + enrollment.remote_control_token = Some(token); + enrollment.expires_at = Some(expires_at); + enrollment.next_refresh_at = None; + Ok(()) +} + +fn remote_control_server_request_error( + err: &io::Error, +) -> Option<&RemoteControlServerRequestError> { + err.get_ref()?.downcast_ref() +} + +fn parse_retry_after(headers: &HeaderMap, received_at: OffsetDateTime) -> Option { + let retry_after = headers + .get(axum::http::header::RETRY_AFTER)? + .to_str() + .ok()?; + let retry_at = if let Ok(seconds) = retry_after.parse::() { + let seconds = i64::try_from(seconds).ok()?; + received_at.checked_add(time::Duration::seconds(seconds))? + } else { + OffsetDateTime::from(httpdate::parse_http_date(retry_after).ok()?) + }; + (retry_at > received_at).then_some(retry_at) +} + +fn refresh_deferral( + retry_at: Option, + now: OffsetDateTime, +) -> (Duration, OffsetDateTime) { + if let Some(retry_at) = retry_at + && let Ok(delay) = Duration::try_from(retry_at - now) + && !delay.is_zero() + { + return (delay, retry_at); + } + let delay = remote_control_server_token_refresh_backoff(); + let next_refresh_at = now + time::Duration::seconds(delay.as_secs() as i64); + (delay, next_refresh_at) +} + +fn remote_control_server_token_refresh_backoff() -> Duration { + Duration::from_secs(rand::rng().random_range( + REMOTE_CONTROL_SERVER_TOKEN_REFRESH_BACKOFF_MIN_SECS + ..=REMOTE_CONTROL_SERVER_TOKEN_REFRESH_BACKOFF_MAX_SECS, + )) +} + +#[cfg(test)] +#[path = "server_api_tests.rs"] +mod tests; diff --git a/codex-rs/app-server-transport/src/transport/remote_control/server_api_tests.rs b/codex-rs/app-server-transport/src/transport/remote_control/server_api_tests.rs new file mode 100644 index 0000000000..786dc65a90 --- /dev/null +++ b/codex-rs/app-server-transport/src/transport/remote_control/server_api_tests.rs @@ -0,0 +1,284 @@ +use super::*; +use crate::transport::remote_control::protocol::normalize_remote_control_url; +use pretty_assertions::assert_eq; +use serde_json::json; +use std::time::SystemTime; +use tokio::io::AsyncWriteExt; +use tokio::net::TcpListener; +use tokio::sync::oneshot; + +const TEST_REQUEST_TIMEOUT: Duration = Duration::from_millis(100); + +fn auth() -> RemoteControlConnectionAuth { + RemoteControlConnectionAuth { + auth_provider: codex_model_provider::unauthenticated_auth_provider(), + account_id: "account-a".to_string(), + } +} + +fn assert_transient_timeout(err: &io::Error, expected_status: Option) { + let request_error = remote_control_server_request_error(err) + .expect("request error should preserve refresh metadata"); + assert_eq!( + ( + err.kind(), + request_error.status, + request_error.is_transient(err.kind()), + ), + (ErrorKind::TimedOut, expected_status, true) + ); +} + +async fn timed_out_request(partial_response: Option<&'static [u8]>) -> io::Error { + let listener = TcpListener::bind("127.0.0.1:0") + .await + .expect("listener should bind"); + let url = format!( + "http://{}/backend-api/wham/remote/control/server/refresh", + listener + .local_addr() + .expect("listener should have a local address") + ); + let (request_done_tx, request_done_rx) = oneshot::channel(); + let server_task = tokio::spawn(async move { + let (mut stream, _) = listener.accept().await.expect("request should connect"); + if let Some(partial_response) = partial_response { + stream + .write_all(partial_response) + .await + .expect("partial response should write"); + } + request_done_rx + .await + .expect("test should report request completion"); + }); + + let err = send_remote_control_server_request::<_, serde_json::Value>( + &url, + &auth(), + "installation-id", + &json!({"server_id": "server-id"}), + "refresh", + "server refresh", + TEST_REQUEST_TIMEOUT, + ) + .await + .expect_err("incomplete response should time out"); + request_done_tx + .send(()) + .expect("server should wait for request completion"); + server_task.await.expect("server task should finish"); + err +} + +fn enrollment(now: OffsetDateTime) -> RemoteControlEnrollment { + RemoteControlEnrollment { + remote_control_target: normalize_remote_control_url("http://localhost/backend-api/") + .expect("target should normalize"), + account_id: "account-a".to_string(), + environment_id: "env_first".to_string(), + server_id: "srv_e_first".to_string(), + server_name: "first-server".to_string(), + remote_control_token: Some("token".to_string()), + expires_at: Some(now + time::Duration::seconds(300)), + next_refresh_at: None, + } +} + +#[test] +fn remote_control_enrollment_classifies_server_token_refresh_requirement() { + let now = + OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("test timestamp should parse"); + let enrollment = enrollment(now); + let cases = [ + ( + enrollment.clone(), + RemoteControlServerTokenRefreshRequirement::Proactive, + ), + ( + RemoteControlEnrollment { + expires_at: Some(now + time::Duration::seconds(301)), + ..enrollment.clone() + }, + RemoteControlServerTokenRefreshRequirement::NotNeeded, + ), + ( + RemoteControlEnrollment { + next_refresh_at: Some(now + time::Duration::seconds(30)), + ..enrollment.clone() + }, + RemoteControlServerTokenRefreshRequirement::NotNeeded, + ), + ( + RemoteControlEnrollment { + next_refresh_at: Some(now), + ..enrollment.clone() + }, + RemoteControlServerTokenRefreshRequirement::Proactive, + ), + ( + RemoteControlEnrollment { + remote_control_token: None, + ..enrollment.clone() + }, + RemoteControlServerTokenRefreshRequirement::Required, + ), + ( + RemoteControlEnrollment { + expires_at: None, + ..enrollment.clone() + }, + RemoteControlServerTokenRefreshRequirement::Required, + ), + ( + RemoteControlEnrollment { + expires_at: Some(now), + next_refresh_at: Some(now + time::Duration::hours(1)), + ..enrollment + }, + RemoteControlServerTokenRefreshRequirement::Required, + ), + ]; + + for (enrollment, expected) in cases { + assert_eq!( + enrollment.server_token_refresh_requirement_at(now), + expected + ); + } +} + +#[test] +fn remote_control_server_request_error_classifies_status_before_timeout() { + let cases = [ + (None, true, ErrorKind::TimedOut, true), + (Some(StatusCode::OK), true, ErrorKind::TimedOut, true), + ( + Some(StatusCode::TOO_MANY_REQUESTS), + false, + ErrorKind::Other, + true, + ), + (Some(StatusCode::BAD_GATEWAY), false, ErrorKind::Other, true), + ( + Some(StatusCode::UNAUTHORIZED), + true, + ErrorKind::PermissionDenied, + false, + ), + ( + Some(StatusCode::FORBIDDEN), + true, + ErrorKind::PermissionDenied, + false, + ), + ( + Some(StatusCode::NOT_FOUND), + true, + ErrorKind::NotFound, + false, + ), + (Some(StatusCode::BAD_REQUEST), true, ErrorKind::Other, false), + (None, false, ErrorKind::Other, true), + ]; + + for (status, timed_out, expected_kind, expected_transient) in cases { + let err = RemoteControlServerRequestError::io_error( + String::new(), + status, + /*retry_at*/ None, + timed_out, + ); + let request_error = remote_control_server_request_error(&err) + .expect("request error should preserve refresh metadata"); + assert_eq!( + (err.kind(), request_error.is_transient(err.kind())), + (expected_kind, expected_transient) + ); + } +} + +#[tokio::test] +async fn request_timeout_before_response_headers_is_transient() { + let err = timed_out_request(/*partial_response*/ None).await; + assert_transient_timeout(&err, /*expected_status*/ None); +} + +#[tokio::test] +async fn response_body_timeout_is_transient() { + let err = timed_out_request(Some(b"HTTP/1.1 200 OK\r\nContent-Length: 20\r\n\r\n{")).await; + assert_transient_timeout(&err, Some(StatusCode::OK)); +} + +#[test] +fn retry_after_supports_delta_seconds_and_http_dates() { + let now = + OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("test timestamp should parse"); + let mut headers = HeaderMap::new(); + headers.insert( + axum::http::header::RETRY_AFTER, + axum::http::HeaderValue::from_static("120"), + ); + assert_eq!( + parse_retry_after(&headers, now), + Some(now + time::Duration::seconds(120)) + ); + + let retry_at = now + time::Duration::seconds(90); + let retry_at_system = SystemTime::UNIX_EPOCH + Duration::from_secs(1_700_000_090); + headers.insert( + axum::http::header::RETRY_AFTER, + httpdate::fmt_http_date(retry_at_system) + .parse() + .expect("HTTP date should be a valid header value"), + ); + assert_eq!(parse_retry_after(&headers, now), Some(retry_at)); +} + +#[test] +fn invalid_or_expired_retry_after_uses_bounded_fallback() { + let now = + OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("test timestamp should parse"); + let mut headers = HeaderMap::new(); + headers.insert( + axum::http::header::RETRY_AFTER, + axum::http::HeaderValue::from_static("invalid"), + ); + assert_eq!(parse_retry_after(&headers, now), None); + + headers.insert( + axum::http::header::RETRY_AFTER, + httpdate::fmt_http_date(SystemTime::UNIX_EPOCH + Duration::from_secs(1_699_999_999)) + .parse() + .expect("HTTP date should be a valid header value"), + ); + assert_eq!(parse_retry_after(&headers, now), None); + + let expired_while_reading_body = Some(now + time::Duration::seconds(1)); + for retry_at in [None, expired_while_reading_body] { + let deferred_at = now + time::Duration::seconds(2); + let (delay, next_refresh_at) = refresh_deferral(retry_at, deferred_at); + assert!( + (Duration::from_secs(REMOTE_CONTROL_SERVER_TOKEN_REFRESH_BACKOFF_MIN_SECS) + ..=Duration::from_secs(REMOTE_CONTROL_SERVER_TOKEN_REFRESH_BACKOFF_MAX_SECS,)) + .contains(&delay) + ); + assert_eq!( + next_refresh_at, + deferred_at + time::Duration::seconds(delay.as_secs() as i64) + ); + } +} + +#[test] +fn http_date_retry_after_preserves_absolute_deadline() { + let received_at = + OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("test timestamp should parse"); + let retry_at = received_at + time::Duration::seconds(120); + let body_read_at = received_at + time::Duration::seconds(30); + + assert_eq!( + refresh_deferral(Some(retry_at), body_read_at), + (Duration::from_secs(90), retry_at) + ); +} diff --git a/codex-rs/app-server-transport/src/transport/remote_control/tests.rs b/codex-rs/app-server-transport/src/transport/remote_control/tests.rs index 3a2d844654..465c6b13ec 100644 --- a/codex-rs/app-server-transport/src/transport/remote_control/tests.rs +++ b/codex-rs/app-server-transport/src/transport/remote_control/tests.rs @@ -1,5 +1,4 @@ use super::auth::REMOTE_CONTROL_ACCOUNT_ID_HEADER; -use super::enroll::REMOTE_CONTROL_INSTALLATION_ID_HEADER; use super::enroll::RemoteControlEnrollment; use super::enroll::load_persisted_remote_control_enrollment; use super::enroll::update_persisted_remote_control_enrollment; @@ -8,6 +7,7 @@ use super::protocol::ClientEvent; use super::protocol::ClientId; use super::protocol::StreamId; use super::protocol::normalize_remote_control_url; +use super::server_api::REMOTE_CONTROL_INSTALLATION_ID_HEADER; use super::websocket::REMOTE_CONTROL_PROTOCOL_VERSION; use super::websocket::RemoteControlWebsocket; use super::websocket::RemoteControlWebsocketConfig; @@ -372,7 +372,7 @@ fn test_server_name() -> String { gethostname().to_string_lossy().trim().to_string() } -fn remote_control_handle_with_current_enrollment( +pub(super) fn remote_control_handle_with_current_enrollment( remote_control_url: &str, auth_manager: Arc, ) -> RemoteControlHandle { @@ -400,6 +400,7 @@ fn remote_control_handle_with_current_enrollment( OffsetDateTime::from_unix_timestamp(33_336_362_096) .expect("future timestamp should parse"), ), + next_refresh_at: None, }, ))); RemoteControlHandle { @@ -1682,6 +1683,7 @@ async fn remote_control_http_mode_refreshes_persisted_enrollment_before_connecti server_name: "persisted-server".to_string(), remote_control_token: None, expires_at: None, + next_refresh_at: None, }; update_persisted_remote_control_enrollment( Some(state_db.as_ref()), @@ -1802,6 +1804,7 @@ async fn remote_control_stdio_mode_waits_for_client_name_before_connecting() { server_name: "persisted-server".to_string(), remote_control_token: None, expires_at: None, + next_refresh_at: None, }; update_persisted_remote_control_enrollment( Some(state_db.as_ref()), @@ -1899,6 +1902,7 @@ async fn remote_control_waits_for_account_id_before_enrolling() { server_name: expected_server_name, remote_control_token: None, expires_at: None, + next_refresh_at: None, }; let (transport_event_tx, _transport_event_rx) = @@ -1995,6 +1999,7 @@ async fn persisted_enable_does_not_follow_auth_to_an_account_without_a_preferenc server_name: "server-a".to_string(), remote_control_token: None, expires_at: None, + next_refresh_at: None, }; update_persisted_remote_control_enrollment( Some(state_db.as_ref()), @@ -2102,6 +2107,7 @@ async fn remote_control_http_mode_reenrolls_when_refresh_reports_stale_enrollmen server_name: "stale-server".to_string(), remote_control_token: None, expires_at: None, + next_refresh_at: None, }; let refreshed_enrollment = RemoteControlEnrollment { remote_control_target: remote_control_target.clone(), @@ -2111,6 +2117,7 @@ async fn remote_control_http_mode_reenrolls_when_refresh_reports_stale_enrollmen server_name: expected_server_name, remote_control_token: None, expires_at: None, + next_refresh_at: None, }; update_persisted_remote_control_enrollment( Some(state_db.as_ref()), @@ -2225,6 +2232,7 @@ async fn remote_control_http_mode_reenrolls_after_explicit_missing_server_404() server_name: "stale-server".to_string(), remote_control_token: None, expires_at: None, + next_refresh_at: None, }; let refreshed_enrollment = RemoteControlEnrollment { remote_control_target: remote_control_target.clone(), @@ -2234,6 +2242,7 @@ async fn remote_control_http_mode_reenrolls_after_explicit_missing_server_404() server_name: expected_server_name, remote_control_token: None, expires_at: None, + next_refresh_at: None, }; update_persisted_remote_control_enrollment( Some(state_db.as_ref()), @@ -2371,6 +2380,7 @@ async fn remote_control_http_mode_preserves_stale_enrollment_when_reenrollment_f server_name: test_server_name(), remote_control_token: None, expires_at: None, + next_refresh_at: None, }; update_persisted_remote_control_enrollment( Some(state_db.as_ref()), @@ -2421,6 +2431,7 @@ async fn remote_control_http_mode_preserves_stale_enrollment_when_reenrollment_f retry_refresh_request.request_line, "POST /backend-api/wham/remote/control/server/refresh HTTP/1.1" ); + let refresh_failed_at = OffsetDateTime::now_utc(); respond_with_status( retry_refresh_request.stream, "500 Internal Server Error", @@ -2428,9 +2439,26 @@ async fn remote_control_http_mode_preserves_stale_enrollment_when_reenrollment_f ) .await; + let current_enrollment = remote_handle + .current_enrollment + .lock() + .await + .clone() + .expect("stale enrollment should remain available"); + let next_refresh_at = current_enrollment + .next_refresh_at + .expect("required refresh failure should set a retry deadline"); + assert!( + (refresh_failed_at + time::Duration::seconds(24) + ..=OffsetDateTime::now_utc() + time::Duration::seconds(36)) + .contains(&next_refresh_at) + ); assert_eq!( - *remote_handle.current_enrollment.lock().await, - Some(stale_enrollment.clone()) + current_enrollment, + RemoteControlEnrollment { + next_refresh_at: Some(next_refresh_at), + ..stale_enrollment.clone() + } ); assert_eq!( state_db @@ -2474,6 +2502,7 @@ async fn remote_control_http_mode_preserves_enrollment_after_generic_websocket_4 server_name: "stale-server".to_string(), remote_control_token: None, expires_at: None, + next_refresh_at: None, }; update_persisted_remote_control_enrollment( Some(state_db.as_ref()), diff --git a/codex-rs/app-server-transport/src/transport/remote_control/tests/pairing_tests.rs b/codex-rs/app-server-transport/src/transport/remote_control/tests/pairing_tests.rs index 380f13984e..6f8b9878ea 100644 --- a/codex-rs/app-server-transport/src/transport/remote_control/tests/pairing_tests.rs +++ b/codex-rs/app-server-transport/src/transport/remote_control/tests/pairing_tests.rs @@ -21,6 +21,69 @@ fn remote_control_enrollment( OffsetDateTime::from_unix_timestamp(33_336_362_096) .expect("future timestamp should parse"), ), + next_refresh_at: None, + } +} + +async fn auth_manager_with_replacement( + codex_home: &TempDir, + replacement_account_id: &str, +) -> Arc { + let mut stale_auth = remote_control_auth_dot_json(Some("account_id")); + stale_auth + .tokens + .as_mut() + .expect("stale auth should include tokens") + .access_token = "stale-token".to_string(); + save_auth( + codex_home.path(), + &stale_auth, + AuthCredentialsStoreMode::File, + AuthKeyringBackendKind::default(), + ) + .expect("stale auth should save"); + let auth_manager = AuthManager::shared( + codex_home.path().to_path_buf(), + /*enable_codex_api_key_env*/ false, + AuthCredentialsStoreMode::File, + /*forced_chatgpt_workspace_id*/ None, + /*chatgpt_base_url*/ None, + AuthKeyringBackendKind::default(), + /*auth_route_config*/ None, + ) + .await; + let mut replacement_auth = remote_control_auth_dot_json(Some(replacement_account_id)); + replacement_auth + .tokens + .as_mut() + .expect("replacement auth should include tokens") + .access_token = "fresh-token".to_string(); + save_auth( + codex_home.path(), + &replacement_auth, + AuthCredentialsStoreMode::File, + AuthKeyringBackendKind::default(), + ) + .expect("replacement auth should save"); + auth_manager +} + +fn pairing_response_json(server_id: &str, environment_id: &str) -> serde_json::Value { + json!({ + "pairing_code": "pairing-code", + "manual_pairing_code": "ABCD-EFGH", + "server_id": server_id, + "environment_id": environment_id, + "expires_at": "3026-05-22T12:34:56Z", + }) +} + +fn pairing_response(environment_id: &str) -> RemoteControlPairingStartResponse { + RemoteControlPairingStartResponse { + pairing_code: "pairing-code".to_string(), + manual_pairing_code: Some("ABCD-EFGH".to_string()), + environment_id: environment_id.to_string(), + expires_at: 33_336_362_096, } } @@ -147,13 +210,7 @@ async fn remote_control_handle_starts_pairing_before_websocket_connects() { ); respond_with_json( pairing_request.stream, - json!({ - "pairing_code": "pairing-code", - "manual_pairing_code": "ABCD-EFGH", - "server_id": "srv_e_test", - "environment_id": "env_test", - "expires_at": "3026-05-22T12:34:56Z", - }), + pairing_response_json("srv_e_test", "env_test"), ) .await; }); @@ -178,15 +235,139 @@ async fn remote_control_handle_starts_pairing_before_websocket_connects() { .expect("pairing should use the current server before websocket connect"); server_task.await.expect("server task should finish"); - assert_eq!( - response, - RemoteControlPairingStartResponse { - pairing_code: "pairing-code".to_string(), - manual_pairing_code: Some("ABCD-EFGH".to_string()), - environment_id: "env_test".to_string(), - expires_at: 33_336_362_096, - } + assert_eq!(response, pairing_response("env_test")); +} + +#[tokio::test] +async fn proactive_refresh_rate_limit_uses_valid_token_for_pairing() { + let listener = TcpListener::bind("127.0.0.1:0") + .await + .expect("listener should bind"); + let remote_control_url = remote_control_url_for_listener(&listener); + let server_task = tokio::spawn(async move { + let refresh_request = accept_http_request(&listener).await; + assert_eq!( + refresh_request.request_line, + "POST /backend-api/wham/remote/control/server/refresh HTTP/1.1" + ); + respond_with_status_and_headers( + refresh_request.stream, + "429 Too Many Requests", + &[], + "rate limited", + ) + .await; + + let pairing_request = accept_http_request(&listener).await; + assert_eq!( + pairing_request.request_line, + "POST /backend-api/wham/remote/control/server/pair HTTP/1.1" + ); + assert_eq!( + pairing_request.headers.get("authorization"), + Some(&format!("Bearer {TEST_REMOTE_CONTROL_SERVER_TOKEN}")) + ); + respond_with_json( + pairing_request.stream, + pairing_response_json("srv_e_test", "env_test"), + ) + .await; + }); + let remote_handle = remote_control_handle_with_current_enrollment( + &remote_control_url, + remote_control_auth_manager(), ); + remote_handle + .current_enrollment + .lock() + .await + .as_mut() + .expect("current enrollment should exist") + .expires_at = Some(OffsetDateTime::now_utc() + time::Duration::minutes(4)); + + let response = remote_handle + .start_pairing( + RemoteControlPairingStartParams::default(), + /*app_server_client_name*/ None, + ) + .await + .expect("valid token should allow pairing after proactive refresh failure"); + server_task.await.expect("server task should finish"); + + assert_eq!(response, pairing_response("env_test")); + assert!( + remote_handle + .current_enrollment + .snapshot() + .and_then(|enrollment| enrollment.next_refresh_at) + .is_some() + ); +} + +#[tokio::test] +async fn required_refresh_deadline_blocks_pairing_without_request() { + let listener = TcpListener::bind("127.0.0.1:0") + .await + .expect("listener should bind"); + let remote_control_url = remote_control_url_for_listener(&listener); + let server_task = tokio::spawn(async move { + let refresh_request = accept_http_request(&listener).await; + assert_eq!( + refresh_request.request_line, + "POST /backend-api/wham/remote/control/server/refresh HTTP/1.1" + ); + respond_with_status_and_headers( + refresh_request.stream, + "502 Bad Gateway", + &[("retry-after", "120")], + "upstream unavailable", + ) + .await; + listener + }); + let remote_handle = remote_control_handle_with_current_enrollment( + &remote_control_url, + remote_control_auth_manager(), + ); + remote_handle + .current_enrollment + .lock() + .await + .as_mut() + .expect("current enrollment should exist") + .expires_at = Some(OffsetDateTime::now_utc() - time::Duration::seconds(1)); + + let refresh_err = remote_handle + .start_pairing( + RemoteControlPairingStartParams::default(), + /*app_server_client_name*/ None, + ) + .await + .expect_err("required refresh failure should block pairing"); + let listener = server_task.await.expect("server task should finish"); + let next_refresh_at = remote_handle + .current_enrollment + .snapshot() + .and_then(|enrollment| enrollment.next_refresh_at) + .expect("required pairing refresh should preserve the retry deadline"); + let deferred_err = remote_handle + .start_pairing( + RemoteControlPairingStartParams::default(), + /*app_server_client_name*/ None, + ) + .await + .expect_err("required refresh deadline should block pairing"); + + assert!(refresh_err.to_string().contains("HTTP 502 Bad Gateway")); + assert_eq!(deferred_err.kind(), io::ErrorKind::WouldBlock); + assert!( + deferred_err + .to_string() + .contains(&next_refresh_at.to_string()) + ); + timeout(Duration::from_millis(100), listener.accept()) + .await + .expect_err("pairing should not issue a request before the refresh deadline"); } #[tokio::test] @@ -423,13 +604,7 @@ async fn remote_control_handle_refreshes_after_pairing_auth_failure() { ); respond_with_json( refreshed_pairing_request.stream, - json!({ - "pairing_code": "pairing-code", - "manual_pairing_code": "ABCD-EFGH", - "server_id": "srv_e_test", - "environment_id": "env_test", - "expires_at": "3026-05-22T12:34:56Z", - }), + pairing_response_json("srv_e_test", "env_test"), ) .await; }); @@ -447,14 +622,54 @@ async fn remote_control_handle_refreshes_after_pairing_auth_failure() { .expect("pairing should refresh after server token auth failure"); server_task.await.expect("server task should finish"); + assert_eq!(response, pairing_response("env_test")); +} + +#[tokio::test] +async fn pairing_auth_failure_preserves_refresh_deadline() { + let listener = TcpListener::bind("127.0.0.1:0") + .await + .expect("listener should bind"); + let remote_control_url = remote_control_url_for_listener(&listener); + let server_task = tokio::spawn(async move { + let pairing_request = accept_http_request(&listener).await; + assert_eq!( + pairing_request.request_line, + "POST /backend-api/wham/remote/control/server/pair HTTP/1.1" + ); + respond_with_status(pairing_request.stream, "401 Unauthorized", "").await; + }); + let remote_handle = remote_control_handle_with_current_enrollment( + &remote_control_url, + remote_control_auth_manager(), + ); + let next_refresh_at = OffsetDateTime::now_utc() + time::Duration::minutes(2); + remote_handle + .current_enrollment + .lock() + .await + .as_mut() + .expect("current enrollment should exist") + .next_refresh_at = Some(next_refresh_at); + let mut expected_enrollment = remote_handle + .current_enrollment + .snapshot() + .expect("current enrollment should exist"); + expected_enrollment.clear_server_token(); + + let err = remote_handle + .start_pairing( + RemoteControlPairingStartParams::default(), + /*app_server_client_name*/ None, + ) + .await + .expect_err("refresh deadline should throttle recovery after token rejection"); + server_task.await.expect("server task should finish"); + + assert_eq!(err.kind(), io::ErrorKind::WouldBlock); assert_eq!( - response, - RemoteControlPairingStartResponse { - pairing_code: "pairing-code".to_string(), - manual_pairing_code: Some("ABCD-EFGH".to_string()), - environment_id: "env_test".to_string(), - expires_at: 33_336_362_096, - } + remote_handle.current_enrollment.snapshot(), + Some(expected_enrollment) ); } @@ -508,53 +723,12 @@ async fn remote_control_handle_recovers_auth_before_refreshing_pairing() { ); respond_with_json( pairing_request.stream, - json!({ - "pairing_code": "pairing-code", - "manual_pairing_code": "ABCD-EFGH", - "server_id": "srv_e_test", - "environment_id": "env_test", - "expires_at": "3026-05-22T12:34:56Z", - }), + pairing_response_json("srv_e_test", "env_test"), ) .await; }); let codex_home = TempDir::new().expect("temp dir should create"); - let mut stale_auth = remote_control_auth_dot_json(Some("account_id")); - stale_auth - .tokens - .as_mut() - .expect("stale auth should include tokens") - .access_token = "stale-token".to_string(); - save_auth( - codex_home.path(), - &stale_auth, - AuthCredentialsStoreMode::File, - AuthKeyringBackendKind::default(), - ) - .expect("stale auth should save"); - let auth_manager = AuthManager::shared( - codex_home.path().to_path_buf(), - /*enable_codex_api_key_env*/ false, - AuthCredentialsStoreMode::File, - /*forced_chatgpt_workspace_id*/ None, - /*chatgpt_base_url*/ None, - AuthKeyringBackendKind::default(), - /*auth_route_config*/ None, - ) - .await; - let mut fresh_auth = remote_control_auth_dot_json(Some("account_id")); - fresh_auth - .tokens - .as_mut() - .expect("fresh auth should include tokens") - .access_token = "fresh-token".to_string(); - save_auth( - codex_home.path(), - &fresh_auth, - AuthCredentialsStoreMode::File, - AuthKeyringBackendKind::default(), - ) - .expect("fresh auth should save"); + let auth_manager = auth_manager_with_replacement(&codex_home, "account_id").await; let remote_handle = remote_control_handle_with_current_enrollment(&remote_control_url, auth_manager); remote_handle @@ -574,14 +748,128 @@ async fn remote_control_handle_recovers_auth_before_refreshing_pairing() { .expect("pairing should refresh after auth recovery"); server_task.await.expect("server task should finish"); + assert_eq!(response, pairing_response("env_test")); +} + +#[tokio::test] +async fn pairing_publishes_refresh_deferral_after_auth_recovery() { + let listener = TcpListener::bind("127.0.0.1:0") + .await + .expect("listener should bind"); + let remote_control_url = remote_control_url_for_listener(&listener); + let server_task = tokio::spawn(async move { + let stale_refresh_request = accept_http_request(&listener).await; + assert_eq!( + stale_refresh_request.headers.get("authorization"), + Some(&"Bearer stale-token".to_string()) + ); + respond_with_status(stale_refresh_request.stream, "401 Unauthorized", "").await; + + let recovered_refresh_request = accept_http_request(&listener).await; + assert_eq!( + recovered_refresh_request.headers.get("authorization"), + Some(&"Bearer fresh-token".to_string()) + ); + respond_with_status_and_headers( + recovered_refresh_request.stream, + "502 Bad Gateway", + &[("retry-after", "120")], + "upstream unavailable", + ) + .await; + }); + let codex_home = TempDir::new().expect("temp dir should create"); + let auth_manager = auth_manager_with_replacement(&codex_home, "account_id").await; + let remote_handle = + remote_control_handle_with_current_enrollment(&remote_control_url, auth_manager); + remote_handle + .current_enrollment + .lock() + .await + .as_mut() + .expect("current enrollment should exist") + .expires_at = Some(OffsetDateTime::now_utc() - time::Duration::seconds(1)); + + let refresh_started_at = OffsetDateTime::now_utc(); + let refresh_err = remote_handle + .start_pairing( + RemoteControlPairingStartParams::default(), + /*app_server_client_name*/ None, + ) + .await + .expect_err("required refresh should remain strict after auth recovery"); + let refresh_completed_at = OffsetDateTime::now_utc(); + let deferred_err = remote_handle + .start_pairing( + RemoteControlPairingStartParams::default(), + /*app_server_client_name*/ None, + ) + .await + .expect_err("published deadline should throttle the next pairing refresh"); + server_task.await.expect("server task should finish"); + + assert!(refresh_err.to_string().contains("HTTP 502 Bad Gateway")); + assert_eq!(deferred_err.kind(), io::ErrorKind::WouldBlock); + let next_refresh_at = remote_handle + .current_enrollment + .snapshot() + .and_then(|enrollment| enrollment.next_refresh_at) + .expect("required refresh failure should publish its retry deadline"); + assert!( + (refresh_started_at + time::Duration::seconds(120) + ..=refresh_completed_at + time::Duration::seconds(120)) + .contains(&next_refresh_at) + ); +} + +#[tokio::test] +async fn pairing_auth_recovery_failure_publishes_cleared_server_token() { + let listener = TcpListener::bind("127.0.0.1:0") + .await + .expect("listener should bind"); + let remote_control_url = remote_control_url_for_listener(&listener); + let server_task = tokio::spawn(async move { + let stale_refresh_request = accept_http_request(&listener).await; + assert_eq!( + stale_refresh_request.request_line, + "POST /backend-api/wham/remote/control/server/refresh HTTP/1.1" + ); + assert_eq!( + stale_refresh_request.headers.get("authorization"), + Some(&"Bearer stale-token".to_string()) + ); + respond_with_status(stale_refresh_request.stream, "401 Unauthorized", "").await; + }); + let codex_home = TempDir::new().expect("temp dir should create"); + let auth_manager = auth_manager_with_replacement(&codex_home, "different_account_id").await; + let remote_handle = + remote_control_handle_with_current_enrollment(&remote_control_url, auth_manager); + remote_handle + .current_enrollment + .lock() + .await + .as_mut() + .expect("current enrollment should exist") + .expires_at = Some(OffsetDateTime::now_utc() + time::Duration::seconds(29)); + let mut expected_enrollment = remote_handle + .current_enrollment + .snapshot() + .expect("current enrollment should exist"); + expected_enrollment.clear_server_token(); + + let err = remote_handle + .start_pairing( + RemoteControlPairingStartParams::default(), + /*app_server_client_name*/ None, + ) + .await + .expect_err("pairing should fail after auth changes account"); + server_task.await.expect("server task should finish"); + + assert_eq!(err.kind(), io::ErrorKind::PermissionDenied); assert_eq!( - response, - RemoteControlPairingStartResponse { - pairing_code: "pairing-code".to_string(), - manual_pairing_code: Some("ABCD-EFGH".to_string()), - environment_id: "env_test".to_string(), - expires_at: 33_336_362_096, - } + remote_handle.current_enrollment.snapshot(), + Some(expected_enrollment) ); } @@ -688,6 +976,7 @@ async fn remote_control_handle_reenrolls_after_stale_pairing_enrollment() { server_name: test_server_name(), remote_control_token: None, expires_at: None, + next_refresh_at: None, }; update_persisted_remote_control_enrollment( Some(state_db.as_ref()), @@ -745,13 +1034,10 @@ async fn remote_control_handle_reenrolls_after_stale_pairing_enrollment() { ); respond_with_json( refreshed_pairing_request.stream, - json!({ - "pairing_code": "pairing-code", - "manual_pairing_code": "ABCD-EFGH", - "server_id": server_refreshed_enrollment.server_id, - "environment_id": server_refreshed_enrollment.environment_id, - "expires_at": "3026-05-22T12:34:56Z", - }), + pairing_response_json( + &server_refreshed_enrollment.server_id, + &server_refreshed_enrollment.environment_id, + ), ) .await; }); @@ -764,15 +1050,7 @@ async fn remote_control_handle_reenrolls_after_stale_pairing_enrollment() { .expect("pairing should re-enroll after stale enrollment"); server_task.await.expect("server task should finish"); - assert_eq!( - response, - RemoteControlPairingStartResponse { - pairing_code: "pairing-code".to_string(), - manual_pairing_code: Some("ABCD-EFGH".to_string()), - environment_id: "env_refreshed".to_string(), - expires_at: 33_336_362_096, - } - ); + assert_eq!(response, pairing_response("env_refreshed")); assert_eq!( state_db .get_remote_control_enrollment( diff --git a/codex-rs/app-server-transport/src/transport/remote_control/websocket.rs b/codex-rs/app-server-transport/src/transport/remote_control/websocket.rs index e34cd34091..7434454499 100644 --- a/codex-rs/app-server-transport/src/transport/remote_control/websocket.rs +++ b/codex-rs/app-server-transport/src/transport/remote_control/websocket.rs @@ -23,12 +23,12 @@ use crate::transport::remote_control::auth::recover_remote_control_auth; use crate::transport::remote_control::client_tracker::ClientTracker; use crate::transport::remote_control::client_tracker::REMOTE_CONTROL_IDLE_SWEEP_INTERVAL; use crate::transport::remote_control::enroll::RemoteControlEnrollment; -use crate::transport::remote_control::enroll::enroll_remote_control_server; use crate::transport::remote_control::enroll::format_headers; use crate::transport::remote_control::enroll::load_persisted_remote_control_enrollment; use crate::transport::remote_control::enroll::preview_remote_control_response_body; -use crate::transport::remote_control::enroll::refresh_remote_control_server; use crate::transport::remote_control::enroll::update_persisted_remote_control_enrollment; +use crate::transport::remote_control::server_api::enroll_remote_control_server; +use crate::transport::remote_control::server_api::refresh_remote_control_server; use axum::http::HeaderValue; use base64::Engine; use codex_app_server_protocol::RemoteControlConnectionStatus; @@ -1566,17 +1566,19 @@ async fn prepare_remote_control_enrollment( ) .await?; } - Err(err) - if err.kind() == ErrorKind::PermissionDenied - && recover_remote_control_auth( - auth_context.auth_recovery, - auth_context.auth_change_rx, - ) - .await => - { - return Err(io::Error::other(format!( - "{err}; retrying after auth recovery" - ))); + Err(err) if err.kind() == ErrorKind::PermissionDenied => { + if recover_remote_control_auth( + auth_context.auth_recovery, + auth_context.auth_change_rx, + ) + .await + { + return Err(io::Error::other(format!( + "{err}; retrying after auth recovery" + ))); + } + enrollment_ref.clear_server_token(); + return Err(err); } Err(err) => return Err(err), } @@ -1681,13 +1683,15 @@ async fn clear_remote_control_server_token_if_matches( enrollment: &RemoteControlEnrollment, ) -> io::Result<()> { let mut current_enrollment = current_enrollment.lock().await; - current_enrollment + let current_enrollment = current_enrollment .as_mut() .filter(|current| same_remote_control_enrollment(current, enrollment)) .ok_or_else(|| { io::Error::other("missing remote control enrollment after websocket auth failure") - })? - .clear_server_token(); + })?; + if current_enrollment.remote_control_token == enrollment.remote_control_token { + current_enrollment.clear_server_token(); + } Ok(()) } @@ -1804,6 +1808,10 @@ fn format_remote_control_websocket_connect_error( message } +#[cfg(test)] +#[path = "websocket_refresh_tests.rs"] +mod refresh_tests; + #[cfg(test)] mod tests { use super::*; @@ -1844,13 +1852,15 @@ mod tests { // Windows Bazel CI can take longer than a few seconds for the websocket // client connection attempt to reach the local test listener. #[cfg(windows)] - const TEST_HTTP_ACCEPT_TIMEOUT: Duration = Duration::from_secs(30); + pub(super) const TEST_HTTP_ACCEPT_TIMEOUT: Duration = Duration::from_secs(30); #[cfg(not(windows))] - const TEST_HTTP_ACCEPT_TIMEOUT: Duration = Duration::from_secs(5); - const TEST_INSTALLATION_ID: &str = "11111111-1111-4111-8111-111111111111"; - const TEST_REMOTE_CONTROL_SERVER_TOKEN: &str = "Remote Control Token"; + pub(super) const TEST_HTTP_ACCEPT_TIMEOUT: Duration = Duration::from_secs(5); + pub(super) const TEST_INSTALLATION_ID: &str = "11111111-1111-4111-8111-111111111111"; + pub(super) const TEST_REMOTE_CONTROL_SERVER_TOKEN: &str = "Remote Control Token"; - fn remote_control_enrollment(remote_control_token: Option<&str>) -> RemoteControlEnrollment { + pub(super) fn remote_control_enrollment( + remote_control_token: Option<&str>, + ) -> RemoteControlEnrollment { RemoteControlEnrollment { remote_control_target: normalize_remote_control_url("http://localhost/backend-api/") .expect("target should normalize"), @@ -1861,10 +1871,11 @@ mod tests { remote_control_token: remote_control_token.map(str::to_string), expires_at: remote_control_token .map(|_| time::OffsetDateTime::now_utc() + time::Duration::hours(1)), + next_refresh_at: None, } } - fn test_current_enrollment( + pub(super) fn test_current_enrollment( enrollment: Option, ) -> CurrentRemoteControlEnrollment { Arc::new(RemoteControlEnrollmentState::new(enrollment)) @@ -1930,7 +1941,7 @@ mod tests { )); } - fn remote_control_status_channel() -> ( + pub(super) fn remote_control_status_channel() -> ( RemoteControlStatusPublisher, watch::Receiver, ) { @@ -1943,7 +1954,7 @@ mod tests { (RemoteControlStatusPublisher::new(status_tx), status_rx) } - fn enabled_desired_state_sender() -> watch::Sender { + pub(super) fn enabled_desired_state_sender() -> watch::Sender { watch::channel(RemoteControlDesiredState::Enabled { persistence_preference: None, }) @@ -1981,17 +1992,17 @@ mod tests { ); } - async fn remote_control_state_runtime(codex_home: &TempDir) -> Arc { + pub(super) async fn remote_control_state_runtime(codex_home: &TempDir) -> Arc { StateRuntime::init(codex_home.path().to_path_buf(), "test-provider".to_string()) .await .expect("state runtime should initialize") } - fn remote_control_auth_manager() -> Arc { + pub(super) fn remote_control_auth_manager() -> Arc { auth_manager_from_auth(CodexAuth::create_dummy_chatgpt_auth_for_testing()) } - fn remote_control_url_for_listener(listener: &TcpListener) -> String { + pub(super) fn remote_control_url_for_listener(listener: &TcpListener) -> String { let addr = listener .local_addr() .expect("listener should have a local addr"); @@ -2126,9 +2137,10 @@ mod tests { let auth_manager = remote_control_auth_manager(); let mut auth_recovery = auth_manager.unauthorized_recovery(); let mut auth_change_rx = auth_manager.auth_change_receiver(); - let current_enrollment = test_current_enrollment(Some(remote_control_enrollment(Some( - TEST_REMOTE_CONTROL_SERVER_TOKEN, - )))); + let next_refresh_at = time::OffsetDateTime::now_utc() + time::Duration::minutes(2); + let mut enrollment = remote_control_enrollment(Some(TEST_REMOTE_CONTROL_SERVER_TOKEN)); + enrollment.next_refresh_at = Some(next_refresh_at); + let current_enrollment = test_current_enrollment(Some(enrollment)); let (status_publisher, status_rx) = remote_control_status_channel(); let server_task = tokio::spawn(async move { @@ -2178,6 +2190,7 @@ mod tests { ); let mut expected_enrollment = remote_control_enrollment(/*remote_control_token*/ None); expected_enrollment.remote_control_target = remote_control_target; + expected_enrollment.next_refresh_at = Some(next_refresh_at); assert_eq!(*current_enrollment.lock().await, Some(expected_enrollment)); } @@ -2318,9 +2331,12 @@ mod tests { .await; let mut auth_recovery = auth_manager.unauthorized_recovery(); let mut auth_change_rx = auth_manager.auth_change_receiver(); - let current_enrollment = test_current_enrollment(Some(remote_control_enrollment( - /*remote_control_token*/ None, - ))); + let mut expected_enrollment = + remote_control_enrollment(Some(TEST_REMOTE_CONTROL_SERVER_TOKEN)); + expected_enrollment.remote_control_target = remote_control_target.clone(); + expected_enrollment.expires_at = + Some(time::OffsetDateTime::now_utc() + time::Duration::minutes(4)); + let current_enrollment = test_current_enrollment(Some(expected_enrollment.clone())); let (status_publisher, status_rx) = remote_control_status_channel(); save_auth( codex_home.path(), @@ -2377,6 +2393,7 @@ mod tests { .expect("token should be readable"), "fresh-token" ); + assert_eq!(current_enrollment.snapshot(), Some(expected_enrollment)); assert!( !auth_change_rx .has_changed() @@ -3411,7 +3428,7 @@ mod tests { state.observe_client_message(envelope, wire_size_bytes) } - async fn accept_http_request(listener: &TcpListener) -> (TcpStream, String) { + pub(super) async fn accept_http_request(listener: &TcpListener) -> (TcpStream, String) { let (stream, _) = timeout(TEST_HTTP_ACCEPT_TIMEOUT, listener.accept()) .await .expect("HTTP request should arrive in time") @@ -3482,7 +3499,7 @@ mod tests { serde_json::from_str(text.as_ref()).expect("server event should deserialize") } - async fn respond_with_status_and_headers( + pub(super) async fn respond_with_status_and_headers( mut stream: TcpStream, status: &str, headers: &[(&str, &str)], diff --git a/codex-rs/app-server-transport/src/transport/remote_control/websocket_refresh_tests.rs b/codex-rs/app-server-transport/src/transport/remote_control/websocket_refresh_tests.rs new file mode 100644 index 0000000000..ee4ea15f83 --- /dev/null +++ b/codex-rs/app-server-transport/src/transport/remote_control/websocket_refresh_tests.rs @@ -0,0 +1,467 @@ +use super::tests::TEST_HTTP_ACCEPT_TIMEOUT; +use super::tests::TEST_INSTALLATION_ID; +use super::tests::TEST_REMOTE_CONTROL_SERVER_TOKEN; +use super::tests::accept_http_request; +use super::tests::enabled_desired_state_sender; +use super::tests::remote_control_auth_manager; +use super::tests::remote_control_enrollment; +use super::tests::remote_control_state_runtime; +use super::tests::remote_control_status_channel; +use super::tests::remote_control_url_for_listener; +use super::tests::respond_with_status_and_headers; +use super::tests::test_current_enrollment; +use super::*; +use crate::transport::remote_control::protocol::normalize_remote_control_url; +use crate::transport::remote_control::tests::remote_control_handle_with_current_enrollment; +use codex_app_server_protocol::RemoteControlPairingStartParams; +use codex_app_server_protocol::RemoteControlPairingStartResponse; +use pretty_assertions::assert_eq; +use tempfile::TempDir; +use tokio::net::TcpListener; +use tokio::net::TcpStream; +use tokio::time::Duration; +use tokio::time::timeout; +use tokio_tungstenite::WebSocketStream; +use tokio_tungstenite::accept_async; + +async fn connect_test_websocket( + remote_control_target: &RemoteControlTarget, + state_db: &StateRuntime, + auth_manager: &Arc, + current_enrollment: &CurrentRemoteControlEnrollment, +) -> io::Result<()> { + let mut auth_recovery = auth_manager.unauthorized_recovery(); + let mut auth_change_rx = auth_manager.auth_change_receiver(); + let (status_publisher, _) = remote_control_status_channel(); + let desired_state_tx = enabled_desired_state_sender(); + let desired_state_persistence_lock = Semaphore::new(1); + connect_remote_control_websocket( + remote_control_target, + Some(state_db), + RemoteControlAuthContext { + auth_manager, + auth_recovery: &mut auth_recovery, + auth_change_rx: &mut auth_change_rx, + }, + current_enrollment, + RemoteControlConnectOptions { + installation_id: TEST_INSTALLATION_ID, + server_name: "test-server", + subscribe_cursor: None, + app_server_client_name: None, + desired_state_tx: &desired_state_tx, + desired_state_persistence_lock: &desired_state_persistence_lock, + }, + &status_publisher, + ) + .await + .map(|_| ()) +} + +#[tokio::test] +async fn proactive_refresh_failure_uses_valid_token_for_websocket_connect() { + let listener = TcpListener::bind("127.0.0.1:0") + .await + .expect("listener should bind"); + let remote_control_url = remote_control_url_for_listener(&listener); + let remote_control_target = + normalize_remote_control_url(&remote_control_url).expect("target should parse"); + let server_task = tokio::spawn(async move { + let (stream, request_line) = accept_http_request(&listener).await; + assert_eq!( + request_line, + "POST /backend-api/wham/remote/control/server/refresh HTTP/1.1" + ); + respond_with_status_and_headers(stream, "502 Bad Gateway", &[], "upstream unavailable") + .await; + accept_test_websocket(&listener).await + }); + let codex_home = TempDir::new().expect("temp dir should create"); + let state_db = remote_control_state_runtime(&codex_home).await; + let auth_manager = remote_control_auth_manager(); + let mut enrollment = remote_control_enrollment(Some(TEST_REMOTE_CONTROL_SERVER_TOKEN)); + enrollment.expires_at = Some(time::OffsetDateTime::now_utc() + time::Duration::minutes(4)); + let current_enrollment = test_current_enrollment(Some(enrollment)); + + let refresh_started_at = time::OffsetDateTime::now_utc(); + connect_test_websocket( + &remote_control_target, + state_db.as_ref(), + &auth_manager, + ¤t_enrollment, + ) + .await + .expect("valid token should allow websocket connect after proactive refresh failure"); + let refresh_completed_at = time::OffsetDateTime::now_utc(); + let server_websocket = server_task.await.expect("server task should succeed"); + + let enrollment = current_enrollment + .lock() + .await + .clone() + .expect("enrollment should remain available"); + assert_eq!( + enrollment.remote_control_token.as_deref(), + Some(TEST_REMOTE_CONTROL_SERVER_TOKEN) + ); + let next_refresh_at = enrollment + .next_refresh_at + .expect("transient refresh should set a retry deadline"); + assert!( + (refresh_started_at + time::Duration::seconds(24) + ..=refresh_completed_at + time::Duration::seconds(36)) + .contains(&next_refresh_at) + ); + drop(server_websocket); +} + +#[tokio::test] +async fn proactive_refresh_connection_failure_uses_valid_token_for_websocket_connect() { + let listener = TcpListener::bind("127.0.0.1:0") + .await + .expect("listener should bind"); + let remote_control_url = remote_control_url_for_listener(&listener); + let remote_control_target = + normalize_remote_control_url(&remote_control_url).expect("target should parse"); + let server_task = tokio::spawn(async move { + let (stream, request_line) = accept_http_request(&listener).await; + assert_eq!( + request_line, + "POST /backend-api/wham/remote/control/server/refresh HTTP/1.1" + ); + drop(stream); + accept_test_websocket(&listener).await + }); + let codex_home = TempDir::new().expect("temp dir should create"); + let state_db = remote_control_state_runtime(&codex_home).await; + let auth_manager = remote_control_auth_manager(); + let mut enrollment = remote_control_enrollment(Some(TEST_REMOTE_CONTROL_SERVER_TOKEN)); + enrollment.expires_at = Some(time::OffsetDateTime::now_utc() + time::Duration::minutes(4)); + let current_enrollment = test_current_enrollment(Some(enrollment)); + + connect_test_websocket( + &remote_control_target, + state_db.as_ref(), + &auth_manager, + ¤t_enrollment, + ) + .await + .expect("valid token should allow websocket connect after refresh connection failure"); + let server_websocket = server_task.await.expect("server task should succeed"); + + assert!( + current_enrollment + .snapshot() + .and_then(|enrollment| enrollment.next_refresh_at) + .is_some(), + "connection failure should set a retry deadline" + ); + drop(server_websocket); +} + +#[tokio::test] +async fn websocket_retry_after_throttles_pairing_refresh() { + let listener = TcpListener::bind("127.0.0.1:0") + .await + .expect("listener should bind"); + let remote_control_url = remote_control_url_for_listener(&listener); + let remote_control_target = + normalize_remote_control_url(&remote_control_url).expect("target should parse"); + let server_task = tokio::spawn(async move { + let (stream, request_line) = accept_http_request(&listener).await; + assert_eq!( + request_line, + "POST /backend-api/wham/remote/control/server/refresh HTTP/1.1" + ); + respond_with_status_and_headers( + stream, + "502 Bad Gateway", + &[("retry-after", "120")], + "upstream unavailable", + ) + .await; + let first_websocket = accept_test_websocket(&listener).await; + let (pairing_stream, request_line) = accept_http_request(&listener).await; + assert_eq!( + request_line, + "POST /backend-api/wham/remote/control/server/pair HTTP/1.1" + ); + respond_with_status_and_headers( + pairing_stream, + "200 OK", + &[], + r#"{"pairing_code":"pairing-code","manual_pairing_code":"ABCD-EFGH","server_id":"srv_e_test","environment_id":"env_test","expires_at":"3026-05-22T12:34:56Z"}"#, + ) + .await; + first_websocket + }); + let codex_home = TempDir::new().expect("temp dir should create"); + let state_db = remote_control_state_runtime(&codex_home).await; + let auth_manager = remote_control_auth_manager(); + let mut remote_handle = + remote_control_handle_with_current_enrollment(&remote_control_url, auth_manager.clone()); + remote_handle.state_db = Some(state_db.clone()); + remote_handle + .current_enrollment + .lock() + .await + .as_mut() + .expect("current enrollment should exist") + .expires_at = Some(time::OffsetDateTime::now_utc() + time::Duration::minutes(4)); + let current_enrollment = remote_handle.current_enrollment.clone(); + let refresh_started_at = time::OffsetDateTime::now_utc(); + connect_test_websocket( + &remote_control_target, + state_db.as_ref(), + &auth_manager, + ¤t_enrollment, + ) + .await + .expect("first websocket should connect after deferred refresh"); + let refresh_completed_at = time::OffsetDateTime::now_utc(); + let next_refresh_at = current_enrollment + .snapshot() + .and_then(|enrollment| enrollment.next_refresh_at) + .expect("Retry-After should set a retry deadline"); + assert!( + (refresh_started_at + time::Duration::seconds(120) + ..=refresh_completed_at + time::Duration::seconds(120)) + .contains(&next_refresh_at) + ); + + let pairing_response = remote_handle + .start_pairing( + RemoteControlPairingStartParams::default(), + /*app_server_client_name*/ None, + ) + .await + .expect("websocket Retry-After should throttle pairing refresh"); + let first_server_websocket = server_task.await.expect("server task should succeed"); + + assert_eq!( + pairing_response, + RemoteControlPairingStartResponse { + pairing_code: "pairing-code".to_string(), + manual_pairing_code: Some("ABCD-EFGH".to_string()), + environment_id: "env_test".to_string(), + expires_at: 33_336_362_096, + } + ); + drop(first_server_websocket); +} + +#[tokio::test] +async fn pairing_http_date_retry_after_throttles_websocket_refresh() { + let listener = TcpListener::bind("127.0.0.1:0") + .await + .expect("listener should bind"); + let remote_control_url = remote_control_url_for_listener(&listener); + let remote_control_target = + normalize_remote_control_url(&remote_control_url).expect("target should parse"); + let retry_after = + httpdate::fmt_http_date(std::time::SystemTime::now() + Duration::from_secs(120)); + let expected_next_refresh_at = time::OffsetDateTime::from( + httpdate::parse_http_date(&retry_after).expect("Retry-After date should parse"), + ); + let server_task = tokio::spawn(async move { + let (refresh_stream, request_line) = accept_http_request(&listener).await; + assert_eq!( + request_line, + "POST /backend-api/wham/remote/control/server/refresh HTTP/1.1" + ); + respond_with_status_and_headers( + refresh_stream, + "502 Bad Gateway", + &[("retry-after", &retry_after)], + "upstream unavailable", + ) + .await; + let (pairing_stream, request_line) = accept_http_request(&listener).await; + assert_eq!( + request_line, + "POST /backend-api/wham/remote/control/server/pair HTTP/1.1" + ); + respond_with_status_and_headers( + pairing_stream, + "200 OK", + &[], + r#"{"pairing_code":"pairing-code","manual_pairing_code":"ABCD-EFGH","server_id":"srv_e_test","environment_id":"env_test","expires_at":"3026-05-22T12:34:56Z"}"#, + ) + .await; + accept_test_websocket(&listener).await + }); + let codex_home = TempDir::new().expect("temp dir should create"); + let state_db = remote_control_state_runtime(&codex_home).await; + let auth_manager = remote_control_auth_manager(); + let mut remote_handle = + remote_control_handle_with_current_enrollment(&remote_control_url, auth_manager.clone()); + remote_handle.state_db = Some(state_db.clone()); + remote_handle + .current_enrollment + .lock() + .await + .as_mut() + .expect("current enrollment should exist") + .expires_at = Some(time::OffsetDateTime::now_utc() + time::Duration::minutes(4)); + let current_enrollment = remote_handle.current_enrollment.clone(); + + let pairing_response = remote_handle + .start_pairing( + RemoteControlPairingStartParams::default(), + /*app_server_client_name*/ None, + ) + .await + .expect("pairing should continue after proactive refresh failure"); + assert_eq!( + current_enrollment + .snapshot() + .and_then(|enrollment| enrollment.next_refresh_at), + Some(expected_next_refresh_at) + ); + connect_test_websocket( + &remote_control_target, + state_db.as_ref(), + &auth_manager, + ¤t_enrollment, + ) + .await + .expect("pairing Retry-After should throttle websocket refresh"); + let server_websocket = server_task.await.expect("server task should succeed"); + + assert_eq!( + pairing_response, + RemoteControlPairingStartResponse { + pairing_code: "pairing-code".to_string(), + manual_pairing_code: Some("ABCD-EFGH".to_string()), + environment_id: "env_test".to_string(), + expires_at: 33_336_362_096, + } + ); + drop(server_websocket); +} + +async fn assert_refresh_failure_blocks_websocket( + expires_in: time::Duration, + response_delay: Duration, +) { + let listener = TcpListener::bind("127.0.0.1:0") + .await + .expect("listener should bind"); + let remote_control_url = remote_control_url_for_listener(&listener); + let remote_control_target = + normalize_remote_control_url(&remote_control_url).expect("target should parse"); + let (connects_done_tx, connects_done_rx) = oneshot::channel(); + let server_task = tokio::spawn(async move { + let (stream, request_line) = accept_http_request(&listener).await; + assert_eq!( + request_line, + "POST /backend-api/wham/remote/control/server/refresh HTTP/1.1" + ); + tokio::time::sleep(response_delay).await; + respond_with_status_and_headers( + stream, + "502 Bad Gateway", + &[("retry-after", "120")], + "upstream unavailable", + ) + .await; + assert_no_connection_until_connect_finishes(&listener, connects_done_rx).await; + }); + let codex_home = TempDir::new().expect("temp dir should create"); + let state_db = remote_control_state_runtime(&codex_home).await; + let auth_manager = remote_control_auth_manager(); + let mut enrollment = remote_control_enrollment(Some(TEST_REMOTE_CONTROL_SERVER_TOKEN)); + enrollment.expires_at = Some(time::OffsetDateTime::now_utc() + expires_in); + let current_enrollment = test_current_enrollment(Some(enrollment)); + + let refresh_started_at = time::OffsetDateTime::now_utc(); + let refresh_err = connect_test_websocket( + &remote_control_target, + state_db.as_ref(), + &auth_manager, + ¤t_enrollment, + ) + .await + .expect_err("required refresh failure should block websocket connect"); + let refresh_completed_at = time::OffsetDateTime::now_utc(); + let deferred_err = connect_test_websocket( + &remote_control_target, + state_db.as_ref(), + &auth_manager, + ¤t_enrollment, + ) + .await + .expect_err("required refresh deadline should block websocket reconnect"); + connects_done_tx + .send(()) + .expect("server should wait for connect attempts to finish"); + + server_task.await.expect("server task should succeed"); + assert!(refresh_err.to_string().contains("HTTP 502 Bad Gateway")); + assert_eq!(deferred_err.kind(), io::ErrorKind::WouldBlock); + assert!(deferred_err.to_string().contains("refresh deferred until")); + let next_refresh_at = current_enrollment + .snapshot() + .and_then(|enrollment| enrollment.next_refresh_at) + .expect("required refresh failure should set a retry deadline"); + assert!( + (refresh_started_at + time::Duration::seconds(120) + ..=refresh_completed_at + time::Duration::seconds(120)) + .contains(&next_refresh_at) + ); +} + +#[tokio::test] +async fn expired_token_refresh_failure_throttles_reconnect_without_websocket() { + assert_refresh_failure_blocks_websocket(-time::Duration::seconds(1), Duration::ZERO).await; +} + +#[tokio::test] +async fn token_expiring_during_refresh_failure_throttles_reconnect_without_websocket() { + assert_refresh_failure_blocks_websocket( + time::Duration::seconds(1), + Duration::from_millis(1_200), + ) + .await; +} + +#[tokio::test] +async fn websocket_auth_failure_does_not_clear_rotated_server_token() { + let attempted_enrollment = remote_control_enrollment(Some("old-token")); + let mut rotated_enrollment = attempted_enrollment.clone(); + rotated_enrollment.remote_control_token = Some("new-token".to_string()); + rotated_enrollment.expires_at = + Some(time::OffsetDateTime::now_utc() + time::Duration::hours(1)); + let current_enrollment = test_current_enrollment(Some(rotated_enrollment.clone())); + + clear_remote_control_server_token_if_matches(¤t_enrollment, &attempted_enrollment) + .await + .expect("matching enrollment identity should remain available"); + + assert_eq!(current_enrollment.snapshot(), Some(rotated_enrollment)); +} + +async fn accept_test_websocket(listener: &TcpListener) -> WebSocketStream { + let (stream, _) = timeout(TEST_HTTP_ACCEPT_TIMEOUT, listener.accept()) + .await + .expect("websocket request should arrive in time") + .expect("listener accept should succeed"); + accept_async(stream) + .await + .expect("websocket handshake should succeed") +} + +async fn assert_no_connection_until_connect_finishes( + listener: &TcpListener, + mut connect_done_rx: oneshot::Receiver<()>, +) { + tokio::select! { + accepted = listener.accept() => { + accepted.expect("unexpected websocket connection should be accepted"); + panic!("required refresh failure must not proceed to websocket connect"); + } + connect_done = &mut connect_done_rx => { + connect_done.expect("connect completion should be reported"); + } + } +} diff --git a/codex-rs/app-server/README.md b/codex-rs/app-server/README.md index 246b2bb0c0..a7986753ee 100644 --- a/codex-rs/app-server/README.md +++ b/codex-rs/app-server/README.md @@ -137,9 +137,9 @@ Example with notification opt-out: ## API Overview -- `thread/start` — create a new thread; emits `thread/started` (including the current `thread.status`) and auto-subscribes you to turn/item events for that thread. When the request includes a `cwd` and the resolved sandbox is `workspace-write` or full access, app-server also marks that project as trusted in the user `config.toml`. Pass `sessionStartSource: "clear"` when starting a replacement thread after clearing the current session so `SessionStart` hooks receive `source: "clear"` instead of the default `"startup"`. Experimental `runtimeWorkspaceRoots` replaces the thread-scoped runtime workspace roots used to materialize `:workspace_roots`; paths must be absolute. For permissions, prefer experimental `permissions` profile selection by id; the legacy `sandbox` shorthand is still accepted but cannot be combined with `permissions`. Deprecated experimental `multiAgentMode` is ignored; use Ultra reasoning effort for proactive multi-agent behavior. Experimental `environments` selects the sticky execution environments for turns on the thread; omit it to use the server default, pass `[]` to disable environments, or pass explicit environment ids with per-environment `cwd`. Experimental `selectedCapabilityRoots` selects environment-owned plugin or standalone-skill roots using environment-native absolute paths. Skills found below those roots are listed and read through the owning environment. Stdio MCP servers declared by selected plugins are also started in that environment; HTTP MCP declarations remain inactive. +- `thread/start` — create a new thread; emits `thread/started` (including the current `thread.status`) and auto-subscribes you to turn/item events for that thread. When the request includes a `cwd` and the resolved sandbox is `workspace-write` or full access, app-server also marks that project as trusted in the user `config.toml`. Pass `sessionStartSource: "clear"` when starting a replacement thread after clearing the current session so `SessionStart` hooks receive `source: "clear"` instead of the default `"startup"`. Experimental `allowProviderModelFallback` lets providers backed by an authoritative static model catalog replace an unavailable requested `model` with the catalog default; dynamic or cached catalogs preserve the requested model. Experimental `runtimeWorkspaceRoots` replaces the thread-scoped runtime workspace roots used to materialize `:workspace_roots`; paths must be absolute. For permissions, prefer experimental `permissions` profile selection by id; the legacy `sandbox` shorthand is still accepted but cannot be combined with `permissions`. Deprecated experimental `multiAgentMode` is ignored; use Ultra reasoning effort for proactive multi-agent behavior. Experimental `environments` selects the sticky execution environments for turns on the thread; omit it to use the server default, pass `[]` to disable environments, or pass explicit environment ids with per-environment `cwd`. Experimental `selectedCapabilityRoots` selects environment-owned plugin or standalone-skill roots using environment-native absolute paths. Skills found below those roots are listed and read through the owning environment. Stdio MCP servers declared by selected plugins are started in that environment, and HTTP MCP connections use that environment's HTTP client. - `thread/resume` — reopen an existing thread by id so subsequent `turn/start` calls append to it. Accepts the same permission override rules as `thread/start`. -- `thread/fork` — fork an existing thread into a new thread id by copying the stored history; if the source thread is currently mid-turn, the fork records the same interruption marker as `turn/interrupt` instead of inheriting an unmarked partial turn suffix. The returned `thread.forkedFromId` points at the source thread when known. Accepts `ephemeral: true` for an in-memory temporary fork, emits `thread/started` (including the current `thread.status`), and auto-subscribes you to turn/item events for the new thread. Experimental clients can pass `excludeTurns: true` when they plan to page fork history via `thread/turns/list` instead of receiving the full turn array immediately. Accepts the same permission override rules as `thread/start`. +- `thread/fork` — fork an existing thread into a new thread id by copying the stored history; pass an optional `lastTurnId` to copy history only through that turn, inclusive, and drop later turns from the fork. An in-progress `lastTurnId` is rejected. If `lastTurnId` is null while the source thread is mid-turn, the fork records the same interruption marker as `turn/interrupt` instead of inheriting an unmarked partial turn suffix. The returned `thread.forkedFromId` points at the source thread when known. Accepts `ephemeral: true` for an in-memory temporary fork, emits `thread/started` (including the current `thread.status`), and auto-subscribes you to turn/item events for the new thread. Experimental clients can pass `excludeTurns: true` when they plan to page fork history via `thread/turns/list` instead of receiving the full turn array immediately. Accepts the same permission override rules as `thread/start`. - `thread/start`, `thread/resume`, and `thread/fork` responses include the legacy `sandbox` compatibility projection. `instructionSources` lists loaded instruction files using each source environment's native absolute path syntax, including files loaded from remote environments. Experimental clients can read `runtimeWorkspaceRoots` for the thread-scoped runtime roots and `activePermissionProfile` for the named or implicit built-in profile identity/provenance when known. Their deprecated experimental `multiAgentMode` field, and the corresponding thread setting, always report `explicitRequestOnly`; Ultra reasoning effort is the source of proactive multi-agent behavior. - `thread/list` — page through stored threads; supports cursor-based pagination and optional `modelProviders`, `sourceKinds`, `archived`, `cwd`, and `searchTerm` filters. Experimental clients can use `parentThreadId` for direct spawned children or `ancestorThreadId` for spawned descendants at any depth; the two filters are mutually exclusive. Review and Guardian threads are not included because they do not participate in that spawn-edge lifecycle. Each returned `thread` includes `status` (`ThreadStatus`), defaulting to `notLoaded` when the thread is not currently loaded. Subagent threads also include `parentThreadId` when the immediate parent is known. - `thread/loaded/list` — list the thread ids currently loaded in memory. @@ -167,7 +167,7 @@ Example with notification opt-out: - `thread/backgroundTerminals/clean` — terminate all running background terminals for a thread (experimental; requires `capabilities.experimentalApi`); returns `{}` when the cleanup request is accepted. - `thread/backgroundTerminals/list` — list running background terminals for a loaded thread (experimental; requires `capabilities.experimentalApi`); returns `data` with the running terminal ids. - `thread/backgroundTerminals/terminate` — terminate one running background terminal by app-server `processId` (experimental; requires `capabilities.experimentalApi`); returns whether a process was terminated. -- `thread/rollback` — drop the last N turns from the agent’s in-memory context and persist a rollback marker in the rollout so future resumes see the pruned history; returns the updated `thread` (with `turns` populated) on success. +- `thread/rollback` — deprecated and will be removed soon. Drop the last N turns from the agent’s in-memory context and persist a rollback marker in the rollout so future resumes see the pruned history; returns the updated `thread` (with `turns` populated) on success. - `turn/start` — add user input to a thread and begin Codex generation; responds with the initial `turn` object and streams `turn/started`, `item/*`, and `turn/completed` notifications. `clientUserMessageId` is optional; when supplied, the corresponding `userMessage` item echoes it as `clientId`. Experimental `runtimeWorkspaceRoots` replaces the thread-scoped runtime workspace roots used to materialize `:workspace_roots`; paths must be absolute. Prefer experimental `permissions` profile selection by id for permission overrides; the legacy `sandboxPolicy` field is still accepted but cannot be combined with `permissions`. For `collaborationMode`, `settings.developer_instructions: null` means "use built-in instructions for the selected mode". Deprecated experimental `multiAgentMode` is ignored; Ultra reasoning effort selects proactive behavior. - `thread/inject_items` — append raw Responses API items to a loaded thread’s model-visible history without starting a user turn; returns `{}` on success. - `turn/steer` — add user input to an already in-flight regular turn without starting a new turn; returns the active `turnId` that accepted the input. `clientUserMessageId` is optional; when supplied, the corresponding `userMessage` item echoes it as `clientId`. Review and manual compaction turns reject `turn/steer`. @@ -205,6 +205,7 @@ Example with notification opt-out: - `permissionProfile/list` — beta; list available permission profile ids with optional display `description` text and an `allowed` flag reflecting effective requirements, using cursor pagination. Pass `cwd` when the caller needs project-local `[permissions.]` entries to be included in the current catalog view. - `experimentalFeature/enablement/set` — patch the in-memory process-wide runtime feature enablement for currently supported feature keys. For each feature, precedence is: cloud requirements > --enable > config.toml > experimentalFeature/enablement/set (new) > code default. Invalid keys will be ignored. - `environment/add` — experimental; add or replace a named remote environment by `environmentId` and `execServerUrl` for later selection by `thread/start` or `turn/start`; optional `connectTimeoutMs` overrides the WebSocket connection timeout; returns `{}` and does not change the default environment. +- `environment/info` — experimental; connect to a configured environment by `environmentId` and return its detected `shell` plus its default `cwd` as a canonical environment-native `file:` URI. Connection failures are returned as request errors. - `collaborationMode/list` — list available collaboration mode presets (experimental, no pagination). Built-in presets do not select a model; the Plan preset selects medium reasoning effort. This response omits built-in developer instructions; clients should either pass `settings.developer_instructions: null` when setting a mode to use Codex's built-in instructions, or provide their own instructions explicitly. - `skills/list` — list skills for one or more `cwd` values (optional `forceReload`). - `skills/extraRoots/set` — replace the app-server process runtime extra standalone skill roots. The roots are not persisted; missing directories are accepted and simply load no skills. @@ -212,7 +213,7 @@ Example with notification opt-out: - `marketplace/add` — add a remote plugin marketplace from an HTTP(S) Git URL, SSH Git URL, or GitHub `owner/repo` shorthand, then persist it into the user marketplace config. Returns the installed root path plus whether the marketplace was already present. - `marketplace/remove` — remove a configured marketplace by name from the user marketplace config, and delete its installed marketplace root when one exists. - `marketplace/upgrade` — upgrade all configured Git plugin marketplaces, or one named marketplace when `marketplaceName` is provided. Returns selected marketplace names, upgraded roots, and per-marketplace errors. -- `plugin/list` — list discovered plugin marketplaces and plugin state, including effective marketplace install/auth policy metadata, plugin `availability` (`AVAILABLE` by default or `DISABLED_BY_ADMIN` for remote plugins blocked upstream), fail-open `marketplaceLoadErrors` entries for marketplace files that could not be parsed or loaded, and best-effort `featuredPluginIds` for the official curated marketplace. Clients can explicitly request the remote `workspace-directory`, `shared-with-me`, or `created-by-me-remote` marketplace kinds. `interface.category` uses the marketplace category when present; otherwise it falls back to the plugin manifest category (**under development; do not call from production clients yet**). +- `plugin/list` — list discovered plugin marketplaces and plugin state, including effective marketplace install/auth policy metadata, the remote marketplace `version` and locally materialized `localVersion` when available, plugin `availability` (`AVAILABLE` by default or `DISABLED_BY_ADMIN` for remote plugins blocked upstream), fail-open `marketplaceLoadErrors` entries for marketplace files that could not be parsed or loaded, and best-effort `featuredPluginIds` for the official curated marketplace. Clients can explicitly request the remote `workspace-directory`, `shared-with-me`, or `created-by-me-remote` marketplace kinds. `interface.category` uses the marketplace category when present; otherwise it falls back to the plugin manifest category (**under development; do not call from production clients yet**). - `plugin/installed` — list installed plugin rows plus any explicitly requested local install-suggestion plugin names, without fetching the broader remote catalog. Mention surfaces can use this narrower view when they need plugin mention payloads rather than plugin-page discovery data (**under development; do not call from production clients yet**). - `plugin/read` — read one plugin by `marketplacePath` plus `pluginName`, returning marketplace info, a list-style `summary`, manifest descriptions/interface metadata, and bundled skills/hooks/apps/MCP server names. Remote plugin details expose the canonical `shareUrl` supplied by the remote catalog when available; it is `null` for local plugins or when the catalog omits it. This field is separate from `summary.shareContext`, which continues to describe user and workspace sharing state. Returned plugin skills include their current `enabled` state after local config filtering; bundled hooks are returned as lightweight declaration summaries keyed for correlation with `hooks/list`. Use `plugin/install`'s `appsNeedingAuth` to drive post-install authentication and `app/list`'s `isAccessible` to determine current connector accessibility (**under development; do not call from production clients yet**). - `plugin/skill/read` — read remote plugin skill markdown on demand by `remoteMarketplaceName`, `remotePluginId`, and `skillName`. This lets clients preview uninstalled remote plugin skills without downloading the plugin bundle. @@ -229,7 +230,7 @@ Example with notification opt-out: - `skills/config/write` — write user-level skill config by name or absolute path. - `plugin/install` — install a plugin from a discovered marketplace entry, rejecting marketplace entries marked unavailable for install, install MCPs if any, and return the effective plugin auth policy plus any apps that still need auth (**under development; do not call from production clients yet**). - `plugin/uninstall` — uninstall a local plugin by `pluginId` in `@` form by removing its cached files and clearing its user-level config entry, or uninstall a remote ChatGPT plugin by backend `pluginId` by forwarding the uninstall to the ChatGPT plugin backend and removing any downloaded remote-plugin cache (**under development; do not call from production clients yet**). -- `mcpServer/oauth/login` — start an OAuth login for a configured MCP server; returns an `authorization_url` and later emits `mcpServer/oauthLogin/completed` once the browser flow finishes. +- `mcpServer/oauth/login` — start an OAuth login for a configured MCP server; pass `threadId` to resolve servers from that thread's selected plugins and executor, and receive an `authorization_url` followed by `mcpServer/oauthLogin/completed` once the browser flow finishes. - `tool/requestUserInput` — prompt the user with 1–3 short questions for a tool call and return their answers (experimental). - `config/mcpServer/reload` — reload MCP server config from disk and queue a refresh for loaded threads (applied on each thread's next active turn); returns `{}`. Use this after editing `config.toml` without restarting the server. - `mcpServerStatus/list` — enumerate configured MCP servers with their tools, auth status, server info, plus resources/resource templates for `full` detail; supports optional `threadId` and cursor+limit pagination. If `threadId` is omitted, the server reads from the latest global config directly. If `detail` is omitted, the server defaults to `full`. @@ -242,7 +243,7 @@ Example with notification opt-out: - `externalAgentConfig/import` — apply selected external-agent migration items by passing explicit `migrationItems` with `cwd` (`null` for home) and any `details` returned by detect. Callers may pass `source` to identify the product that initiated the import; omitted or `null` means unspecified. The response acknowledges the synchronous import phase with an `importId`. Expected migration failures are reported as per-item failures rather than JSON-RPC errors, so the server still returns that `importId` and emits `externalAgentConfig/import/completed` with the same ID once all synchronous and background work finishes. The completion notification contains type-level `itemTypeResults` with successes and failures, including raw failure messages for the client to report separately. - `config/value/write` — write a single config key/value to the user's config.toml on disk; dotted paths such as `desktop.someKey` use the same generic write surface. - `config/batchWrite` — apply multiple config edits atomically to the user's config.toml on disk, with optional `reloadUserConfig: true` to hot-reload loaded threads, including multiple `desktop.*` edits. -- `configRequirements/read` — fetch loaded requirements constraints from `requirements.toml` and/or MDM (or `null` if none are configured), including allow-lists (`allowedApprovalPolicies`, `allowedSandboxModes`, `allowedWebSearchModes`), the layered permission-profile allow map (`allowedPermissionProfiles`), the managed permission-profile default (`defaultPermissions`), lifecycle hook lockdown (`allowManagedHooksOnly`), remote-control policy (`allowRemoteControl`; `false` force-disables remote control while `true` or `null` preserves existing behavior), computer use policy (`computerUse`), pinned feature values (`featureRequirements`), managed lifecycle hooks (`hooks`), `enforceResidency`, and `network` constraints such as canonical domain/socket permissions plus `managedAllowedDomainsOnly` and `dangerFullAccessDenylistOnly`. +- `configRequirements/read` — fetch loaded requirements constraints from `requirements.toml` and/or MDM (or `null` if none are configured), including allow-lists (`allowedApprovalPolicies`, `allowedSandboxModes`, `allowedWebSearchModes`), the layered permission-profile allow map (`allowedPermissionProfiles`), the managed permission-profile default (`defaultPermissions`), lifecycle hook lockdown (`allowManagedHooksOnly`), remote-control policy (`allowRemoteControl`; `false` force-disables remote control while `true` or `null` preserves existing behavior), computer use policy (`computerUse`), pinned feature values (`featureRequirements`), managed lifecycle hooks (`hooks`), `enforceResidency`, managed new-thread defaults (`models.newThread.model`, `models.newThread.modelReasoningEffort`, and `models.newThread.serviceTier`), and `network` constraints such as canonical domain/socket permissions plus `managedAllowedDomainsOnly` and `dangerFullAccessDenylistOnly`. ### Example: Start or resume a thread @@ -1300,7 +1301,7 @@ Event notifications are the server-initiated event stream for thread lifecycles, Thread realtime uses a separate thread-scoped notification surface. `thread/realtime/*` notifications are ephemeral transport events, not `ThreadItem`s, and are not returned by `thread/read`, `thread/resume`, or `thread/fork`. -Recoverable configuration and initialization warnings use the existing `configWarning` notification: `{ summary, details?, path?, range? }`. App-server may emit it during initialization for config parsing and related setup diagnostics. +Recoverable configuration and initialization warnings use the existing `configWarning` notification: `{ summary, details?, path?, range? }`. App-server may emit it during initialization for config parsing and related setup diagnostics, or to the requesting connection during `thread/start` when that thread's exec-policy rules fail to parse. Generic runtime warnings use the `warning` notification: `{ threadId?, message }`. App-server emits this for non-fatal warnings from the core event stream, including cases where not all enabled skills are included in the model-visible skills list for a session. @@ -1345,7 +1346,7 @@ Because audio is intentionally separate from `ThreadItem`, clients can opt out o ### MCP server startup events -- `mcpServer/startupStatus/updated` — `{ threadId, name, status, error }` when app-server observes an MCP server startup transition. `threadId` identifies the owning thread when startup is thread-scoped and is `null` when startup is app-scoped. `status` is one of `starting`, `ready`, `failed`, or `cancelled`. `error` is `null` except for `failed`. +- `mcpServer/startupStatus/updated` — `{ threadId, name, status, error, failureReason }` when app-server observes an MCP server startup transition. `threadId` identifies the owning thread when startup is thread-scoped and is `null` when startup is app-scoped. `status` is one of `starting`, `ready`, `failed`, or `cancelled`. `error` and `failureReason` are `null` except for `failed`; `failureReason` is `reauthenticationRequired` when stored OAuth credentials have expired and cannot be refreshed, so clients can prompt the user to reconnect the named server. ### Turn events @@ -1372,7 +1373,7 @@ Today both notifications carry an empty `items` array even when item events were - `reasoning` — `{id, summary, content}` where `summary` holds streamed reasoning summaries (applicable for most OpenAI models) and `content` holds raw reasoning blocks (applicable for e.g. open source models). - `commandExecution` — `{id, command, cwd, status, commandActions, aggregatedOutput?, exitCode?, durationMs?}` for sandboxed commands; `status` is `inProgress`, `completed`, `failed`, or `declined`. - `fileChange` — `{id, changes, status}` describing proposed edits; `changes` list `{path, kind, diff}` and `status` is `inProgress`, `completed`, `failed`, or `declined`. -- `mcpToolCall` — `{id, server, tool, status, arguments, appContext, mcpAppResourceUri?, pluginId, result?, error?}` describing MCP calls; `appContext` is `{connectorId, linkId, resourceUri}` for calls through a trusted MCP app, where `connectorId` identifies the connector that owns the tool, `linkId` identifies the app link, and `resourceUri` points to the widget template. The top-level `mcpAppResourceUri` is deprecated and temporarily duplicated for client migration. `tool` identifies the invoked action. `status` is `inProgress`, `completed`, or `failed`. +- `mcpToolCall` — `{id, server, tool, status, arguments, appContext, mcpAppResourceUri?, pluginId, result?, error?}` describing MCP calls; `appContext` is `{connectorId, linkId, resourceUri, appName, templateId, actionName}` for calls through a trusted MCP app, where `connectorId` identifies the connector that owns the tool, `linkId` identifies the app link, `resourceUri` points to the widget template, `appName` is the connector's display name, `templateId` identifies the app template, and `actionName` is the stable connector `Action.name`. `appName`, `templateId`, and `actionName` may be null for older rollout entries. The top-level `mcpAppResourceUri` is deprecated and temporarily duplicated for client migration. `tool` identifies the raw MCP tool. `status` is `inProgress`, `completed`, or `failed`. - `collabToolCall` — `{id, tool, status, senderThreadId, receiverThreadId?, newThreadId?, prompt?, agentStatus?}` describing collab tool calls (`spawn_agent`, `send_input`, `resume_agent`, `wait`, `close_agent`); `status` is `inProgress`, `completed`, or `failed`. - `webSearch` — `{id, query, action?}` for a web search request issued by the agent; `action` mirrors the Responses API web_search action payload (`search`, `open_page`, `find_in_page`) and may be omitted until completion. - `imageView` — `{id, path}` emitted when the agent invokes the image viewer tool. @@ -1928,14 +1929,14 @@ Codex supports these authentication modes. The current mode is surfaced in `acco - `account/login/cancel` — cancel a pending managed ChatGPT login by `loginId`. - `account/logout` — sign out; triggers `account/updated`. - `account/updated` (notify) — emitted whenever auth mode changes (`authMode`: `apikey`, `chatgpt`, `personalAccessToken`, or `null`) and includes the current ChatGPT `planType` when available. -- `account/rateLimits/read` — fetch ChatGPT rate limits, an optional effective monthly credit limit, and the number of earned rate-limit resets currently available. Rate-limit updates arrive via `account/rateLimits/updated` (notify); the reset count is snapshot-only. -- `account/rateLimitResetCredit/consume` — consume one earned reset using a caller-provided idempotency key. +- `account/rateLimits/read` — fetch ChatGPT rate limits, an optional effective monthly credit limit, and the earned rate-limit resets currently available, including expiry details when provided by the backend. Rate-limit updates arrive via `account/rateLimits/updated` (notify); reset-credit data is snapshot-only. +- `account/rateLimitResetCredit/consume` — consume one earned reset using a caller-provided idempotency key, optionally selecting a reset-credit ID returned by `account/rateLimits/read`. - `account/usage/read` — fetch ChatGPT account token-activity summary and daily buckets. - `account/workspaceMessages/read` — fetch active workspace messages, including workspace notification headlines when available. - `account/rateLimits/updated` (notify) — emitted whenever a user's ChatGPT rate limits change. This is a sparse rolling update; merge available values into the most recent `account/rateLimits/read` response or refetch that snapshot. - `account/sendAddCreditsNudgeEmail` — ask ChatGPT to email the workspace owner about depleted credits or a reached usage limit. -- `mcpServer/oauthLogin/completed` (notify) — emitted after a `mcpServer/oauth/login` flow finishes for a server; payload includes `{ name, success, error? }`. -- `mcpServer/startupStatus/updated` (notify) — emitted when a configured MCP server's startup status changes; payload includes `{ threadId, name, status, error }`, where `threadId` is the owning thread when startup is thread-scoped and `null` when it is app-scoped, and `status` is `starting`, `ready`, `failed`, or `cancelled`. +- `mcpServer/oauthLogin/completed` (notify) — emitted after a `mcpServer/oauth/login` flow finishes for a server; payload includes `{ name, threadId, success, error? }`. +- `mcpServer/startupStatus/updated` (notify) — emitted when a configured MCP server's startup status changes; payload includes `{ threadId, name, status, error, failureReason }`, where `threadId` is the owning thread when startup is thread-scoped and `null` when it is app-scoped, and `status` is `starting`, `ready`, `failed`, or `cancelled`. `failureReason` is `reauthenticationRequired` when stored OAuth credentials have expired and cannot be refreshed, so clients can prompt the user to reconnect the named server. ### 1) Check auth state @@ -2031,7 +2032,30 @@ Field notes: ```json { "method": "account/rateLimits/read", "id": 7 } -{ "id": 7, "result": { "rateLimits": { "primary": { "usedPercent": 25, "windowDurationMins": 15, "resetsAt": 1730947200 }, "secondary": null, "rateLimitReachedType": null }, "rateLimitResetCredits": { "availableCount": 2 } } } +{ + "id": 7, + "result": { + "rateLimits": { + "primary": { "usedPercent": 25, "windowDurationMins": 15, "resetsAt": 1730947200 }, + "secondary": null, + "rateLimitReachedType": null + }, + "rateLimitResetCredits": { + "availableCount": 2, + "credits": [ + { + "id": "RateLimitResetCredit_1", + "resetType": "codexRateLimits", + "status": "available", + "grantedAt": 1781654400, + "expiresAt": 1784246400, + "title": "Full reset (Weekly + 5 hr)", + "description": "Ready to redeem" + } + ] + } + } +} { "method": "account/rateLimits/updated", "params": { "rateLimits": { … } } } ``` @@ -2042,23 +2066,27 @@ Field notes: - `resetsAt` is a Unix timestamp (seconds) for the next reset. - `rateLimitReachedType` identifies the backend-classified limit state when one has been reached. - `individualLimit` describes the effective monthly credit limit when available. In an `account/rateLimits/read` response, `null` means no monthly limit is available. In a sparse `account/rateLimits/updated` notification, nullable account metadata may be unavailable and does not clear a previously observed value. -- `rateLimitResetCredits` contains the available earned-reset count when the backend provides it; otherwise it is `null`. Refetch `account/rateLimits/read` after consuming a reset. +- `rateLimitResetCredits` contains the available earned-reset count when the backend provides it; otherwise it is `null`. +- `rateLimitResetCredits.credits` is `null` when only the count is available. An empty array means details were fetched and no available credits were returned. +- The backend may cap `rateLimitResetCredits.credits`, so `availableCount` is the authoritative total and can be greater than the number of detail rows. +- Refetch `account/rateLimits/read` after consuming a reset. ### 8) Earned rate-limit resets (ChatGPT) ```json -{ "method": "account/rateLimitResetCredit/consume", "id": 8, "params": { "idempotencyKey": "8ae96ff3-3425-4f4c-8772-b6fd61502868" } } +{ "method": "account/rateLimitResetCredit/consume", "id": 8, "params": { "idempotencyKey": "8ae96ff3-3425-4f4c-8772-b6fd61502868", "creditId": "RateLimitResetCredit_1" } } { "id": 8, "result": { "outcome": "reset" } } ``` Field notes: - `idempotencyKey` must be non-empty. A UUID is recommended for each logical redemption attempt; reuse the same value when retrying that attempt. +- `creditId` is optional. When provided, it must be a non-empty opaque ID returned by `account/rateLimits/read`; when omitted, the backend selects the next available credit. - `reset` means a credit was consumed. - `alreadyRedeemed` means the same redemption completed previously. Treat it as an idempotent success and refresh account limits. - `nothingToReset` means there is no eligible rate-limit window to reset. - `noCredit` means the account has no earned reset credits available. -- Refetch `account/rateLimits/read` after consuming a reset instead of inferring updated windows from this response. +- Refetch `account/rateLimits/read` after consuming a reset instead of inferring updated state from this response. ### 9) Workspace messages (ChatGPT) diff --git a/codex-rs/app-server/src/bespoke_event_handling.rs b/codex-rs/app-server/src/bespoke_event_handling.rs index 36dbdd9d3b..c63103161f 100644 --- a/codex-rs/app-server/src/bespoke_event_handling.rs +++ b/codex-rs/app-server/src/bespoke_event_handling.rs @@ -200,18 +200,20 @@ pub(crate) async fn apply_bespoke_event_handling( .await; } EventMsg::McpStartupUpdate(update) => { - let (status, error) = match update.status { + let (status, error, failure_reason) = match update.status { codex_protocol::protocol::McpStartupStatus::Starting => { - (McpServerStartupState::Starting, None) + (McpServerStartupState::Starting, None, None) } codex_protocol::protocol::McpStartupStatus::Ready => { - (McpServerStartupState::Ready, None) - } - codex_protocol::protocol::McpStartupStatus::Failed { error } => { - (McpServerStartupState::Failed, Some(error)) + (McpServerStartupState::Ready, None, None) } + codex_protocol::protocol::McpStartupStatus::Failed { error, reason } => ( + McpServerStartupState::Failed, + Some(error), + reason.map(Into::into), + ), codex_protocol::protocol::McpStartupStatus::Cancelled => { - (McpServerStartupState::Cancelled, None) + (McpServerStartupState::Cancelled, None, None) } }; let notification = McpServerStatusUpdatedNotification { @@ -219,6 +221,7 @@ pub(crate) async fn apply_bespoke_event_handling( name: update.server, status, error, + failure_reason, }; outgoing .send_server_notification(ServerNotification::McpServerStatusUpdated(notification)) @@ -2284,6 +2287,7 @@ mod tests { cwd: test_path_buf("/tmp").abs().into(), cli_version: "0.0.0".to_string(), source: SessionSource::Cli, + history_mode: Default::default(), thread_source: None, agent_nickname: None, agent_role: None, diff --git a/codex-rs/app-server/src/current_time.rs b/codex-rs/app-server/src/current_time.rs index 205068be2b..57be618af3 100644 --- a/codex-rs/app-server/src/current_time.rs +++ b/codex-rs/app-server/src/current_time.rs @@ -22,7 +22,8 @@ use crate::outgoing_message::ConnectionId; use crate::outgoing_message::OutgoingMessageSender; use crate::thread_state::ThreadStateManager; -const CURRENT_TIME_REQUEST_TIMEOUT: Duration = Duration::from_secs(5); +const CURRENT_TIME_REQUEST_TIMEOUT: Duration = Duration::from_secs(10); +const CURRENT_TIME_POLL_INTERVAL: Duration = Duration::from_secs(1); pub(crate) fn app_server_time_provider( outgoing: Arc, @@ -51,10 +52,32 @@ impl TimeProvider for AppServerTimeProvider { }) } - fn sleep(&self, _thread_id: ThreadId, duration: Duration) -> SleepFuture<'_> { + fn sleep(&self, thread_id: ThreadId, duration: Duration) -> SleepFuture<'_> { + let outgoing = self.outgoing.clone(); + let thread_state_manager = self.thread_state_manager.clone(); Box::pin(async move { - tokio::time::sleep(duration).await; - Ok(()) + let outgoing = outgoing + .upgrade() + .context("app-server current-time provider is unavailable")?; + let started_at = + request_current_time(outgoing.clone(), thread_state_manager.clone(), thread_id) + .await?; + let wake_at = started_at + .checked_add_signed( + chrono::Duration::from_std(duration) + .context("external sleep duration is outside the supported range")?, + ) + .context("external sleep deadline is outside the supported range")?; + + loop { + tokio::time::sleep(CURRENT_TIME_POLL_INTERVAL).await; + if request_current_time(outgoing.clone(), thread_state_manager.clone(), thread_id) + .await? + >= wake_at + { + return Ok(()); + } + } }) } } diff --git a/codex-rs/app-server/src/in_process.rs b/codex-rs/app-server/src/in_process.rs index 938d539a9f..d8a2bc2276 100644 --- a/codex-rs/app-server/src/in_process.rs +++ b/codex-rs/app-server/src/in_process.rs @@ -80,6 +80,7 @@ use codex_arg0::Arg0DispatchPaths; use codex_config::CloudConfigBundleLoader; use codex_config::LoaderOverrides; use codex_config::ThreadConfigLoader; +use codex_core::check_execpolicy_for_warnings; use codex_core::config::Config; use codex_core::resolve_installation_id; use codex_exec_server::EnvironmentManager; @@ -349,7 +350,16 @@ impl InProcessClientHandle { /// This function sends `initialize` followed by `initialized` before returning /// the handle, so callers receive a ready-to-use runtime. If initialize fails, /// the runtime is shut down and an `InvalidData` error is returned. -pub async fn start(args: InProcessStartArgs) -> IoResult { +pub async fn start(mut args: InProcessStartArgs) -> IoResult { + if let Ok(Some(err)) = check_execpolicy_for_warnings(&args.config.config_layer_stack).await { + let (path, range) = crate::exec_policy_warning_location(&err); + args.config_warnings.push(ConfigWarningNotification { + summary: "Error parsing rules; custom rules not applied.".to_string(), + details: Some(err.to_string()), + path, + range, + }); + } let initialize = args.initialize.clone(); let client = start_uninitialized(args).await?; diff --git a/codex-rs/app-server/src/lib.rs b/codex-rs/app-server/src/lib.rs index beed35848b..fa6b6cd811 100644 --- a/codex-rs/app-server/src/lib.rs +++ b/codex-rs/app-server/src/lib.rs @@ -320,6 +320,16 @@ fn exec_policy_warning_location(err: &ExecPolicyError) -> (Option, Optio } } +fn exec_policy_config_warning(err: &ExecPolicyError) -> ConfigWarningNotification { + let (path, range) = exec_policy_warning_location(err); + ConfigWarningNotification { + summary: "Error parsing rules; custom rules not applied.".to_string(), + details: Some(err.to_string()), + path, + range, + } +} + fn app_text_range(range: &CoreTextRange) -> AppTextRange { AppTextRange { start: AppTextPosition { @@ -612,14 +622,7 @@ pub async fn run_main_with_transport_options( } if let Ok(Some(err)) = check_execpolicy_for_warnings(&config.config_layer_stack).await { - let (path, range) = exec_policy_warning_location(&err); - let message = ConfigWarningNotification { - summary: "Error parsing rules; custom rules not applied.".to_string(), - details: Some(err.to_string()), - path, - range, - }; - config_warnings.push(message); + config_warnings.push(exec_policy_config_warning(&err)); } if let Some(warning) = project_config_warning(&config) { @@ -912,7 +915,7 @@ pub async fn run_main_with_transport_options( async move { let mut listen_for_threads = true; let mut shutdown_state = ShutdownState::default(); - loop { + let exit_reason = loop { let running_turn_count = { let running_turn_count = running_turn_count_rx.borrow(); *running_turn_count @@ -925,7 +928,7 @@ pub async fn run_main_with_transport_options( let _ = outbound_control_tx .send(OutboundControlEvent::DisconnectAll) .await; - break; + break "shutdown_requested"; } tokio::select! { @@ -947,7 +950,7 @@ pub async fn run_main_with_transport_options( } event = transport_event_rx.recv() => { let Some(event) = event else { - break; + break "transport_channel_closed"; }; match event { TransportEvent::ConnectionOpened { @@ -977,7 +980,7 @@ pub async fn run_main_with_transport_options( .await .is_err() { - break; + break "outbound_router_closed"; } connections.insert( connection_id, @@ -1005,10 +1008,10 @@ pub async fn run_main_with_transport_options( .await; }); if !outbound_closed { - break; + break "outbound_router_closed"; } if shutdown_when_no_connections && connections.is_empty() { - break; + break "last_connection_closed"; } } TransportEvent::IncomingMessage { connection_id, message } => { @@ -1147,7 +1150,7 @@ pub async fn run_main_with_transport_options( } } } - } + }; if !shutdown_state.forced() { futures::future::join_all( @@ -1162,7 +1165,12 @@ pub async fn run_main_with_transport_options( } else { connection_cleanup_tasks.abort(); } - info!("processor task exited (channel closed)"); + info!( + exit_reason, + remaining_connection_count = connections.len(), + shutdown_forced = shutdown_state.forced(), + "processor task exited" + ); } }); diff --git a/codex-rs/app-server/src/message_processor.rs b/codex-rs/app-server/src/message_processor.rs index 41ecbfe561..6ae73784ff 100644 --- a/codex-rs/app-server/src/message_processor.rs +++ b/codex-rs/app-server/src/message_processor.rs @@ -447,7 +447,7 @@ impl MessageProcessor { outgoing.clone(), analytics_events_client.clone(), Arc::clone(&config), - config_warnings, + config_warnings.clone(), rpc_transport, ); let marketplace_processor = MarketplaceRequestProcessor::new( @@ -495,6 +495,7 @@ impl MessageProcessor { state_db.clone(), log_db, Arc::clone(&skills_watcher), + config_warnings, ); let turn_processor = TurnRequestProcessor::new( auth_manager.clone(), @@ -1037,6 +1038,9 @@ impl MessageProcessor { ClientRequest::EnvironmentAdd { params, .. } => { self.environment_processor.environment_add(params).await } + ClientRequest::EnvironmentInfo { params, .. } => { + self.environment_processor.environment_info(params).await + } ClientRequest::FsReadFile { params, .. } => self .fs_processor .read_file(params) @@ -1205,7 +1209,7 @@ impl MessageProcessor { } ClientRequest::ThreadRollback { params, .. } => { self.thread_processor - .thread_rollback(&request_id, params) + .thread_rollback(&request_id, params, app_server_client_name.as_deref()) .await } ClientRequest::ThreadList { params, .. } => { diff --git a/codex-rs/app-server/src/outgoing_message.rs b/codex-rs/app-server/src/outgoing_message.rs index 9ce815bafe..16d5c2ff9d 100644 --- a/codex-rs/app-server/src/outgoing_message.rs +++ b/codex-rs/app-server/src/outgoing_message.rs @@ -504,13 +504,36 @@ impl OutgoingMessageSender { where T: Into, { - self.send_response_as(request_id, response.into()).await; + self.send_response_as_inner(request_id, response.into(), /*thread_originator*/ None) + .await; + } + + pub(crate) async fn send_response_with_thread_originator( + &self, + request_id: ConnectionRequestId, + response: T, + thread_originator: String, + ) where + T: Into, + { + self.send_response_as_inner(request_id, response.into(), Some(thread_originator)) + .await; } pub(crate) async fn send_response_as( &self, request_id: ConnectionRequestId, response: ClientResponsePayload, + ) { + self.send_response_as_inner(request_id, response, /*thread_originator*/ None) + .await; + } + + async fn send_response_as_inner( + &self, + request_id: ConnectionRequestId, + response: ClientResponsePayload, + thread_originator: Option, ) { let connection_id = request_id.connection_id; let request_id_for_analytics = request_id.request_id.clone(); @@ -518,11 +541,24 @@ impl OutgoingMessageSender { .into_jsonrpc_parts_and_payload(request_id.request_id.clone()) .map(|(id, result, response)| { if let Some(response) = response { - self.analytics_events_client.track_response( - connection_id.0, - request_id_for_analytics, - response, - ); + match thread_originator { + Some(thread_originator) => { + self.analytics_events_client + .track_response_with_thread_originator( + connection_id.0, + request_id_for_analytics, + response, + thread_originator, + ); + } + None => { + self.analytics_events_client.track_response( + connection_id.0, + request_id_for_analytics, + response, + ); + } + } } (id, result) }); diff --git a/codex-rs/app-server/src/request_processors.rs b/codex-rs/app-server/src/request_processors.rs index d73a41706c..f03351468f 100644 --- a/codex-rs/app-server/src/request_processors.rs +++ b/codex-rs/app-server/src/request_processors.rs @@ -56,11 +56,15 @@ use codex_app_server_protocol::ConsumeAccountRateLimitResetCreditParams; use codex_app_server_protocol::ConsumeAccountRateLimitResetCreditResponse; use codex_app_server_protocol::ConversationGitInfo; use codex_app_server_protocol::ConversationSummary; +use codex_app_server_protocol::DeprecationNoticeNotification; use codex_app_server_protocol::DynamicToolFunctionSpec; use codex_app_server_protocol::DynamicToolNamespaceTool; use codex_app_server_protocol::DynamicToolSpec; use codex_app_server_protocol::EnvironmentAddParams; use codex_app_server_protocol::EnvironmentAddResponse; +use codex_app_server_protocol::EnvironmentInfoParams; +use codex_app_server_protocol::EnvironmentInfoResponse; +use codex_app_server_protocol::EnvironmentShellInfo; use codex_app_server_protocol::ExperimentalFeature as ApiExperimentalFeature; use codex_app_server_protocol::ExperimentalFeatureListParams; use codex_app_server_protocol::ExperimentalFeatureListResponse; @@ -152,7 +156,10 @@ use codex_app_server_protocol::PluginSource; use codex_app_server_protocol::PluginSummary; use codex_app_server_protocol::PluginUninstallParams; use codex_app_server_protocol::PluginUninstallResponse; +use codex_app_server_protocol::RateLimitResetCredit; +use codex_app_server_protocol::RateLimitResetCreditStatus; use codex_app_server_protocol::RateLimitResetCreditsSummary; +use codex_app_server_protocol::RateLimitResetType; use codex_app_server_protocol::RequestId; use codex_app_server_protocol::ReviewDelivery as ApiReviewDelivery; use codex_app_server_protocol::ReviewStartParams; @@ -205,6 +212,8 @@ use codex_app_server_protocol::ThreadGoalSetResponse; use codex_app_server_protocol::ThreadGoalStatus; use codex_app_server_protocol::ThreadGoalUpdatedNotification; use codex_app_server_protocol::ThreadHistoryBuilder; +#[cfg(test)] +use codex_app_server_protocol::ThreadHistoryMode; use codex_app_server_protocol::ThreadIncrementElicitationParams; use codex_app_server_protocol::ThreadIncrementElicitationResponse; use codex_app_server_protocol::ThreadInjectItemsParams; @@ -292,6 +301,8 @@ use codex_backend_client::CodexWorkspaceMessage as BackendWorkspaceMessage; use codex_backend_client::CodexWorkspaceMessageType as BackendWorkspaceMessageType; use codex_backend_client::CodexWorkspaceMessagesResponse as BackendWorkspaceMessagesResponse; use codex_backend_client::ConsumeRateLimitResetCreditCode as BackendConsumeRateLimitResetCreditCode; +use codex_backend_client::RateLimitResetCreditDetails as BackendRateLimitResetCreditDetails; +use codex_backend_client::RateLimitResetCreditsDetails as BackendRateLimitResetCreditsDetails; use codex_backend_client::RequestError as BackendRequestError; use codex_backend_client::TokenUsageProfile; use codex_chatgpt::connectors; @@ -327,6 +338,7 @@ use codex_core::path_utils; #[cfg(test)] use codex_core::read_head_for_summary; use codex_core::sandboxing::SandboxPermissions; +use codex_core::truncate_rollout_after_turn_id; use codex_core::windows_sandbox::WindowsSandboxLevelExt; use codex_core::windows_sandbox::WindowsSandboxSetupMode as CoreWindowsSandboxSetupMode; use codex_core::windows_sandbox::WindowsSandboxSetupRequest; @@ -380,7 +392,7 @@ use codex_mcp::McpRuntimeContext; use codex_mcp::McpServerStatusSnapshot; use codex_mcp::McpSnapshotDetail; use codex_mcp::collect_mcp_server_status_snapshot_with_detail; -use codex_mcp::discover_supported_scopes; +use codex_mcp::discover_supported_scopes_with_http_client; use codex_mcp::read_mcp_resource as read_mcp_resource_without_thread; use codex_mcp::resolve_oauth_scopes; use codex_memories_write::clear_memory_roots_contents; @@ -428,7 +440,7 @@ use codex_protocol::protocol::USER_MESSAGE_BEGIN; use codex_protocol::protocol::W3cTraceContext; use codex_protocol::user_input::MAX_USER_INPUT_TEXT_CHARS; use codex_protocol::user_input::UserInput as CoreInputItem; -use codex_rmcp_client::perform_oauth_login_return_url; +use codex_rmcp_client::perform_oauth_login_return_url_with_http_client; use codex_rollout::is_persisted_rollout_item; use codex_rollout::state_db::StateDbHandle; use codex_rollout::state_db::reconcile_rollout; diff --git a/codex-rs/app-server/src/request_processors/account_processor.rs b/codex-rs/app-server/src/request_processors/account_processor.rs index 4691e22204..9592563bec 100644 --- a/codex-rs/app-server/src/request_processors/account_processor.rs +++ b/codex-rs/app-server/src/request_processors/account_processor.rs @@ -891,9 +891,11 @@ impl AccountRequestProcessor { let client = BackendClient::from_auth(self.config.chatgpt_base_url.clone(), &auth) .map_err(|err| internal_error(format!("failed to construct backend client: {err}")))?; - let response = client - .get_rate_limits_with_reset_credits() - .await + let (response, detailed_rate_limit_reset_credits) = tokio::join!( + client.get_rate_limits_with_reset_credits(), + Self::detailed_rate_limit_reset_credits(&client), + ); + let response = response .map_err(|err| internal_error(format!("failed to fetch codex rate limits: {err}")))?; if response.rate_limits.is_empty() { return Err(internal_error( @@ -920,6 +922,15 @@ impl AccountRequestProcessor { .cloned() .unwrap_or_else(|| response.rate_limits[0].clone()); + let rate_limit_reset_credits = detailed_rate_limit_reset_credits.or_else(|| { + response + .rate_limit_reset_credits + .map(|summary| RateLimitResetCreditsSummary { + available_count: summary.available_count, + credits: None, + }) + }); + Ok(GetAccountRateLimitsResponse { rate_limits: rate_limits.into(), rate_limits_by_limit_id: Some( @@ -928,11 +939,7 @@ impl AccountRequestProcessor { .map(|(limit_id, snapshot)| (limit_id, snapshot.into())) .collect(), ), - rate_limit_reset_credits: response.rate_limit_reset_credits.map(|summary| { - RateLimitResetCreditsSummary { - available_count: summary.available_count, - } - }), + rate_limit_reset_credits, }) } diff --git a/codex-rs/app-server/src/request_processors/account_processor/rate_limit_resets.rs b/codex-rs/app-server/src/request_processors/account_processor/rate_limit_resets.rs index c8a397d3a7..919d30d770 100644 --- a/codex-rs/app-server/src/request_processors/account_processor/rate_limit_resets.rs +++ b/codex-rs/app-server/src/request_processors/account_processor/rate_limit_resets.rs @@ -1,11 +1,47 @@ use super::*; const RATE_LIMIT_RESET_REQUEST_TIMEOUT: Duration = Duration::from_secs(/*secs*/ 10); +const RATE_LIMIT_RESET_DETAILS_REQUEST_TIMEOUT: Duration = Duration::from_secs(/*secs*/ 5); #[cfg(debug_assertions)] const RATE_LIMIT_RESET_REQUEST_TIMEOUT_ENV_VAR: &str = "CODEX_TEST_RATE_LIMIT_RESET_REQUEST_TIMEOUT_MS"; impl AccountRequestProcessor { + pub(super) async fn detailed_rate_limit_reset_credits( + client: &BackendClient, + ) -> Option { + let details = match tokio::time::timeout( + RATE_LIMIT_RESET_DETAILS_REQUEST_TIMEOUT, + client.list_rate_limit_reset_credits(), + ) + .await + { + Ok(Ok(details)) => details, + Ok(Err(err)) => { + tracing::warn!( + "failed to fetch rate limit reset credit details; falling back to the usage response: {err}" + ); + return None; + } + Err(_) => { + tracing::warn!( + "rate limit reset credit detail request timed out; falling back to the usage response" + ); + return None; + } + }; + + match rate_limit_reset_credits_from_backend(details) { + Ok(summary) => Some(summary), + Err(err) => { + tracing::warn!( + "failed to parse rate limit reset credit details; falling back to the usage response: {err}" + ); + None + } + } + } + pub(crate) async fn consume_account_rate_limit_reset_credit( &self, params: ConsumeAccountRateLimitResetCreditParams, @@ -13,6 +49,9 @@ impl AccountRequestProcessor { if params.idempotency_key.is_empty() { return Err(invalid_request("idempotencyKey must not be empty")); } + if params.credit_id.as_deref().is_some_and(str::is_empty) { + return Err(invalid_request("creditId must not be empty")); + } let client = self.rate_limit_reset_backend_client().await?; let request_timeout = RATE_LIMIT_RESET_REQUEST_TIMEOUT; @@ -22,10 +61,20 @@ impl AccountRequestProcessor { .and_then(|value| value.parse::().ok()) .map(Duration::from_millis) .unwrap_or(request_timeout); - let response = tokio::time::timeout( - request_timeout, - client.consume_rate_limit_reset_credit(¶ms.idempotency_key), - ) + let response = tokio::time::timeout(request_timeout, async { + match params.credit_id.as_deref() { + Some(credit_id) => { + client + .consume_rate_limit_reset_credit_by_id(¶ms.idempotency_key, credit_id) + .await + } + None => { + client + .consume_rate_limit_reset_credit(¶ms.idempotency_key) + .await + } + } + }) .await .map_err(|_| internal_error("rate limit reset consume timed out"))? .map_err(|err| internal_error(format!("failed to consume rate limit reset: {err}")))?; @@ -64,3 +113,56 @@ impl AccountRequestProcessor { .map_err(|err| internal_error(format!("failed to construct backend client: {err}"))) } } + +fn rate_limit_reset_credits_from_backend( + details: BackendRateLimitResetCreditsDetails, +) -> Result { + let credits = details + .credits + .into_iter() + .map(rate_limit_reset_credit_from_backend) + .collect::, _>>()?; + Ok(RateLimitResetCreditsSummary { + available_count: details.available_count, + credits: Some(credits), + }) +} + +fn rate_limit_reset_credit_from_backend( + credit: BackendRateLimitResetCreditDetails, +) -> Result { + let reset_type = match credit.reset_type.as_str() { + "codex_rate_limits" => RateLimitResetType::CodexRateLimits, + _ => RateLimitResetType::Unknown, + }; + let status = match credit.status.as_str() { + "available" => RateLimitResetCreditStatus::Available, + "redeeming" => RateLimitResetCreditStatus::Redeeming, + "redeemed" => RateLimitResetCreditStatus::Redeemed, + _ => RateLimitResetCreditStatus::Unknown, + }; + let granted_at = rate_limit_reset_credit_timestamp(&credit.granted_at) + .map_err(|err| format!("invalid granted_at for credit `{}`: {err}", credit.id))?; + let expires_at = credit + .expires_at + .as_deref() + .map(rate_limit_reset_credit_timestamp) + .transpose() + .map_err(|err| format!("invalid expires_at for credit `{}`: {err}", credit.id))?; + + Ok(RateLimitResetCredit { + id: credit.id, + reset_type, + status, + granted_at, + expires_at, + title: credit.title, + description: credit.description, + }) +} + +fn rate_limit_reset_credit_timestamp(timestamp: &str) -> Result { + DateTime::parse_from_rfc3339(timestamp) + .map(|timestamp| timestamp.timestamp()) + .map_err(|err| format!("failed to parse timestamp `{timestamp}`: {err}")) +} diff --git a/codex-rs/app-server/src/request_processors/config_processor.rs b/codex-rs/app-server/src/request_processors/config_processor.rs index 50bba54009..b6dbb30b08 100644 --- a/codex-rs/app-server/src/request_processors/config_processor.rs +++ b/codex-rs/app-server/src/request_processors/config_processor.rs @@ -24,9 +24,11 @@ use codex_app_server_protocol::ExperimentalFeatureEnablementSetResponse; use codex_app_server_protocol::JSONRPCErrorError; use codex_app_server_protocol::ManagedHooksRequirements; use codex_app_server_protocol::ModelProviderCapabilitiesReadResponse; +use codex_app_server_protocol::ModelsRequirements; use codex_app_server_protocol::NetworkDomainPermission; use codex_app_server_protocol::NetworkRequirements; use codex_app_server_protocol::NetworkUnixSocketPermission; +use codex_app_server_protocol::NewThreadModelDefaults; use codex_app_server_protocol::SandboxMode; use codex_app_server_protocol::WindowsSandboxSetupMode; use codex_config::ConfigRequirementsToml; @@ -375,6 +377,13 @@ fn map_requirements_toml_to_api(requirements: ConfigRequirementsToml) -> ConfigR .enforce_residency .map(map_residency_requirement_to_api), network: requirements.network.map(map_network_requirements_to_api), + models: requirements.models.map(|models| ModelsRequirements { + new_thread: models.new_thread.map(|new_thread| NewThreadModelDefaults { + model: new_thread.model, + model_reasoning_effort: new_thread.model_reasoning_effort, + service_tier: new_thread.service_tier, + }), + }), } } @@ -568,7 +577,10 @@ mod tests { use codex_app_server_protocol::WindowsSandboxSetupMode; use codex_config::ComputerUseRequirementsToml; use codex_config::ConfigRequirementsToml; + use codex_config::ModelsRequirementsToml; + use codex_config::NewThreadModelDefaultsToml; use codex_config::WindowsRequirementsToml; + use codex_protocol::openai_models::ReasoningEffort; use pretty_assertions::assert_eq; use std::collections::BTreeMap; @@ -628,6 +640,31 @@ mod tests { assert_eq!(mapped.allow_remote_control, Some(false)); } + #[test] + fn requirements_api_includes_new_thread_model_defaults() { + let mapped = map_requirements_toml_to_api(ConfigRequirementsToml { + models: Some(ModelsRequirementsToml { + new_thread: Some(NewThreadModelDefaultsToml { + model: Some("gpt-managed".to_string()), + model_reasoning_effort: Some(ReasoningEffort::Medium), + service_tier: Some("fast".to_string()), + }), + }), + ..ConfigRequirementsToml::default() + }); + + let defaults = mapped + .models + .and_then(|models| models.new_thread) + .expect("new-thread defaults"); + assert_eq!(defaults.model.as_deref(), Some("gpt-managed")); + assert_eq!( + defaults.model_reasoning_effort, + Some(ReasoningEffort::Medium) + ); + assert_eq!(defaults.service_tier.as_deref(), Some("fast")); + } + #[test] fn requirements_api_includes_computer_use_requirements() { let mapped = map_requirements_toml_to_api(ConfigRequirementsToml { diff --git a/codex-rs/app-server/src/request_processors/environment_processor.rs b/codex-rs/app-server/src/request_processors/environment_processor.rs index 36533aa231..635e1732ec 100644 --- a/codex-rs/app-server/src/request_processors/environment_processor.rs +++ b/codex-rs/app-server/src/request_processors/environment_processor.rs @@ -26,4 +26,30 @@ impl EnvironmentRequestProcessor { .map_err(|err| invalid_request(err.to_string()))?; Ok(Some(EnvironmentAddResponse {}.into())) } + + pub(crate) async fn environment_info( + &self, + params: EnvironmentInfoParams, + ) -> Result, JSONRPCErrorError> { + let environment_id = params.environment_id; + let environment = self + .environment_manager + .get_environment(&environment_id) + .ok_or_else(|| invalid_request(format!("unknown environment id `{environment_id}`")))?; + let info = environment.info().await.map_err(|err| { + internal_error(format!( + "failed to get info for environment `{environment_id}`: {err}" + )) + })?; + Ok(Some( + EnvironmentInfoResponse { + shell: EnvironmentShellInfo { + name: info.shell.name, + path: info.shell.path, + }, + cwd: info.cwd, + } + .into(), + )) + } } diff --git a/codex-rs/app-server/src/request_processors/external_agent_session_import.rs b/codex-rs/app-server/src/request_processors/external_agent_session_import.rs index 77b9bfcbf3..e0a53a9c98 100644 --- a/codex-rs/app-server/src/request_processors/external_agent_session_import.rs +++ b/codex-rs/app-server/src/request_processors/external_agent_session_import.rs @@ -15,6 +15,7 @@ use codex_models_manager::manager::RefreshStrategy; use codex_protocol::ThreadId; use codex_protocol::models::BaseInstructions; use codex_protocol::protocol::MultiAgentVersion; +use codex_protocol::protocol::ThreadHistoryMode; use codex_protocol::protocol::ThreadMemoryMode; use codex_rollout::is_persisted_rollout_item; use codex_thread_store::AppendThreadItemsParams; @@ -184,7 +185,11 @@ impl ExternalAgentSessionImporter { .map_err(|err| format!("failed to load imported session config: {err}"))?; let models_manager = self.thread_manager.get_models_manager(); let model = models_manager - .get_default_model(&config.model, RefreshStrategy::Offline) + .get_default_model( + &config.model, + /*allow_provider_model_fallback*/ false, + RefreshStrategy::Offline, + ) .await; let model_info = models_manager .get_model_info(model.as_str(), &config.to_models_manager_config()) @@ -215,7 +220,9 @@ impl ExternalAgentSessionImporter { .unwrap_or_else(|| model_info.get_model_instructions(config.personality)), }, dynamic_tools: Vec::new(), + selected_capability_roots: Vec::new(), multi_agent_version: Some(MultiAgentVersion::V1), + history_mode: ThreadHistoryMode::Legacy, initial_window_id: uuid::Uuid::now_v7().to_string(), metadata: ThreadPersistenceMetadata { cwd: Some(cwd.clone()), diff --git a/codex-rs/app-server/src/request_processors/mcp_processor.rs b/codex-rs/app-server/src/request_processors/mcp_processor.rs index 08d464598b..ac2125c2cf 100644 --- a/codex-rs/app-server/src/request_processors/mcp_processor.rs +++ b/codex-rs/app-server/src/request_processors/mcp_processor.rs @@ -113,19 +113,35 @@ impl McpRequestProcessor { &self, params: McpServerOauthLoginParams, ) -> Result { - let config = self.load_latest_config(/*fallback_cwd*/ None).await?; let McpServerOauthLoginParams { name, + thread_id, scopes, timeout_secs, } = params; let auth = self.auth_manager.auth().await; - let effective_servers = self - .thread_manager - .mcp_manager() - .effective_servers(&config, auth.as_ref()) - .await; + let (mcp_config, runtime_context) = match thread_id.as_deref() { + Some(thread_id) => { + let (_, thread) = self.load_thread(thread_id).await?; + let runtime = thread.current_mcp_runtime().await; + (runtime.config().clone(), runtime.runtime_context().clone()) + } + None => { + let config = self.load_latest_config(/*fallback_cwd*/ None).await?; + let mcp_config = self + .thread_manager + .mcp_manager() + .runtime_config(&config) + .await; + let runtime_context = McpRuntimeContext::new( + self.thread_manager.environment_manager(), + config.cwd.to_path_buf(), + ); + (mcp_config, runtime_context) + } + }; + let effective_servers = codex_mcp::effective_mcp_servers(&mcp_config, auth.as_ref()); let Some(server) = effective_servers .get(&name) .and_then(codex_mcp::EffectiveMcpServer::configured_config) @@ -149,32 +165,41 @@ impl McpRequestProcessor { } }; + let http_client = runtime_context + .resolve_http_client(&name, server) + .map_err(|err| { + internal_error(format!("failed to resolve MCP server runtime: {err}")) + })?; + let discovered_scopes = if scopes.is_none() && server.scopes.is_none() { - discover_supported_scopes(&server.transport).await + discover_supported_scopes_with_http_client(&server.transport, Arc::clone(&http_client)) + .await } else { None }; let resolved_scopes = resolve_oauth_scopes(scopes, server.scopes.clone(), discovered_scopes); - let handle = perform_oauth_login_return_url( + let handle = perform_oauth_login_return_url_with_http_client( &name, &url, - config.mcp_oauth_credentials_store_mode, - config.auth_keyring_backend_kind(), + mcp_config.mcp_oauth_credentials_store_mode, + mcp_config.auth_keyring_backend_kind, http_headers, env_http_headers, &resolved_scopes.scopes, server.oauth_client_id(), server.oauth_resource.as_deref(), timeout_secs, - config.mcp_oauth_callback_port, - config.mcp_oauth_callback_url.as_deref(), + mcp_config.mcp_oauth_callback_port, + mcp_config.mcp_oauth_callback_url.as_deref(), + http_client, ) .await .map_err(|err| internal_error(format!("failed to login to MCP server '{name}': {err}")))?; let authorization_url = handle.authorization_url().to_string(); let notification_name = name.clone(); + let notification_thread_id = thread_id; let outgoing = Arc::clone(&self.outgoing); tokio::spawn(async move { @@ -186,6 +211,7 @@ impl McpRequestProcessor { let notification = ServerNotification::McpServerOauthLoginCompleted( McpServerOauthLoginCompletedNotification { name: notification_name, + thread_id: notification_thread_id, success, error, }, @@ -217,22 +243,24 @@ impl McpRequestProcessor { } None => (self.load_latest_config(/*fallback_cwd*/ None).await?, None), }; - let mcp_config = match thread { - Some(thread) => thread.runtime_mcp_config(&config).await, + let mcp_manager = self.thread_manager.mcp_manager(); + let codex_apps_tools_cache = mcp_manager.codex_apps_tools_cache(); + let auth = self.auth_manager.auth().await; + let (mcp_config, runtime_context) = match thread { + Some(thread) => { + let mcp_config = thread.runtime_mcp_config(&config).await; + let runtime = thread.current_mcp_runtime().await; + (mcp_config, runtime.runtime_context().clone()) + } None => { - self.thread_manager - .mcp_manager() - .runtime_config(&config) - .await + let mcp_config = mcp_manager.runtime_config(&config).await; + let runtime_context = McpRuntimeContext::new( + self.thread_manager.environment_manager(), + config.cwd.to_path_buf(), + ); + (mcp_config, runtime_context) } }; - let auth = self.auth_manager.auth().await; - let environment_manager = self.thread_manager.environment_manager(); - // This status path has no turn-selected environment. Use config cwd - // as the local stdio fallback; named environment stdio MCPs must - // declare their own absolute cwd. - let runtime_context = - McpRuntimeContext::new(Arc::clone(&environment_manager), config.cwd.to_path_buf()); tokio::spawn(async move { Self::list_mcp_server_status_task( @@ -242,6 +270,7 @@ impl McpRequestProcessor { mcp_config, auth, runtime_context, + codex_apps_tools_cache, ) .await; }); @@ -255,6 +284,7 @@ impl McpRequestProcessor { mcp_config: codex_mcp::McpConfig, auth: Option, runtime_context: McpRuntimeContext, + codex_apps_tools_cache: codex_mcp::CodexAppsToolsCache, ) { let result = Self::list_mcp_server_status_response( request_id.request_id.to_string(), @@ -262,6 +292,7 @@ impl McpRequestProcessor { mcp_config, auth, runtime_context, + codex_apps_tools_cache, ) .await; outgoing.send_result(request_id, result).await; @@ -273,6 +304,7 @@ impl McpRequestProcessor { mcp_config: codex_mcp::McpConfig, auth: Option, runtime_context: McpRuntimeContext, + codex_apps_tools_cache: codex_mcp::CodexAppsToolsCache, ) -> Result { let detail = match params.detail.unwrap_or(McpServerStatusDetail::Full) { McpServerStatusDetail::Full => McpSnapshotDetail::Full, @@ -284,6 +316,7 @@ impl McpRequestProcessor { auth.as_ref(), request_id, runtime_context, + codex_apps_tools_cache, detail, ) .await; @@ -374,11 +407,9 @@ impl McpRequestProcessor { } let config = self.load_latest_config(/*fallback_cwd*/ None).await?; - let mcp_config = self - .thread_manager - .mcp_manager() - .runtime_config(&config) - .await; + let mcp_manager = self.thread_manager.mcp_manager(); + let mcp_config = mcp_manager.runtime_config(&config).await; + let codex_apps_tools_cache = mcp_manager.codex_apps_tools_cache(); let auth = self.auth_manager.auth().await; let environment_manager = self.thread_manager.environment_manager(); // This threadless resource-read path has no turn cwd or turn-selected @@ -393,6 +424,7 @@ impl McpRequestProcessor { &mcp_config, auth.as_ref(), runtime_context, + codex_apps_tools_cache, &server, &uri, ) diff --git a/codex-rs/app-server/src/request_processors/plugins.rs b/codex-rs/app-server/src/request_processors/plugins.rs index 4aa8352d15..35c12d49c1 100644 --- a/codex-rs/app-server/src/request_processors/plugins.rs +++ b/codex-rs/app-server/src/request_processors/plugins.rs @@ -101,6 +101,15 @@ fn marketplace_plugin_source_to_info(source: MarketplacePluginSource) -> PluginS ref_name, sha, }, + MarketplacePluginSource::Npm { + package, + version, + registry, + } => PluginSource::Npm { + package, + version, + registry, + }, } } @@ -134,7 +143,7 @@ fn share_context_for_source( creator_name: None, share_principals: None, }), - MarketplacePluginSource::Git { .. } => None, + MarketplacePluginSource::Git { .. } | MarketplacePluginSource::Npm { .. } => None, } } @@ -146,6 +155,7 @@ fn convert_configured_marketplace_plugin_to_plugin_summary( PluginSummary { id: plugin.id, remote_plugin_id: None, + version: None, local_version: plugin.local_version, installed: plugin.installed, enabled: plugin.enabled, @@ -1086,6 +1096,7 @@ impl PluginRequestProcessor { summary: PluginSummary { id: outcome.plugin.id, remote_plugin_id: None, + version: None, local_version: outcome.plugin.local_version, name: outcome.plugin.name, share_context, @@ -1883,6 +1894,7 @@ impl PluginRequestProcessor { let notification = ServerNotification::McpServerOauthLoginCompleted( McpServerOauthLoginCompletedNotification { name: notification_name, + thread_id: None, success, error, }, @@ -2158,6 +2170,7 @@ fn remote_plugin_summary_to_info(summary: RemoteCatalogPluginSummary) -> PluginS PluginSummary { id: summary.id, remote_plugin_id: Some(summary.remote_plugin_id), + version: summary.version, local_version: summary.local_version, name: summary.name, share_context: summary diff --git a/codex-rs/app-server/src/request_processors/thread_lifecycle.rs b/codex-rs/app-server/src/request_processors/thread_lifecycle.rs index 03031c87f8..82a49bae0b 100644 --- a/codex-rs/app-server/src/request_processors/thread_lifecycle.rs +++ b/codex-rs/app-server/src/request_processors/thread_lifecycle.rs @@ -640,6 +640,7 @@ pub(super) async fn handle_pending_thread_resume_request( active_permission_profile, workspace_roots, reasoning_effort, + originator, .. } = config_snapshot; let instruction_sources = pending.instruction_sources; @@ -665,7 +666,9 @@ pub(super) async fn handle_pending_thread_resume_request( multi_agent_mode: MultiAgentMode::ExplicitRequestOnly, initial_turns_page, }; - outgoing.send_response(request_id, response).await; + outgoing + .send_response_with_thread_originator(request_id, response, originator) + .await; // Match cold resume: metadata-only resume should attach the listener without // paying the cost of turn reconstruction for historical usage replay. if let Some(token_usage_thread) = token_usage_thread { diff --git a/codex-rs/app-server/src/request_processors/thread_processor.rs b/codex-rs/app-server/src/request_processors/thread_processor.rs index 7242e816be..af8525873f 100644 --- a/codex-rs/app-server/src/request_processors/thread_processor.rs +++ b/codex-rs/app-server/src/request_processors/thread_processor.rs @@ -5,9 +5,13 @@ use codex_extension_api::ExtensionDataInit; use codex_protocol::config_types::MultiAgentMode; use codex_protocol::models::BUILT_IN_PERMISSION_PROFILE_DANGER_FULL_ACCESS; use codex_protocol::models::BUILT_IN_PERMISSION_PROFILE_WORKSPACE; +use codex_protocol::protocol::ThreadHistoryMode; const THREAD_LIST_DEFAULT_LIMIT: usize = 25; const THREAD_LIST_MAX_LIMIT: usize = 100; +const CODEX_TUI_CLIENT_NAME: &str = "codex-tui"; +const THREAD_ROLLBACK_DEPRECATION_SUMMARY: &str = + "thread/rollback is deprecated and will be removed soon"; struct ThreadListFilters { model_providers: Option>, @@ -360,6 +364,7 @@ pub(crate) struct ThreadRequestProcessor { pub(super) log_db: Option, pub(super) background_tasks: TaskTracker, pub(super) skills_watcher: Arc, + pub(super) initial_config_warnings: Arc>, } /// Outcome of trying to satisfy a resume request from an already loaded thread. @@ -391,6 +396,7 @@ impl ThreadRequestProcessor { state_db: Option, log_db: Option, skills_watcher: Arc, + initial_config_warnings: Vec, ) -> Self { Self { auth_manager, @@ -409,6 +415,7 @@ impl ThreadRequestProcessor { log_db, background_tasks: TaskTracker::new(), skills_watcher, + initial_config_warnings: Arc::new(initial_config_warnings), } } @@ -632,12 +639,29 @@ impl ThreadRequestProcessor { &self, request_id: &ConnectionRequestId, params: ThreadRollbackParams, + app_server_client_name: Option<&str>, ) -> Result, JSONRPCErrorError> { + if app_server_client_name != Some(CODEX_TUI_CLIENT_NAME) { + self.send_thread_rollback_deprecation_notice(request_id.connection_id) + .await; + } self.thread_rollback_inner(request_id, params) .await .map(|()| None) } + async fn send_thread_rollback_deprecation_notice(&self, connection_id: ConnectionId) { + self.outgoing + .send_server_notification_to_connections( + &[connection_id], + ServerNotification::DeprecationNotice(DeprecationNoticeNotification { + summary: THREAD_ROLLBACK_DEPRECATION_SUMMARY.to_string(), + details: None, + }), + ) + .await; + } + pub(crate) async fn thread_list( &self, params: ThreadListParams, @@ -887,6 +911,7 @@ impl ThreadRequestProcessor { let ThreadStartParams { model, model_provider, + allow_provider_model_fallback, service_tier, cwd, runtime_workspace_roots, @@ -905,6 +930,7 @@ impl ThreadRequestProcessor { personality, multi_agent_mode: _multi_agent_mode, ephemeral, + history_mode, session_start_source, thread_source, environments, @@ -945,6 +971,7 @@ impl ThreadRequestProcessor { }; let request_trace = request_context.request_trace(); let config_manager = self.config_manager.clone(); + let initial_config_warnings = Arc::clone(&self.initial_config_warnings); let outgoing = Arc::clone(&listener_task_context.outgoing); let error_request_id = request_id.clone(); let thread_start_task = async move { @@ -959,12 +986,15 @@ impl ThreadRequestProcessor { typesafe_overrides, dynamic_tools, selected_capability_roots.unwrap_or_default(), + history_mode.map(Into::into), session_start_source, thread_source.map(Into::into), environment_selections, service_name, + allow_provider_model_fallback, experimental_raw_events, request_trace, + initial_config_warnings, ) .await { @@ -1033,12 +1063,15 @@ impl ThreadRequestProcessor { typesafe_overrides: ConfigOverrides, dynamic_tools: Option>, selected_capability_roots: Vec, + history_mode: Option, session_start_source: Option, thread_source: Option, environments: Option>, service_name: Option, + allow_provider_model_fallback: bool, experimental_raw_events: bool, request_trace: Option, + initial_config_warnings: Arc>, ) -> Result<(), JSONRPCErrorError> { let thread_start_started_at = std::time::Instant::now(); let requested_cwd = typesafe_overrides.cwd.clone(); @@ -1112,6 +1145,21 @@ impl ThreadRequestProcessor { .map_err(|err| config_load_error(&err))?; } + if let Ok(Some(err)) = + codex_core::check_execpolicy_for_warnings(&config.config_layer_stack).await + { + let notification = crate::exec_policy_config_warning(&err); + if !initial_config_warnings.contains(¬ification) { + listener_task_context + .outgoing + .send_server_notification_to_connections( + &[request_id.connection_id], + ServerNotification::ConfigWarning(notification), + ) + .await; + } + } + let environments = environments.unwrap_or_else(|| { listener_task_context .thread_manager @@ -1132,7 +1180,6 @@ impl ThreadRequestProcessor { let mut thread_extension_init = ExtensionDataInit::new(); if !selected_capability_roots.is_empty() { thread_extension_init.insert(selected_capability_roots); - codex_mcp_extension::initialize_executor_plugin_thread_data(&mut thread_extension_init); } let create_thread_started_at = std::time::Instant::now(); let NewThread { @@ -1144,12 +1191,14 @@ impl ThreadRequestProcessor { .thread_manager .start_thread_with_options(StartThreadOptions { config, + allow_provider_model_fallback, initial_history: match session_start_source .unwrap_or(codex_app_server_protocol::ThreadStartSource::Startup) { codex_app_server_protocol::ThreadStartSource::Startup => InitialHistory::New, codex_app_server_protocol::ThreadStartSource::Clear => InitialHistory::Cleared, }, + history_mode, session_source: None, thread_source, dynamic_tools, @@ -1167,6 +1216,7 @@ impl ThreadRequestProcessor { .await .map_err(|err| match err { CodexErr::InvalidRequest(message) => invalid_request(message), + CodexErr::UnsupportedOperation(message) => method_not_found(message), err => internal_error(format!("error creating thread: {err}")), })?; let session_telemetry = thread.session_telemetry(); @@ -1245,6 +1295,7 @@ impl ThreadRequestProcessor { let cwd = config_snapshot.cwd().clone(); let active_permission_profile = thread_response_active_permission_profile(config_snapshot.active_permission_profile); + let thread_originator = config_snapshot.originator.clone(); let response = ThreadStartResponse { thread: thread.clone(), @@ -1264,7 +1315,7 @@ impl ThreadRequestProcessor { let notif = thread_started_notification(thread); listener_task_context .outgoing - .send_response(request_id, response) + .send_response_with_thread_originator(request_id, response, thread_originator) .instrument(tracing::info_span!( "app_server.thread_start.send_response", otel.name = "app_server.thread_start.send_response", @@ -2275,6 +2326,9 @@ impl ThreadRequestProcessor { Err(ThreadStoreError::InvalidRequest { message }) => { Err(ThreadReadViewError::InvalidRequest(message)) } + Err(ThreadStoreError::Unsupported { operation }) => { + Err(ThreadReadViewError::Unsupported(operation)) + } Err(err) => Err(ThreadReadViewError::Internal(format!( "failed to read thread: {err}" ))), @@ -2477,6 +2531,9 @@ impl ThreadRequestProcessor { Err(ThreadStoreError::InvalidRequest { message }) => { return Err(ThreadReadViewError::InvalidRequest(message)); } + Err(ThreadStoreError::Unsupported { operation }) => { + return Err(ThreadReadViewError::Unsupported(operation)); + } Err(err) => { return Err(ThreadReadViewError::Internal(format!( "failed to read thread: {err}" @@ -2840,6 +2897,7 @@ impl ThreadRequestProcessor { } } + let thread_originator = config_snapshot.originator.clone(); let response = ThreadResumeResponse { thread, model: session_configured.model, @@ -2858,7 +2916,9 @@ impl ThreadRequestProcessor { }; let connection_id = request_id.connection_id; - self.outgoing.send_response(request_id, response).await; + self.outgoing + .send_response_with_thread_originator(request_id, response, thread_originator) + .await; // `excludeTurns` is explicitly the cheap resume path, so avoid // rebuilding history only to attribute a replayed usage update. if let Some(token_usage_thread) = token_usage_thread { @@ -3348,6 +3408,7 @@ impl ThreadRequestProcessor { ) -> Result<(), JSONRPCErrorError> { let ThreadForkParams { thread_id, + last_turn_id, path, model, model_provider, @@ -3382,12 +3443,20 @@ impl ThreadRequestProcessor { let history_items = source_thread .history .take() - .map(|history| Arc::new(history.items)) + .map(|history| history.items) .ok_or_else(|| { internal_error(format!( "thread {source_thread_id} did not include persisted history" )) })?; + let history_items = if let Some(last_turn_id) = last_turn_id.as_deref() { + Arc::new( + truncate_rollout_after_turn_id(&history_items, last_turn_id) + .map_err(|err| core_thread_write_error("truncate thread for fork", err))?, + ) + } else { + Arc::new(history_items) + }; let history_cwd = Some(source_thread.cwd.clone()); // Persist Windows sandbox mode. @@ -3559,6 +3628,7 @@ impl ThreadRequestProcessor { ); let active_permission_profile = thread_response_active_permission_profile(config_snapshot.active_permission_profile); + let thread_originator = config_snapshot.originator.clone(); let response = ThreadForkResponse { thread: thread.clone(), @@ -3579,7 +3649,9 @@ impl ThreadRequestProcessor { let notif = thread_started_notification(thread); let connection_id = request_id.connection_id; let token_usage_thread = include_turns.then(|| response.thread.clone()); - self.outgoing.send_response(request_id, response).await; + self.outgoing + .send_response_with_thread_originator(request_id, response, thread_originator) + .await; // `excludeTurns` is the cheap fork path, so skip restored usage replay // instead of rebuilding history only to attribute a historical update. if let Some(token_usage_thread) = token_usage_thread { @@ -4224,6 +4296,7 @@ pub(crate) fn thread_from_stored_thread( parent_thread_id: thread.parent_thread_id.map(|id| id.to_string()), preview: thread.preview, ephemeral: false, + history_mode: thread.history_mode.into(), model_provider: if thread.model_provider.is_empty() { fallback_provider.to_string() } else { @@ -4435,6 +4508,7 @@ fn build_thread_from_snapshot( parent_thread_id: config_snapshot.parent_thread_id.map(|id| id.to_string()), preview: String::new(), ephemeral: config_snapshot.ephemeral, + history_mode: config_snapshot.history_mode.into(), model_provider: config_snapshot.model_provider_id.clone(), created_at: now, updated_at: now, diff --git a/codex-rs/app-server/src/request_processors/thread_processor_tests.rs b/codex-rs/app-server/src/request_processors/thread_processor_tests.rs index 0d447011b8..8c0d59efed 100644 --- a/codex-rs/app-server/src/request_processors/thread_processor_tests.rs +++ b/codex-rs/app-server/src/request_processors/thread_processor_tests.rs @@ -483,6 +483,7 @@ mod thread_processor_behavior_tests { cwd: PathBuf::from("/tmp"), cli_version: "0.0.0".to_string(), source: SessionSource::Cli, + history_mode: Default::default(), thread_source: Some(codex_protocol::protocol::ThreadSource::User), agent_nickname: None, agent_role: None, @@ -777,6 +778,7 @@ mod thread_processor_behavior_tests { }, }, session_source: SessionSource::Cli, + history_mode: Default::default(), forked_from_thread_id: None, parent_thread_id: None, thread_source: None, diff --git a/codex-rs/app-server/src/request_processors/thread_resume_redaction.rs b/codex-rs/app-server/src/request_processors/thread_resume_redaction.rs index 586828e8dd..90d25ff593 100644 --- a/codex-rs/app-server/src/request_processors/thread_resume_redaction.rs +++ b/codex-rs/app-server/src/request_processors/thread_resume_redaction.rs @@ -83,6 +83,9 @@ mod tests { connector_id: "calendar".to_string(), link_id: Some("link_calendar".to_string()), resource_uri: Some("ui://widget/lookup.html".to_string()), + app_name: Some("Calendar".to_string()), + template_id: Some("calendar_template".to_string()), + action_name: Some("lookup".to_string()), }), mcp_app_resource_uri: Some("ui://widget/lookup.html".to_string()), plugin_id: Some("sample@test".to_string()), @@ -130,6 +133,9 @@ mod tests { connector_id: "calendar".to_string(), link_id: Some("link_calendar".to_string()), resource_uri: Some("ui://widget/lookup.html".to_string()), + app_name: Some("Calendar".to_string()), + template_id: Some("calendar_template".to_string()), + action_name: Some("lookup".to_string()), }), mcp_app_resource_uri: Some("ui://widget/lookup.html".to_string()), plugin_id: Some("sample@test".to_string()), @@ -189,6 +195,7 @@ mod tests { parent_thread_id: None, preview: "preview".to_string(), ephemeral: false, + history_mode: Default::default(), model_provider: "mock_provider".to_string(), created_at: 0, updated_at: 0, diff --git a/codex-rs/app-server/src/request_processors/thread_summary.rs b/codex-rs/app-server/src/request_processors/thread_summary.rs index 2bacaf6446..37c77a198b 100644 --- a/codex-rs/app-server/src/request_processors/thread_summary.rs +++ b/codex-rs/app-server/src/request_processors/thread_summary.rs @@ -320,6 +320,7 @@ pub(crate) fn summary_to_thread( parent_thread_id: None, preview, ephemeral: false, + history_mode: ThreadHistoryMode::Legacy, model_provider, created_at: created_at.map(|dt| dt.timestamp()).unwrap_or(0), updated_at: updated_at.map(|dt| dt.timestamp()).unwrap_or(0), diff --git a/codex-rs/app-server/src/request_processors/turn_processor.rs b/codex-rs/app-server/src/request_processors/turn_processor.rs index bb074e457b..3ff64cd34e 100644 --- a/codex-rs/app-server/src/request_processors/turn_processor.rs +++ b/codex-rs/app-server/src/request_processors/turn_processor.rs @@ -1002,6 +1002,9 @@ impl TurnRequestProcessor { thread.as_ref(), Op::RealtimeConversationStart(ConversationStartParams { client_managed_handoffs: params.client_managed_handoffs.unwrap_or(false), + flush_transcript_tail_on_session_end: params + .flush_transcript_tail_on_session_end + .unwrap_or(false), codex_responses_as_items: params.codex_responses_as_items.unwrap_or(false), codex_response_item_prefix: params.codex_response_item_prefix, codex_response_handoff_prefix: params.codex_response_handoff_prefix, diff --git a/codex-rs/app-server/src/thread_status.rs b/codex-rs/app-server/src/thread_status.rs index 32b79e1f69..a29b649788 100644 --- a/codex-rs/app-server/src/thread_status.rs +++ b/codex-rs/app-server/src/thread_status.rs @@ -895,6 +895,7 @@ mod tests { parent_thread_id: None, preview: String::new(), ephemeral: false, + history_mode: Default::default(), model_provider: "mock-provider".to_string(), created_at: 0, updated_at: 0, diff --git a/codex-rs/app-server/tests/common/json_logging.rs b/codex-rs/app-server/tests/common/json_logging.rs new file mode 100644 index 0000000000..8683d1bdaf --- /dev/null +++ b/codex-rs/app-server/tests/common/json_logging.rs @@ -0,0 +1,137 @@ +use std::path::Path; +use std::process::Command; +use std::process::Stdio; +use std::sync::Arc; +use std::sync::Mutex; +use std::time::Duration; + +use anyhow::Context; +use anyhow::Result; +use serde_json::Value; +use serde_json::json; +use tokio::sync::Notify; + +#[derive(Clone, Default)] +pub(crate) struct JsonLogCapture { + lines: Arc>>, + updated: Arc, +} + +impl JsonLogCapture { + pub(crate) fn record(&self, line: String) { + self.lines + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .push(line); + self.updated.notify_one(); + } + + pub(crate) async fn wait_for_event(&self, event_name: &str) -> Result { + let mut events = self.wait_for_events(event_name, /*count*/ 1).await?; + Ok(events.remove(0)) + } + + pub(crate) async fn wait_for_events( + &self, + event_name: &str, + count: usize, + ) -> Result> { + let result = tokio::time::timeout(Duration::from_secs(10), async { + loop { + let updated = self.updated.notified(); + let events = self + .events()? + .into_iter() + .filter(|event| event["fields"]["event.name"].as_str() == Some(event_name)) + .collect::>(); + if events.len() >= count { + return Ok(events); + } + updated.await; + } + }) + .await; + match result { + Ok(result) => result, + Err(_) => { + let lines = self + .lines + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .join("\n"); + anyhow::bail!( + "timed out waiting for {count} JSON log event(s) named `{event_name}`; captured stderr:\n{lines}" + ) + } + } + } + + pub(crate) fn events(&self) -> Result> { + let lines = self + .lines + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + json_log_events(lines.iter().map(String::as_str)) + } +} + +pub fn app_server_json_shutdown_event( + binary: &str, + args: &[&str], + codex_home: &Path, +) -> Result { + std::fs::write( + codex_home.join("config.toml"), + "[features]\nplugins = false\n", + )?; + let output = Command::new(codex_utils_cargo_bin::cargo_bin(binary)?) + .stdin(Stdio::null()) + .env("CODEX_HOME", codex_home) + .env( + "CODEX_APP_SERVER_MANAGED_CONFIG_PATH", + codex_home.join("managed_config.toml"), + ) + .env("LOG_FORMAT", "json") + .env("RUST_LOG", "codex_app_server=info") + .args(args) + .output()?; + + let stderr = String::from_utf8(output.stderr)?; + anyhow::ensure!(output.status.success(), "app-server failed: {stderr}"); + + let events = json_log_events(stderr.lines()) + .with_context(|| format!("app-server stderr was not valid JSONL: {stderr}"))?; + let event = events + .iter() + .find(|event| event["fields"]["message"] == "processor task exited") + .context("missing INFO shutdown event in app-server JSON logs")?; + Ok(json!({ + "level": event["level"], + "fields": event["fields"], + "target": event["target"], + })) +} + +fn json_log_events<'a>(lines: impl IntoIterator) -> Result> { + lines + .into_iter() + .filter(|line| !line.is_empty()) + .map(|line| { + let event = serde_json::from_str::(line) + .with_context(|| format!("log line was not JSON: {line}"))?; + anyhow::ensure!( + event["level"].is_string() + && event["fields"].is_object() + && event["target"].is_string(), + "JSON log event did not include level, fields, and target: {line}" + ); + let timestamp = event["timestamp"] + .as_str() + .with_context(|| format!("JSON log event did not include a timestamp: {line}"))?; + chrono::DateTime::parse_from_rfc3339(timestamp).with_context(|| { + format!("JSON log event timestamp was not RFC 3339: {timestamp}") + })?; + Ok(event) + }) + .collect() +} diff --git a/codex-rs/app-server/tests/common/lib.rs b/codex-rs/app-server/tests/common/lib.rs index 76c4c85f26..fbc674806a 100644 --- a/codex-rs/app-server/tests/common/lib.rs +++ b/codex-rs/app-server/tests/common/lib.rs @@ -3,6 +3,7 @@ mod analytics_server; mod auth_fixtures; mod config; +mod json_logging; mod mock_model_server; mod models_cache; mod responses; @@ -26,6 +27,7 @@ pub use core_test_support::test_absolute_path; pub use core_test_support::test_path_buf_with_windows; pub use core_test_support::test_tmp_path; pub use core_test_support::test_tmp_path_buf; +pub use json_logging::app_server_json_shutdown_event; pub use mock_model_server::create_mock_responses_server_repeating_assistant; pub use mock_model_server::create_mock_responses_server_sequence; pub use mock_model_server::create_mock_responses_server_sequence_unchecked; diff --git a/codex-rs/app-server/tests/common/models_cache.rs b/codex-rs/app-server/tests/common/models_cache.rs index 2a0c10a27c..b34515302e 100644 --- a/codex-rs/app-server/tests/common/models_cache.rs +++ b/codex-rs/app-server/tests/common/models_cache.rs @@ -34,6 +34,7 @@ fn preset_to_info(preset: &ModelPreset, priority: i32) -> ModelInfo { upgrade: preset.upgrade.as_ref().map(Into::into), base_instructions: "base instructions".to_string(), model_messages: None, + include_skills_usage_instructions: false, supports_reasoning_summaries: false, default_reasoning_summary: ReasoningSummary::Auto, support_verbosity: false, diff --git a/codex-rs/app-server/tests/common/rollout.rs b/codex-rs/app-server/tests/common/rollout.rs index ec01dacc47..c13f2355d4 100644 --- a/codex-rs/app-server/tests/common/rollout.rs +++ b/codex-rs/app-server/tests/common/rollout.rs @@ -200,7 +200,9 @@ fn create_fake_rollout_with_source_and_parent_thread_id( model_provider: model_provider.map(str::to_string), base_instructions: None, dynamic_tools: None, + selected_capability_roots: Vec::new(), memory_mode: None, + history_mode: Default::default(), multi_agent_version: None, context_window: None, }; @@ -288,7 +290,9 @@ pub fn create_fake_rollout_with_text_elements( model_provider: model_provider.map(str::to_string), base_instructions: None, dynamic_tools: None, + selected_capability_roots: Vec::new(), memory_mode: None, + history_mode: Default::default(), multi_agent_version: None, context_window: None, }; diff --git a/codex-rs/app-server/tests/common/test_app_server.rs b/codex-rs/app-server/tests/common/test_app_server.rs index fbe43e7689..cbb8ff161e 100644 --- a/codex-rs/app-server/tests/common/test_app_server.rs +++ b/codex-rs/app-server/tests/common/test_app_server.rs @@ -123,6 +123,8 @@ use core_test_support::test_codex::TestEnv; use core_test_support::test_codex::test_env; use tokio::process::Command; +use crate::json_logging::JsonLogCapture; + pub struct TestAppServer { next_request_id: AtomicI64, /// Retain this child process until the client is dropped. The Tokio runtime @@ -134,6 +136,7 @@ pub struct TestAppServer { stdout: BufReader, pending_messages: VecDeque, auto_env: Option, + json_logs: JsonLogCapture, } pub const DEFAULT_CLIENT_NAME: &str = "codex-app-server-tests"; @@ -160,6 +163,31 @@ impl TestAppServer { /// URL-based configuration, this helper rejects a `codex_home` containing /// that file. pub async fn new_with_auto_env(codex_home: &Path) -> anyhow::Result { + Self::new_with_auto_env_and_env(codex_home, &[]).await + } + + /// Starts an auto-environment app server that emits JSON logs. + /// + /// `rust_log` is the value to use for the `RUST_LOG` environment variable. + pub async fn new_with_auto_env_and_json_logging( + codex_home: &Path, + rust_log: impl Into, + ) -> anyhow::Result { + let rust_log = rust_log.into(); + Self::new_with_auto_env_and_env( + codex_home, + &[ + ("LOG_FORMAT", Some("json")), + ("RUST_LOG", Some(rust_log.as_str())), + ], + ) + .await + } + + async fn new_with_auto_env_and_env( + codex_home: &Path, + extra_env_overrides: &[(&str, Option<&str>)], + ) -> anyhow::Result { let environments_toml = codex_home.join("environments.toml"); ensure!( !environments_toml @@ -172,7 +200,7 @@ impl TestAppServer { let auto_env = test_env().await?; // Noise registry configuration takes precedence over the URL-based // provider, so clear inherited values to keep the selection hermetic. - let env_overrides = [ + let mut env_overrides = vec![ ( CODEX_EXEC_SERVER_URL_ENV_VAR, auto_env.environment().exec_server_url(), @@ -182,6 +210,7 @@ impl TestAppServer { (CODEX_EXEC_SERVER_NOISE_AUTH_TOKEN_ENV_VAR, None), (CODEX_EXEC_SERVER_NOISE_CHATGPT_ACCOUNT_ID_ENV_VAR, None), ]; + env_overrides.extend_from_slice(extra_env_overrides); let mut app_server = Self::new_with_env(codex_home, &env_overrides).await?; app_server.auto_env = Some(auto_env); Ok(app_server) @@ -202,6 +231,28 @@ impl TestAppServer { }) } + /// Waits for a JSON stderr event whose structured `event.name` field matches. + pub async fn wait_for_json_log_event( + &self, + event_name: &str, + ) -> anyhow::Result { + self.json_logs.wait_for_event(event_name).await + } + + /// Waits for the requested number of JSON stderr events with the same `event.name` field. + pub async fn wait_for_json_log_events( + &self, + event_name: &str, + count: usize, + ) -> anyhow::Result> { + self.json_logs.wait_for_events(event_name, count).await + } + + /// Returns every stderr line parsed and validated as a JSON log event. + pub fn json_log_events(&self) -> anyhow::Result> { + self.json_logs.events() + } + pub async fn new_without_managed_config(codex_home: &Path) -> anyhow::Result { Self::new_with_env(codex_home, &[(DISABLE_MANAGED_CONFIG_ENV_VAR, Some("1"))]).await } @@ -322,10 +373,13 @@ impl TestAppServer { // Forward child's stderr to our stderr so failures are visible even // when stdout/stderr are captured by the test harness. + let json_logs = JsonLogCapture::default(); if let Some(stderr) = process.stderr.take() { + let json_logs = json_logs.clone(); let mut stderr_reader = BufReader::new(stderr).lines(); tokio::spawn(async move { while let Ok(Some(line)) = stderr_reader.next_line().await { + json_logs.record(line.clone()); eprintln!("[mcp stderr] {line}"); } }); @@ -337,6 +391,7 @@ impl TestAppServer { stdout, pending_messages: VecDeque::new(), auto_env: None, + json_logs, }) } diff --git a/codex-rs/app-server/tests/suite/conversation_summary.rs b/codex-rs/app-server/tests/suite/conversation_summary.rs index 7f964d3147..64a0e62249 100644 --- a/codex-rs/app-server/tests/suite/conversation_summary.rs +++ b/codex-rs/app-server/tests/suite/conversation_summary.rs @@ -131,7 +131,9 @@ async fn get_conversation_summary_by_thread_id_reads_pathless_store_thread() -> originator: "test_originator".to_string(), base_instructions: BaseInstructions::default(), dynamic_tools: Vec::new(), + selected_capability_roots: Vec::new(), multi_agent_version: None, + history_mode: Default::default(), initial_window_id: Uuid::now_v7().to_string(), metadata: ThreadPersistenceMetadata { cwd: None, diff --git a/codex-rs/app-server/tests/suite/logging.rs b/codex-rs/app-server/tests/suite/logging.rs new file mode 100644 index 0000000000..8d39a485d1 --- /dev/null +++ b/codex-rs/app-server/tests/suite/logging.rs @@ -0,0 +1,172 @@ +use anyhow::Context; +use anyhow::Result; +use app_test_support::TestAppServer; +use app_test_support::app_server_json_shutdown_event; +use app_test_support::create_exec_command_sse_response; +use app_test_support::create_final_assistant_message_sse_response; +use app_test_support::create_mock_responses_server_sequence; +use app_test_support::to_response; +use app_test_support::write_mock_responses_config_toml; +use codex_app_server_protocol::JSONRPCResponse; +use codex_app_server_protocol::RequestId; +use codex_app_server_protocol::ThreadStartParams; +use codex_app_server_protocol::ThreadStartResponse; +use codex_app_server_protocol::TurnStartParams; +use codex_app_server_protocol::TurnStartResponse; +use codex_app_server_protocol::UserInput; +use codex_features::Feature; +use core_test_support::skip_if_no_network; +use pretty_assertions::assert_eq; +use serde_json::Value; +use serde_json::json; +use std::collections::BTreeMap; +use tempfile::TempDir; +use tokio::time::Duration; +use tokio::time::timeout; + +const READ_TIMEOUT: Duration = Duration::from_secs(10); + +#[test] +fn standalone_app_server_emits_json_info_events() -> Result<()> { + let codex_home = TempDir::new()?; + let event = app_server_json_shutdown_event("codex-app-server", &[], codex_home.path())?; + + assert_eq!( + event, + json!({ + "level": "INFO", + "fields": { + "message": "processor task exited", + "exit_reason": "last_connection_closed", + "remaining_connection_count": 0, + "shutdown_forced": false, + }, + "target": "codex_app_server", + }) + ); + + Ok(()) +} + +#[tokio::test] +async fn app_server_emits_structured_tool_call_timing_event() -> Result<()> { + skip_if_no_network!(Ok(())); + + let server = create_mock_responses_server_sequence(vec![ + create_exec_command_sse_response("exec-call-1")?, + create_final_assistant_message_sse_response("done")?, + ]) + .await; + let codex_home = TempDir::new()?; + write_mock_responses_config_toml( + codex_home.path(), + &server.uri(), + &BTreeMap::from([(Feature::UnifiedExec, true)]), + /*auto_compact_limit*/ 100_000, + /*requires_openai_auth*/ None, + "mock_provider", + "compact", + )?; + + let mut app_server = TestAppServer::new_with_auto_env_and_json_logging( + codex_home.path(), + "warn,codex_core::tools::parallel=info", + ) + .await?; + timeout(READ_TIMEOUT, app_server.initialize()).await??; + + let thread_start_id = app_server + .send_thread_start_request_with_auto_env(ThreadStartParams { + model: Some("mock-model".to_string()), + ..Default::default() + }) + .await?; + let thread_start_response: JSONRPCResponse = timeout( + READ_TIMEOUT, + app_server.read_stream_until_response_message(RequestId::Integer(thread_start_id)), + ) + .await??; + let ThreadStartResponse { thread, .. } = to_response(thread_start_response)?; + + let turn_start_id = app_server + .send_turn_start_request(TurnStartParams { + thread_id: thread.id.clone(), + input: vec![UserInput::Text { + text: "run a command".to_string(), + text_elements: Vec::new(), + }], + ..Default::default() + }) + .await?; + let turn_start_response: JSONRPCResponse = timeout( + READ_TIMEOUT, + app_server.read_stream_until_response_message(RequestId::Integer(turn_start_id)), + ) + .await??; + let TurnStartResponse { turn } = to_response(turn_start_response)?; + + timeout( + READ_TIMEOUT, + app_server.read_stream_until_notification_message("turn/completed"), + ) + .await??; + + let mut tool_call = app_server + .wait_for_json_log_event("codex.tool_call") + .await?; + let tool_call_object = tool_call + .as_object_mut() + .context("tool call log event must be an object")?; + // JsonLogCapture already validates the timestamp as RFC 3339. + tool_call_object + .remove("timestamp") + .context("tool call log event must include a timestamp")?; + let fields = tool_call_object + .get_mut("fields") + .and_then(Value::as_object_mut) + .context("tool call log event fields must be an object")?; + let trace_id = fields + .remove("trace_id") + .context("tool call log event must include trace_id")?; + anyhow::ensure!(trace_id.is_string(), "trace_id must be a string"); + let dispatch_duration_ms = fields + .remove("dispatch_duration_ms") + .and_then(|duration| duration.as_u64()) + .context("dispatch_duration_ms must be a nonnegative integer")?; + let handler_duration_ms = fields + .remove("handler_duration_ms") + .and_then(|duration| duration.as_u64()) + .context("handler_duration_ms must be a nonnegative integer")?; + let total_duration_ms = fields + .remove("total_duration_ms") + .and_then(|duration| duration.as_u64()) + .context("total_duration_ms must be a nonnegative integer")?; + let accounted_duration_ms = dispatch_duration_ms + .checked_add(handler_duration_ms) + .context("dispatch and handler durations must not overflow")?; + anyhow::ensure!( + total_duration_ms >= accounted_duration_ms + && total_duration_ms - accounted_duration_ms <= 1, + "dispatch and handler durations must account for total duration within integer truncation" + ); + + assert_eq!( + tool_call, + json!({ + "level": "INFO", + "fields": { + "message": "tool call completed", + "event.name": "codex.tool_call", + "conversation.id": thread.id, + "turn_id": turn.id, + "tool_name": "exec_command", + "call_id": "exec-call-1", + "tool_source": "direct", + "execution_started": true, + }, + "target": "codex_core::tools::parallel", + }) + ); + + Ok(()) +} diff --git a/codex-rs/app-server/tests/suite/mod.rs b/codex-rs/app-server/tests/suite/mod.rs index bac309a930..09c40493f6 100644 --- a/codex-rs/app-server/tests/suite/mod.rs +++ b/codex-rs/app-server/tests/suite/mod.rs @@ -1,5 +1,6 @@ mod auth; mod conversation_summary; mod fuzzy_file_search; +mod logging; mod strict_config; mod v2; diff --git a/codex-rs/app-server/tests/suite/v2/analytics.rs b/codex-rs/app-server/tests/suite/v2/analytics.rs index ea876940ab..2acc86caee 100644 --- a/codex-rs/app-server/tests/suite/v2/analytics.rs +++ b/codex-rs/app-server/tests/suite/v2/analytics.rs @@ -145,7 +145,7 @@ pub(crate) async fn wait_for_goal_event( .await } -async fn wait_for_matching_analytics_event( +pub(crate) async fn wait_for_matching_analytics_event( server: &MockServer, read_timeout: Duration, matches: impl Fn(&Value) -> bool, @@ -191,6 +191,7 @@ pub(crate) fn assert_basic_thread_initialized_event( event: &Value, thread_id: &str, session_id: &str, + expected_product_client_id: &str, expected_model: &str, initialization_mode: &str, expected_thread_source: &str, @@ -199,7 +200,7 @@ pub(crate) fn assert_basic_thread_initialized_event( assert_eq!(event["event_params"]["session_id"], session_id); assert_eq!( event["event_params"]["app_server_client"]["product_client_id"], - DEFAULT_CLIENT_NAME + expected_product_client_id ); assert_eq!( event["event_params"]["app_server_client"]["client_name"], diff --git a/codex-rs/app-server/tests/suite/v2/app_list.rs b/codex-rs/app-server/tests/suite/v2/app_list.rs index 1ddf355269..5fad04929f 100644 --- a/codex-rs/app-server/tests/suite/v2/app_list.rs +++ b/codex-rs/app-server/tests/suite/v2/app_list.rs @@ -1533,7 +1533,7 @@ impl ServerHandler for AppListMcpServer { } } -async fn start_apps_server_with_delays( +pub(super) async fn start_apps_server_with_delays( connectors: Vec, tools: Vec, directory_delay: Duration, @@ -1693,7 +1693,7 @@ async fn list_directory_connectors( } } -fn connector_tool(connector_id: &str, connector_name: &str) -> Result { +pub(super) fn connector_tool(connector_id: &str, connector_name: &str) -> Result { let schema: JsonObject = serde_json::from_value(json!({ "type": "object", "additionalProperties": false diff --git a/codex-rs/app-server/tests/suite/v2/config_rpc.rs b/codex-rs/app-server/tests/suite/v2/config_rpc.rs index 81ef145ef0..07b8ccc2e8 100644 --- a/codex-rs/app-server/tests/suite/v2/config_rpc.rs +++ b/codex-rs/app-server/tests/suite/v2/config_rpc.rs @@ -75,6 +75,43 @@ async fn config_requirements_read_includes_allow_remote_control() -> Result<()> Ok(()) } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn config_requirements_read_includes_new_thread_model_defaults() -> Result<()> { + let codex_home = TempDir::new()?; + std::fs::write( + codex_home.path().join("requirements.toml"), + r#" +[models.new_thread] +model = "gpt-managed" +model_reasoning_effort = "medium" +service_tier = "fast" +"#, + )?; + let mut mcp = TestAppServer::new(codex_home.path()).await?; + timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; + + let request_id = mcp.send_config_requirements_read_request().await?; + let response = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(request_id)), + ) + .await??; + let response: ConfigRequirementsReadResponse = to_response(response)?; + + let defaults = response + .requirements + .and_then(|requirements| requirements.models) + .and_then(|models| models.new_thread) + .expect("managed new-thread defaults"); + assert_eq!(defaults.model.as_deref(), Some("gpt-managed")); + assert_eq!( + defaults.model_reasoning_effort, + Some(ReasoningEffort::Medium) + ); + assert_eq!(defaults.service_tier.as_deref(), Some("fast")); + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn config_read_returns_effective_and_layers() -> Result<()> { let codex_home = TempDir::new()?; diff --git a/codex-rs/app-server/tests/suite/v2/connection_handling_websocket.rs b/codex-rs/app-server/tests/suite/v2/connection_handling_websocket.rs index 648d60f3b5..ece66c8405 100644 --- a/codex-rs/app-server/tests/suite/v2/connection_handling_websocket.rs +++ b/codex-rs/app-server/tests/suite/v2/connection_handling_websocket.rs @@ -7,6 +7,7 @@ use app_test_support::to_response; use base64::Engine; use base64::engine::general_purpose::URL_SAFE_NO_PAD; use codex_app_server_protocol::ClientInfo; +use codex_app_server_protocol::ConfigWarningNotification; use codex_app_server_protocol::InitializeParams; use codex_app_server_protocol::JSONRPCError; use codex_app_server_protocol::JSONRPCMessage; @@ -18,6 +19,8 @@ use codex_app_server_protocol::ThreadLoadedListParams; use codex_app_server_protocol::ThreadLoadedListResponse; use codex_app_server_protocol::ThreadStartParams; use codex_app_server_protocol::ThreadStartResponse; +use codex_core::config::set_project_trust_level; +use codex_protocol::config_types::TrustLevel; use futures::SinkExt; use futures::StreamExt; use hmac::Hmac; @@ -104,6 +107,106 @@ async fn websocket_transport_routes_per_connection_handshake_and_responses() -> Ok(()) } +#[tokio::test] +async fn thread_start_routes_project_exec_policy_warning_to_requester() -> Result<()> { + let server = create_mock_responses_server_sequence_unchecked(Vec::new()).await; + let codex_home = TempDir::new()?; + create_config_toml(codex_home.path(), &server.uri(), "never")?; + + let project = TempDir::new()?; + std::fs::create_dir(project.path().join(".git"))?; + let rules_dir = project.path().join(".codex/rules"); + std::fs::create_dir_all(&rules_dir)?; + let rules_path = rules_dir.join("broken.rules"); + std::fs::write(&rules_path, "prefix_rule(")?; + set_project_trust_level(codex_home.path(), project.path(), TrustLevel::Trusted)?; + + let (mut process, bind_addr) = spawn_websocket_server(codex_home.path()).await?; + let mut requester = connect_websocket(bind_addr).await?; + let mut other_client = connect_websocket(bind_addr).await?; + + send_initialize_request(&mut requester, /*id*/ 1, "requester").await?; + read_response_for_id(&mut requester, /*id*/ 1).await?; + send_initialize_request(&mut other_client, /*id*/ 2, "other_client").await?; + read_response_for_id(&mut other_client, /*id*/ 2).await?; + + send_request( + &mut requester, + "thread/start", + /*id*/ 3, + Some(serde_json::to_value(ThreadStartParams { + cwd: Some(project.path().display().to_string()), + model: Some("mock-model".to_string()), + ..Default::default() + })?), + ) + .await?; + + let target_id = RequestId::Integer(3); + let warning_summary = "Error parsing rules; custom rules not applied."; + let is_exec_policy_warning = |notification: &JSONRPCNotification| { + notification.method == "configWarning" + && notification + .params + .as_ref() + .and_then(|params| params.get("summary")) + .and_then(serde_json::Value::as_str) + == Some(warning_summary) + }; + let mut response = None; + let mut warning = None; + while response.is_none() || warning.is_none() { + match read_jsonrpc_message(&mut requester).await? { + JSONRPCMessage::Response(candidate) if candidate.id == target_id => { + response = Some(candidate); + } + JSONRPCMessage::Notification(candidate) if is_exec_policy_warning(&candidate) => { + warning = Some(candidate); + } + _ => {} + } + } + + let _: ThreadStartResponse = to_response(response.context("missing thread/start response")?)?; + let warning: ConfigWarningNotification = serde_json::from_value( + warning + .context("missing exec-policy configWarning")? + .params + .context("configWarning should include params")?, + )?; + assert_eq!( + warning + .path + .as_deref() + .map(Path::new) + .and_then(Path::file_name), + Some(std::ffi::OsStr::new("broken.rules")) + ); + + match timeout(Duration::from_millis(250), async { + loop { + let message = read_jsonrpc_message(&mut other_client).await?; + if let JSONRPCMessage::Notification(notification) = message + && is_exec_policy_warning(¬ification) + { + return Ok::<_, anyhow::Error>(notification); + } + } + }) + .await + { + Ok(Ok(_)) => bail!("exec-policy configWarning leaked to another connection"), + Ok(Err(err)) => return Err(err), + Err(_) => {} + } + + process + .kill() + .await + .context("failed to stop websocket app-server process")?; + Ok(()) +} + #[tokio::test] async fn websocket_transport_serves_health_endpoints_on_same_listener() -> Result<()> { let server = create_mock_responses_server_sequence_unchecked(Vec::new()).await; diff --git a/codex-rs/app-server/tests/suite/v2/environment_info.rs b/codex-rs/app-server/tests/suite/v2/environment_info.rs new file mode 100644 index 0000000000..8b0e82518f --- /dev/null +++ b/codex-rs/app-server/tests/suite/v2/environment_info.rs @@ -0,0 +1,208 @@ +use std::time::Duration; + +use anyhow::Result; +use app_test_support::TestAppServer; +use app_test_support::to_response; +use codex_app_server_protocol::EnvironmentAddResponse; +use codex_app_server_protocol::EnvironmentInfoResponse; +use codex_app_server_protocol::EnvironmentShellInfo; +use codex_app_server_protocol::JSONRPCError; +use codex_app_server_protocol::JSONRPCErrorError; +use codex_app_server_protocol::JSONRPCResponse; +use codex_app_server_protocol::RequestId; +use codex_utils_path_uri::PathUri; +use pretty_assertions::assert_eq; +use serde_json::json; +use tempfile::TempDir; +use tokio::net::TcpListener; +use tokio::time::timeout; + +use super::exec_server_test_support::accept_exec_server_environment; + +const RPC_TIMEOUT: Duration = Duration::from_secs(10); +const INVALID_REQUEST_ERROR_CODE: i64 = -32600; +const INTERNAL_ERROR_CODE: i64 = -32603; + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn environment_info_returns_remote_environment_info() -> Result<()> { + let listener = TcpListener::bind("127.0.0.1:0").await?; + let exec_server_url = format!("ws://{}", listener.local_addr()?); + let exec_server = tokio::spawn(async move { + accept_exec_server_environment( + listener, + json!({ + "shell": {"name": "zsh", "path": "/bin/zsh"}, + "cwd": "file:///workspace", + }), + ) + .await?; + Ok::<_, anyhow::Error>(()) + }); + + let codex_home = TempDir::new()?; + let mut app_server = TestAppServer::new(codex_home.path()).await?; + timeout(RPC_TIMEOUT, app_server.initialize()).await??; + add_environment( + &mut app_server, + &exec_server_url, + /*connect_timeout_ms*/ None, + ) + .await?; + + let request_id = app_server + .send_raw_request( + "environment/info", + Some(json!({"environmentId": "remote-a"})), + ) + .await?; + let response: JSONRPCResponse = timeout( + RPC_TIMEOUT, + app_server.read_stream_until_response_message(RequestId::Integer(request_id)), + ) + .await??; + assert_eq!( + to_response::(response)?, + EnvironmentInfoResponse { + shell: EnvironmentShellInfo { + name: "zsh".to_string(), + path: "/bin/zsh".to_string(), + }, + cwd: Some(PathUri::parse("file:///workspace")?), + } + ); + timeout(RPC_TIMEOUT, exec_server).await???; + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn environment_info_accepts_missing_cwd() -> Result<()> { + let listener = TcpListener::bind("127.0.0.1:0").await?; + let exec_server_url = format!("ws://{}", listener.local_addr()?); + let exec_server = tokio::spawn(async move { + accept_exec_server_environment( + listener, + json!({"shell": {"name": "zsh", "path": "/bin/zsh"}}), + ) + .await?; + Ok::<_, anyhow::Error>(()) + }); + + let codex_home = TempDir::new()?; + let mut app_server = TestAppServer::new(codex_home.path()).await?; + timeout(RPC_TIMEOUT, app_server.initialize()).await??; + add_environment( + &mut app_server, + &exec_server_url, + /*connect_timeout_ms*/ None, + ) + .await?; + + let request_id = app_server + .send_raw_request( + "environment/info", + Some(json!({"environmentId": "remote-a"})), + ) + .await?; + let response: JSONRPCResponse = timeout( + RPC_TIMEOUT, + app_server.read_stream_until_response_message(RequestId::Integer(request_id)), + ) + .await??; + assert_eq!( + to_response::(response)?, + EnvironmentInfoResponse { + shell: EnvironmentShellInfo { + name: "zsh".to_string(), + path: "/bin/zsh".to_string(), + }, + cwd: None, + } + ); + timeout(RPC_TIMEOUT, exec_server).await???; + Ok(()) +} + +#[tokio::test] +async fn environment_info_rejects_unknown_environment() -> Result<()> { + let codex_home = TempDir::new()?; + let mut app_server = TestAppServer::new(codex_home.path()).await?; + timeout(RPC_TIMEOUT, app_server.initialize()).await??; + + let request_id = app_server + .send_raw_request( + "environment/info", + Some(json!({"environmentId": "missing"})), + ) + .await?; + let error = timeout( + RPC_TIMEOUT, + app_server.read_stream_until_error_message(RequestId::Integer(request_id)), + ) + .await??; + assert_eq!( + error, + JSONRPCError { + id: RequestId::Integer(request_id), + error: JSONRPCErrorError { + code: INVALID_REQUEST_ERROR_CODE, + message: "unknown environment id `missing`".to_string(), + data: None, + }, + } + ); + Ok(()) +} + +#[tokio::test] +async fn environment_info_reports_connection_failure() -> Result<()> { + let listener = TcpListener::bind("127.0.0.1:0").await?; + let exec_server_url = format!("ws://{}", listener.local_addr()?); + let codex_home = TempDir::new()?; + let mut app_server = TestAppServer::new(codex_home.path()).await?; + timeout(RPC_TIMEOUT, app_server.initialize()).await??; + add_environment(&mut app_server, &exec_server_url, Some(50)).await?; + + let request_id = app_server + .send_raw_request( + "environment/info", + Some(json!({"environmentId": "remote-a"})), + ) + .await?; + let error = timeout( + RPC_TIMEOUT, + app_server.read_stream_until_error_message(RequestId::Integer(request_id)), + ) + .await??; + assert_eq!(error.error.code, INTERNAL_ERROR_CODE); + assert!( + error + .error + .message + .contains("failed to get info for environment `remote-a`") + ); + Ok(()) +} + +async fn add_environment( + app_server: &mut TestAppServer, + exec_server_url: &str, + connect_timeout_ms: Option, +) -> Result<()> { + let request_id = app_server + .send_raw_request( + "environment/add", + Some(json!({ + "environmentId": "remote-a", + "execServerUrl": exec_server_url, + "connectTimeoutMs": connect_timeout_ms, + })), + ) + .await?; + let response: JSONRPCResponse = timeout( + RPC_TIMEOUT, + app_server.read_stream_until_response_message(RequestId::Integer(request_id)), + ) + .await??; + let _: EnvironmentAddResponse = to_response(response)?; + Ok(()) +} diff --git a/codex-rs/app-server/tests/suite/v2/exec_server_test_support.rs b/codex-rs/app-server/tests/suite/v2/exec_server_test_support.rs new file mode 100644 index 0000000000..f28e06e9fa --- /dev/null +++ b/codex-rs/app-server/tests/suite/v2/exec_server_test_support.rs @@ -0,0 +1,73 @@ +use anyhow::Result; +use futures::SinkExt; +use futures::StreamExt; +use serde_json::Value; +use serde_json::json; +use tokio::net::TcpListener; +use tokio::net::TcpStream; +use tokio_tungstenite::WebSocketStream; +use tokio_tungstenite::accept_async; +use tokio_tungstenite::tungstenite::Message; + +pub(crate) async fn accept_exec_server_environment( + listener: TcpListener, + environment_info: Value, +) -> Result> { + let mut websocket = accept_initialized_exec_server(listener).await?; + + let request = read_exec_server_json(&mut websocket).await?; + assert_eq!(request["method"], "environment/info"); + websocket + .send(Message::Text( + json!({ + "id": request["id"], + "result": environment_info, + }) + .to_string() + .into(), + )) + .await?; + + Ok(websocket) +} + +pub(crate) async fn accept_initialized_exec_server( + listener: TcpListener, +) -> Result> { + let (stream, _) = listener.accept().await?; + let mut websocket = accept_async(stream).await?; + + let initialize = read_exec_server_json(&mut websocket).await?; + assert_eq!(initialize["method"], "initialize"); + websocket + .send(Message::Text( + json!({ + "id": initialize["id"], + "result": {"sessionId": "test-session"}, + }) + .to_string() + .into(), + )) + .await?; + let initialized = read_exec_server_json(&mut websocket).await?; + assert_eq!(initialized["method"], "initialized"); + + Ok(websocket) +} + +pub(crate) async fn read_exec_server_json( + websocket: &mut WebSocketStream, +) -> Result { + loop { + match websocket + .next() + .await + .ok_or_else(|| anyhow::anyhow!("exec-server websocket closed"))?? + { + Message::Text(text) => return Ok(serde_json::from_str(text.as_ref())?), + Message::Binary(bytes) => return Ok(serde_json::from_slice(bytes.as_ref())?), + Message::Ping(_) | Message::Pong(_) => {} + message => anyhow::bail!("expected JSON-RPC message, got {message:?}"), + } + } +} diff --git a/codex-rs/app-server/tests/suite/v2/executor_mcp.rs b/codex-rs/app-server/tests/suite/v2/executor_mcp.rs index 299ca81aa5..aa650c3ecf 100644 --- a/codex-rs/app-server/tests/suite/v2/executor_mcp.rs +++ b/codex-rs/app-server/tests/suite/v2/executor_mcp.rs @@ -2,9 +2,16 @@ use anyhow::Result; use app_test_support::TestAppServer; use app_test_support::to_response; use app_test_support::write_mock_responses_config_toml; +use axum::Json; +use axum::Router; +use axum::body::Bytes; +use axum::routing::get; +use axum::routing::post; use codex_app_server_protocol::CapabilityRootLocation; use codex_app_server_protocol::ListMcpServerStatusParams; use codex_app_server_protocol::ListMcpServerStatusResponse; +use codex_app_server_protocol::McpServerOauthLoginCompletedNotification; +use codex_app_server_protocol::McpServerOauthLoginResponse; use codex_app_server_protocol::McpServerToolCallParams; use codex_app_server_protocol::McpServerToolCallResponse; use codex_app_server_protocol::RequestId; @@ -17,14 +24,36 @@ use codex_utils_path_uri::PathUri; use core_test_support::responses; use core_test_support::stdio_server_bin; use pretty_assertions::assert_eq; +use rmcp::handler::server::ServerHandler; +use rmcp::model::CallToolRequestParams; +use rmcp::model::CallToolResult; +use rmcp::model::JsonObject; +use rmcp::model::ListToolsResult; +use rmcp::model::ServerCapabilities; +use rmcp::model::ServerInfo; +use rmcp::model::Tool; +use rmcp::model::ToolAnnotations; +use rmcp::service::RequestContext; +use rmcp::service::RoleServer; +use rmcp::transport::StreamableHttpServerConfig; +use rmcp::transport::StreamableHttpService; +use rmcp::transport::streamable_http_server::session::local::LocalSessionManager; use serde_json::json; +use std::borrow::Cow; use std::collections::BTreeMap; +use std::sync::Arc; use std::time::Duration; use tempfile::TempDir; +use tokio::net::TcpListener; +use tokio::sync::mpsc; use tokio::time::timeout; const DEFAULT_READ_TIMEOUT: Duration = Duration::from_secs(20); +const EXECUTOR_HTTP_MCP_URL: &str = "http://executor-only.invalid/mcp"; +const HTTP_MCP_SERVER_NAME: &str = "executor_http"; const MCP_SERVER_NAME: &str = "executor_demo"; +const OAUTH_MCP_SERVER_NAME: &str = "executor_oauth"; +const EXECUTOR_OAUTH_MCP_URL: &str = "http://oauth-only.invalid/oauth-mcp"; const EXECUTOR_ENV_NAME: &str = "MCP_EXECUTOR_MARKER"; const EXECUTOR_ENV_VALUE: &str = "executor-only"; const EXECUTOR_ID: &str = "executor-1"; @@ -32,8 +61,58 @@ const REFRESH_PROBE_SERVER_NAME: &str = "refresh_probe"; const TOOL_CALL_ID: &str = "executor-mcp-call"; #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn selected_executor_plugin_exposes_its_stdio_mcp_only_to_that_thread() -> Result<()> { +async fn selected_executor_plugin_exposes_its_mcps_only_to_that_thread() -> Result<()> { let responses_server = responses::start_mock_server().await; + let http_listener = TcpListener::bind("127.0.0.1:0").await?; + let http_addr = http_listener.local_addr()?; + let http_server_config = StreamableHttpServerConfig::default() + .with_allowed_hosts(["executor-only.invalid", "oauth-only.invalid"]); + let http_mcp_service = StreamableHttpService::new( + || Ok(ExecutorHttpMcpServer), + Arc::new(LocalSessionManager::default()), + http_server_config.clone(), + ); + let oauth_mcp_service = StreamableHttpService::new( + || Ok(ExecutorHttpMcpServer), + Arc::new(LocalSessionManager::default()), + http_server_config, + ); + let (token_request_tx, mut token_request_rx) = mpsc::unbounded_channel(); + let oauth_metadata = json!({ + "authorization_endpoint": "https://oauth-only.invalid/authorize", + "token_endpoint": "http://oauth-only.invalid/token", + "scopes_supported": ["read", "write"], + "response_types_supported": ["code"], + "code_challenge_methods_supported": ["S256"], + }); + let http_router = Router::new() + .route( + "/.well-known/oauth-authorization-server/oauth-mcp", + get(move || { + let metadata = oauth_metadata.clone(); + async move { Json(metadata) } + }), + ) + .route( + "/token", + post(move |body: Bytes| { + let token_request_tx = token_request_tx.clone(); + async move { + let _ = token_request_tx.send(String::from_utf8_lossy(&body).into_owned()); + Json(json!({ + "access_token": "executor-access-token", + "token_type": "Bearer", + "expires_in": 3600, + "refresh_token": "executor-refresh-token", + })) + } + }), + ) + .nest_service("/mcp", http_mcp_service) + .nest_service("/oauth-mcp", oauth_mcp_service); + let http_server_handle = tokio::spawn(async move { + let _ = axum::serve(http_listener, http_router).await; + }); let codex_home = TempDir::new()?; write_mock_responses_config_toml( codex_home.path(), @@ -44,6 +123,12 @@ async fn selected_executor_plugin_exposes_its_stdio_mcp_only_to_that_thread() -> "mock_provider", "compact", )?; + let codex_bin = toml::Value::String( + codex_utils_cargo_bin::cargo_bin("codex")? + .to_string_lossy() + .into_owned(), + ); + let http_proxy = toml::Value::String(format!("http://{http_addr}")); std::fs::write( codex_home.path().join("environments.toml"), format!( @@ -52,16 +137,12 @@ include_local = true [[environments]] id = "{EXECUTOR_ID}" -program = {} +program = {codex_bin} args = ["exec-server", "--listen", "stdio"] [environments.env] {EXECUTOR_ENV_NAME} = "{EXECUTOR_ENV_VALUE}" -"#, - toml::Value::String( - codex_utils_cargo_bin::cargo_bin("codex")? - .to_string_lossy() - .into_owned() - ) +HTTP_PROXY = {http_proxy} +"# ), )?; @@ -79,6 +160,17 @@ args = ["exec-server", "--listen", "stdio"] "command": stdio_server_bin()?, "env_vars": [EXECUTOR_ENV_NAME], "startup_timeout_sec": 10, + }, + (HTTP_MCP_SERVER_NAME): { + "url": EXECUTOR_HTTP_MCP_URL, + "environment_id": "local", + "startup_timeout_sec": 10, + }, + (OAUTH_MCP_SERVER_NAME): { + "url": EXECUTOR_OAUTH_MCP_URL, + "environment_id": "local", + "oauth": {"clientId": "executor-oauth-client"}, + "startup_timeout_sec": 10, } } }))?, @@ -99,7 +191,6 @@ args = ["exec-server", "--listen", "stdio"] ) .await?; - std::fs::write(plugin.path().join(".mcp.json"), r#"{"mcpServers":{}}"#)?; let config_path = codex_home.path().join("config.toml"); let mut config = std::fs::read_to_string(&config_path)?; config.push_str(&format!( @@ -120,6 +211,76 @@ startup_timeout_sec = 10 ) .await??; + let request_id = app_server + .send_raw_request( + "mcpServer/oauth/login", + Some(json!({ + "name": OAUTH_MCP_SERVER_NAME, + "threadId": selected_thread.clone(), + "timeoutSecs": 10, + })), + ) + .await?; + let response = timeout( + DEFAULT_READ_TIMEOUT, + app_server.read_stream_until_response_message(RequestId::Integer(request_id)), + ) + .await??; + let response: McpServerOauthLoginResponse = to_response(response)?; + assert!( + response + .authorization_url + .starts_with("https://oauth-only.invalid/authorize?") + ); + assert!( + response + .authorization_url + .contains("client_id=executor-oauth-client") + ); + let authorization_url = reqwest::Url::parse(&response.authorization_url)?; + let state = authorization_url + .query_pairs() + .find_map(|(key, value)| (key == "state").then(|| value.into_owned())) + .expect("authorization URL should include state"); + let redirect_uri = authorization_url + .query_pairs() + .find_map(|(key, value)| (key == "redirect_uri").then(|| value.into_owned())) + .expect("authorization URL should include redirect_uri"); + let mut callback_url = reqwest::Url::parse(&redirect_uri)?; + callback_url + .query_pairs_mut() + .append_pair("code", "executor-test-code") + .append_pair("state", &state); + reqwest::Client::builder() + .no_proxy() + .build()? + .get(callback_url) + .send() + .await? + .error_for_status()?; + let token_request = timeout(DEFAULT_READ_TIMEOUT, token_request_rx.recv()) + .await? + .expect("executor token endpoint should receive a request"); + assert!(token_request.contains("grant_type=authorization_code")); + assert!(token_request.contains("code=executor-test-code")); + assert!(token_request.contains("code_verifier=")); + let notification = timeout( + DEFAULT_READ_TIMEOUT, + app_server.read_stream_until_notification_message("mcpServer/oauthLogin/completed"), + ) + .await??; + let completed: McpServerOauthLoginCompletedNotification = + serde_json::from_value(notification.params.expect("notification params"))?; + assert_eq!( + completed, + McpServerOauthLoginCompletedNotification { + name: OAUTH_MCP_SERVER_NAME.to_string(), + thread_id: Some(selected_thread.clone()), + success: true, + error: None, + } + ); + let namespace = format!("mcp__{MCP_SERVER_NAME}"); let response_mock = responses::mount_sse_sequence( &responses_server, @@ -178,6 +339,26 @@ startup_timeout_sec = 10 assert!(output.contains("ECHOING: hello from executor")); assert!(output.contains(EXECUTOR_ENV_VALUE)); + let request_id = app_server + .send_mcp_server_tool_call_request(McpServerToolCallParams { + thread_id: selected_thread.clone(), + server: HTTP_MCP_SERVER_NAME.to_string(), + tool: "echo".to_string(), + arguments: Some(json!({"message": "hello over executor HTTP"})), + meta: None, + }) + .await?; + let response = timeout( + DEFAULT_READ_TIMEOUT, + app_server.read_stream_until_response_message(RequestId::Integer(request_id)), + ) + .await??; + let response: McpServerToolCallResponse = to_response(response)?; + assert_eq!( + response.structured_content, + Some(json!({"echo": "ECHOING: hello over executor HTTP"})) + ); + let request_id = app_server .send_mcp_server_tool_call_request(McpServerToolCallParams { thread_id: selected_thread.clone(), @@ -200,25 +381,87 @@ startup_timeout_sec = 10 Some(json!("ECHOING: refresh applied")) ); + let selected_server_names = mcp_server_names(&mut app_server, selected_thread).await?; assert!( - mcp_server_names(&mut app_server, selected_thread) - .await? + selected_server_names .iter() .any(|name| name == MCP_SERVER_NAME) ); + assert!( + selected_server_names + .iter() + .any(|name| name == HTTP_MCP_SERVER_NAME) + ); + assert!( + selected_server_names + .iter() + .any(|name| name == OAUTH_MCP_SERVER_NAME) + ); let unselected_thread = start_thread(&mut app_server, /*selected_capability_roots*/ None).await?; - assert!( - mcp_server_names(&mut app_server, unselected_thread) - .await? - .iter() - .all(|name| name != MCP_SERVER_NAME) - ); + let unselected_server_names = mcp_server_names(&mut app_server, unselected_thread).await?; + assert!(unselected_server_names.iter().all(|name| { + name != MCP_SERVER_NAME && name != HTTP_MCP_SERVER_NAME && name != OAUTH_MCP_SERVER_NAME + })); + + http_server_handle.abort(); + let _ = http_server_handle.await; Ok(()) } +#[derive(Clone, Copy)] +struct ExecutorHttpMcpServer; + +impl ServerHandler for ExecutorHttpMcpServer { + fn get_info(&self) -> ServerInfo { + ServerInfo::new(ServerCapabilities::builder().enable_tools().build()) + } + + async fn list_tools( + &self, + _request: Option, + _context: RequestContext, + ) -> Result { + let input_schema: JsonObject = serde_json::from_value(json!({ + "type": "object", + "properties": {"message": {"type": "string"}}, + "required": ["message"], + "additionalProperties": false + })) + .map_err(|err| rmcp::ErrorData::internal_error(err.to_string(), None))?; + let mut tool = Tool::new( + Cow::Borrowed("echo"), + Cow::Borrowed("Echo a message."), + Arc::new(input_schema), + ); + tool.annotations = Some(ToolAnnotations::new().read_only(true)); + + Ok(ListToolsResult { + tools: vec![tool], + next_cursor: None, + meta: None, + }) + } + + async fn call_tool( + &self, + request: CallToolRequestParams, + _context: RequestContext, + ) -> Result { + let message = request + .arguments + .as_ref() + .and_then(|arguments| arguments.get("message")) + .and_then(serde_json::Value::as_str) + .unwrap_or_default(); + Ok(CallToolResult::structured(json!({ + "echo": format!("ECHOING: {message}") + }))) + } +} + async fn mcp_server_names( app_server: &mut TestAppServer, thread_id: String, diff --git a/codex-rs/app-server/tests/suite/v2/experimental_api.rs b/codex-rs/app-server/tests/suite/v2/experimental_api.rs index d9d3469936..80fdb03171 100644 --- a/codex-rs/app-server/tests/suite/v2/experimental_api.rs +++ b/codex-rs/app-server/tests/suite/v2/experimental_api.rs @@ -82,6 +82,7 @@ async fn realtime_conversation_start_requires_experimental_api_capability() -> R let request_id = mcp .send_thread_realtime_start_request(ThreadRealtimeStartParams { client_managed_handoffs: None, + flush_transcript_tail_on_session_end: None, codex_responses_as_items: None, codex_response_item_prefix: None, codex_response_handoff_prefix: None, @@ -198,6 +199,7 @@ async fn realtime_webrtc_start_requires_experimental_api_capability() -> Result< let request_id = mcp .send_thread_realtime_start_request(ThreadRealtimeStartParams { client_managed_handoffs: None, + flush_transcript_tail_on_session_end: None, codex_responses_as_items: None, codex_response_item_prefix: None, codex_response_handoff_prefix: None, diff --git a/codex-rs/app-server/tests/suite/v2/mcp_resource.rs b/codex-rs/app-server/tests/suite/v2/mcp_resource.rs index 98b0fa3512..1bdf06161f 100644 --- a/codex-rs/app-server/tests/suite/v2/mcp_resource.rs +++ b/codex-rs/app-server/tests/suite/v2/mcp_resource.rs @@ -138,7 +138,7 @@ async fn orchestrator_skill_can_read_referenced_resource_without_an_executor() - let thread_start_id = mcp .send_thread_start_request(ThreadStartParams { - model: Some("mock-model".to_string()), + model: Some("gpt-5.5".to_string()), environments: Some(Vec::new()), ..Default::default() }) diff --git a/codex-rs/app-server/tests/suite/v2/mcp_tool.rs b/codex-rs/app-server/tests/suite/v2/mcp_tool.rs index 3b5afec3fd..1f3289e439 100644 --- a/codex-rs/app-server/tests/suite/v2/mcp_tool.rs +++ b/codex-rs/app-server/tests/suite/v2/mcp_tool.rs @@ -10,6 +10,8 @@ use app_test_support::create_mock_responses_server_sequence; use app_test_support::to_response; use app_test_support::write_mock_responses_config_toml; use axum::Router; +use codex_app_server_protocol::CapabilityRootLocation; +use codex_app_server_protocol::EnvironmentAddResponse; use codex_app_server_protocol::ItemCompletedNotification; use codex_app_server_protocol::JSONRPCError; use codex_app_server_protocol::JSONRPCResponse; @@ -22,15 +24,20 @@ use codex_app_server_protocol::McpServerToolCallParams; use codex_app_server_protocol::McpServerToolCallResponse; use codex_app_server_protocol::McpToolCallStatus; use codex_app_server_protocol::RequestId; +use codex_app_server_protocol::SelectedCapabilityRoot; use codex_app_server_protocol::ServerRequest; use codex_app_server_protocol::ThreadItem; use codex_app_server_protocol::ThreadStartParams; use codex_app_server_protocol::ThreadStartResponse; +use codex_app_server_protocol::TurnEnvironmentParams; use codex_app_server_protocol::TurnStartParams; use codex_app_server_protocol::TurnStartResponse; use codex_app_server_protocol::UserInput as V2UserInput; +use codex_features::Feature; +use codex_utils_path_uri::PathUri; use codex_utils_pty::DEFAULT_OUTPUT_BYTES_CAP; use core_test_support::responses; +use futures::SinkExt; use pretty_assertions::assert_eq; use rmcp::handler::server::ServerHandler; use rmcp::model::BooleanSchema; @@ -56,8 +63,13 @@ use rmcp::transport::streamable_http_server::session::local::LocalSessionManager use serde_json::json; use tempfile::TempDir; use tokio::net::TcpListener; +use tokio::sync::oneshot; use tokio::task::JoinHandle; use tokio::time::timeout; +use tokio_tungstenite::tungstenite::Message; + +use super::exec_server_test_support::accept_exec_server_environment; +use super::exec_server_test_support::read_exec_server_json; const DEFAULT_READ_TIMEOUT: Duration = Duration::from_secs(10); const TEST_SERVER_NAME: &str = "tool_server"; @@ -68,6 +80,7 @@ const ELICITATION_MESSAGE: &str = "Allow this request?"; const URL_ELICITATION_TRIGGER_MESSAGE: &str = "auth"; const URL_ELICITATION_MESSAGE: &str = "Sign in to GitHub to continue."; const URL_ELICITATION_URL: &str = "https://github.example/login/device"; +const LATE_ENVIRONMENT_ID: &str = "late-environment"; #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn mcp_server_tool_call_returns_tool_result() -> Result<()> { @@ -297,6 +310,145 @@ url = "{mcp_server_url}/mcp" Ok(()) } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn mcp_server_elicitation_survives_environment_runtime_refresh() -> Result<()> { + let responses_server = responses::start_mock_server().await; + let (mcp_server_url, mcp_server_handle) = start_mcp_server().await?; + let exec_listener = TcpListener::bind("127.0.0.1:0").await?; + let exec_server_url = format!("ws://{}", exec_listener.local_addr()?); + let codex_home = TempDir::new()?; + write_mock_responses_config_toml( + codex_home.path(), + &responses_server.uri(), + &BTreeMap::from([(Feature::DeferredExecutor, true)]), + /*auto_compact_limit*/ 1024, + /*requires_openai_auth*/ None, + "mock_provider", + "compact", + )?; + let config_path = codex_home.path().join("config.toml"); + let mut config_toml = std::fs::read_to_string(&config_path)?; + config_toml.push_str(&format!( + r#" +[mcp_servers.{TEST_SERVER_NAME}] +url = "{mcp_server_url}/mcp" +"# + )); + std::fs::write(config_path, config_toml)?; + + let mut mcp = TestAppServer::new(codex_home.path()).await?; + timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; + let add_environment_id = mcp + .send_raw_request( + "environment/add", + Some(json!({ + "environmentId": LATE_ENVIRONMENT_ID, + "execServerUrl": exec_server_url, + "connectTimeoutMs": 10_000, + })), + ) + .await?; + let add_environment_response = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(add_environment_id)), + ) + .await??; + let _: EnvironmentAddResponse = to_response(add_environment_response)?; + + let capability_root = TempDir::new()?; + let thread_start_id = mcp + .send_thread_start_request(ThreadStartParams { + model: Some("mock-model".to_string()), + approval_policy: Some(codex_app_server_protocol::AskForApproval::UnlessTrusted), + environments: Some(vec![TurnEnvironmentParams { + environment_id: LATE_ENVIRONMENT_ID.to_string(), + cwd: codex_utils_absolute_path::AbsolutePathBuf::try_from( + capability_root.path().to_path_buf(), + )? + .into(), + }]), + selected_capability_roots: Some(vec![SelectedCapabilityRoot { + id: "late-plugin@1".to_string(), + location: CapabilityRootLocation::Environment { + environment_id: LATE_ENVIRONMENT_ID.to_string(), + path: PathUri::from_host_native_path(capability_root.path())?, + }, + }]), + ..Default::default() + }) + .await?; + let thread_start_response = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(thread_start_id)), + ) + .await??; + let ThreadStartResponse { thread, .. } = to_response(thread_start_response)?; + + let tool_call_request_id = mcp + .send_mcp_server_tool_call_request(McpServerToolCallParams { + thread_id: thread.id.clone(), + server: TEST_SERVER_NAME.to_string(), + tool: TEST_TOOL_NAME.to_string(), + arguments: Some(json!({"message": ELICITATION_TRIGGER_MESSAGE})), + meta: None, + }) + .await?; + let server_request = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_request_message(), + ) + .await??; + let ServerRequest::McpServerElicitationRequest { request_id, .. } = server_request else { + panic!("expected MCP elicitation request, got: {server_request:?}"); + }; + + let (filesystem_request_tx, filesystem_request_rx) = oneshot::channel(); + let (shutdown_tx, shutdown_rx) = oneshot::channel(); + let exec_server_handle = tokio::spawn(serve_environment_until_shutdown( + exec_listener, + filesystem_request_tx, + shutdown_rx, + )); + let mut filesystem_request_rx = filesystem_request_rx; + timeout(DEFAULT_READ_TIMEOUT, async { + loop { + let status_request_id = mcp + .send_raw_request("mcpServerStatus/list", Some(json!({"threadId": thread.id}))) + .await?; + mcp.read_stream_until_response_message(RequestId::Integer(status_request_id)) + .await?; + if filesystem_request_rx.try_recv().is_ok() { + return Ok::<_, anyhow::Error>(()); + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await??; + + mcp.send_response( + request_id, + serde_json::to_value(McpServerElicitationRequestResponse { + action: McpServerElicitationAction::Accept, + content: Some(json!({"confirmed": true})), + meta: None, + })?, + ) + .await?; + let tool_call_response = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(tool_call_request_id)), + ) + .await??; + let response: McpServerToolCallResponse = to_response(tool_call_response)?; + assert_eq!(response.content[0].get("text"), Some(&json!("accepted"))); + + let _ = shutdown_tx.send(()); + exec_server_handle.await??; + mcp_server_handle.abort(); + let _ = mcp_server_handle.await; + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn mcp_server_tool_call_forwards_url_elicitation() -> Result<()> { let responses_server = responses::start_mock_server().await; @@ -683,6 +835,45 @@ async fn start_mcp_server() -> Result<(String, JoinHandle<()>)> { Ok((format!("http://{addr}"), handle)) } +async fn serve_environment_until_shutdown( + listener: TcpListener, + filesystem_request_tx: oneshot::Sender<()>, + mut shutdown_rx: oneshot::Receiver<()>, +) -> Result<()> { + let mut websocket = accept_exec_server_environment( + listener, + json!({"shell": {"name": "zsh", "path": "/bin/zsh"}}), + ) + .await?; + + let mut filesystem_request_tx = Some(filesystem_request_tx); + loop { + let request = tokio::select! { + request = read_exec_server_json(&mut websocket) => request?, + _ = &mut shutdown_rx => return Ok(()), + }; + if request["method"] + .as_str() + .is_some_and(|method| method.starts_with("fs/")) + && let Some(tx) = filesystem_request_tx.take() + { + let _ = tx.send(()); + } + if request.get("id").is_some() { + websocket + .send(Message::Text( + json!({ + "id": request["id"], + "error": {"code": -32004, "message": "not found"}, + }) + .to_string() + .into(), + )) + .await?; + } + } +} + async fn wait_for_mcp_tool_call_completed( mcp: &mut TestAppServer, call_id: &str, diff --git a/codex-rs/app-server/tests/suite/v2/mod.rs b/codex-rs/app-server/tests/suite/v2/mod.rs index bbd9952754..bdb03271d0 100644 --- a/codex-rs/app-server/tests/suite/v2/mod.rs +++ b/codex-rs/app-server/tests/suite/v2/mod.rs @@ -15,6 +15,8 @@ mod connection_handling_websocket_unix; mod current_time; mod dynamic_tools; mod environment_add; +mod environment_info; +mod exec_server_test_support; #[cfg(not(target_os = "windows"))] mod executor_mcp; mod executor_skills; @@ -56,6 +58,8 @@ mod request_user_input; mod request_validation; mod review; mod safety_check_downgrade; +#[cfg(not(target_os = "windows"))] +mod selected_capability_stack; mod skills_list; mod sleep; mod thread_archive; diff --git a/codex-rs/app-server/tests/suite/v2/plugin_install.rs b/codex-rs/app-server/tests/suite/v2/plugin_install.rs index 34fc9b18a4..975338e11d 100644 --- a/codex-rs/app-server/tests/suite/v2/plugin_install.rs +++ b/codex-rs/app-server/tests/suite/v2/plugin_install.rs @@ -2027,7 +2027,6 @@ chatgpt_base_url = "{base_url}" [features] plugins = true -remote_plugin = true "# ), ) @@ -2057,7 +2056,6 @@ chatgpt_base_url = "{}/backend-api/" [features] plugins = true -remote_plugin = true connectors = true "#, server.uri() diff --git a/codex-rs/app-server/tests/suite/v2/plugin_list.rs b/codex-rs/app-server/tests/suite/v2/plugin_list.rs index a301714cf7..ae03b3a1a9 100644 --- a/codex-rs/app-server/tests/suite/v2/plugin_list.rs +++ b/codex-rs/app-server/tests/suite/v2/plugin_list.rs @@ -71,6 +71,23 @@ plugins = true ) } +fn write_remote_plugins_disabled_config_with_base_url( + codex_home: &std::path::Path, + base_url: &str, +) -> std::io::Result<()> { + std::fs::write( + codex_home.join("config.toml"), + format!( + r#"chatgpt_base_url = "{base_url}" + +[features] +plugins = true +remote_plugin = false +"#, + ), + ) +} + #[tokio::test] async fn plugin_list_skips_invalid_marketplace_file_and_reports_error() -> Result<()> { let codex_home = TempDir::new()?; @@ -197,7 +214,6 @@ async fn plugin_installed_prefers_remote_curated_conflicts_when_remote_plugin_en [features] plugins = true -remote_plugin = true plugin_sharing = false [plugins."linear@openai-curated"] @@ -436,6 +452,7 @@ async fn plugin_list_keeps_valid_marketplaces_when_another_marketplace_fails_to_ plugins: vec![PluginSummary { id: "valid-plugin@valid-marketplace".to_string(), remote_plugin_id: None, + version: None, local_version: None, name: "valid-plugin".to_string(), share_context: None, @@ -727,6 +744,7 @@ async fn plugin_list_uses_alternate_discoverable_manifest_and_keeps_undiscoverab PluginSummary { id: "valid-plugin@alternate-marketplace".to_string(), remote_plugin_id: None, + version: None, local_version: None, name: "valid-plugin".to_string(), share_context: None, @@ -764,6 +782,7 @@ async fn plugin_list_uses_alternate_discoverable_manifest_and_keeps_undiscoverab PluginSummary { id: "missing-plugin@alternate-marketplace".to_string(), remote_plugin_id: None, + version: None, local_version: None, name: "missing-plugin".to_string(), share_context: None, @@ -1667,6 +1686,7 @@ async fn plugin_list_includes_remote_marketplaces_when_remote_plugin_enabled() - "authentication_policy": "ON_USE", "status": "ENABLED", "release": { + "version": "1.2.3", "display_name": "Linear", "description": "Track work in Linear", "app_ids": [], @@ -1815,6 +1835,10 @@ async fn plugin_list_includes_remote_marketplaces_when_remote_plugin_enabled() - ); assert_eq!(remote_marketplace.plugins[0].name, "linear"); assert_eq!(remote_marketplace.plugins[0].source, PluginSource::Remote); + assert_eq!( + remote_marketplace.plugins[0].version.as_deref(), + Some("1.2.3") + ); assert_eq!( remote_marketplace.plugins[0].local_version.as_deref(), Some("1.2.3") @@ -1987,10 +2011,11 @@ async fn plugin_list_uses_cached_global_remote_catalog_and_refreshes_it() -> Res } #[tokio::test] -async fn plugin_list_includes_openai_curated_remote_collection_when_requested() -> Result<()> { +async fn plugin_list_includes_openai_curated_remote_collection_when_remote_plugin_disabled_and_requested() +-> Result<()> { let codex_home = TempDir::new()?; let server = MockServer::start().await; - write_plugins_enabled_config_with_base_url( + write_remote_plugins_disabled_config_with_base_url( codex_home.path(), &format!("{}/backend-api/", server.uri()), )?; @@ -2013,6 +2038,7 @@ async fn plugin_list_includes_openai_curated_remote_collection_when_requested() "authentication_policy": "ON_USE", "status": "ENABLED", "release": { + "version": "1.2.3", "display_name": "Linear", "description": "Track work in Linear", "app_ids": [], @@ -2073,6 +2099,7 @@ async fn plugin_list_includes_openai_curated_remote_collection_when_requested() ); assert_eq!(plugin.name, "linear"); assert_eq!(plugin.source, PluginSource::Remote); + assert_eq!(plugin.version.as_deref(), Some("1.2.3")); assert_eq!(plugin.installed, false); assert_eq!(plugin.enabled, false); @@ -2092,10 +2119,11 @@ async fn plugin_list_includes_openai_curated_remote_collection_when_requested() } #[tokio::test] -async fn plugin_list_propagates_explicit_openai_curated_remote_collection_errors() -> Result<()> { +async fn plugin_list_propagates_openai_curated_remote_collection_errors_when_remote_plugin_disabled() +-> Result<()> { let codex_home = TempDir::new()?; let server = MockServer::start().await; - write_plugins_enabled_config_with_base_url( + write_remote_plugins_disabled_config_with_base_url( codex_home.path(), &format!("{}/backend-api/", server.uri()), )?; @@ -2148,10 +2176,11 @@ async fn plugin_list_propagates_explicit_openai_curated_remote_collection_errors } #[tokio::test] -async fn plugin_list_skips_explicit_openai_curated_remote_collection_for_api_auth() -> Result<()> { +async fn plugin_list_skips_openai_curated_remote_collection_for_api_auth_when_remote_plugin_disabled() +-> Result<()> { let codex_home = TempDir::new()?; let server = MockServer::start().await; - write_plugins_enabled_config_with_base_url( + write_remote_plugins_disabled_config_with_base_url( codex_home.path(), &format!("{}/backend-api/", server.uri()), )?; @@ -2391,7 +2420,8 @@ async fn plugin_list_does_not_append_global_remote_when_marketplace_kinds_are_ex } #[tokio::test] -async fn plugin_installed_includes_remote_shared_with_me_plugins() -> Result<()> { +async fn plugin_installed_includes_remote_shared_with_me_plugins_when_remote_plugin_disabled() +-> Result<()> { let codex_home = TempDir::new()?; let server = MockServer::start().await; std::fs::write( @@ -2472,16 +2502,25 @@ plugin_sharing = true marketplace .plugins .iter() - .map(|plugin| (plugin.id.clone(), plugin.installed, plugin.enabled)) + .map(|plugin| { + ( + plugin.id.clone(), + plugin.version.clone(), + plugin.installed, + plugin.enabled, + ) + }) .collect::>(), vec![ ( "shared-linear@workspace-shared-with-me".to_string(), + Some("1.2.3".to_string()), true, true ), ( "unlisted-linear@workspace-shared-with-me".to_string(), + Some("1.2.3".to_string()), true, false ) @@ -2493,7 +2532,8 @@ plugin_sharing = true } #[tokio::test] -async fn plugin_installed_includes_workspace_directory_without_plugin_sharing() -> Result<()> { +async fn plugin_installed_includes_workspace_directory_without_plugin_sharing_when_remote_plugin_disabled() +-> Result<()> { let codex_home = TempDir::new()?; let server = MockServer::start().await; std::fs::write( @@ -2590,7 +2630,6 @@ async fn plugin_installed_includes_created_by_me_when_remote_plugins_enabled() - [features] plugins = true -remote_plugin = true plugin_sharing = false "#, server.uri() @@ -2681,7 +2720,6 @@ async fn plugin_installed_starts_remote_installed_bundle_sync() -> Result<()> { [features] plugins = true -remote_plugin = true plugin_sharing = false "#, server.uri() @@ -2750,10 +2788,10 @@ plugin_sharing = false } #[tokio::test] -async fn plugin_list_fetches_workspace_directory_kind_without_remote_plugin_flag() -> Result<()> { +async fn plugin_list_fetches_workspace_directory_kind_when_remote_plugin_disabled() -> Result<()> { let codex_home = TempDir::new()?; let server = MockServer::start().await; - write_plugins_enabled_config_with_base_url( + write_remote_plugins_disabled_config_with_base_url( codex_home.path(), &format!("{}/backend-api/", server.uri()), )?; @@ -2848,7 +2886,6 @@ async fn plugin_list_fetches_user_plugins_in_created_by_me_remote_marketplace() [features] plugins = true -remote_plugin = true plugin_sharing = false "#, server.uri() @@ -4032,7 +4069,6 @@ chatgpt_base_url = "{base_url}" [features] plugins = true -remote_plugin = true "# ), ) diff --git a/codex-rs/app-server/tests/suite/v2/plugin_read.rs b/codex-rs/app-server/tests/suite/v2/plugin_read.rs index 011cbec52e..41b7990ec8 100644 --- a/codex-rs/app-server/tests/suite/v2/plugin_read.rs +++ b/codex-rs/app-server/tests/suite/v2/plugin_read.rs @@ -2132,7 +2132,6 @@ chatgpt_base_url = "{base_url}" [features] plugins = true -remote_plugin = true "# ), ) diff --git a/codex-rs/app-server/tests/suite/v2/plugin_share.rs b/codex-rs/app-server/tests/suite/v2/plugin_share.rs index 5257ea03bc..a82b363806 100644 --- a/codex-rs/app-server/tests/suite/v2/plugin_share.rs +++ b/codex-rs/app-server/tests/suite/v2/plugin_share.rs @@ -171,6 +171,7 @@ async fn plugin_share_save_uploads_local_plugin() -> Result<()> { plugin: PluginSummary { id: "demo-plugin@workspace-shared-with-me".to_string(), remote_plugin_id: Some("plugins_123".to_string()), + version: Some("0.1.0".to_string()), local_version: Some("0.1.0".to_string()), name: "demo-plugin".to_string(), share_context: Some(expected_share_context("plugins_123")), @@ -344,7 +345,6 @@ chatgpt_base_url = "{}/backend-api" [features] plugins = true -remote_plugin = true plugin_sharing = false "#, server.uri() @@ -575,6 +575,7 @@ async fn plugin_share_list_returns_created_workspace_plugins() -> Result<()> { plugin: PluginSummary { id: "demo-plugin@workspace-shared-with-me".to_string(), remote_plugin_id: Some("plugins_123".to_string()), + version: Some("0.1.0".to_string()), local_version: Some("0.1.0".to_string()), name: "demo-plugin".to_string(), share_context: Some(expected_share_context("plugins_123")), @@ -1050,7 +1051,6 @@ chatgpt_base_url = "{}/backend-api" [features] plugins = true -remote_plugin = true plugin_sharing = false "#, server.uri() @@ -1176,6 +1176,7 @@ async fn plugin_share_delete_removes_created_workspace_plugin() -> Result<()> { plugin: PluginSummary { id: "demo-plugin@workspace-shared-with-me".to_string(), remote_plugin_id: Some("plugins_123".to_string()), + version: Some("0.1.0".to_string()), local_version: Some("0.1.0".to_string()), name: "demo-plugin".to_string(), share_context: Some(expected_share_context("plugins_123")), @@ -1204,7 +1205,6 @@ chatgpt_base_url = "{base_url}" [features] plugins = true -remote_plugin = true "# ), ) diff --git a/codex-rs/app-server/tests/suite/v2/plugin_uninstall.rs b/codex-rs/app-server/tests/suite/v2/plugin_uninstall.rs index 6f9041a74e..ea82bca08c 100644 --- a/codex-rs/app-server/tests/suite/v2/plugin_uninstall.rs +++ b/codex-rs/app-server/tests/suite/v2/plugin_uninstall.rs @@ -622,7 +622,6 @@ chatgpt_base_url = "{base_url}" [features] plugins = true -remote_plugin = true "# ), ) diff --git a/codex-rs/app-server/tests/suite/v2/rate_limit_reset_credits.rs b/codex-rs/app-server/tests/suite/v2/rate_limit_reset_credits.rs index 86ea09504c..fbb4e6e940 100644 --- a/codex-rs/app-server/tests/suite/v2/rate_limit_reset_credits.rs +++ b/codex-rs/app-server/tests/suite/v2/rate_limit_reset_credits.rs @@ -44,6 +44,7 @@ async fn consume_rate_limit_reset_credit_requires_chatgpt_auth() -> Result<()> { .send_consume_account_rate_limit_reset_credit_request( ConsumeAccountRateLimitResetCreditParams { idempotency_key: "request-1".to_string(), + credit_id: None, }, ) .await?; @@ -120,6 +121,44 @@ async fn consume_account_rate_limit_reset_credit_maps_backend_outcomes() -> Resu Ok(()) } +#[tokio::test] +async fn consume_account_rate_limit_reset_credit_forwards_selected_credit_id() -> Result<()> { + let (codex_home, server) = chatgpt_test_context().await?; + Mock::given(method("POST")) + .and(path("/api/codex/rate-limit-reset-credits/consume")) + .and(header("authorization", "Bearer chatgpt-token")) + .and(header("chatgpt-account-id", "account-123")) + .and(body_json(json!({ + "redeem_request_id": "request-selected", + "credit_id": "credit-123", + }))) + .respond_with( + ResponseTemplate::new(200) + .set_body_json(json!({ "code": "reset", "windows_reset": 2 })), + ) + .expect(1) + .mount(&server) + .await; + + let mut mcp = initialized_app_server(codex_home.path()).await?; + let request_id = mcp + .send_consume_account_rate_limit_reset_credit_request( + ConsumeAccountRateLimitResetCreditParams { + idempotency_key: "request-selected".to_string(), + credit_id: Some("credit-123".to_string()), + }, + ) + .await?; + + assert_eq!( + read_response::(&mut mcp, request_id).await?, + ConsumeAccountRateLimitResetCreditResponse { + outcome: ConsumeAccountRateLimitResetCreditOutcome::Reset, + } + ); + Ok(()) +} + #[tokio::test] async fn consume_account_rate_limit_reset_credit_rejects_empty_idempotency_key() -> Result<()> { let (codex_home, _server) = chatgpt_test_context().await?; @@ -129,6 +168,7 @@ async fn consume_account_rate_limit_reset_credit_rejects_empty_idempotency_key() .send_consume_account_rate_limit_reset_credit_request( ConsumeAccountRateLimitResetCreditParams { idempotency_key: String::new(), + credit_id: None, }, ) .await?; @@ -139,6 +179,26 @@ async fn consume_account_rate_limit_reset_credit_rejects_empty_idempotency_key() Ok(()) } +#[tokio::test] +async fn consume_account_rate_limit_reset_credit_rejects_empty_credit_id() -> Result<()> { + let (codex_home, _server) = chatgpt_test_context().await?; + let mut mcp = initialized_app_server(codex_home.path()).await?; + + let request_id = mcp + .send_consume_account_rate_limit_reset_credit_request( + ConsumeAccountRateLimitResetCreditParams { + idempotency_key: "request-1".to_string(), + credit_id: Some(String::new()), + }, + ) + .await?; + let error = read_error_response(&mut mcp, request_id).await?; + + assert_eq!(error.error.code, INVALID_REQUEST_ERROR_CODE); + assert_eq!(error.error.message, "creditId must not be empty"); + Ok(()) +} + #[tokio::test] async fn consume_account_rate_limit_reset_credit_surfaces_backend_failure() -> Result<()> { let (codex_home, server) = chatgpt_test_context().await?; @@ -244,6 +304,7 @@ async fn send_consume_reset_credit(mcp: &mut TestAppServer, idempotency_key: &st mcp.send_consume_account_rate_limit_reset_credit_request( ConsumeAccountRateLimitResetCreditParams { idempotency_key: idempotency_key.to_string(), + credit_id: None, }, ) .await diff --git a/codex-rs/app-server/tests/suite/v2/rate_limits.rs b/codex-rs/app-server/tests/suite/v2/rate_limits.rs index 58c31c453b..423385cea9 100644 --- a/codex-rs/app-server/tests/suite/v2/rate_limits.rs +++ b/codex-rs/app-server/tests/suite/v2/rate_limits.rs @@ -10,7 +10,10 @@ use codex_app_server_protocol::JSONRPCError; use codex_app_server_protocol::JSONRPCResponse; use codex_app_server_protocol::LoginAccountResponse; use codex_app_server_protocol::RateLimitReachedType; +use codex_app_server_protocol::RateLimitResetCredit; +use codex_app_server_protocol::RateLimitResetCreditStatus; use codex_app_server_protocol::RateLimitResetCreditsSummary; +use codex_app_server_protocol::RateLimitResetType; use codex_app_server_protocol::RateLimitSnapshot; use codex_app_server_protocol::RateLimitWindow; use codex_app_server_protocol::RequestId; @@ -109,6 +112,16 @@ async fn get_account_rate_limits_returns_snapshot() -> Result<()> { let secondary_reset_timestamp = chrono::DateTime::parse_from_rfc3339("2025-01-01T01:00:00Z") .expect("parse secondary reset timestamp") .timestamp(); + let reset_credit_granted_at = chrono::DateTime::parse_from_rfc3339("2026-06-17T00:00:00Z") + .expect("parse reset credit grant timestamp") + .timestamp(); + let reset_credit_expires_at = chrono::DateTime::parse_from_rfc3339("2026-07-17T00:00:00Z") + .expect("parse reset credit expiry timestamp") + .timestamp(); + let second_reset_credit_granted_at = + chrono::DateTime::parse_from_rfc3339("2026-06-18T00:00:00Z") + .expect("parse second reset credit grant timestamp") + .timestamp(); let response_body = json!({ "plan_type": "pro", "rate_limit": { @@ -170,6 +183,35 @@ async fn get_account_rate_limits_returns_snapshot() -> Result<()> { .expect(1) .mount(&server) .await; + Mock::given(method("GET")) + .and(path("/api/codex/rate-limit-reset-credits")) + .and(header("authorization", "Bearer chatgpt-token")) + .and(header("chatgpt-account-id", "account-123")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "credits": [ + { + "id": "credit-1", + "reset_type": "codex_rate_limits", + "status": "available", + "granted_at": "2026-06-17T00:00:00Z", + "expires_at": "2026-07-17T00:00:00Z", + "title": "Full reset (Weekly + 5 hr)", + "description": "Ready to redeem" + }, + { + "id": "credit-2", + "reset_type": "future_reset_type", + "status": "future_status", + "granted_at": "2026-06-18T00:00:00Z", + "expires_at": null + } + ], + "available_count": 2, + "total_earned_count": 4 + }))) + .expect(1) + .mount(&server) + .await; let mut mcp = TestAppServer::new_with_env(codex_home.path(), &[("OPENAI_API_KEY", None)]).await?; @@ -260,13 +302,98 @@ async fn get_account_rate_limits_returns_snapshot() -> Result<()> { .into_iter() .collect(), ), - rate_limit_reset_credits: Some(RateLimitResetCreditsSummary { available_count: 3 }), + rate_limit_reset_credits: Some(RateLimitResetCreditsSummary { + available_count: 2, + credits: Some(vec![ + RateLimitResetCredit { + id: "credit-1".to_string(), + reset_type: RateLimitResetType::CodexRateLimits, + status: RateLimitResetCreditStatus::Available, + granted_at: reset_credit_granted_at, + expires_at: Some(reset_credit_expires_at), + title: Some("Full reset (Weekly + 5 hr)".to_string()), + description: Some("Ready to redeem".to_string()), + }, + RateLimitResetCredit { + id: "credit-2".to_string(), + reset_type: RateLimitResetType::Unknown, + status: RateLimitResetCreditStatus::Unknown, + granted_at: second_reset_credit_granted_at, + expires_at: None, + title: None, + description: None, + }, + ]), + }), }; assert_eq!(received, expected); Ok(()) } +#[tokio::test] +async fn get_account_rate_limits_preserves_count_when_reset_credit_details_fail() -> Result<()> { + let codex_home = TempDir::new()?; + write_chatgpt_auth( + codex_home.path(), + ChatGptAuthFixture::new("chatgpt-token") + .account_id("account-123") + .plan_type("pro"), + AuthCredentialsStoreMode::File, + )?; + + let server = MockServer::start().await; + write_chatgpt_base_url(codex_home.path(), &server.uri())?; + + Mock::given(method("GET")) + .and(path("/api/codex/usage")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "plan_type": "pro", + "rate_limit": { + "allowed": true, + "limit_reached": false, + "primary_window": { + "used_percent": 42, + "limit_window_seconds": 3600, + "reset_after_seconds": 120, + "reset_at": 1735689720 + } + }, + "rate_limit_reset_credits": { "available_count": 3 } + }))) + .expect(1) + .mount(&server) + .await; + Mock::given(method("GET")) + .and(path("/api/codex/rate-limit-reset-credits")) + .respond_with(ResponseTemplate::new(500).set_body_string("boom")) + .expect(1) + .mount(&server) + .await; + + let mut mcp = + TestAppServer::new_with_env(codex_home.path(), &[("OPENAI_API_KEY", None)]).await?; + timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; + + let request_id = mcp.send_get_account_rate_limits_request().await?; + let response: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(request_id)), + ) + .await??; + let received: GetAccountRateLimitsResponse = to_response(response)?; + + assert_eq!( + received.rate_limit_reset_credits, + Some(RateLimitResetCreditsSummary { + available_count: 3, + credits: None, + }) + ); + + Ok(()) +} + #[tokio::test] async fn send_add_credits_nudge_email_requires_auth() -> Result<()> { let codex_home = TempDir::new()?; diff --git a/codex-rs/app-server/tests/suite/v2/realtime_conversation.rs b/codex-rs/app-server/tests/suite/v2/realtime_conversation.rs index 26b46c4317..1d8bd90e6d 100644 --- a/codex-rs/app-server/tests/suite/v2/realtime_conversation.rs +++ b/codex-rs/app-server/tests/suite/v2/realtime_conversation.rs @@ -349,6 +349,7 @@ impl RealtimeE2eHarness { .mcp .send_thread_realtime_start_request(ThreadRealtimeStartParams { client_managed_handoffs, + flush_transcript_tail_on_session_end: None, thread_id: self.thread_id.clone(), codex_response_item_prefix: codex_responses_as_items .unwrap_or(false) @@ -410,6 +411,7 @@ impl RealtimeE2eHarness { .send_thread_realtime_start_request(ThreadRealtimeStartParams { thread_id: self.thread_id.clone(), client_managed_handoffs: None, + flush_transcript_tail_on_session_end: None, codex_response_item_prefix: codex_responses_as_items .unwrap_or(false) .then(|| RESPONSE_ITEM_PREFIX.to_string()), @@ -672,6 +674,7 @@ async fn realtime_conversation_streams_v2_notifications() -> Result<()> { let start_request_id = mcp .send_thread_realtime_start_request(ThreadRealtimeStartParams { client_managed_handoffs: None, + flush_transcript_tail_on_session_end: None, codex_responses_as_items: None, codex_response_item_prefix: None, codex_response_handoff_prefix: None, @@ -963,6 +966,7 @@ async fn realtime_start_can_skip_startup_context() -> Result<()> { let start_request_id = mcp .send_thread_realtime_start_request(ThreadRealtimeStartParams { client_managed_handoffs: None, + flush_transcript_tail_on_session_end: None, codex_responses_as_items: None, codex_response_item_prefix: None, codex_response_handoff_prefix: None, @@ -1061,6 +1065,7 @@ async fn realtime_text_output_modality_requests_text_output_and_final_transcript let start_request_id = mcp .send_thread_realtime_start_request(ThreadRealtimeStartParams { client_managed_handoffs: None, + flush_transcript_tail_on_session_end: None, codex_responses_as_items: None, codex_response_item_prefix: None, codex_response_handoff_prefix: None, @@ -1242,6 +1247,7 @@ async fn realtime_conversation_stop_emits_closed_notification() -> Result<()> { let start_request_id = mcp .send_thread_realtime_start_request(ThreadRealtimeStartParams { client_managed_handoffs: None, + flush_transcript_tail_on_session_end: None, codex_responses_as_items: None, codex_response_item_prefix: None, codex_response_handoff_prefix: None, @@ -1346,6 +1352,7 @@ async fn realtime_webrtc_start_emits_sdp_notification() -> Result<()> { let start_request_id = mcp .send_thread_realtime_start_request(ThreadRealtimeStartParams { client_managed_handoffs: None, + flush_transcript_tail_on_session_end: None, codex_responses_as_items: None, codex_response_item_prefix: None, codex_response_handoff_prefix: None, @@ -2701,6 +2708,7 @@ async fn realtime_webrtc_start_surfaces_backend_error() -> Result<()> { let start_request_id = mcp .send_thread_realtime_start_request(ThreadRealtimeStartParams { client_managed_handoffs: None, + flush_transcript_tail_on_session_end: None, codex_responses_as_items: None, codex_response_item_prefix: None, codex_response_handoff_prefix: None, @@ -2767,6 +2775,7 @@ async fn realtime_conversation_requires_feature_flag() -> Result<()> { let start_request_id = mcp .send_thread_realtime_start_request(ThreadRealtimeStartParams { client_managed_handoffs: None, + flush_transcript_tail_on_session_end: None, codex_responses_as_items: None, codex_response_item_prefix: None, codex_response_handoff_prefix: None, diff --git a/codex-rs/app-server/tests/suite/v2/recommended_plugins.rs b/codex-rs/app-server/tests/suite/v2/recommended_plugins.rs index 59748a1980..d440f56903 100644 --- a/codex-rs/app-server/tests/suite/v2/recommended_plugins.rs +++ b/codex-rs/app-server/tests/suite/v2/recommended_plugins.rs @@ -68,9 +68,7 @@ async fn first_turn_after_external_login_waits_for_recommended_plugins() -> Resu let config = std::fs::read_to_string(&config_path)?; std::fs::write( config_path, - format!( - "{config}\n[features]\napps = true\nplugins = true\nremote_plugin = true\ntool_suggest = true\n" - ), + format!("{config}\n[features]\napps = true\nplugins = true\ntool_suggest = true\n"), )?; let sqlite_home = codex_home.path().to_string_lossy(); diff --git a/codex-rs/app-server/tests/suite/v2/remote_thread_store.rs b/codex-rs/app-server/tests/suite/v2/remote_thread_store.rs index 8fdb38e5ea..2002b08a88 100644 --- a/codex-rs/app-server/tests/suite/v2/remote_thread_store.rs +++ b/codex-rs/app-server/tests/suite/v2/remote_thread_store.rs @@ -158,7 +158,9 @@ async fn thread_delete_with_non_local_thread_store_does_not_create_local_persist originator: "test_originator".to_string(), base_instructions: BaseInstructions::default(), dynamic_tools: Vec::new(), + selected_capability_roots: Vec::new(), multi_agent_version: None, + history_mode: Default::default(), initial_window_id: Uuid::now_v7().to_string(), metadata: ThreadPersistenceMetadata { cwd: Some(codex_home.path().to_path_buf()), diff --git a/codex-rs/app-server/tests/suite/v2/selected_capability_stack.rs b/codex-rs/app-server/tests/suite/v2/selected_capability_stack.rs new file mode 100644 index 0000000000..22153f1af7 --- /dev/null +++ b/codex-rs/app-server/tests/suite/v2/selected_capability_stack.rs @@ -0,0 +1,755 @@ +use std::process::Stdio; +use std::time::Duration; + +use anyhow::Context; +use anyhow::Result; +use app_test_support::ChatGptAuthFixture; +use app_test_support::TestAppServer; +use app_test_support::to_response; +use app_test_support::write_chatgpt_auth; +use app_test_support::write_mock_responses_config_toml_with_chatgpt_base_url; +use codex_app_server_protocol::AppInfo; +use codex_app_server_protocol::CapabilityRootLocation; +use codex_app_server_protocol::EnvironmentAddResponse; +use codex_app_server_protocol::ListMcpServerStatusParams; +use codex_app_server_protocol::ListMcpServerStatusResponse; +use codex_app_server_protocol::RequestId; +use codex_app_server_protocol::SelectedCapabilityRoot; +use codex_app_server_protocol::ServerRequest; +use codex_app_server_protocol::ThreadResumeParams; +use codex_app_server_protocol::ThreadResumeResponse; +use codex_app_server_protocol::ThreadStartParams; +use codex_app_server_protocol::ThreadStartResponse; +use codex_app_server_protocol::TurnEnvironmentParams; +use codex_app_server_protocol::TurnStartParams; +use codex_app_server_protocol::UserInput; +use codex_config::types::AuthCredentialsStoreMode; +use codex_exec_server::LOCAL_ENVIRONMENT_ID; +use codex_protocol::config_types::CollaborationMode; +use codex_protocol::config_types::ModeKind; +use codex_protocol::config_types::Settings; +use codex_protocol::protocol::PLUGINS_INSTRUCTIONS_OPEN_TAG; +use codex_utils_absolute_path::AbsolutePathBuf; +use codex_utils_path_uri::PathUri; +use core_test_support::process::wait_for_pid_file; +use core_test_support::responses; +use core_test_support::responses::ResponsesRequest; +use core_test_support::stdio_server_bin; +use pretty_assertions::assert_eq; +use pretty_assertions::assert_ne; +use serde_json::json; +use tempfile::TempDir; +use tokio::io::AsyncBufReadExt; +use tokio::io::BufReader; +use tokio::process::Child; +use tokio::process::Command; +use tokio::time::timeout; + +use super::app_list::connector_tool; +use super::app_list::start_apps_server_with_delays; + +const READ_TIMEOUT: Duration = Duration::from_secs(20); +const EXECUTOR_ID: &str = "executor-1"; +const EXECUTOR_ENV_NAME: &str = "MCP_EXECUTOR_MARKER"; +const EXECUTOR_ENV_VALUE: &str = "executor-only"; +const PLUGIN_ID: &str = "executor-demo@1"; +const PLUGIN_DISPLAY_NAME: &str = "Executor Demo"; +const SKILL_NAME: &str = "executor-demo:deploy"; +const SKILL_DESCRIPTION: &str = "Deploy through the selected executor."; +const SKILL_BODY_MARKER: &str = "SELECTED_EXECUTOR_SKILL_BODY"; +const LOCAL_SKILL_BODY_MARKER: &str = "COLLIDING_LOCAL_SKILL_BODY"; +const NO_SELECTED_SKILLS_MESSAGE: &str = "No selected-environment skills are currently available."; +const MCP_SERVER_NAME: &str = "executor_probe"; +const MCP_CALL_ID: &str = "selected-executor-mcp-call"; +const CONNECTOR_ID: &str = "calendar"; + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn selected_capability_stack_tracks_environment_availability_and_resume() -> Result<()> { + let responses_server = responses::start_mock_server().await; + let (apps_url, apps_server_handle) = start_apps_server_with_delays( + vec![AppInfo { + id: CONNECTOR_ID.to_string(), + name: "Calendar".to_string(), + description: None, + logo_url: None, + logo_url_dark: None, + icon_assets: None, + icon_dark_assets: None, + distribution_channel: None, + branding: None, + app_metadata: None, + labels: None, + install_url: None, + is_accessible: false, + is_enabled: true, + plugin_display_names: Vec::new(), + }], + vec![connector_tool(CONNECTOR_ID, "Calendar")?], + Duration::ZERO, + Duration::ZERO, + ) + .await?; + let fixture = selected_capability_fixture(&responses_server.uri(), &apps_url)?; + + let response_mock = responses::mount_sse_sequence( + &responses_server, + vec![ + responses::sse(vec![ + responses::ev_response_created("environment-unavailable"), + responses::ev_assistant_message("unavailable-message", "Waiting"), + responses::ev_completed("environment-unavailable"), + ]), + responses::sse(vec![ + responses::ev_response_created("environment-available-call"), + responses::ev_function_call_with_namespace( + MCP_CALL_ID, + &format!("mcp__{MCP_SERVER_NAME}"), + "echo", + &json!({ + "message": "hello from the selected executor", + "env_var": EXECUTOR_ENV_NAME, + }) + .to_string(), + ), + responses::ev_completed("environment-available-call"), + ]), + responses::sse(vec![ + responses::ev_response_created("environment-available-done"), + responses::ev_assistant_message("available-message", "Done"), + responses::ev_completed("environment-available-done"), + ]), + responses::sse(vec![ + responses::ev_response_created("unchanged-step"), + responses::ev_assistant_message("unchanged-message", "Still ready"), + responses::ev_completed("unchanged-step"), + ]), + responses::sse(vec![ + responses::ev_response_created("resumed-unavailable-step"), + responses::ev_assistant_message( + "resumed-unavailable-message", + "Unavailable after resume", + ), + responses::ev_completed("resumed-unavailable-step"), + ]), + responses::sse(vec![ + responses::ev_response_created("reattached-step"), + responses::ev_assistant_message("reattached-message", "Ready after reattach"), + responses::ev_completed("reattached-step"), + ]), + ], + ) + .await; + + let mut app_server = TestAppServer::new(fixture.codex_home.path()).await?; + timeout(READ_TIMEOUT, app_server.initialize()).await??; + let thread_id = start_thread( + &mut app_server, + fixture.selected_root.clone(), + fixture.environment_cwd.clone(), + ) + .await?; + + run_turn( + &mut app_server, + &thread_id, + "Inspect the current capabilities", + fixture.environment_cwd.clone(), + ) + .await?; + let initial_requests = response_mock.requests(); + assert_selected_capabilities_absent(&initial_requests[0]); + + let mut exec_server = + spawn_exec_server(fixture.codex_home.path(), &fixture.exec_server_url).await?; + add_environment(&mut app_server, &fixture.exec_server_url).await?; + wait_for_selected_mcp_server(&mut app_server, &thread_id).await?; + + run_turn( + &mut app_server, + &thread_id, + &format!("Use ${SKILL_NAME} and call its selected executor MCP"), + fixture.environment_cwd.clone(), + ) + .await?; + let first_mcp_pid = wait_for_pid_file(&fixture.pid_file).await?; + + run_turn( + &mut app_server, + &thread_id, + "Continue with the same selected capabilities", + fixture.environment_cwd.clone(), + ) + .await?; + assert_eq!(first_mcp_pid, wait_for_pid_file(&fixture.pid_file).await?); + + exec_server.kill().await?; + drop(app_server); + std::fs::remove_file(&fixture.pid_file)?; + + let mut app_server = TestAppServer::new(fixture.codex_home.path()).await?; + timeout(READ_TIMEOUT, app_server.initialize()).await??; + let request_id = app_server + .send_thread_resume_request(ThreadResumeParams { + thread_id: thread_id.clone(), + ..Default::default() + }) + .await?; + let response = timeout( + READ_TIMEOUT, + app_server.read_stream_until_response_message(RequestId::Integer(request_id)), + ) + .await??; + let ThreadResumeResponse { thread, .. } = to_response(response)?; + assert_eq!(thread_id, thread.id); + + run_turn( + &mut app_server, + &thread_id, + "Inspect capabilities while the selected executor is unavailable", + fixture.environment_cwd.clone(), + ) + .await?; + let requests = response_mock.requests(); + assert_eq!(5, requests.len()); + assert_selected_plugin_tools_absent(&requests[4]); + assert!( + latest_selected_skill_update(&requests[4]) + .is_some_and(|text| text.contains(NO_SELECTED_SKILLS_MESSAGE)) + ); + + exec_server = spawn_exec_server(fixture.codex_home.path(), &fixture.exec_server_url).await?; + add_environment(&mut app_server, &fixture.exec_server_url).await?; + wait_for_selected_mcp_server(&mut app_server, &thread_id).await?; + + run_turn( + &mut app_server, + &thread_id, + &format!("Use ${SKILL_NAME} after reattaching the selected executor"), + fixture.environment_cwd, + ) + .await?; + let resumed_mcp_pid = wait_for_pid_file(&fixture.pid_file).await?; + assert_ne!(first_mcp_pid, resumed_mcp_pid); + + let requests = response_mock.requests(); + assert_eq!(6, requests.len()); + for request in &requests[1..4] { + assert_selected_skill_is_injected(request, /*expected_count*/ 1); + assert_selected_plugin_tools(request); + assert_plugin_guidance_count(request, /*expected_count*/ 1); + } + assert_plugin_guidance_count(&requests[4], /*expected_count*/ 1); + assert_selected_skill_is_injected(&requests[5], /*expected_count*/ 2); + assert_selected_plugin_tools(&requests[5]); + let output = requests[2].function_call_output(MCP_CALL_ID); + let output = output["output"] + .as_str() + .expect("MCP function output should be text"); + assert!(output.contains("ECHOING: hello from the selected executor")); + assert!(output.contains(EXECUTOR_ENV_VALUE)); + + exec_server.kill().await?; + apps_server_handle.abort(); + let _ = apps_server_handle.await; + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn selected_capabilities_become_available_between_samples_in_one_turn() -> Result<()> { + const USER_INPUT_CALL_ID: &str = "pause-for-environment"; + + let responses_server = responses::start_mock_server().await; + let (apps_url, apps_server_handle) = start_apps_server_with_delays( + vec![AppInfo { + id: CONNECTOR_ID.to_string(), + name: "Calendar".to_string(), + description: None, + logo_url: None, + logo_url_dark: None, + icon_assets: None, + icon_dark_assets: None, + distribution_channel: None, + branding: None, + app_metadata: None, + labels: None, + install_url: None, + is_accessible: false, + is_enabled: true, + plugin_display_names: Vec::new(), + }], + vec![connector_tool(CONNECTOR_ID, "Calendar")?], + Duration::ZERO, + Duration::ZERO, + ) + .await?; + let fixture = selected_capability_fixture(&responses_server.uri(), &apps_url)?; + let response_mock = responses::mount_sse_sequence( + &responses_server, + vec![ + responses::sse(vec![ + responses::ev_response_created("environment-pending"), + responses::ev_function_call( + USER_INPUT_CALL_ID, + "request_user_input", + &json!({ + "questions": [{ + "id": "continue", + "header": "Continue", + "question": "Continue after the executor is attached?", + "options": [{ + "label": "Yes (Recommended)", + "description": "Continue the same turn." + }, { + "label": "No", + "description": "Stop here." + }] + }], + "autoResolutionMs": 60_000 + }) + .to_string(), + ), + responses::ev_completed("environment-pending"), + ]), + responses::sse(vec![ + responses::ev_response_created("environment-ready-call"), + responses::ev_function_call_with_namespace( + MCP_CALL_ID, + &format!("mcp__{MCP_SERVER_NAME}"), + "echo", + &json!({ + "message": "same turn", + "env_var": EXECUTOR_ENV_NAME, + }) + .to_string(), + ), + responses::ev_completed("environment-ready-call"), + ]), + responses::sse(vec![ + responses::ev_response_created("same-turn-done"), + responses::ev_assistant_message("same-turn-message", "Done"), + responses::ev_completed("same-turn-done"), + ]), + ], + ) + .await; + + let mut app_server = TestAppServer::new(fixture.codex_home.path()).await?; + timeout(READ_TIMEOUT, app_server.initialize()).await??; + let thread_id = start_thread( + &mut app_server, + fixture.selected_root, + fixture.environment_cwd.clone(), + ) + .await?; + let turn_start_id = app_server + .send_turn_start_request(TurnStartParams { + thread_id, + input: vec![UserInput::Text { + text: "Use the executor when it becomes ready.".to_string(), + text_elements: Vec::new(), + }], + environments: Some(vec![TurnEnvironmentParams { + environment_id: LOCAL_ENVIRONMENT_ID.to_string(), + cwd: fixture.environment_cwd.into(), + }]), + collaboration_mode: Some(CollaborationMode { + mode: ModeKind::Plan, + settings: Settings { + model: "mock-model".to_string(), + reasoning_effort: None, + developer_instructions: None, + }, + }), + ..Default::default() + }) + .await?; + timeout( + READ_TIMEOUT, + app_server.read_stream_until_response_message(RequestId::Integer(turn_start_id)), + ) + .await??; + + let request = timeout(READ_TIMEOUT, app_server.read_stream_until_request_message()).await??; + let ServerRequest::ToolRequestUserInput { request_id, .. } = request else { + panic!("expected request_user_input, got {request:?}"); + }; + let requests = response_mock.requests(); + assert_eq!(1, requests.len()); + assert_selected_capabilities_absent(&requests[0]); + + let mut exec_server = + spawn_exec_server(fixture.codex_home.path(), &fixture.exec_server_url).await?; + add_environment(&mut app_server, &fixture.exec_server_url).await?; + tokio::time::sleep(Duration::from_millis(200)).await; + app_server + .send_response( + request_id, + json!({ + "answers": { + "continue": { "answers": ["yes"] } + } + }), + ) + .await?; + timeout( + READ_TIMEOUT, + app_server.read_stream_until_notification_message("turn/completed"), + ) + .await??; + + let requests = response_mock.requests(); + assert_eq!(3, requests.len()); + assert_selected_skill_catalog_available(&requests[1]); + assert_selected_plugin_tools(&requests[1]); + assert_plugin_guidance_count(&requests[1], /*expected_count*/ 1); + assert_selected_plugin_tools(&requests[2]); + assert_plugin_guidance_count(&requests[2], /*expected_count*/ 1); + let output = requests[2].function_call_output(MCP_CALL_ID); + let output = output["output"] + .as_str() + .expect("MCP function output should be text"); + assert!(output.contains("ECHOING: same turn")); + assert!(output.contains(EXECUTOR_ENV_VALUE)); + wait_for_pid_file(&fixture.pid_file).await?; + + exec_server.kill().await?; + apps_server_handle.abort(); + let _ = apps_server_handle.await; + Ok(()) +} + +struct SelectedCapabilityFixture { + codex_home: TempDir, + _plugin: TempDir, + pid_file: std::path::PathBuf, + exec_server_url: String, + selected_root: SelectedCapabilityRoot, + environment_cwd: AbsolutePathBuf, +} + +fn selected_capability_fixture( + responses_server_uri: &str, + apps_url: &str, +) -> Result { + let codex_home = TempDir::new()?; + write_mock_responses_config_toml_with_chatgpt_base_url( + codex_home.path(), + responses_server_uri, + apps_url, + )?; + let config_path = codex_home.path().join("config.toml"); + let config = std::fs::read_to_string(&config_path)?.replacen( + "model_provider = \"mock_provider\"", + "mcp_oauth_credentials_store = \"file\"\nmodel_provider = \"mock_provider\"", + 1, + ); + std::fs::write( + config_path, + format!( + "{config}\n[features]\napps = true\ndeferred_executor = true\n\n[skills]\ninclude_instructions = true\n" + ), + )?; + write_chatgpt_auth( + codex_home.path(), + ChatGptAuthFixture::new("chatgpt-token") + .account_id("account-123") + .email("selected-capability-stack@example.com") + .plan_type("pro") + .chatgpt_account_id("account-123"), + AuthCredentialsStoreMode::File, + )?; + + // Reserve the URL before app-server starts. The configured environment initially fails to + // connect, then environment/add points the same stable ID at the same URL once it is live. + let listener = std::net::TcpListener::bind("127.0.0.1:0")?; + let exec_server_url = format!("ws://{}", listener.local_addr()?); + drop(listener); + std::fs::write( + codex_home.path().join("environments.toml"), + format!( + "default = \"{EXECUTOR_ID}\"\ninclude_local = true\n\n[[environments]]\nid = \"{EXECUTOR_ID}\"\nurl = \"{exec_server_url}\"\nconnect_timeout_sec = 0.05\n" + ), + )?; + + let local_skill_dir = codex_home.path().join("skills/local-deploy"); + std::fs::create_dir_all(&local_skill_dir)?; + std::fs::write( + local_skill_dir.join("SKILL.md"), + format!( + "---\nname: {SKILL_NAME}\ndescription: Colliding local skill.\n---\n\n{LOCAL_SKILL_BODY_MARKER}\n" + ), + )?; + + let plugin = TempDir::new()?; + let manifest_dir = plugin.path().join(".codex-plugin"); + let skill_dir = plugin.path().join("skills/deploy"); + let pid_file = plugin.path().join("executor-mcp.pid"); + std::fs::create_dir_all(&manifest_dir)?; + std::fs::create_dir_all(&skill_dir)?; + std::fs::write( + manifest_dir.join("plugin.json"), + r#"{"name":"executor-demo","apps":"./.app.json","interface":{"displayName":"Executor Demo"}}"#, + )?; + std::fs::write( + skill_dir.join("SKILL.md"), + format!( + "---\nname: deploy\ndescription: {SKILL_DESCRIPTION}\n---\n\n{SKILL_BODY_MARKER}\n" + ), + )?; + std::fs::write( + plugin.path().join(".app.json"), + format!(r#"{{"apps":{{"calendar":{{"id":"{CONNECTOR_ID}"}}}}}}"#), + )?; + std::fs::write( + plugin.path().join(".mcp.json"), + serde_json::to_vec_pretty(&json!({ + "mcpServers": { + (MCP_SERVER_NAME): { + "command": stdio_server_bin()?, + "env": { + "MCP_TEST_PID_FILE": pid_file.to_string_lossy(), + }, + "env_vars": [EXECUTOR_ENV_NAME], + "startup_timeout_sec": 10, + } + } + }))?, + )?; + + let selected_root = SelectedCapabilityRoot { + id: PLUGIN_ID.to_string(), + location: CapabilityRootLocation::Environment { + environment_id: EXECUTOR_ID.to_string(), + path: PathUri::from_host_native_path(plugin.path())?, + }, + }; + let environment_cwd = AbsolutePathBuf::try_from(plugin.path().to_path_buf())?; + Ok(SelectedCapabilityFixture { + codex_home, + _plugin: plugin, + pid_file, + exec_server_url, + selected_root, + environment_cwd, + }) +} + +fn assert_selected_capabilities_absent(request: &ResponsesRequest) { + assert!( + request + .message_input_texts("developer") + .into_iter() + .all(|text| !text.contains(SKILL_DESCRIPTION)) + ); + assert_selected_plugin_tools_absent(request); + assert_plugin_guidance_count(request, /*expected_count*/ 0); +} + +fn assert_selected_plugin_tools_absent(request: &ResponsesRequest) { + assert!( + request + .tool_by_name(&format!("mcp__{MCP_SERVER_NAME}"), "echo") + .is_none() + ); + let connector = request + .tool_by_name("mcp__codex_apps__calendar", "connector_calendar") + .expect("host connector should remain model-visible"); + assert!( + connector["description"] + .as_str() + .is_some_and(|description| !description.contains(PLUGIN_DISPLAY_NAME)) + ); +} + +fn assert_plugin_guidance_count(request: &ResponsesRequest, expected_count: usize) { + assert_eq!( + expected_count, + request + .message_input_texts("developer") + .into_iter() + .filter(|text| text.starts_with(PLUGINS_INSTRUCTIONS_OPEN_TAG)) + .count() + ); +} + +fn assert_selected_skill_is_injected(request: &ResponsesRequest, expected_count: usize) { + assert_selected_skill_catalog_available(request); + + let skill_fragments = request + .message_input_texts("user") + .into_iter() + .filter(|text| text.starts_with("")) + .collect::>(); + assert_eq!(expected_count, skill_fragments.len()); + for fragment in skill_fragments { + assert!(fragment.contains(&format!("{SKILL_NAME}"))); + assert!(fragment.contains(SKILL_BODY_MARKER)); + assert!(!fragment.contains(LOCAL_SKILL_BODY_MARKER)); + } +} + +fn assert_selected_skill_catalog_available(request: &ResponsesRequest) { + let catalog_fragment = latest_selected_skill_update(request) + .expect("selected skill catalog update should be model-visible"); + assert!(catalog_fragment.contains(SKILL_DESCRIPTION)); + assert!(catalog_fragment.contains("environment resource:")); +} + +fn latest_selected_skill_update(request: &ResponsesRequest) -> Option { + request + .message_input_texts("developer") + .into_iter() + .rfind(|text| text.contains(SKILL_DESCRIPTION) || text.contains(NO_SELECTED_SKILLS_MESSAGE)) +} + +fn assert_selected_plugin_tools(request: &ResponsesRequest) { + assert!( + request + .tool_by_name(&format!("mcp__{MCP_SERVER_NAME}"), "echo") + .is_some() + ); + let connector = request + .tool_by_name("mcp__codex_apps__calendar", "connector_calendar") + .expect("selected connector should be model-visible"); + assert!( + connector["description"] + .as_str() + .is_some_and(|description| description.contains(PLUGIN_DISPLAY_NAME)) + ); +} + +async fn start_thread( + app_server: &mut TestAppServer, + selected_root: SelectedCapabilityRoot, + environment_cwd: AbsolutePathBuf, +) -> Result { + let request_id = app_server + .send_thread_start_request(ThreadStartParams { + model: Some("mock-model".to_string()), + environments: Some(vec![TurnEnvironmentParams { + environment_id: LOCAL_ENVIRONMENT_ID.to_string(), + cwd: environment_cwd.into(), + }]), + selected_capability_roots: Some(vec![selected_root]), + ..Default::default() + }) + .await?; + let response = timeout( + READ_TIMEOUT, + app_server.read_stream_until_response_message(RequestId::Integer(request_id)), + ) + .await??; + let ThreadStartResponse { thread, .. } = to_response(response)?; + Ok(thread.id) +} + +async fn run_turn( + app_server: &mut TestAppServer, + thread_id: &str, + text: &str, + environment_cwd: AbsolutePathBuf, +) -> Result<()> { + let request_id = app_server + .send_turn_start_request(TurnStartParams { + thread_id: thread_id.to_string(), + input: vec![UserInput::Text { + text: text.to_string(), + text_elements: Vec::new(), + }], + environments: Some(vec![TurnEnvironmentParams { + environment_id: LOCAL_ENVIRONMENT_ID.to_string(), + cwd: environment_cwd.into(), + }]), + ..Default::default() + }) + .await?; + timeout( + READ_TIMEOUT, + app_server.read_stream_until_response_message(RequestId::Integer(request_id)), + ) + .await??; + timeout( + READ_TIMEOUT, + app_server.read_stream_until_notification_message("turn/completed"), + ) + .await??; + Ok(()) +} + +async fn add_environment(app_server: &mut TestAppServer, exec_server_url: &str) -> Result<()> { + let request_id = app_server + .send_raw_request( + "environment/add", + Some(json!({ + "environmentId": EXECUTOR_ID, + "execServerUrl": exec_server_url, + "connectTimeoutMs": 10_000, + })), + ) + .await?; + let response = timeout( + READ_TIMEOUT, + app_server.read_stream_until_response_message(RequestId::Integer(request_id)), + ) + .await??; + let _: EnvironmentAddResponse = to_response(response)?; + Ok(()) +} + +async fn wait_for_selected_mcp_server( + app_server: &mut TestAppServer, + thread_id: &str, +) -> Result<()> { + timeout(READ_TIMEOUT, async { + loop { + let request_id = app_server + .send_list_mcp_server_status_request(ListMcpServerStatusParams { + cursor: None, + limit: None, + detail: None, + thread_id: Some(thread_id.to_string()), + }) + .await?; + let response = app_server + .read_stream_until_response_message(RequestId::Integer(request_id)) + .await?; + let response: ListMcpServerStatusResponse = to_response(response)?; + if response + .data + .iter() + .any(|server| server.name == MCP_SERVER_NAME) + { + return Ok::<_, anyhow::Error>(()); + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + }) + .await??; + Ok(()) +} + +async fn spawn_exec_server(codex_home: &std::path::Path, url: &str) -> Result { + let mut child = Command::new(codex_utils_cargo_bin::cargo_bin("codex")?) + .args(["exec-server", "--listen", url]) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::inherit()) + .kill_on_drop(true) + .env("CODEX_HOME", codex_home) + .env(EXECUTOR_ENV_NAME, EXECUTOR_ENV_VALUE) + .spawn()?; + let stdout = child + .stdout + .take() + .context("exec-server stdout was not captured")?; + let mut lines = BufReader::new(stdout).lines(); + loop { + let line = timeout(READ_TIMEOUT, lines.next_line()) + .await + .context("timed out waiting for exec-server URL")?? + .context("exec-server exited before printing its URL")?; + if line.trim() == url { + return Ok(child); + } + } +} diff --git a/codex-rs/app-server/tests/suite/v2/skills_list.rs b/codex-rs/app-server/tests/suite/v2/skills_list.rs index 075cc007ac..5993bf6da2 100644 --- a/codex-rs/app-server/tests/suite/v2/skills_list.rs +++ b/codex-rs/app-server/tests/suite/v2/skills_list.rs @@ -79,23 +79,6 @@ plugins = true ) } -fn write_remote_plugins_enabled_config_with_base_url( - codex_home: &std::path::Path, - base_url: &str, -) -> std::io::Result<()> { - std::fs::write( - codex_home.join("config.toml"), - format!( - r#"chatgpt_base_url = "{base_url}" - -[features] -plugins = true -remote_plugin = true -"#, - ), - ) -} - fn write_plugin_with_skill( repo_root: &std::path::Path, plugin_name: &str, @@ -175,7 +158,7 @@ fn write_cached_local_curated_plugin_with_skill(codex_home: &std::path::Path) -> } #[tokio::test] -async fn runtime_remote_plugin_enablement_excludes_local_curated_plugin_skills() -> Result<()> { +async fn runtime_remote_plugin_toggle_updates_local_curated_plugin_skills() -> Result<()> { let codex_home = TempDir::new()?; let cwd = TempDir::new()?; let server = MockServer::start().await; @@ -206,6 +189,18 @@ enabled = true let mut mcp = TestAppServer::new(codex_home.path()).await?; timeout(DEFAULT_TIMEOUT, mcp.initialize()).await??; + let disablement_request_id = mcp + .send_experimental_feature_enablement_set_request(ExperimentalFeatureEnablementSetParams { + enablement: BTreeMap::from([("remote_plugin".to_string(), false)]), + }) + .await?; + let disablement_response: JSONRPCResponse = timeout( + DEFAULT_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(disablement_request_id)), + ) + .await??; + let _: ExperimentalFeatureEnablementSetResponse = to_response(disablement_response)?; + let initial_skills_list_request_id = mcp .send_skills_list_request(SkillsListParams { cwds: vec![cwd.path().to_path_buf()], @@ -266,7 +261,7 @@ async fn skills_list_loads_remote_installed_plugin_skills_from_cache() -> Result let server = MockServer::start().await; let expected_skill_path = std::fs::canonicalize(write_cached_remote_plugin_with_skill(codex_home.path())?)?; - write_remote_plugins_enabled_config_with_base_url( + write_plugins_enabled_config_with_base_url( codex_home.path(), &format!("{}/backend-api/", server.uri()), )?; @@ -882,6 +877,7 @@ async fn skills_changed_notification_is_emitted_after_skill_change() -> Result<( .send_thread_start_request(ThreadStartParams { model: None, model_provider: None, + allow_provider_model_fallback: false, service_tier: None, cwd: None, runtime_workspace_roots: None, @@ -896,6 +892,7 @@ async fn skills_changed_notification_is_emitted_after_skill_change() -> Result<( personality: None, multi_agent_mode: None, ephemeral: None, + history_mode: None, session_start_source: None, thread_source: None, dynamic_tools: None, diff --git a/codex-rs/app-server/tests/suite/v2/sleep.rs b/codex-rs/app-server/tests/suite/v2/sleep.rs index 872983f76d..8d88816fde 100644 --- a/codex-rs/app-server/tests/suite/v2/sleep.rs +++ b/codex-rs/app-server/tests/suite/v2/sleep.rs @@ -1,11 +1,12 @@ use anyhow::Result; use app_test_support::TestAppServer; use app_test_support::to_response; +use codex_app_server_protocol::CurrentTimeReadResponse; use codex_app_server_protocol::ItemCompletedNotification; use codex_app_server_protocol::ItemStartedNotification; -use codex_app_server_protocol::JSONRPCMessage; use codex_app_server_protocol::JSONRPCResponse; use codex_app_server_protocol::RequestId; +use codex_app_server_protocol::ServerRequest; use codex_app_server_protocol::ThreadItem; use codex_app_server_protocol::ThreadStartParams; use codex_app_server_protocol::ThreadStartResponse; @@ -19,12 +20,16 @@ use std::time::Duration; use tempfile::TempDir; use tokio::time::timeout; +#[cfg(windows)] +const DEFAULT_READ_TIMEOUT: Duration = Duration::from_secs(25); +#[cfg(not(windows))] const DEFAULT_READ_TIMEOUT: Duration = Duration::from_secs(10); +const CURRENT_TIME_AT: i64 = 1_781_717_655; #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn sleep_emits_started_and_completed_items() -> Result<()> { +async fn external_sleep_polls_current_time_and_emits_items() -> Result<()> { const CALL_ID: &str = "sleep-1"; - const DURATION_MS: u64 = 1; + const DURATION_MS: u64 = 2_000; let server = responses::start_mock_server().await; responses::mount_sse_sequence( @@ -85,38 +90,19 @@ async fn sleep_emits_started_and_completed_items() -> Result<()> { .await??; let TurnStartResponse { turn, .. } = to_response(turn_start_response)?; - let (started, completed) = timeout(DEFAULT_READ_TIMEOUT, async { - let mut started = None; - let mut completed = None; - while started.is_none() || completed.is_none() { - let JSONRPCMessage::Notification(notification) = mcp.read_next_message().await? else { - continue; - }; - match notification.method.as_str() { - "item/started" => { - let payload: ItemStartedNotification = - serde_json::from_value(notification.params.expect("item/started params"))?; - if matches!(&payload.item, ThreadItem::Sleep { .. }) { - started = Some(payload); - } - } - "item/completed" => { - let payload: ItemCompletedNotification = serde_json::from_value( - notification.params.expect("item/completed params"), - )?; - if matches!(&payload.item, ThreadItem::Sleep { .. }) { - completed = Some(payload); - } - } - _ => {} - } - } - Ok::<_, anyhow::Error>(( - started.expect("sleep started"), - completed.expect("sleep completed"), - )) - }) - .await??; + // Read once for the initial reminder, then once to establish the sleep deadline. + respond_to_current_time_read(&mut mcp, &thread.id, CURRENT_TIME_AT).await?; + let started = wait_for_sleep_started(&mut mcp, CALL_ID).await?; + respond_to_current_time_read(&mut mcp, &thread.id, CURRENT_TIME_AT).await?; + + // The first poll remains below the deadline, so the provider must request time again. + respond_to_current_time_read(&mut mcp, &thread.id, CURRENT_TIME_AT + 1).await?; + respond_to_current_time_read(&mut mcp, &thread.id, CURRENT_TIME_AT + 2).await?; + + let completed = wait_for_sleep_completed(&mut mcp, CALL_ID).await?; + + // The next inference boundary reads the same external clock after the sleep completes. + respond_to_current_time_read(&mut mcp, &thread.id, CURRENT_TIME_AT + 2).await?; timeout( DEFAULT_READ_TIMEOUT, mcp.read_stream_until_notification_message("turn/completed"), @@ -150,6 +136,64 @@ async fn sleep_emits_started_and_completed_items() -> Result<()> { Ok(()) } +async fn wait_for_sleep_started( + mcp: &mut TestAppServer, + call_id: &str, +) -> Result { + loop { + let notification = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_notification_message("item/started"), + ) + .await??; + let started: ItemStartedNotification = + serde_json::from_value(notification.params.expect("item/started params"))?; + if matches!(&started.item, ThreadItem::Sleep { id, .. } if id == call_id) { + return Ok(started); + } + } +} + +async fn wait_for_sleep_completed( + mcp: &mut TestAppServer, + call_id: &str, +) -> Result { + loop { + let notification = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_notification_message("item/completed"), + ) + .await??; + let completed: ItemCompletedNotification = + serde_json::from_value(notification.params.expect("item/completed params"))?; + if matches!(&completed.item, ThreadItem::Sleep { id, .. } if id == call_id) { + return Ok(completed); + } + } +} + +async fn respond_to_current_time_read( + mcp: &mut TestAppServer, + thread_id: &str, + current_time_at: i64, +) -> Result<()> { + let request = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_request_message(), + ) + .await??; + let ServerRequest::CurrentTimeRead { request_id, params } = request else { + panic!("expected CurrentTimeRead request, got: {request:?}"); + }; + assert_eq!(params.thread_id, thread_id); + mcp.send_response( + request_id, + serde_json::to_value(CurrentTimeReadResponse { current_time_at })?, + ) + .await?; + Ok(()) +} + fn create_config_toml(codex_home: &Path, server_uri: &str) -> std::io::Result<()> { std::fs::write( codex_home.join("config.toml"), @@ -170,6 +214,7 @@ stream_max_retries = 0 [features.current_time_reminder] enabled = true sleep_tool = true +clock_source = "external" "# ), ) diff --git a/codex-rs/app-server/tests/suite/v2/thread_fork.rs b/codex-rs/app-server/tests/suite/v2/thread_fork.rs index 214a678678..d7c1707a32 100644 --- a/codex-rs/app-server/tests/suite/v2/thread_fork.rs +++ b/codex-rs/app-server/tests/suite/v2/thread_fork.rs @@ -251,6 +251,119 @@ async fn thread_fork_creates_new_thread_and_emits_started() -> Result<()> { Ok(()) } +#[tokio::test] +async fn thread_fork_at_last_turn_id_keeps_only_terminal_prefix() -> Result<()> { + let server = create_mock_responses_server_repeating_assistant("Done").await; + let codex_home = TempDir::new()?; + create_config_toml(codex_home.path(), &server.uri())?; + + let mut mcp = TestAppServer::new(codex_home.path()).await?; + timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; + + let start_id = mcp + .send_thread_start_request(ThreadStartParams::default()) + .await?; + let start_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(start_id)), + ) + .await??; + let ThreadStartResponse { + thread: source_thread, + .. + } = to_response::(start_resp)?; + let source_thread_id = source_thread.id.clone(); + let source_path = source_thread.path.expect("source thread path"); + + let mut turn_ids = Vec::new(); + for text in ["first", "second", "third"] { + let turn_request_id = mcp + .send_turn_start_request(TurnStartParams { + thread_id: source_thread_id.clone(), + client_user_message_id: None, + input: vec![UserInput::Text { + text: text.to_string(), + text_elements: Vec::new(), + }], + ..Default::default() + }) + .await?; + let turn_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(turn_request_id)), + ) + .await??; + let TurnStartResponse { turn } = to_response::(turn_resp)?; + turn_ids.push(turn.id); + timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_notification_message("turn/completed"), + ) + .await??; + } + + let original_contents = std::fs::read_to_string(source_path.as_path())?; + let fork_id = mcp + .send_thread_fork_request(ThreadForkParams { + thread_id: source_thread_id.clone(), + last_turn_id: Some(turn_ids[1].clone()), + ..Default::default() + }) + .await?; + let fork_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(fork_id)), + ) + .await??; + let ThreadForkResponse { + thread: forked_thread, + .. + } = to_response::(fork_resp)?; + + assert_eq!( + forked_thread + .turns + .iter() + .map(|turn| turn.id.clone()) + .collect::>(), + turn_ids[..2] + ); + assert!( + forked_thread + .turns + .iter() + .all(|turn| turn.status == TurnStatus::Completed) + ); + assert_eq!(forked_thread.forked_from_id, Some(source_thread_id)); + assert_eq!(forked_thread.preview, "first"); + assert_eq!( + std::fs::read_to_string(source_path.as_path())?, + original_contents, + "forking at a turn must not mutate the source rollout" + ); + + let forked_path = forked_thread.path.clone().expect("forked thread path"); + let forked_contents = std::fs::read_to_string(forked_path.as_path())?; + assert!(forked_contents.contains(turn_ids[1].as_str())); + assert!(!forked_contents.contains(turn_ids[2].as_str())); + + let started = loop { + let notification = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_notification_message("thread/started"), + ) + .await??; + let started: ThreadStartedNotification = + serde_json::from_value(notification.params.expect("params must be present"))?; + if started.thread.id == forked_thread.id { + break started; + } + }; + assert!(started.thread.turns.is_empty()); + + Ok(()) +} + #[tokio::test] async fn thread_fork_inherits_explicit_source_name_from_session_index() -> Result<()> { let server = create_mock_responses_server_repeating_assistant("Done").await; @@ -488,6 +601,7 @@ async fn thread_fork_tracks_thread_initialized_analytics() -> Result<()> { event, &thread.id, &thread.session_id, + "codex", "mock-model", "forked", "user", diff --git a/codex-rs/app-server/tests/suite/v2/thread_read.rs b/codex-rs/app-server/tests/suite/v2/thread_read.rs index 6b427b0d97..22adba37c8 100644 --- a/codex-rs/app-server/tests/suite/v2/thread_read.rs +++ b/codex-rs/app-server/tests/suite/v2/thread_read.rs @@ -18,6 +18,7 @@ use codex_app_server_protocol::SessionSource; use codex_app_server_protocol::SortDirection; use codex_app_server_protocol::ThreadForkParams; use codex_app_server_protocol::ThreadForkResponse; +use codex_app_server_protocol::ThreadHistoryMode; use codex_app_server_protocol::ThreadItem; use codex_app_server_protocol::ThreadItemsListParams; use codex_app_server_protocol::ThreadListParams; @@ -196,6 +197,121 @@ async fn thread_read_can_include_turns() -> Result<()> { Ok(()) } +#[tokio::test] +async fn paginated_stored_thread_allows_metadata_discovery_and_rejects_legacy_history_paths() +-> Result<()> { + let server = create_mock_responses_server_repeating_assistant("Done").await; + let codex_home = TempDir::new()?; + create_config_toml(codex_home.path(), &server.uri())?; + + let conversation_id = create_fake_rollout_with_text_elements( + codex_home.path(), + "2025-01-05T12-00-00", + "2025-01-05T12:00:00Z", + "Saved user message", + Vec::new(), + Some("mock_provider"), + /*git_info*/ None, + )?; + set_rollout_history_mode( + rollout_path(codex_home.path(), "2025-01-05T12-00-00", &conversation_id).as_path(), + ThreadHistoryMode::Paginated, + )?; + + let mut mcp = TestAppServer::new(codex_home.path()).await?; + timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; + + let read_id = mcp + .send_thread_read_request(ThreadReadParams { + thread_id: conversation_id.clone(), + include_turns: false, + }) + .await?; + let read_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(read_id)), + ) + .await??; + let ThreadReadResponse { thread } = to_response::(read_resp)?; + assert_eq!(thread.history_mode, ThreadHistoryMode::Paginated); + assert!(thread.turns.is_empty()); + + let list_id = mcp + .send_thread_list_request(ThreadListParams { + cursor: None, + limit: Some(50), + sort_key: None, + sort_direction: None, + model_providers: Some(vec!["mock_provider".to_string()]), + source_kinds: None, + archived: None, + cwd: None, + use_state_db_only: false, + search_term: None, + parent_thread_id: None, + ancestor_thread_id: None, + }) + .await?; + let list_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(list_id)), + ) + .await??; + let ThreadListResponse { data, .. } = to_response::(list_resp)?; + let listed = data + .iter() + .find(|thread| thread.id == conversation_id) + .expect("thread/list should include paginated thread"); + assert_eq!(listed.history_mode, ThreadHistoryMode::Paginated); + + let read_id = mcp + .send_thread_read_request(ThreadReadParams { + thread_id: conversation_id.clone(), + include_turns: true, + }) + .await?; + assert_paginated_threads_unsupported( + timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_error_message(RequestId::Integer(read_id)), + ) + .await??, + ); + + let turns_list_id = mcp + .send_thread_turns_list_request(ThreadTurnsListParams { + thread_id: conversation_id.clone(), + cursor: None, + limit: None, + sort_direction: None, + items_view: None, + }) + .await?; + assert_paginated_threads_unsupported( + timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_error_message(RequestId::Integer(turns_list_id)), + ) + .await??, + ); + + let resume_id = mcp + .send_thread_resume_request(ThreadResumeParams { + thread_id: conversation_id, + ..Default::default() + }) + .await?; + assert_paginated_threads_unsupported( + timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_error_message(RequestId::Integer(resume_id)), + ) + .await??, + ); + + Ok(()) +} + #[tokio::test] async fn thread_turns_list_can_page_backward_and_forward() -> Result<()> { let server = create_mock_responses_server_repeating_assistant("Done").await; @@ -1369,7 +1485,9 @@ async fn seed_pathless_store_thread( originator: "test_originator".to_string(), base_instructions: BaseInstructions::default(), dynamic_tools: Vec::new(), + selected_capability_roots: Vec::new(), multi_agent_version: None, + history_mode: Default::default(), initial_window_id: Uuid::now_v7().to_string(), metadata: ThreadPersistenceMetadata { cwd: None, @@ -1410,6 +1528,26 @@ fn store_history_items() -> Vec { ))] } +fn set_rollout_history_mode(path: &Path, history_mode: ThreadHistoryMode) -> Result<()> { + let mut lines = std::fs::read_to_string(path)? + .lines() + .map(serde_json::from_str::) + .collect::, _>>()?; + lines[0]["payload"]["history_mode"] = serde_json::to_value(history_mode)?; + let contents = lines + .into_iter() + .map(|line| line.to_string()) + .collect::>() + .join("\n"); + std::fs::write(path, format!("{contents}\n"))?; + Ok(()) +} + +fn assert_paginated_threads_unsupported(err: JSONRPCError) { + assert_eq!(err.error.code, -32601); + assert_eq!(err.error.message, "paginated_threads is not supported yet"); +} + fn create_config_toml_with_thread_store(codex_home: &Path, store_id: &str) -> std::io::Result<()> { let config_toml = codex_home.join("config.toml"); std::fs::write( diff --git a/codex-rs/app-server/tests/suite/v2/thread_resume.rs b/codex-rs/app-server/tests/suite/v2/thread_resume.rs index ce95f4810c..a6e851f1d7 100644 --- a/codex-rs/app-server/tests/suite/v2/thread_resume.rs +++ b/codex-rs/app-server/tests/suite/v2/thread_resume.rs @@ -107,6 +107,7 @@ use super::analytics::mount_analytics_capture; use super::analytics::thread_initialized_event; use super::analytics::wait_for_analytics_payload; use super::analytics::wait_for_goal_event; +use super::analytics::wait_for_matching_analytics_event; #[cfg(windows)] const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(25); @@ -573,11 +574,12 @@ async fn thread_resume_tracks_thread_initialized_analytics() -> Result<()> { Some("mock_provider"), /*git_info*/ None, )?; - set_thread_source_on_fake_rollout( + set_session_meta_on_fake_rollout( codex_home.path(), "2025-01-05T12-00-00", &conversation_id, "user", + "codex_work_desktop", )?; let mut mcp = TestAppServer::new_without_managed_config(codex_home.path()).await?; @@ -607,6 +609,7 @@ async fn thread_resume_tracks_thread_initialized_analytics() -> Result<()> { event, &thread.id, &thread.session_id, + "codex_work_desktop", "gpt-5.3-codex", "resumed", "user", @@ -615,11 +618,94 @@ async fn thread_resume_tracks_thread_initialized_analytics() -> Result<()> { Ok(()) } -fn set_thread_source_on_fake_rollout( +#[tokio::test] +async fn thread_resume_running_thread_tracks_thread_originator_in_analytics() -> Result<()> { + let server = create_mock_responses_server_repeating_assistant("Done").await; + + let codex_home = TempDir::new()?; + create_config_toml_with_chatgpt_base_url(codex_home.path(), &server.uri(), &server.uri())?; + mount_analytics_capture(&server, codex_home.path()).await?; + + let mut mcp = TestAppServer::new_without_managed_config(codex_home.path()).await?; + timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; + + let start_id = mcp + .send_thread_start_request(ThreadStartParams { + model: Some("mock-model".to_string()), + thread_source: Some(ThreadSource::User), + service_name: Some("codex_work_desktop".to_string()), + ..Default::default() + }) + .await?; + let start_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(start_id)), + ) + .await??; + let ThreadStartResponse { thread, .. } = to_response::(start_resp)?; + + let turn_id = mcp + .send_turn_start_request(TurnStartParams { + thread_id: thread.id.clone(), + client_user_message_id: None, + input: vec![UserInput::Text { + text: "materialize rollout".to_string(), + text_elements: Vec::new(), + }], + ..Default::default() + }) + .await?; + timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(turn_id)), + ) + .await??; + timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_notification_message("turn/completed"), + ) + .await??; + + let resume_id = mcp + .send_thread_resume_request(ThreadResumeParams { + thread_id: thread.id.clone(), + exclude_turns: true, + ..Default::default() + }) + .await?; + let resume_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(resume_id)), + ) + .await??; + let ThreadResumeResponse { + thread: resumed, .. + } = to_response::(resume_resp)?; + + let event = wait_for_matching_analytics_event(&server, DEFAULT_READ_TIMEOUT, |event| { + event["event_type"] == "codex_thread_initialized" + && event["event_params"]["thread_id"] == resumed.id + && event["event_params"]["initialization_mode"] == "resumed" + }) + .await?; + assert_basic_thread_initialized_event( + &event, + &resumed.id, + &resumed.session_id, + "codex_work_desktop", + "mock-model", + "resumed", + "user", + ); + Ok(()) +} + +fn set_session_meta_on_fake_rollout( codex_home: &std::path::Path, filename_ts: &str, thread_id: &str, thread_source: &str, + originator: &str, ) -> Result<()> { let path = rollout_path(codex_home, filename_ts, thread_id); let contents = std::fs::read_to_string(&path)?; @@ -629,6 +715,7 @@ fn set_thread_source_on_fake_rollout( .ok_or_else(|| anyhow::anyhow!("fake rollout missing session meta"))?; let mut session_meta: serde_json::Value = serde_json::from_str(session_meta)?; session_meta["payload"]["thread_source"] = serde_json::json!(thread_source); + session_meta["payload"]["originator"] = serde_json::json!(originator); let remaining = lines.collect::>().join("\n"); std::fs::write(&path, format!("{session_meta}\n{remaining}\n"))?; Ok(()) @@ -747,6 +834,9 @@ async fn thread_resume_redacts_payloads_for_chatgpt_remote_clients() -> Result<( connector_id: "calendar".to_string(), link_id: Some("link_calendar".to_string()), resource_uri: Some("ui://widget/lookup.html".to_string()), + app_name: Some("Calendar".to_string()), + template_id: Some("calendar_template".to_string()), + action_name: Some("lookup".to_string()), }) ); let result = result.as_ref().expect("redacted MCP result"); @@ -797,6 +887,9 @@ async fn thread_resume_redacts_payloads_for_chatgpt_remote_clients() -> Result<( connector_id: "calendar".to_string(), link_id: Some("link_calendar".to_string()), resource_uri: Some("ui://widget/lookup.html".to_string()), + app_name: Some("Calendar".to_string()), + template_id: Some("calendar_template".to_string()), + action_name: Some("lookup".to_string()), }) ); let result = result.as_ref().expect("normal MCP result"); @@ -903,6 +996,9 @@ fn append_resume_redaction_history( connector_id: Some("calendar".to_string()), mcp_app_resource_uri: Some("ui://widget/lookup.html".to_string()), link_id: Some("link_calendar".to_string()), + app_name: Some("Calendar".to_string()), + template_id: Some("calendar_template".to_string()), + action_name: Some("lookup".to_string()), plugin_id: None, duration: Duration::from_millis(8), result: Ok(CallToolResult { @@ -2070,7 +2166,9 @@ stream_max_retries = 0 model_provider: Some("mock_provider".to_string()), base_instructions: None, dynamic_tools: None, + selected_capability_roots: Vec::new(), memory_mode: None, + history_mode: Default::default(), multi_agent_version: None, context_window: None, }; diff --git a/codex-rs/app-server/tests/suite/v2/thread_rollback.rs b/codex-rs/app-server/tests/suite/v2/thread_rollback.rs index 9f0df0ff2d..328ae529c8 100644 --- a/codex-rs/app-server/tests/suite/v2/thread_rollback.rs +++ b/codex-rs/app-server/tests/suite/v2/thread_rollback.rs @@ -3,6 +3,9 @@ use app_test_support::TestAppServer; use app_test_support::create_final_assistant_message_sse_response; use app_test_support::create_mock_responses_server_sequence_unchecked; use app_test_support::to_response; +use codex_app_server_protocol::ClientInfo; +use codex_app_server_protocol::DeprecationNoticeNotification; +use codex_app_server_protocol::JSONRPCMessage; use codex_app_server_protocol::JSONRPCResponse; use codex_app_server_protocol::RequestId; use codex_app_server_protocol::ThreadItem; @@ -22,6 +25,48 @@ use tokio::time::timeout; const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); +#[tokio::test] +async fn thread_rollback_does_not_emit_deprecation_notice_to_codex_tui() -> Result<()> { + let codex_home = TempDir::new()?; + let mut mcp = TestAppServer::new_with_auto_env(codex_home.path()).await?; + let initialized = timeout( + DEFAULT_READ_TIMEOUT, + mcp.initialize_with_client_info(ClientInfo { + name: "codex-tui".to_string(), + title: None, + version: "0.1.0".to_string(), + }), + ) + .await??; + let JSONRPCMessage::Response(_) = initialized else { + panic!("expected initialize response, got {initialized:?}"); + }; + mcp.clear_message_buffer(); + + let rollback_id = mcp + .send_thread_rollback_request(ThreadRollbackParams { + thread_id: "00000000-0000-0000-0000-000000000001".to_string(), + num_turns: 1, + }) + .await?; + loop { + let message = timeout(DEFAULT_READ_TIMEOUT, mcp.read_next_message()).await??; + match message { + JSONRPCMessage::Notification(notification) => { + assert_ne!(notification.method, "deprecationNotice"); + } + JSONRPCMessage::Error(error) if error.id == RequestId::Integer(rollback_id) => { + break; + } + message => { + panic!("expected rollback error response, got {message:?}"); + } + } + } + + Ok(()) +} + #[tokio::test] async fn thread_rollback_drops_last_turns_and_persists_to_rollout() -> Result<()> { // Three Codex turns hit the mock model (session start + two turn/start calls). @@ -97,6 +142,7 @@ async fn thread_rollback_drops_last_turns_and_persists_to_rollout() -> Result<() mcp.read_stream_until_notification_message("turn/completed"), ) .await??; + mcp.clear_message_buffer(); // Roll back the last turn. let rollback_id = mcp @@ -105,6 +151,23 @@ async fn thread_rollback_drops_last_turns_and_persists_to_rollout() -> Result<() num_turns: 1, }) .await?; + let deprecation_notice = timeout(DEFAULT_READ_TIMEOUT, mcp.read_next_message()).await??; + let JSONRPCMessage::Notification(deprecation_notice) = deprecation_notice else { + panic!("thread/rollback should emit deprecationNotice before its response"); + }; + assert_eq!(deprecation_notice.method, "deprecationNotice"); + let deprecation_notice: DeprecationNoticeNotification = serde_json::from_value( + deprecation_notice + .params + .expect("deprecationNotice params should be present"), + )?; + assert_eq!( + deprecation_notice, + DeprecationNoticeNotification { + summary: "thread/rollback is deprecated and will be removed soon".to_string(), + details: None, + } + ); let rollback_resp: JSONRPCResponse = timeout( DEFAULT_READ_TIMEOUT, mcp.read_stream_until_response_message(RequestId::Integer(rollback_id)), diff --git a/codex-rs/app-server/tests/suite/v2/thread_start.rs b/codex-rs/app-server/tests/suite/v2/thread_start.rs index 39f8935a93..f96653b02a 100644 --- a/codex-rs/app-server/tests/suite/v2/thread_start.rs +++ b/codex-rs/app-server/tests/suite/v2/thread_start.rs @@ -7,14 +7,19 @@ use app_test_support::create_mock_responses_server_repeating_assistant; use app_test_support::to_response; use app_test_support::write_chatgpt_auth; use codex_app_server_protocol::AskForApproval; +use codex_app_server_protocol::ConfigWarningNotification; use codex_app_server_protocol::JSONRPCError; use codex_app_server_protocol::JSONRPCMessage; +use codex_app_server_protocol::JSONRPCNotification; use codex_app_server_protocol::JSONRPCResponse; use codex_app_server_protocol::McpServerStartupState; use codex_app_server_protocol::McpServerStatusUpdatedNotification; use codex_app_server_protocol::RequestId; use codex_app_server_protocol::SandboxMode; use codex_app_server_protocol::ServerNotification; +use codex_app_server_protocol::TextPosition; +use codex_app_server_protocol::TextRange; +use codex_app_server_protocol::ThreadHistoryMode; use codex_app_server_protocol::ThreadSource; use codex_app_server_protocol::ThreadStartParams; use codex_app_server_protocol::ThreadStartResponse; @@ -53,6 +58,244 @@ use super::analytics::wait_for_analytics_payload; const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); const INVALID_REQUEST_ERROR_CODE: i64 = -32600; +const EXEC_POLICY_PARSE_WARNING_SUMMARY: &str = "Error parsing rules; custom rules not applied."; + +fn is_exec_policy_config_warning(notification: &JSONRPCNotification) -> bool { + notification.method == "configWarning" + && notification + .params + .as_ref() + .and_then(|params| params.get("summary")) + .and_then(Value::as_str) + == Some(EXEC_POLICY_PARSE_WARNING_SUMMARY) +} + +async fn start_thread_with_model( + mcp: &mut TestAppServer, + model: &str, + allow_provider_model_fallback: bool, +) -> Result { + let request_id = mcp + .send_thread_start_request_with_auto_env(ThreadStartParams { + model: Some(model.to_string()), + allow_provider_model_fallback, + ..Default::default() + }) + .await?; + let response: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(request_id)), + ) + .await??; + to_response(response) +} + +#[tokio::test] +async fn thread_start_provider_model_fallback_applies_to_configured_model() -> Result<()> { + let codex_home = TempDir::new()?; + std::fs::write( + codex_home.path().join("config.toml"), + r#"model_provider = "amazon-bedrock" +model = "gpt-5.4-mini" +"#, + )?; + let mut mcp = TestAppServer::new_with_auto_env(codex_home.path()).await?; + timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; + + let request_id = mcp + .send_thread_start_request_with_auto_env(ThreadStartParams { + allow_provider_model_fallback: true, + ..Default::default() + }) + .await?; + let response: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(request_id)), + ) + .await??; + let response: ThreadStartResponse = to_response(response)?; + + assert_eq!(response.model, "openai.gpt-5.5"); + Ok(()) +} + +#[tokio::test] +async fn thread_start_warns_for_exec_policy_parse_failure_after_initialize() -> Result<()> { + let codex_home = TempDir::new()?; + let mut mcp = TestAppServer::new_with_auto_env(codex_home.path()).await?; + timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; + + let rules_dir = codex_home.path().join("rules"); + std::fs::create_dir_all(&rules_dir)?; + let rules_path = rules_dir.join("broken.rules"); + std::fs::write(&rules_path, "prefix_rule(")?; + let rules_path = std::fs::canonicalize(rules_path)?; + + let request_id = mcp + .send_thread_start_request_with_auto_env(ThreadStartParams::default()) + .await?; + let response = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(request_id)), + ) + .await??; + let _: ThreadStartResponse = to_response(response)?; + + let notification = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_matching_notification( + "exec-policy configWarning", + is_exec_policy_config_warning, + ), + ) + .await??; + let notification: ServerNotification = notification.try_into()?; + let ServerNotification::ConfigWarning(warning) = notification else { + anyhow::bail!("unexpected notification variant"); + }; + let ConfigWarningNotification { + summary, + details, + path, + range, + } = warning; + assert_eq!( + (summary, range), + ( + "Error parsing rules; custom rules not applied.".to_string(), + Some(TextRange { + start: TextPosition { + line: 1, + column: 13, + }, + end: TextPosition { + line: 1, + column: 13, + }, + }), + ) + ); + let path = path.context("warning should include a path")?; + assert_eq!( + normalize_path_for_comparison(path), + normalize_path_for_comparison(&rules_path) + ); + let details = details.context("warning should include details")?; + assert!( + details.contains("failed to parse rules file") && details.contains("broken.rules"), + "unexpected warning details: {details}" + ); + assert!( + details.contains("Parse error"), + "unexpected warning details: {details}" + ); + + Ok(()) +} + +#[tokio::test] +async fn thread_start_does_not_repeat_initialize_exec_policy_warning() -> Result<()> { + let codex_home = TempDir::new()?; + let rules_dir = codex_home.path().join("rules"); + std::fs::create_dir_all(&rules_dir)?; + std::fs::write(rules_dir.join("broken.rules"), "prefix_rule(")?; + + let mut mcp = TestAppServer::new_with_auto_env(codex_home.path()).await?; + timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; + timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_matching_notification( + "initialize exec-policy configWarning", + is_exec_policy_config_warning, + ), + ) + .await??; + + let request_id = mcp + .send_thread_start_request_with_auto_env(ThreadStartParams::default()) + .await?; + let response = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(request_id)), + ) + .await??; + let _: ThreadStartResponse = to_response(response)?; + + let duplicate_warning = timeout( + std::time::Duration::from_millis(250), + mcp.read_stream_until_matching_notification( + "duplicate exec-policy configWarning", + is_exec_policy_config_warning, + ), + ) + .await; + assert!( + duplicate_warning.is_err(), + "thread/start repeated the initialize exec-policy warning" + ); + + Ok(()) +} + +#[tokio::test] +async fn thread_start_provider_model_fallback_uses_bedrock_static_catalog() -> Result<()> { + let codex_home = TempDir::new()?; + std::fs::write( + codex_home.path().join("config.toml"), + r#"model_provider = "amazon-bedrock" +"#, + )?; + let mut mcp = TestAppServer::new_with_auto_env(codex_home.path()).await?; + timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; + + let unsupported_with_fallback = start_thread_with_model( + &mut mcp, + "gpt-5.4-mini", + /*allow_provider_model_fallback*/ true, + ) + .await?; + let supported_with_fallback = start_thread_with_model( + &mut mcp, + "openai.gpt-5.4", + /*allow_provider_model_fallback*/ true, + ) + .await?; + let unsupported_without_fallback = start_thread_with_model( + &mut mcp, + "gpt-5.4-mini", + /*allow_provider_model_fallback*/ false, + ) + .await?; + + assert_eq!( + vec![ + unsupported_with_fallback.model, + supported_with_fallback.model, + unsupported_without_fallback.model, + ], + vec!["openai.gpt-5.5", "openai.gpt-5.4", "gpt-5.4-mini"] + ); + Ok(()) +} + +#[tokio::test] +async fn thread_start_provider_model_fallback_ignores_dynamic_catalog() -> Result<()> { + let server = create_mock_responses_server_repeating_assistant("Done").await; + let codex_home = TempDir::new()?; + create_config_toml_without_approval_policy(codex_home.path(), &server.uri())?; + let mut mcp = TestAppServer::new_with_auto_env(codex_home.path()).await?; + timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; + + let response = start_thread_with_model( + &mut mcp, + "unlisted-dynamic-model", + /*allow_provider_model_fallback*/ true, + ) + .await?; + + assert_eq!(response.model, "unlisted-dynamic-model"); + Ok(()) +} #[tokio::test] async fn thread_start_creates_thread_and_emits_started() -> Result<()> { @@ -139,6 +382,11 @@ async fn thread_start_creates_thread_and_emits_started() -> Result<()> { Some(false), "new persistent threads should serialize `ephemeral: false`" ); + assert_eq!( + thread_json.get("historyMode").and_then(Value::as_str), + Some("legacy"), + "new threads should serialize `historyMode: legacy`" + ); assert_eq!( thread_json.get("threadSource").and_then(Value::as_str), Some("user"), @@ -199,6 +447,50 @@ async fn thread_start_creates_thread_and_emits_started() -> Result<()> { Ok(()) } +#[tokio::test] +async fn thread_start_history_mode_accepts_legacy_and_rejects_paginated() -> Result<()> { + let server = create_mock_responses_server_repeating_assistant("Done").await; + let codex_home = TempDir::new()?; + create_config_toml_without_approval_policy(codex_home.path(), &server.uri())?; + + let mut mcp = TestAppServer::new_with_auto_env(codex_home.path()).await?; + timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; + + let request_id = mcp + .send_thread_start_request_with_auto_env(ThreadStartParams { + history_mode: Some(ThreadHistoryMode::Legacy), + ..Default::default() + }) + .await?; + let response: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(request_id)), + ) + .await??; + let ThreadStartResponse { thread, .. } = to_response(response)?; + + assert_eq!(thread.history_mode, ThreadHistoryMode::Legacy); + + let request_id = mcp + .send_thread_start_request_with_auto_env(ThreadStartParams { + history_mode: Some(ThreadHistoryMode::Paginated), + ..Default::default() + }) + .await?; + let error: JSONRPCError = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_error_message(RequestId::Integer(request_id)), + ) + .await??; + + assert_eq!(error.error.code, -32601); + assert_eq!( + error.error.message, + "paginated_threads is not supported yet" + ); + Ok(()) +} + #[tokio::test] async fn thread_start_accepts_absolute_runtime_workspace_roots() -> Result<()> { let server = create_mock_responses_server_repeating_assistant("Done").await; @@ -546,6 +838,7 @@ async fn thread_start_tracks_thread_initialized_analytics() -> Result<()> { let req_id = mcp .send_thread_start_request(ThreadStartParams { thread_source: Some(ThreadSource::User), + service_name: Some("codex_work_desktop".to_string()), ..Default::default() }) .await?; @@ -563,6 +856,7 @@ async fn thread_start_tracks_thread_initialized_analytics() -> Result<()> { event, &thread.id, &thread.session_id, + "codex_work_desktop", "mock-model", "new", "user", @@ -836,6 +1130,7 @@ async fn thread_start_emits_mcp_server_status_updated_notifications() -> Result< name: "optional_broken".to_string(), status: McpServerStartupState::Starting, error: None, + failure_reason: None, } ); @@ -868,6 +1163,7 @@ async fn thread_start_emits_mcp_server_status_updated_notifications() -> Result< assert_eq!(failed.thread_id, Some(start_response.thread.id)); assert_eq!(failed.name, "optional_broken"); assert_eq!(failed.status, McpServerStartupState::Failed); + assert_eq!(failed.failure_reason, None); assert!( failed .error diff --git a/codex-rs/app-server/tests/suite/v2/thread_unarchive.rs b/codex-rs/app-server/tests/suite/v2/thread_unarchive.rs index f647dc2cfd..07999e1650 100644 --- a/codex-rs/app-server/tests/suite/v2/thread_unarchive.rs +++ b/codex-rs/app-server/tests/suite/v2/thread_unarchive.rs @@ -218,7 +218,9 @@ async fn thread_unarchive_preserves_pathless_store_metadata() -> Result<()> { originator: "test_originator".to_string(), base_instructions: BaseInstructions::default(), dynamic_tools: Vec::new(), + selected_capability_roots: Vec::new(), multi_agent_version: None, + history_mode: Default::default(), initial_window_id: Uuid::now_v7().to_string(), metadata: ThreadPersistenceMetadata { cwd: None, diff --git a/codex-rs/app-server/tests/suite/v2/turn_start.rs b/codex-rs/app-server/tests/suite/v2/turn_start.rs index a528298580..b42a645a33 100644 --- a/codex-rs/app-server/tests/suite/v2/turn_start.rs +++ b/codex-rs/app-server/tests/suite/v2/turn_start.rs @@ -1,6 +1,5 @@ use anyhow::Context; use anyhow::Result; -use app_test_support::DEFAULT_CLIENT_NAME; use app_test_support::TestAppServer; use app_test_support::create_apply_patch_sse_response; use app_test_support::create_exec_command_sse_response; @@ -840,7 +839,7 @@ async fn thread_start_omits_empty_instruction_overrides_from_model_request() -> } #[tokio::test] -async fn turn_start_tracks_turn_event_analytics() -> Result<()> { +async fn turn_start_tracks_thread_originator_in_analytics() -> Result<()> { let server = responses::start_mock_server().await; let response_mock = responses::mount_response_sequence( &server, @@ -875,6 +874,7 @@ async fn turn_start_tracks_turn_event_analytics() -> Result<()> { .send_thread_start_request(ThreadStartParams { model: Some("mock-model".to_string()), thread_source: Some(ThreadSource::User), + service_name: Some("codex_work_desktop".to_string()), ..Default::default() }) .await?; @@ -919,7 +919,7 @@ async fn turn_start_tracks_turn_event_analytics() -> Result<()> { assert_eq!(event["event_params"]["turn_id"], turn.id); assert_eq!( event["event_params"]["app_server_client"]["product_client_id"], - DEFAULT_CLIENT_NAME + "codex_work_desktop" ); assert_eq!(event["event_params"]["model"], "mock-model"); assert_eq!(event["event_params"]["model_provider"], "mock_provider"); @@ -1817,7 +1817,9 @@ async fn turn_start_ignores_deprecated_multi_agent_mode() -> Result<()> { .single_request() .message_input_texts("developer"); assert!(developer_texts.iter().any(|text| { - text.contains("Do not spawn sub-agents unless the user explicitly asks for sub-agents") + text.contains( + "Do not spawn sub-agents unless the user or applicable AGENTS.md/skill instructions explicitly ask for sub-agents", + ) })); assert!( !developer_texts @@ -1898,8 +1900,9 @@ async fn thread_start_ignores_deprecated_multi_agent_mode() -> Result<()> { .message_input_texts("developer"); assert!(developer_texts.iter().any(|text| { text.contains(MULTI_AGENT_MODE_OPEN_TAG) - && text - .contains("Do not spawn sub-agents unless the user explicitly asks for sub-agents") + && text.contains( + "Do not spawn sub-agents unless the user or applicable AGENTS.md/skill instructions explicitly ask for sub-agents", + ) })); assert!( !developer_texts diff --git a/codex-rs/backend-client/src/client/rate_limit_resets.rs b/codex-rs/backend-client/src/client/rate_limit_resets.rs index b22a4ec91a..bed56ba648 100644 --- a/codex-rs/backend-client/src/client/rate_limit_resets.rs +++ b/codex-rs/backend-client/src/client/rate_limit_resets.rs @@ -3,6 +3,7 @@ use super::Client; use super::PathStyle; use crate::types::ConsumeRateLimitResetCreditResponse; +use crate::types::RateLimitResetCreditsDetails; use crate::types::RateLimitStatusWithResetCredits; use crate::types::RateLimitsWithResetCredits; use anyhow::Result; @@ -13,6 +14,8 @@ use serde::Serialize; #[derive(Serialize)] struct ConsumeRateLimitResetCreditRequest<'a> { redeem_request_id: &'a str, + #[serde(skip_serializing_if = "Option::is_none")] + credit_id: Option<&'a str>, } impl Client { @@ -31,9 +34,34 @@ impl Client { self.decode_json(&url, &ct, &body) } + pub async fn list_rate_limit_reset_credits(&self) -> Result { + let url = self.rate_limit_reset_credits_url(); + let req = self.http.get(&url).headers(self.headers()); + let (body, ct) = self.exec_request(req, "GET", &url).await?; + self.decode_json(&url, &ct, &body) + } + pub async fn consume_rate_limit_reset_credit( &self, redeem_request_id: &str, + ) -> Result { + self.consume_rate_limit_reset_credit_request(redeem_request_id, /*credit_id*/ None) + .await + } + + pub async fn consume_rate_limit_reset_credit_by_id( + &self, + redeem_request_id: &str, + credit_id: &str, + ) -> Result { + self.consume_rate_limit_reset_credit_request(redeem_request_id, Some(credit_id)) + .await + } + + async fn consume_rate_limit_reset_credit_request( + &self, + redeem_request_id: &str, + credit_id: Option<&str>, ) -> Result { let url = self.consume_rate_limit_reset_credit_url(); let req = self @@ -41,7 +69,10 @@ impl Client { .post(&url) .headers(self.headers()) .header(CONTENT_TYPE, HeaderValue::from_static("application/json")) - .json(&ConsumeRateLimitResetCreditRequest { redeem_request_id }); + .json(&ConsumeRateLimitResetCreditRequest { + redeem_request_id, + credit_id, + }); let (body, ct) = self.exec_request(req, "POST", &url).await?; self.decode_json(&url, &ct, &body) } @@ -53,6 +84,17 @@ impl Client { } } + fn rate_limit_reset_credits_url(&self) -> String { + match self.path_style { + PathStyle::CodexApi => { + format!("{}/api/codex/rate-limit-reset-credits", self.base_url) + } + PathStyle::ChatGptApi => { + format!("{}/wham/rate-limit-reset-credits", self.base_url) + } + } + } + fn consume_rate_limit_reset_credit_url(&self) -> String { match self.path_style { PathStyle::CodexApi => { diff --git a/codex-rs/backend-client/src/client/rate_limit_resets_tests.rs b/codex-rs/backend-client/src/client/rate_limit_resets_tests.rs index 89313dbe5d..f706be5da9 100644 --- a/codex-rs/backend-client/src/client/rate_limit_resets_tests.rs +++ b/codex-rs/backend-client/src/client/rate_limit_resets_tests.rs @@ -1,5 +1,7 @@ use super::*; use crate::types::ConsumeRateLimitResetCreditCode; +use crate::types::RateLimitResetCreditDetails; +use crate::types::RateLimitResetCreditsDetails; use crate::types::RateLimitResetCreditsSummary; use pretty_assertions::assert_eq; @@ -9,6 +11,10 @@ fn rate_limit_reset_contract_uses_expected_paths_and_payloads() { test_client("https://example.test", PathStyle::CodexApi).rate_limit_status_url(), "https://example.test/api/codex/usage" ); + assert_eq!( + test_client("https://example.test", PathStyle::CodexApi).rate_limit_reset_credits_url(), + "https://example.test/api/codex/rate-limit-reset-credits" + ); assert_eq!( test_client("https://example.test", PathStyle::CodexApi) .consume_rate_limit_reset_credit_url(), @@ -19,6 +25,11 @@ fn rate_limit_reset_contract_uses_expected_paths_and_payloads() { .rate_limit_status_url(), "https://chatgpt.com/backend-api/wham/usage" ); + assert_eq!( + test_client("https://chatgpt.com/backend-api", PathStyle::ChatGptApi) + .rate_limit_reset_credits_url(), + "https://chatgpt.com/backend-api/wham/rate-limit-reset-credits" + ); assert_eq!( test_client("https://chatgpt.com/backend-api", PathStyle::ChatGptApi) .consume_rate_limit_reset_credit_url(), @@ -28,10 +39,22 @@ fn rate_limit_reset_contract_uses_expected_paths_and_payloads() { assert_eq!( serde_json::to_value(ConsumeRateLimitResetCreditRequest { redeem_request_id: "redeem-123", + credit_id: None, }) .unwrap(), serde_json::json!({ "redeem_request_id": "redeem-123" }) ); + assert_eq!( + serde_json::to_value(ConsumeRateLimitResetCreditRequest { + redeem_request_id: "redeem-456", + credit_id: Some("credit-123"), + }) + .unwrap(), + serde_json::json!({ + "redeem_request_id": "redeem-456", + "credit_id": "credit-123", + }) + ); let status: RateLimitStatusWithResetCredits = serde_json::from_value(serde_json::json!({ "plan_type": "plus", @@ -43,6 +66,60 @@ fn rate_limit_reset_contract_uses_expected_paths_and_payloads() { Some(RateLimitResetCreditsSummary { available_count: 3 }) ); + let details: RateLimitResetCreditsDetails = serde_json::from_value(serde_json::json!({ + "credits": [ + { + "id": "credit-1", + "reset_type": "codex_rate_limits", + "status": "available", + "granted_at": "2026-06-17T00:00:00Z", + "expires_at": "2026-07-17T00:00:00Z", + "redeem_started_at": null, + "redeemed_at": null, + "profile_image_url": "https://example.test/avatar.png", + "profile_user_id": "@friend", + "title": "Full reset (Weekly + 5 hr)", + "description": "Ready to redeem" + }, + { + "id": "credit-2", + "reset_type": "codex_rate_limits", + "status": "available", + "granted_at": "2026-06-18T00:00:00Z", + "expires_at": null + } + ], + "available_count": 2, + "total_earned_count": 4 + })) + .unwrap(); + assert_eq!( + details, + RateLimitResetCreditsDetails { + credits: vec![ + RateLimitResetCreditDetails { + id: "credit-1".to_string(), + reset_type: "codex_rate_limits".to_string(), + status: "available".to_string(), + granted_at: "2026-06-17T00:00:00Z".to_string(), + expires_at: Some("2026-07-17T00:00:00Z".to_string()), + title: Some("Full reset (Weekly + 5 hr)".to_string()), + description: Some("Ready to redeem".to_string()), + }, + RateLimitResetCreditDetails { + id: "credit-2".to_string(), + reset_type: "codex_rate_limits".to_string(), + status: "available".to_string(), + granted_at: "2026-06-18T00:00:00Z".to_string(), + expires_at: None, + title: None, + description: None, + }, + ], + available_count: 2, + } + ); + let response: ConsumeRateLimitResetCreditResponse = serde_json::from_value(serde_json::json!({ "code": "reset", "credit": { "id": "ignored-by-cli" }, diff --git a/codex-rs/backend-client/src/lib.rs b/codex-rs/backend-client/src/lib.rs index 6cbeaf30b2..06f2cd442a 100644 --- a/codex-rs/backend-client/src/lib.rs +++ b/codex-rs/backend-client/src/lib.rs @@ -18,6 +18,8 @@ pub use types::DeliveredConfigToml; pub use types::DeliveredRequirementsToml; pub use types::DeliveredTomlFragment; pub use types::PaginatedListTaskListItem; +pub use types::RateLimitResetCreditDetails; +pub use types::RateLimitResetCreditsDetails; pub use types::RateLimitResetCreditsSummary; pub use types::RateLimitsWithResetCredits; pub use types::TaskListItem; diff --git a/codex-rs/backend-client/src/types.rs b/codex-rs/backend-client/src/types.rs index 18ab323986..7fb4346a92 100644 --- a/codex-rs/backend-client/src/types.rs +++ b/codex-rs/backend-client/src/types.rs @@ -23,6 +23,23 @@ pub struct RateLimitResetCreditsSummary { pub available_count: i64, } +#[derive(Clone, Debug, Deserialize, PartialEq, Eq)] +pub struct RateLimitResetCreditsDetails { + pub credits: Vec, + pub available_count: i64, +} + +#[derive(Clone, Debug, Deserialize, PartialEq, Eq)] +pub struct RateLimitResetCreditDetails { + pub id: String, + pub reset_type: String, + pub status: String, + pub granted_at: String, + pub expires_at: Option, + pub title: Option, + pub description: Option, +} + #[derive(Clone, Debug, PartialEq)] pub struct RateLimitsWithResetCredits { pub rate_limits: Vec, diff --git a/codex-rs/cli/Cargo.toml b/codex-rs/cli/Cargo.toml index 7be1146df5..9a8312ced2 100644 --- a/codex-rs/cli/Cargo.toml +++ b/codex-rs/cli/Cargo.toml @@ -95,6 +95,7 @@ windows-sys = { version = "0.52", features = [ ] } [dev-dependencies] +app_test_support = { workspace = true } assert_cmd = { workspace = true } assert_matches = { workspace = true } codex-utils-cargo-bin = { workspace = true } diff --git a/codex-rs/cli/src/exec_server_telemetry.rs b/codex-rs/cli/src/exec_server_telemetry.rs index 4d688a0159..b5af769e6a 100644 --- a/codex-rs/cli/src/exec_server_telemetry.rs +++ b/codex-rs/cli/src/exec_server_telemetry.rs @@ -1,3 +1,5 @@ +use std::future::Future; + use tracing_subscriber::EnvFilter; use tracing_subscriber::prelude::*; @@ -7,7 +9,7 @@ const OTEL_SERVICE_NAME: &str = "codex-exec-server"; pub(crate) fn init( config: Option<&codex_core::config::Config>, -) -> Result> { +) -> (impl Send + Sync, codex_exec_server::ExecServerTelemetry) { let fmt_layer = tracing_subscriber::fmt::layer() .with_writer(std::io::stderr) .with_filter(stderr_env_filter()); @@ -17,21 +19,94 @@ pub(crate) fn init( env!("CARGO_PKG_VERSION"), Some(OTEL_SERVICE_NAME), DEFAULT_ANALYTICS_ENABLED, - ), - None => Ok(None), + ) + .unwrap_or_else(|error| { + eprintln!("Could not create otel exporter: {error}"); + None + }), + None => None, }; - let provider = otel.as_ref().ok().and_then(Option::as_ref); + let provider = otel.as_ref(); codex_core::otel_init::record_process_start(provider, OTEL_SERVICE_NAME); let otel_logger_layer = provider.and_then(|otel| otel.logger_layer()); let otel_tracing_layer = provider.and_then(|otel| otel.tracing_layer()); + let telemetry = provider + .and_then(|otel| otel.metrics()) + .cloned() + .map(codex_exec_server::ExecServerTelemetry::new) + .unwrap_or_default(); let _ = tracing_subscriber::registry() .with(fmt_layer) .with(otel_tracing_layer) .with(otel_logger_layer) .try_init(); tracing::callsite::rebuild_interest_cache(); - otel + (otel, telemetry) +} + +pub(crate) async fn run_until_shutdown(run: F) -> Result<(), E> +where + F: Future>, +{ + let shutdown_signal = match shutdown_signal() { + Ok(signal) => Some(signal), + Err(error) => { + eprintln!("Could not listen for exec-server shutdown signal: {error}"); + None + } + }; + tokio::pin!(run); + + if let Some(shutdown_signal) = shutdown_signal { + tokio::select! { + result = &mut run => result, + signal = wait_for_shutdown_signal(shutdown_signal) => { + match signal { + Ok(()) => Ok(()), + Err(error) => { + eprintln!("Could not listen for exec-server shutdown signal: {error}"); + run.await + } + } + } + } + } else { + run.await + } +} + +#[cfg(unix)] +struct ShutdownSignal { + terminate: tokio::signal::unix::Signal, +} + +#[cfg(unix)] +fn shutdown_signal() -> std::io::Result { + Ok(ShutdownSignal { + terminate: tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())?, + }) +} + +#[cfg(unix)] +async fn wait_for_shutdown_signal(mut shutdown_signal: ShutdownSignal) -> std::io::Result<()> { + tokio::select! { + result = tokio::signal::ctrl_c() => result, + _ = shutdown_signal.terminate.recv() => Ok(()), + } +} + +#[cfg(not(unix))] +struct ShutdownSignal; + +#[cfg(not(unix))] +fn shutdown_signal() -> std::io::Result { + Ok(ShutdownSignal) +} + +#[cfg(not(unix))] +async fn wait_for_shutdown_signal(_: ShutdownSignal) -> std::io::Result<()> { + tokio::signal::ctrl_c().await } fn stderr_env_filter() -> EnvFilter { diff --git a/codex-rs/cli/src/lib.rs b/codex-rs/cli/src/lib.rs index 1c0a2e7e02..c008567b96 100644 --- a/codex-rs/cli/src/lib.rs +++ b/codex-rs/cli/src/lib.rs @@ -53,7 +53,12 @@ pub struct SeatbeltCommand { pub sandbox_state: SandboxStateArgs, /// Named permissions profile to apply from the active configuration stack. - #[arg(long = "permissions-profile", short = 'P', value_name = "NAME")] + #[arg( + long = "permission-profile", + alias = "permissions-profile", + short = 'P', + value_name = "NAME" + )] pub permissions_profile: Option, /// Layer $CODEX_HOME/.config.toml on top of the base user config. @@ -104,7 +109,12 @@ pub struct LandlockCommand { pub sandbox_state: SandboxStateArgs, /// Named permissions profile to apply from the active configuration stack. - #[arg(long = "permissions-profile", short = 'P', value_name = "NAME")] + #[arg( + long = "permission-profile", + alias = "permissions-profile", + short = 'P', + value_name = "NAME" + )] pub permissions_profile: Option, /// Layer $CODEX_HOME/.config.toml on top of the base user config. @@ -142,7 +152,12 @@ pub struct WindowsCommand { pub sandbox_state: SandboxStateArgs, /// Named permissions profile to apply from the active configuration stack. - #[arg(long = "permissions-profile", short = 'P', value_name = "NAME")] + #[arg( + long = "permission-profile", + alias = "permissions-profile", + short = 'P', + value_name = "NAME" + )] pub permissions_profile: Option, /// Layer $CODEX_HOME/.config.toml on top of the base user config. diff --git a/codex-rs/cli/src/main.rs b/codex-rs/cli/src/main.rs index 959ec3a3b3..c766baa048 100644 --- a/codex-rs/cli/src/main.rs +++ b/codex-rs/cli/src/main.rs @@ -1693,9 +1693,7 @@ async fn run_exec_server_command( .environment_id .ok_or_else(|| anyhow::anyhow!("--environment-id is required when --remote is set"))?; let config = load_exec_server_config(root_config_overrides, strict_config).await?; - let _otel = exec_server_telemetry::init(Some(&config)) - .inspect_err(|err| eprintln!("Could not create otel exporter: {err}")) - .ok(); + let (_otel, telemetry) = exec_server_telemetry::init(Some(&config)); let auth_provider = load_exec_server_remote_auth_provider(&config, &base_url, cmd.use_agent_identity_auth) .await?; @@ -1707,7 +1705,11 @@ async fn run_exec_server_command( if let Some(name) = cmd.name { remote_config.name = name; } - codex_exec_server::run_remote_environment(remote_config, runtime_paths).await?; + let remote_config = remote_config.with_telemetry(telemetry); + exec_server_telemetry::run_until_shutdown(async move { + codex_exec_server::run_remote_environment(remote_config, runtime_paths).await + }) + .await?; Ok(()) } else { let config_result = load_exec_server_config(root_config_overrides, strict_config).await; @@ -1716,16 +1718,15 @@ async fn run_exec_server_command( } else { config_result.ok() }; - let _otel = exec_server_telemetry::init(config.as_ref()) - .inspect_err(|err| eprintln!("Could not create otel exporter: {err}")) - .ok(); + let (_otel, telemetry) = exec_server_telemetry::init(config.as_ref()); let listen_url = cmd .listen - .as_deref() - .unwrap_or(codex_exec_server::DEFAULT_LISTEN_URL); - codex_exec_server::run_main(listen_url, runtime_paths) - .await - .map_err(anyhow::Error::from_boxed) + .unwrap_or_else(|| codex_exec_server::DEFAULT_LISTEN_URL.to_string()); + exec_server_telemetry::run_until_shutdown(async move { + codex_exec_server::run_main_with_telemetry(&listen_url, runtime_paths, telemetry).await + }) + .await + .map_err(anyhow::Error::from_boxed) } } @@ -2965,7 +2966,28 @@ mod tests { #[cfg(any(target_os = "macos", target_os = "linux", target_os = "windows"))] #[test] - fn sandbox_parses_permissions_profile() { + fn sandbox_parses_permission_profile() { + let cli = MultitoolCli::try_parse_from([ + "codex", + "sandbox", + "--permission-profile", + ":workspace", + "--", + "echo", + ]) + .expect("parse"); + + let Some(Subcommand::Sandbox(command)) = cli.subcommand else { + panic!("expected sandbox command"); + }; + + assert_eq!(command.permissions_profile.as_deref(), Some(":workspace")); + assert_eq!(command.command, vec!["echo"]); + } + + #[cfg(any(target_os = "macos", target_os = "linux", target_os = "windows"))] + #[test] + fn sandbox_parses_legacy_permissions_profile_alias() { let cli = MultitoolCli::try_parse_from([ "codex", "sandbox", @@ -2984,6 +3006,14 @@ mod tests { assert_eq!(command.command, vec!["echo"]); } + #[cfg(any(target_os = "macos", target_os = "linux", target_os = "windows"))] + #[test] + fn sandbox_help_only_shows_singular_permission_profile() { + let help = help_from_args(&["codex", "sandbox", "--help"]); + assert!(help.contains("--permission-profile"), "{help}"); + assert!(!help.contains("--permissions-profile"), "{help}"); + } + #[cfg(any(target_os = "macos", target_os = "linux", target_os = "windows"))] #[test] fn sandbox_parses_permissions_profile_short_alias() { diff --git a/codex-rs/cli/src/marketplace_cmd.rs b/codex-rs/cli/src/marketplace_cmd.rs index e4e2b63378..16aeb092e9 100644 --- a/codex-rs/cli/src/marketplace_cmd.rs +++ b/codex-rs/cli/src/marketplace_cmd.rs @@ -341,7 +341,7 @@ fn configured_marketplace_sources_by_root( codex_home: &Path, plugins_input: &PluginsConfigInput, ) -> HashMap { - let marketplace_sources = configured_marketplace_sources(plugins_input); + let marketplace_sources = configured_marketplace_sources(plugins_input, codex_home); let Some(user_config) = plugins_input.config_layer_stack.effective_user_config() else { return HashMap::new(); }; diff --git a/codex-rs/cli/src/mcp_cmd.rs b/codex-rs/cli/src/mcp_cmd.rs index 15f0b0bb4d..fd0b1a0e4f 100644 --- a/codex-rs/cli/src/mcp_cmd.rs +++ b/codex-rs/cli/src/mcp_cmd.rs @@ -18,8 +18,10 @@ use codex_core::config::edit::ConfigEditsBuilder; use codex_core::config::find_codex_home; use codex_core::config::load_global_mcp_servers; use codex_core_plugins::PluginsManager; +use codex_exec_server::EnvironmentManager; use codex_login::AuthManager; use codex_mcp::McpOAuthLoginSupport; +use codex_mcp::McpRuntimeContext; use codex_mcp::ResolvedMcpOAuthScopes; use codex_mcp::compute_auth_statuses; use codex_mcp::discover_supported_scopes; @@ -558,6 +560,10 @@ async fn run_list(config_overrides: &CliConfigOverrides, list_args: ListArgs) -> config.mcp_oauth_credentials_store_mode, config.auth_keyring_backend_kind(), auth.as_ref(), + &McpRuntimeContext::new( + Arc::new(EnvironmentManager::without_environments()), + config.cwd.to_path_buf(), + ), ) .await; @@ -567,7 +573,7 @@ async fn run_list(config_overrides: &CliConfigOverrides, list_args: ListArgs) -> .map(|(name, cfg)| { let auth_status = auth_statuses .get(name.as_str()) - .map(|entry| entry.auth_status) + .map(|entry| McpAuthStatus::from(entry.auth_state)) .unwrap_or(McpAuthStatus::Unsupported); let transport = match &cfg.transport { McpServerTransportConfig::Stdio { @@ -651,7 +657,7 @@ async fn run_list(config_overrides: &CliConfigOverrides, list_args: ListArgs) -> let status = format_mcp_status(cfg); let auth_status = auth_statuses .get(name.as_str()) - .map(|entry| entry.auth_status) + .map(|entry| McpAuthStatus::from(entry.auth_state)) .unwrap_or(McpAuthStatus::Unsupported) .to_string(); stdio_rows.push([ @@ -672,7 +678,7 @@ async fn run_list(config_overrides: &CliConfigOverrides, list_args: ListArgs) -> let status = format_mcp_status(cfg); let auth_status = auth_statuses .get(name.as_str()) - .map(|entry| entry.auth_status) + .map(|entry| McpAuthStatus::from(entry.auth_state)) .unwrap_or(McpAuthStatus::Unsupported) .to_string(); let bearer_token_display = diff --git a/codex-rs/cli/src/plugin_cmd.rs b/codex-rs/cli/src/plugin_cmd.rs index 8c9144dfff..056c256b39 100644 --- a/codex-rs/cli/src/plugin_cmd.rs +++ b/codex-rs/cli/src/plugin_cmd.rs @@ -10,6 +10,7 @@ use codex_core_plugins::PluginInstallOutcome; use codex_core_plugins::PluginInstallRequest; use codex_core_plugins::PluginsConfigInput; use codex_core_plugins::PluginsManager; +use codex_core_plugins::allowed_configured_marketplace_names; use codex_core_plugins::installed_marketplaces::marketplace_install_root; use codex_core_plugins::installed_marketplaces::resolve_configured_marketplace_root; use codex_core_plugins::marketplace::MarketplaceListError; @@ -228,7 +229,7 @@ pub async fn run_plugin_list( .is_none_or(|name| marketplace.name == *name) }) .collect::>(); - let marketplace_sources = configured_marketplace_sources(&plugins_input); + let marketplace_sources = configured_marketplace_sources(&plugins_input, codex_home.as_path()); if args.json { let output = JsonPluginListOutput::from_marketplaces( @@ -285,6 +286,20 @@ pub async fn run_plugin_list( } parts.join(", ") } + codex_core_plugins::marketplace::MarketplacePluginSource::Npm { + package, + version, + registry, + } => { + let mut parts = vec![package.clone()]; + if let Some(version) = version { + parts.push(format!("version `{version}`")); + } + if let Some(registry) = registry { + parts.push(format!("registry `{registry}`")); + } + parts.join(", ") + } }; plugin_width = plugin_width.max(plugin.id.len()); status_width = status_width.max(state.len()); @@ -412,6 +427,13 @@ enum JsonPluginSource { #[serde(skip_serializing_if = "Option::is_none")] sha: Option, }, + Npm { + package: String, + #[serde(skip_serializing_if = "Option::is_none")] + version: Option, + #[serde(skip_serializing_if = "Option::is_none")] + registry: Option, + }, } impl JsonPluginSource { @@ -437,6 +459,15 @@ impl JsonPluginSource { ref_name, sha, } => Self::Git { url, ref_name, sha }, + MarketplacePluginSource::Npm { + package, + version, + registry, + } => Self::Npm { + package, + version, + registry, + }, } } } @@ -450,6 +481,7 @@ pub(crate) struct JsonMarketplaceSource { pub(crate) fn configured_marketplace_sources( plugins_input: &PluginsConfigInput, + codex_home: &Path, ) -> HashMap { let Some(user_config) = plugins_input.config_layer_stack.effective_user_config() else { return HashMap::new(); @@ -460,9 +492,12 @@ pub(crate) fn configured_marketplace_sources( else { return HashMap::new(); }; + let allowed_marketplace_names = + allowed_configured_marketplace_names(&plugins_input.config_layer_stack, codex_home); marketplaces .iter() + .filter(|(marketplace_name, _)| allowed_marketplace_names.contains(*marketplace_name)) .filter_map(|(marketplace_name, marketplace)| { let source_type = marketplace .get("source_type") @@ -716,11 +751,16 @@ pub(crate) fn configured_marketplace_snapshot_issues( else { return Vec::new(); }; + let allowed_marketplace_names = + allowed_configured_marketplace_names(&plugins_input.config_layer_stack, codex_home); let default_install_root = marketplace_install_root(codex_home); let mut manifest_paths = Vec::new(); let mut issues = Vec::new(); for (configured_name, marketplace) in configured_marketplaces { + if !allowed_marketplace_names.contains(configured_name) { + continue; + } if marketplace_name.is_some_and(|name| configured_name != name) { continue; } diff --git a/codex-rs/cli/tests/app_server.rs b/codex-rs/cli/tests/app_server.rs index 4a2642379d..6815c2be99 100644 --- a/codex-rs/cli/tests/app_server.rs +++ b/codex-rs/cli/tests/app_server.rs @@ -1,7 +1,10 @@ use std::path::Path; use anyhow::Result; +use app_test_support::app_server_json_shutdown_event; use predicates::str::contains; +use pretty_assertions::assert_eq; +use serde_json::json; use tempfile::TempDir; fn codex_command(codex_home: &Path) -> Result { @@ -28,3 +31,25 @@ foo = "bar" Ok(()) } + +#[test] +fn app_server_emits_json_info_events() -> Result<()> { + let codex_home = TempDir::new()?; + let event = app_server_json_shutdown_event("codex", &["app-server"], codex_home.path())?; + + assert_eq!( + event, + json!({ + "level": "INFO", + "fields": { + "message": "processor task exited", + "exit_reason": "last_connection_closed", + "remaining_connection_count": 0, + "shutdown_forced": false, + }, + "target": "codex_app_server", + }) + ); + + Ok(()) +} diff --git a/codex-rs/cli/tests/exec_server.rs b/codex-rs/cli/tests/exec_server.rs index 5e412a9de1..aabe9c0453 100644 --- a/codex-rs/cli/tests/exec_server.rs +++ b/codex-rs/cli/tests/exec_server.rs @@ -1,9 +1,34 @@ +#[cfg(unix)] +use std::io::BufRead as _; +#[cfg(unix)] +use std::io::BufReader as StdBufReader; +#[cfg(unix)] +use std::io::Read as _; +#[cfg(unix)] +use std::io::Write as _; +#[cfg(unix)] +use std::net::TcpStream; use std::path::Path; +use std::process::Stdio; +#[cfg(unix)] +use std::thread; +use std::time::Duration; +#[cfg(unix)] +use std::time::Instant; use anyhow::Result; use predicates::prelude::PredicateBooleanExt; use predicates::str::contains; use tempfile::TempDir; +use tokio::io::AsyncBufReadExt; +use tokio::io::AsyncReadExt; +use tokio::io::AsyncWriteExt; +use tokio::io::BufReader; +use wiremock::Mock; +use wiremock::MockServer; +use wiremock::ResponseTemplate; +use wiremock::matchers::method; +use wiremock::matchers::path; fn codex_command(codex_home: &Path) -> Result { let mut cmd = assert_cmd::Command::new(codex_utils_cargo_bin::cargo_bin("codex")?); @@ -48,3 +73,273 @@ fn local_exec_server_ignores_invalid_config_without_strict_config() -> Result<() Ok(()) } + +#[tokio::test] +async fn local_exec_server_flushes_telemetry_on_stdio_disconnect() -> Result<()> { + let collector = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/v1/metrics")) + .respond_with(ResponseTemplate::new(202)) + .mount(&collector) + .await; + let codex_home = TempDir::new()?; + let base_url = collector.uri(); + std::fs::write( + codex_home.path().join("config.toml"), + format!( + r#" +[analytics] +enabled = true + +[otel] +environment = "test" +metrics_exporter = {{ otlp-http = {{ endpoint = "{base_url}/v1/metrics", protocol = "json" }} }} +"# + ), + )?; + + let cwd = url::Url::from_directory_path(std::env::current_dir()?) + .map_err(|()| anyhow::anyhow!("could not convert cwd to file URL"))?; + #[cfg(windows)] + let argv = vec!["ping.exe", "-n", "61", "127.0.0.1"]; + #[cfg(not(windows))] + let argv = vec!["/bin/sleep", "60"]; + let codex_bin = codex_utils_cargo_bin::cargo_bin("codex")?; + let codex_home = codex_home.path().to_path_buf(); + let subprocess = async move { + let mut command = tokio::process::Command::new(codex_bin); + command + .env("CODEX_HOME", codex_home) + .env("NO_PROXY", "127.0.0.1,localhost") + .env("no_proxy", "127.0.0.1,localhost") + .args(["exec-server", "--listen", "stdio"]) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .kill_on_drop(true); + let mut child = command.spawn()?; + let mut stdin = child + .stdin + .take() + .ok_or_else(|| anyhow::anyhow!("exec-server stdin was not piped"))?; + let stdout = child + .stdout + .take() + .ok_or_else(|| anyhow::anyhow!("exec-server stdout was not piped"))?; + let mut stdout = BufReader::new(stdout); + send_json_line( + &mut stdin, + &serde_json::json!({ + "id": 1, + "method": "initialize", + "params": {"clientName": "otel-test", "resumeSessionId": null} + }), + ) + .await?; + wait_for_response(&mut stdout, /*expected_id*/ 1).await?; + send_json_line( + &mut stdin, + &serde_json::json!({"method": "initialized", "params": {}}), + ) + .await?; + send_json_line( + &mut stdin, + &serde_json::json!({ + "id": 2, + "method": "process/start", + "params": { + "processId": "otel-process", + "argv": argv, + "cwd": cwd, + "env": {}, + "tty": false, + "pipeStdin": false, + "arg0": null + } + }), + ) + .await?; + wait_for_response(&mut stdout, /*expected_id*/ 2).await?; + drop(stdin); + let mut remaining_stdout = String::new(); + stdout.read_to_string(&mut remaining_stdout).await?; + let status = child.wait().await?; + anyhow::ensure!( + status.success(), + "exec-server exited with {status}; remaining stdout: {remaining_stdout}" + ); + Ok::<(), anyhow::Error>(()) + }; + let subprocess_result = tokio::time::timeout(Duration::from_secs(30), subprocess) + .await + .map_err(|_| anyhow::anyhow!("exec-server subprocess timed out"))?; + subprocess_result?; + + let requests = collector + .received_requests() + .await + .ok_or_else(|| anyhow::anyhow!("failed to read OTLP collector requests"))?; + let metrics = requests + .iter() + .filter(|request| request.url.path() == "/v1/metrics") + .map(|request| serde_json::from_slice::(&request.body)) + .collect::>>()?; + assert_metric_point( + &metrics, + "exec_server_connections_active", + &[("transport", "stdio")], + Some(0), + ); + assert_metric_point( + &metrics, + "exec_server_connections_total", + &[("transport", "stdio")], + Some(1), + ); + assert_metric_point( + &metrics, + "exec_server_requests_total", + &[("method", "process/start"), ("result", "success")], + Some(1), + ); + assert_metric_point(&metrics, "exec_server_processes_active", &[], Some(0)); + assert_metric_point( + &metrics, + "exec_server_processes_finished_total", + &[("result", "terminated")], + Some(1), + ); + assert_metric_point( + &metrics, + "exec_server_request_duration_seconds", + &[("method", "process/start"), ("result", "success")], + /*value*/ None, + ); + assert_metric_point( + &metrics, + "exec_server_process_duration_seconds", + &[("result", "terminated")], + /*value*/ None, + ); + Ok(()) +} + +async fn send_json_line( + stdin: &mut (impl tokio::io::AsyncWrite + Unpin), + message: &serde_json::Value, +) -> Result<()> { + let mut encoded = serde_json::to_vec(message)?; + encoded.push(b'\n'); + stdin.write_all(&encoded).await?; + stdin.flush().await?; + Ok(()) +} + +#[cfg(unix)] +#[test] +fn local_exec_server_exits_successfully_on_sigterm() -> Result<()> { + let codex_home = TempDir::new()?; + let mut child = std::process::Command::new(codex_utils_cargo_bin::cargo_bin("codex")?) + .env("CODEX_HOME", codex_home.path()) + .args(["exec-server", "--listen", "ws://127.0.0.1:0"]) + .stdout(Stdio::piped()) + .spawn()?; + let mut listen_url = String::new(); + StdBufReader::new(child.stdout.take().expect("child stdout")).read_line(&mut listen_url)?; + assert!(listen_url.starts_with("ws://127.0.0.1:"), "{listen_url}"); + + let listen_addr = listen_url + .trim() + .strip_prefix("ws://") + .expect("listen URL should use ws://") + .parse()?; + let deadline = Instant::now() + Duration::from_secs(5); + let mut ready = false; + while let Some(remaining) = deadline.checked_duration_since(Instant::now()) { + if let Ok(mut stream) = + TcpStream::connect_timeout(&listen_addr, remaining.min(Duration::from_millis(100))) + { + let _ = stream.set_read_timeout(Some(Duration::from_secs(1))); + let request = + format!("GET /readyz HTTP/1.1\r\nHost: {listen_addr}\r\nConnection: close\r\n\r\n"); + let mut response = String::new(); + if stream.write_all(request.as_bytes()).is_ok() + && stream.read_to_string(&mut response).is_ok() + && response.starts_with("HTTP/1.1 200") + { + ready = true; + break; + } + } + thread::sleep(Duration::from_millis(10)); + } + assert!(ready, "exec-server did not become ready at {listen_url}"); + + // SAFETY: `child.id()` is the live process spawned above. + let result = unsafe { libc::kill(child.id() as libc::pid_t, libc::SIGTERM) }; + assert_eq!(result, 0); + let status = child.wait()?; + assert!(status.success(), "{status}"); + Ok(()) +} + +async fn wait_for_response( + stdout: &mut (impl tokio::io::AsyncBufRead + Unpin), + expected_id: i64, +) -> Result<()> { + loop { + let mut line = String::new(); + if stdout.read_line(&mut line).await? == 0 { + anyhow::bail!("exec-server stdout closed before response {expected_id}"); + } + let message: serde_json::Value = serde_json::from_str(&line)?; + if message["id"].as_i64() == Some(expected_id) { + anyhow::ensure!( + message.get("error").is_none(), + "exec-server request {expected_id} failed: {message}" + ); + return Ok(()); + } + } +} + +fn assert_metric_point( + payloads: &[serde_json::Value], + name: &str, + attributes: &[(&str, &str)], + value: Option, +) { + let found = payloads + .iter() + .flat_map(|payload| payload["resourceMetrics"].as_array().into_iter().flatten()) + .flat_map(|resource| resource["scopeMetrics"].as_array().into_iter().flatten()) + .flat_map(|scope| scope["metrics"].as_array().into_iter().flatten()) + .filter(|metric| metric["name"].as_str() == Some(name)) + .flat_map(|metric| { + ["gauge", "sum", "histogram"] + .into_iter() + .find_map(|kind| metric[kind]["dataPoints"].as_array()) + .into_iter() + .flatten() + }) + .any(|point| { + let actual_attributes = point["attributes"] + .as_array() + .map(Vec::as_slice) + .unwrap_or_default(); + let attributes_match = actual_attributes.len() == attributes.len() + && attributes.iter().all(|(expected_key, expected_value)| { + actual_attributes.iter().any(|actual| { + actual["key"].as_str() == Some(*expected_key) + && actual["value"]["stringValue"].as_str() == Some(*expected_value) + }) + }); + let actual_value = point["asInt"] + .as_i64() + .or_else(|| point["asInt"].as_str()?.parse().ok()); + attributes_match && value.is_none_or(|expected| actual_value == Some(expected)) + }); + assert!( + found, + "metric {name} with attributes {attributes:?} and value {value:?} missing" + ); +} diff --git a/codex-rs/cli/tests/plugin_cli.rs b/codex-rs/cli/tests/plugin_cli.rs index c8bd7b5429..f9d73c1ce8 100644 --- a/codex-rs/cli/tests/plugin_cli.rs +++ b/codex-rs/cli/tests/plugin_cli.rs @@ -205,6 +205,7 @@ fn setup_local_marketplace_with_implicit_system_roots() -> Result<(TempDir, Temp let cache_home = TempDir::new()?; let runtime_root = cache_home .path() + .join(".cache") .join("codex-runtimes") .join("codex-primary-runtime") .join("plugins") @@ -844,7 +845,8 @@ async fn plugin_list_ignores_implicit_system_marketplace_roots_without_manifests let (codex_home, source, cache_home) = setup_local_marketplace_with_implicit_system_roots()?; codex_command(codex_home.path())? - .env("XDG_CACHE_HOME", cache_home.path()) + .env("HOME", cache_home.path()) + .env("USERPROFILE", cache_home.path()) .args(["plugin", "list"]) .assert() .success() diff --git a/codex-rs/cli/tests/sandbox_network_proxy.rs b/codex-rs/cli/tests/sandbox_network_proxy.rs index 22bdcce49d..9ec1014688 100644 --- a/codex-rs/cli/tests/sandbox_network_proxy.rs +++ b/codex-rs/cli/tests/sandbox_network_proxy.rs @@ -35,7 +35,7 @@ mode = "full" .env("CODEX_HOME", codex_home.path()) .args([ "sandbox", - "--permissions-profile", + "--permission-profile", "network-test", "--", "curl", diff --git a/codex-rs/code-mode-host/Cargo.toml b/codex-rs/code-mode-host/Cargo.toml index a2c384d010..86a36d2a27 100644 --- a/codex-rs/code-mode-host/Cargo.toml +++ b/codex-rs/code-mode-host/Cargo.toml @@ -8,5 +8,24 @@ license.workspace = true name = "codex-code-mode-host" path = "src/main.rs" +[lib] +doctest = false +name = "codex_code_mode_host" +path = "src/lib.rs" + [lints] workspace = true + +[dependencies] +anyhow = { workspace = true } +codex-code-mode = { workspace = true } +codex-code-mode-protocol = { workspace = true } +tokio = { workspace = true, features = ["io-std", "io-util", "macros", "rt", "sync", "time"] } +tokio-util = { workspace = true, features = ["rt"] } + +[dev-dependencies] +codex-protocol = { workspace = true } +codex-utils-cargo-bin = { workspace = true } +pretty_assertions = { workspace = true } +serde_json = { workspace = true } +tempfile = { workspace = true } diff --git a/codex-rs/code-mode-host/src/delegate.rs b/codex-rs/code-mode-host/src/delegate.rs new file mode 100644 index 0000000000..50d45e7de7 --- /dev/null +++ b/codex-rs/code-mode-host/src/delegate.rs @@ -0,0 +1,85 @@ +use std::sync::Arc; + +use codex_code_mode_protocol::CellId; +use codex_code_mode_protocol::CodeModeNestedToolCall; +use codex_code_mode_protocol::CodeModeSessionDelegate; +use codex_code_mode_protocol::NotificationFuture; +use codex_code_mode_protocol::ToolInvocationFuture; +use codex_code_mode_protocol::host::DelegateRequest; +use codex_code_mode_protocol::host::DelegateResponse; +use codex_code_mode_protocol::host::SessionId; +use tokio_util::sync::CancellationToken; + +use crate::peer::HostPeer; + +pub(super) struct RemoteDelegate { + session_id: SessionId, + peer: Arc, +} + +impl RemoteDelegate { + pub(super) fn new(session_id: SessionId, peer: Arc) -> Self { + Self { session_id, peer } + } +} + +impl CodeModeSessionDelegate for RemoteDelegate { + fn invoke_tool<'a>( + &'a self, + invocation: CodeModeNestedToolCall, + cancellation_token: CancellationToken, + ) -> ToolInvocationFuture<'a> { + Box::pin(async move { + match self + .peer + .call( + self.session_id.clone(), + DelegateRequest::InvokeTool { + invocation: invocation.into(), + }, + cancellation_token, + ) + .await? + { + DelegateResponse::ToolResult { result } => Ok(result), + DelegateResponse::NotificationDelivered => { + Err("code-mode client returned an invalid tool result".to_string()) + } + } + }) + } + + fn notify<'a>( + &'a self, + call_id: String, + cell_id: CellId, + text: String, + cancellation_token: CancellationToken, + ) -> NotificationFuture<'a> { + Box::pin(async move { + match self + .peer + .call( + self.session_id.clone(), + DelegateRequest::Notify { + call_id, + cell_id: cell_id.into(), + text, + }, + cancellation_token, + ) + .await? + { + DelegateResponse::NotificationDelivered => Ok(()), + DelegateResponse::ToolResult { .. } => { + Err("code-mode client returned an invalid notification result".to_string()) + } + } + }) + } + + fn cell_closed(&self, cell_id: &CellId) { + self.peer + .close_cell(self.session_id.clone(), cell_id.clone()); + } +} diff --git a/codex-rs/code-mode-host/src/host_tests.rs b/codex-rs/code-mode-host/src/host_tests.rs new file mode 100644 index 0000000000..b189b89e09 --- /dev/null +++ b/codex-rs/code-mode-host/src/host_tests.rs @@ -0,0 +1,486 @@ +use codex_code_mode_protocol::host::Capability; +use codex_code_mode_protocol::host::CapabilitySet; +use codex_code_mode_protocol::host::ClientHello; +use codex_code_mode_protocol::host::ClientToHost; +use codex_code_mode_protocol::host::EncodedFrame; +use codex_code_mode_protocol::host::FramedReader; +use codex_code_mode_protocol::host::FramedWriter; +use codex_code_mode_protocol::host::HandshakeRejectReason; +use codex_code_mode_protocol::host::HostHello; +use codex_code_mode_protocol::host::HostRequest; +use codex_code_mode_protocol::host::HostResponse; +use codex_code_mode_protocol::host::HostToClient; +use codex_code_mode_protocol::host::ProtocolVersion; +use codex_code_mode_protocol::host::RequestId; +use codex_code_mode_protocol::host::SessionId; +use codex_code_mode_protocol::host::SupportedProtocolVersions; +use codex_code_mode_protocol::host::WireExecuteRequest; +use codex_code_mode_protocol::host::WireResult; +use pretty_assertions::assert_eq; +use tokio::sync::Semaphore; +use tokio::sync::mpsc; +use tokio_util::sync::CancellationToken; +use tokio_util::task::TaskTracker; + +use super::HostState; +use super::MAX_ACTIVE_CELLS; +use super::MAX_IN_FLIGHT_REQUESTS; +use super::MAX_RECENT_REQUEST_IDS; +use super::RequestKind; +use super::RequestRegistry; +use super::SeenSessionIds; +use super::peer::HostPeer; +use super::run; + +fn client_hello( + versions: impl IntoIterator, + required_capabilities: CapabilitySet, +) -> ClientToHost { + ClientToHost::ClientHello( + ClientHello::new( + SupportedProtocolVersions::try_new(versions).expect("supported versions"), + required_capabilities, + CapabilitySet::empty(), + ) + .expect("client hello"), + ) +} + +fn session_id(value: &str) -> SessionId { + SessionId::new(value).expect("session ID") +} + +fn request_id(value: i64) -> RequestId { + RequestId::new(value) +} + +async fn decode_frame(frame: EncodedFrame) -> HostToClient { + let (reader, writer) = tokio::io::duplex(/*max_buf_size*/ 4096); + let writer = tokio::spawn(async move { + FramedWriter::new(writer) + .write_frame(&frame) + .await + .expect("write encoded frame"); + }); + let message = FramedReader::new(reader) + .read() + .await + .expect("read encoded frame") + .expect("encoded frame message"); + writer.await.expect("frame writer task"); + message +} + +fn execute_request(source: &str) -> WireExecuteRequest { + WireExecuteRequest { + tool_call_id: "call-1".to_string(), + enabled_tools: Vec::new(), + source: source.to_string(), + yield_time_ms: Some(60_000), + max_output_tokens: Some(1_000), + } +} + +#[tokio::test] +async fn handshake_and_multiple_session_lifecycles_are_ordered() { + let (host_stream, client_stream) = tokio::io::duplex(/*max_buf_size*/ 4096); + let (host_reader, host_writer) = tokio::io::split(host_stream); + let (client_reader, client_writer) = tokio::io::split(client_stream); + let host = tokio::spawn(run(host_reader, host_writer)); + let mut reader = FramedReader::new(client_reader); + let mut writer = FramedWriter::new(client_writer); + + writer + .write(&client_hello([ProtocolVersion::V1], CapabilitySet::empty())) + .await + .expect("write hello"); + assert_eq!( + reader.read::().await.expect("read hello"), + Some(HostToClient::HostHello(HostHello::new( + ProtocolVersion::V1, + CapabilitySet::empty(), + ))) + ); + + for (request_id, id) in [ + (request_id(/*value*/ 1), "session-1"), + (request_id(/*value*/ 2), "session-2"), + ] { + writer + .write(&ClientToHost::Request { + id: request_id, + request: HostRequest::OpenSession { + session_id: session_id(id), + }, + }) + .await + .expect("open session"); + assert_eq!( + reader.read::().await.expect("session ready"), + Some(HostToClient::Response { + id: request_id, + result: WireResult::Ok { + value: HostResponse::SessionReady { + session_id: session_id(id), + }, + }, + }) + ); + } + + for (request_id, id) in [ + (request_id(/*value*/ 3), "session-1"), + (request_id(/*value*/ 4), "session-2"), + ] { + writer + .write(&ClientToHost::Request { + id: request_id, + request: HostRequest::ShutdownSession { + session_id: session_id(id), + }, + }) + .await + .expect("shutdown session"); + assert_eq!( + reader.read::().await.expect("session closed"), + Some(HostToClient::Response { + id: request_id, + result: WireResult::Ok { + value: HostResponse::SessionClosed { + session_id: session_id(id), + }, + }, + }) + ); + } + + drop(writer); + drop(reader); + host.await.expect("host task").expect("host connection"); +} + +#[tokio::test] +async fn incompatible_or_invalid_handshake_is_rejected() { + let (host_stream, client_stream) = tokio::io::duplex(/*max_buf_size*/ 1024); + let (host_reader, host_writer) = tokio::io::split(host_stream); + let (client_reader, client_writer) = tokio::io::split(client_stream); + let host = tokio::spawn(run(host_reader, host_writer)); + let mut reader = FramedReader::new(client_reader); + let mut writer = FramedWriter::new(client_writer); + let version_two = ProtocolVersion::new(/*value*/ 2).expect("protocol version"); + + writer + .write(&client_hello([version_two], CapabilitySet::empty())) + .await + .expect("write hello"); + assert_eq!( + reader.read::().await.expect("rejection"), + Some(HostToClient::HandshakeRejected { + reason: HandshakeRejectReason::NoCompatibleVersion { + supported_versions: SupportedProtocolVersions::try_new([ProtocolVersion::V1]) + .expect("host versions"), + }, + }) + ); + host.await.expect("host task").expect("host connection"); + + let (host_stream, client_stream) = tokio::io::duplex(/*max_buf_size*/ 1024); + let (host_reader, host_writer) = tokio::io::split(host_stream); + let (client_reader, client_writer) = tokio::io::split(client_stream); + let host = tokio::spawn(run(host_reader, host_writer)); + let mut reader = FramedReader::new(client_reader); + let mut writer = FramedWriter::new(client_writer); + writer + .write(&ClientToHost::Request { + id: request_id(/*value*/ 1), + request: HostRequest::OpenSession { + session_id: session_id("session-1"), + }, + }) + .await + .expect("write invalid first message"); + assert_eq!( + reader.read::().await.expect("rejection"), + Some(HostToClient::HandshakeRejected { + reason: HandshakeRejectReason::InvalidHello { + message: "first message must be connection/hello".to_string(), + }, + }) + ); + host.await.expect("host task").expect("host connection"); +} + +#[tokio::test] +async fn unsupported_required_capability_is_rejected() { + let (host_stream, client_stream) = tokio::io::duplex(/*max_buf_size*/ 1024); + let (host_reader, host_writer) = tokio::io::split(host_stream); + let (client_reader, client_writer) = tokio::io::split(client_stream); + let host = tokio::spawn(run(host_reader, host_writer)); + let mut reader = FramedReader::new(client_reader); + let mut writer = FramedWriter::new(client_writer); + let capability = Capability::new("required").expect("capability"); + + writer + .write(&client_hello( + [ProtocolVersion::V1], + CapabilitySet::try_new([capability.clone()]).expect("capabilities"), + )) + .await + .expect("write hello"); + assert_eq!( + reader.read::().await.expect("rejection"), + Some(HostToClient::HandshakeRejected { + reason: HandshakeRejectReason::MissingRequiredCapability { capability }, + }) + ); + host.await.expect("host task").expect("host connection"); +} + +#[tokio::test] +async fn session_id_cannot_be_reused_after_shutdown() { + let (host_stream, client_stream) = tokio::io::duplex(/*max_buf_size*/ 2048); + let (host_reader, host_writer) = tokio::io::split(host_stream); + let (client_reader, client_writer) = tokio::io::split(client_stream); + let host = tokio::spawn(run(host_reader, host_writer)); + let mut reader = FramedReader::new(client_reader); + let mut writer = FramedWriter::new(client_writer); + writer + .write(&client_hello([ProtocolVersion::V1], CapabilitySet::empty())) + .await + .expect("write hello"); + reader + .read::() + .await + .expect("read hello") + .expect("host hello"); + + let id = session_id("session-1"); + for (request_id, request) in [ + ( + request_id(/*value*/ 1), + HostRequest::OpenSession { + session_id: id.clone(), + }, + ), + ( + request_id(/*value*/ 2), + HostRequest::ShutdownSession { + session_id: id.clone(), + }, + ), + ] { + writer + .write(&ClientToHost::Request { + id: request_id, + request, + }) + .await + .expect("session request"); + reader + .read::() + .await + .expect("session response") + .expect("session response message"); + } + writer + .write(&ClientToHost::Request { + id: request_id(/*value*/ 3), + request: HostRequest::OpenSession { session_id: id }, + }) + .await + .expect("reuse session ID"); + assert_eq!( + reader.read::().await.expect("reuse response"), + Some(HostToClient::Response { + id: request_id(/*value*/ 3), + result: WireResult::Err { + message: "code-mode session ID `session-1` was reused".to_string(), + }, + }) + ); + drop(writer); + drop(reader); + host.await.expect("host task").expect("host connection"); +} + +#[test] +fn request_cancellation_tombstones_are_bounded() { + let mut requests = RequestRegistry::default(); + let duplicate = request_id(/*value*/ -1); + requests + .start(duplicate, RequestKind::OpenSession) + .expect("start duplicate probe"); + assert!(requests.start(duplicate, RequestKind::OpenSession).is_err()); + requests.finish(duplicate); + for value in 1..=MAX_RECENT_REQUEST_IDS as i64 + 100 { + let id = request_id(value); + requests + .start(id, RequestKind::Wait) + .expect("start request"); + requests.cancel(id); + requests.finish(id); + } + for value in 10_000..20_000 { + requests.cancel(request_id(value)); + } + + assert!(requests.active.is_empty()); + assert_eq!(requests.recent.len(), MAX_RECENT_REQUEST_IDS); + assert_eq!(requests.recent_order.len(), MAX_RECENT_REQUEST_IDS); +} + +#[tokio::test] +async fn request_task_panic_disconnects_host() { + let (outgoing_tx, _outgoing_rx) = mpsc::channel(/*max_capacity*/ 1); + let peer = Arc::new(HostPeer::new(outgoing_tx)); + let state = HostState { + sessions: Mutex::new(HashMap::new()), + seen_session_ids: Mutex::new(SeenSessionIds::default()), + requests: Mutex::new(RequestRegistry::default()), + request_tasks: TaskTracker::new(), + request_permits: Arc::new(Semaphore::new(MAX_IN_FLIGHT_REQUESTS)), + active_cell_permits: Arc::new(Semaphore::new(MAX_ACTIVE_CELLS)), + closing: AtomicBool::new(false), + peer: Arc::clone(&peer), + }; + let task = state.request_tasks.spawn(async { + panic!("request panic probe"); + }); + state.supervise_request_task(task); + + tokio::time::timeout(Duration::from_secs(1), peer.disconnected()) + .await + .expect("request panic should disconnect host"); + assert!( + peer.failure() + .expect("request failure") + .contains("request task failed") + ); +} + +#[tokio::test] +async fn execute_request_id_remains_active_until_initial_response() { + let (outgoing_tx, mut outgoing_rx) = mpsc::channel(/*max_capacity*/ 4); + let peer = Arc::new(HostPeer::new(outgoing_tx)); + let state = Arc::new(HostState { + sessions: Mutex::new(HashMap::new()), + seen_session_ids: Mutex::new(SeenSessionIds::default()), + requests: Mutex::new(RequestRegistry::default()), + request_tasks: TaskTracker::new(), + request_permits: Arc::new(Semaphore::new(MAX_IN_FLIGHT_REQUESTS)), + active_cell_permits: Arc::new(Semaphore::new(MAX_ACTIVE_CELLS)), + closing: AtomicBool::new(false), + peer, + }); + let session_id = session_id("session-1"); + state + .open_session(session_id.clone()) + .expect("open session"); + let request_id = request_id(/*value*/ 1); + + state + .spawn_request( + request_id, + HostRequest::Execute { + session_id: session_id.clone(), + request: execute_request("await new Promise(() => {});"), + }, + ) + .expect("spawn execute request"); + let started = decode_frame(outgoing_rx.recv().await.expect("execution started frame")).await; + let HostToClient::Response { + id, + result: + WireResult::Ok { + value: HostResponse::ExecutionStarted { cell_id }, + }, + } = started + else { + panic!("expected execution started response"); + }; + assert_eq!(id, request_id); + assert!( + state + .requests + .lock() + .unwrap_or_else(PoisonError::into_inner) + .active + .contains_key(&request_id) + ); + + state + .session(&session_id) + .expect("session") + .terminate(cell_id.into()) + .await + .expect("terminate cell"); + state.disconnect().await; +} + +#[tokio::test] +async fn active_cell_limit_rejects_execute_without_disconnecting() { + let (outgoing_tx, mut outgoing_rx) = mpsc::channel(/*max_capacity*/ 1); + let peer = Arc::new(HostPeer::new(outgoing_tx)); + let state = HostState { + sessions: Mutex::new(HashMap::new()), + seen_session_ids: Mutex::new(SeenSessionIds::default()), + requests: Mutex::new(RequestRegistry::default()), + request_tasks: TaskTracker::new(), + request_permits: Arc::new(Semaphore::new(MAX_IN_FLIGHT_REQUESTS)), + active_cell_permits: Arc::new(Semaphore::new(/*permits*/ 0)), + closing: AtomicBool::new(false), + peer: Arc::clone(&peer), + }; + let session_id = session_id("session-1"); + state + .open_session(session_id.clone()) + .expect("open session"); + let request_id = request_id(/*value*/ 1); + + state + .handle_request( + request_id, + HostRequest::Execute { + session_id, + request: execute_request("text(\"hello\");"), + }, + CancellationToken::new(), + ) + .await; + + assert_eq!( + decode_frame(outgoing_rx.recv().await.expect("execute response frame")).await, + HostToClient::Response { + id: request_id, + result: WireResult::Err { + message: "code-mode host has too many active cells".to_string(), + }, + } + ); + assert!(!peer.is_disconnected()); + state.disconnect().await; +} + +#[tokio::test] +async fn cell_forwarding_panic_disconnects_host() { + let (outgoing_tx, _outgoing_rx) = mpsc::channel(/*max_capacity*/ 1); + let peer = Arc::new(HostPeer::new(outgoing_tx)); + peer.spawn_critical("cell forwarding", async { + panic!("cell forwarding panic probe"); + }); + + tokio::time::timeout(Duration::from_secs(1), peer.disconnected()) + .await + .expect("cell panic should disconnect host"); + assert!( + peer.failure() + .expect("cell failure") + .contains("cell forwarding task failed") + ); +} +use std::collections::HashMap; +use std::sync::Arc; +use std::sync::Mutex; +use std::sync::PoisonError; +use std::sync::atomic::AtomicBool; +use std::time::Duration; diff --git a/codex-rs/code-mode-host/src/lib.rs b/codex-rs/code-mode-host/src/lib.rs new file mode 100644 index 0000000000..d2c1f507b4 --- /dev/null +++ b/codex-rs/code-mode-host/src/lib.rs @@ -0,0 +1,605 @@ +use std::collections::HashMap; +use std::collections::HashSet; +use std::collections::VecDeque; +use std::sync::Arc; +use std::sync::Mutex; +use std::sync::PoisonError; +use std::sync::atomic::AtomicBool; +use std::sync::atomic::Ordering; +use std::time::Duration; + +use anyhow::Context; +use anyhow::Result; +use codex_code_mode::InProcessCodeModeSession; +use codex_code_mode_protocol::host::CapabilitySet; +use codex_code_mode_protocol::host::ClientToHost; +use codex_code_mode_protocol::host::EncodedFrame; +use codex_code_mode_protocol::host::FramedReader; +use codex_code_mode_protocol::host::FramedWriter; +use codex_code_mode_protocol::host::HandshakeRejectReason; +use codex_code_mode_protocol::host::HostHello; +use codex_code_mode_protocol::host::HostRequest; +use codex_code_mode_protocol::host::HostResponse; +use codex_code_mode_protocol::host::HostToClient; +use codex_code_mode_protocol::host::ProtocolVersion; +use codex_code_mode_protocol::host::RequestId; +use codex_code_mode_protocol::host::SessionId; +use codex_code_mode_protocol::host::SupportedProtocolVersions; +use tokio::io::AsyncRead; +use tokio::io::AsyncWrite; +use tokio::sync::Semaphore; +use tokio::sync::mpsc; +use tokio_util::sync::CancellationToken; +use tokio_util::task::TaskTracker; + +use self::delegate::RemoteDelegate; +use self::peer::HostPeer; + +mod delegate; +mod peer; + +const MAX_IN_FLIGHT_REQUESTS: usize = 256; +const MAX_ACTIVE_CELLS: usize = 128; +const MAX_RECENT_REQUEST_IDS: usize = 4096; +const MAX_RECENT_SESSION_IDS: usize = 4096; +const SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(5); + +/// Runs one code-mode host connection over the process standard streams. +pub async fn run_stdio() -> Result<()> { + run(tokio::io::stdin(), tokio::io::stdout()).await +} + +/// Runs one code-mode host connection over an ordered input/output pair. +async fn run(reader: R, writer: W) -> Result<()> +where + R: AsyncRead + Send + Unpin + 'static, + W: AsyncWrite + Send + Unpin + 'static, +{ + let mut reader = FramedReader::new(reader); + let mut writer = FramedWriter::new(writer); + if !negotiate(&mut reader, &mut writer).await? { + return Ok(()); + } + + let (outgoing_tx, mut outgoing_rx) = mpsc::channel::(/*max_capacity*/ 128); + let peer = Arc::new(HostPeer::new(outgoing_tx)); + let state = Arc::new(HostState { + sessions: Mutex::new(HashMap::new()), + seen_session_ids: Mutex::new(SeenSessionIds::default()), + requests: Mutex::new(RequestRegistry::default()), + request_tasks: TaskTracker::new(), + request_permits: Arc::new(Semaphore::new(MAX_IN_FLIGHT_REQUESTS)), + active_cell_permits: Arc::new(Semaphore::new(MAX_ACTIVE_CELLS)), + closing: AtomicBool::new(false), + peer: Arc::clone(&peer), + }); + let writer_disconnected = peer.disconnection_token(); + let writer_task = tokio::spawn(async move { + loop { + tokio::select! { + _ = writer_disconnected.cancelled() => return Ok::<(), anyhow::Error>(()), + frame = outgoing_rx.recv() => { + let Some(frame) = frame else { + return Ok(()); + }; + if let Err(err) = writer.write_frame(&frame).await { + return Err( + anyhow::Error::new(err) + .context("failed to write code-mode host message") + ); + } + } + } + } + }); + let writer_peer = Arc::clone(&peer); + let writer_supervisor = tokio::spawn(async move { + match writer_task.await { + Ok(Ok(())) if !writer_peer.is_disconnected() => { + writer_peer.fail("code-mode writer task exited unexpectedly".to_string()); + } + Ok(Ok(())) => {} + Ok(Err(err)) => { + writer_peer.fail(format!("code-mode writer task failed: {err:#}")); + } + Err(err) => { + writer_peer.fail(format!("code-mode writer task failed: {err}")); + } + } + }); + + let input_result = async { + loop { + let message = tokio::select! { + _ = peer.disconnected() => break, + message = reader.read::() => message + .context("failed to read code-mode client message")?, + }; + let Some(message) = message else { + break; + }; + match message { + ClientToHost::ClientHello(_) => { + anyhow::bail!("received a second code-mode client hello"); + } + ClientToHost::Request { id, request } => { + state.spawn_request(id, request)?; + } + ClientToHost::CancelRequest { id } => { + state.cancel_request(id); + } + ClientToHost::DelegateResponse { id, result } => { + peer.complete(id, result.into_result()).await; + } + } + } + Ok::<(), anyhow::Error>(()) + } + .await; + + peer.disconnect(); + if tokio::time::timeout(SHUTDOWN_TIMEOUT, state.disconnect()) + .await + .is_err() + { + peer.fail("timed out shutting down code-mode host state".to_string()); + } + drop(state); + tokio::time::timeout(SHUTDOWN_TIMEOUT, writer_supervisor) + .await + .context("timed out supervising code-mode writer task")? + .context("code-mode writer supervisor task failed")?; + let failure = peer.failure(); + drop(peer); + input_result?; + if let Some(failure) = failure { + anyhow::bail!(failure); + } + Ok(()) +} + +async fn negotiate(reader: &mut FramedReader, writer: &mut FramedWriter) -> Result +where + R: AsyncRead + Unpin, + W: AsyncWrite + Unpin, +{ + let Some(first_message) = reader + .read::() + .await + .context("failed to read code-mode client hello")? + else { + return Ok(false); + }; + let ClientToHost::ClientHello(client_hello) = first_message else { + writer + .write(&HostToClient::HandshakeRejected { + reason: HandshakeRejectReason::InvalidHello { + message: "first message must be connection/hello".to_string(), + }, + }) + .await + .context("failed to reject invalid code-mode client hello")?; + return Ok(false); + }; + + let supported_versions = SupportedProtocolVersions::try_new([ProtocolVersion::V1])?; + if !client_hello + .supported_versions() + .contains(ProtocolVersion::V1) + { + writer + .write(&HostToClient::HandshakeRejected { + reason: HandshakeRejectReason::NoCompatibleVersion { supported_versions }, + }) + .await + .context("failed to reject incompatible code-mode client")?; + return Ok(false); + } + + let host_capabilities = CapabilitySet::empty(); + if let Some(capability) = client_hello + .required_capabilities() + .iter() + .find(|capability| !host_capabilities.contains(capability)) + { + writer + .write(&HostToClient::HandshakeRejected { + reason: HandshakeRejectReason::MissingRequiredCapability { + capability: capability.clone(), + }, + }) + .await + .context("failed to reject unsupported code-mode capability")?; + return Ok(false); + } + + writer + .write(&HostToClient::HostHello(HostHello::new( + ProtocolVersion::V1, + host_capabilities, + ))) + .await + .context("failed to write code-mode host hello")?; + Ok(true) +} + +struct HostState { + sessions: Mutex>>, + seen_session_ids: Mutex, + requests: Mutex, + request_tasks: TaskTracker, + request_permits: Arc, + active_cell_permits: Arc, + closing: AtomicBool, + peer: Arc, +} + +impl HostState { + fn spawn_request( + self: &Arc, + request_id: RequestId, + request: HostRequest, + ) -> Result<(), anyhow::Error> { + let cancellation = self + .requests + .lock() + .unwrap_or_else(PoisonError::into_inner) + .start(request_id, RequestKind::from(&request))?; + let Ok(permit) = Arc::clone(&self.request_permits).try_acquire_owned() else { + self.respond( + request_id, + Err("code-mode host has too many in-flight requests".to_string()), + ); + self.finish_request(request_id); + return Ok(()); + }; + let state = Arc::clone(self); + let request_task = self.request_tasks.spawn(async move { + let _permit = permit; + state + .handle_request(request_id, request, cancellation) + .await; + state.finish_request(request_id); + }); + self.supervise_request_task(request_task); + Ok(()) + } + + fn supervise_request_task(&self, task: tokio::task::JoinHandle<()>) { + let peer = Arc::clone(&self.peer); + tokio::spawn(async move { + if let Err(err) = task.await { + peer.fail(format!("code-mode request task failed: {err}")); + } + }); + } + + async fn handle_request( + &self, + request_id: RequestId, + request: HostRequest, + cancellation: CancellationToken, + ) { + if self.closing.load(Ordering::Acquire) { + self.respond( + request_id, + Err("code-mode host is shutting down".to_string()), + ); + return; + } + match request { + HostRequest::OpenSession { session_id } => { + let result = self + .open_session(session_id.clone()) + .map(|()| HostResponse::SessionReady { session_id }); + self.respond(request_id, result); + } + HostRequest::Execute { + session_id, + request, + } => { + if cancellation.is_cancelled() { + self.respond(request_id, Err("code-mode request cancelled".to_string())); + return; + } + let request = match request.try_into() { + Ok(request) => request, + Err(err) => { + self.respond( + request_id, + Err(format!("invalid code-mode execute request: {err}")), + ); + return; + } + }; + let session = match self.session(&session_id) { + Ok(session) => session, + Err(err) => { + self.respond(request_id, Err(err)); + return; + } + }; + let Ok(active_cell_permit) = + Arc::clone(&self.active_cell_permits).try_acquire_owned() + else { + self.respond( + request_id, + Err("code-mode host has too many active cells".to_string()), + ); + return; + }; + let result = session.execute(request).await; + match result { + Ok(started) => { + let cell_id = started.cell_id.clone(); + self.respond( + request_id, + Ok(HostResponse::ExecutionStarted { + cell_id: cell_id.into(), + }), + ); + let initial_response_sent = self.peer.start_cell( + session_id, + request_id, + started, + active_cell_permit, + ); + let _ = initial_response_sent.await; + } + Err(err) => self.respond(request_id, Err(err)), + } + } + HostRequest::Wait { + session_id, + request, + } => { + let result = match self.session(&session_id) { + Ok(session) => { + tokio::select! { + biased; + _ = cancellation.cancelled() => { + Err("code-mode request cancelled".to_string()) + } + result = session.wait(request.into()) => result.map(|outcome| { + HostResponse::WaitCompleted { + outcome: outcome.into(), + } + }), + } + } + Err(err) => Err(err), + }; + self.respond(request_id, result); + } + HostRequest::Terminate { + session_id, + cell_id, + } => { + let result = match self.session(&session_id) { + Ok(session) => session.terminate(cell_id.into()).await.map(|outcome| { + HostResponse::WaitCompleted { + outcome: outcome.into(), + } + }), + Err(err) => Err(err), + }; + self.respond(request_id, result); + } + HostRequest::ShutdownSession { session_id } => { + let session = self + .sessions + .lock() + .unwrap_or_else(PoisonError::into_inner) + .remove(&session_id); + let result = match session { + Some(session) => match session.shutdown().await { + Ok(()) => { + self.peer.wait_for_session_cells(&session_id).await; + Ok(HostResponse::SessionClosed { session_id }) + } + Err(err) => Err(err), + }, + None => Err(format!("unknown code-mode session {session_id}")), + }; + self.respond(request_id, result); + } + } + } + + fn open_session(&self, session_id: SessionId) -> Result<(), String> { + let mut sessions = self.sessions.lock().unwrap_or_else(PoisonError::into_inner); + if sessions.contains_key(&session_id) { + return Err(format!( + "code-mode session ID `{session_id}` is already open" + )); + } + if self.closing.load(Ordering::Acquire) { + return Err("code-mode host is shutting down".to_string()); + } + if !self + .seen_session_ids + .lock() + .unwrap_or_else(PoisonError::into_inner) + .remember(session_id.clone()) + { + return Err(format!("code-mode session ID `{session_id}` was reused")); + } + let delegate = Arc::new(RemoteDelegate::new( + session_id.clone(), + Arc::clone(&self.peer), + )); + let peer = Arc::downgrade(&self.peer); + let task_failure_handler = Arc::new(move |reason| { + if let Some(peer) = peer.upgrade() { + peer.fail(reason); + } + }); + sessions.insert( + session_id, + Arc::new( + InProcessCodeModeSession::with_delegate_and_task_failure_handler( + delegate, + task_failure_handler, + ), + ), + ); + Ok(()) + } + + fn session(&self, session_id: &SessionId) -> Result, String> { + self.sessions + .lock() + .unwrap_or_else(PoisonError::into_inner) + .get(session_id) + .cloned() + .ok_or_else(|| format!("unknown code-mode session {session_id}")) + } + + fn respond(&self, id: RequestId, result: Result) { + self.peer.respond(id, result); + } + + fn cancel_request(&self, request_id: RequestId) { + self.requests + .lock() + .unwrap_or_else(PoisonError::into_inner) + .cancel(request_id); + } + + fn finish_request(&self, request_id: RequestId) { + self.requests + .lock() + .unwrap_or_else(PoisonError::into_inner) + .finish(request_id); + } + + async fn disconnect(&self) { + self.closing.store(true, Ordering::Release); + self.requests + .lock() + .unwrap_or_else(PoisonError::into_inner) + .cancel_all(); + self.request_tasks.close(); + self.request_tasks.wait().await; + let sessions = self + .sessions + .lock() + .unwrap_or_else(PoisonError::into_inner) + .drain() + .map(|(_, session)| session) + .collect::>(); + for session in sessions { + let _ = session.shutdown().await; + } + } +} + +#[derive(Clone, Copy)] +enum RequestKind { + OpenSession, + Execute, + Wait, + Terminate, + ShutdownSession, +} + +impl RequestKind { + fn from(request: &HostRequest) -> Self { + match request { + HostRequest::OpenSession { .. } => Self::OpenSession, + HostRequest::Execute { .. } => Self::Execute, + HostRequest::Wait { .. } => Self::Wait, + HostRequest::Terminate { .. } => Self::Terminate, + HostRequest::ShutdownSession { .. } => Self::ShutdownSession, + } + } + + fn is_cancellable(self) -> bool { + matches!(self, Self::Execute | Self::Wait) + } +} + +struct ActiveRequest { + kind: RequestKind, + cancellation: CancellationToken, +} + +#[derive(Default)] +struct RequestRegistry { + active: HashMap, + recent: HashSet, + recent_order: VecDeque, +} + +impl RequestRegistry { + fn start( + &mut self, + request_id: RequestId, + kind: RequestKind, + ) -> Result { + if self.active.contains_key(&request_id) || self.recent.contains(&request_id) { + anyhow::bail!("duplicate code-mode request ID {request_id:?}"); + } + let cancellation = CancellationToken::new(); + self.active.insert( + request_id, + ActiveRequest { + kind, + cancellation: cancellation.clone(), + }, + ); + Ok(cancellation) + } + + fn cancel(&self, request_id: RequestId) { + if let Some(request) = self.active.get(&request_id) + && request.kind.is_cancellable() + { + request.cancellation.cancel(); + } + } + + fn finish(&mut self, request_id: RequestId) { + if self.active.remove(&request_id).is_none() { + return; + } + self.recent.insert(request_id); + self.recent_order.push_back(request_id); + while self.recent_order.len() > MAX_RECENT_REQUEST_IDS { + if let Some(expired) = self.recent_order.pop_front() { + self.recent.remove(&expired); + } + } + } + + fn cancel_all(&self) { + for request in self.active.values() { + request.cancellation.cancel(); + } + } +} + +#[derive(Default)] +struct SeenSessionIds { + ids: HashSet, + order: VecDeque, +} + +impl SeenSessionIds { + fn remember(&mut self, session_id: SessionId) -> bool { + if !self.ids.insert(session_id.clone()) { + return false; + } + self.order.push_back(session_id); + while self.order.len() > MAX_RECENT_SESSION_IDS { + if let Some(expired) = self.order.pop_front() { + self.ids.remove(&expired); + } + } + true + } +} + +#[cfg(test)] +#[path = "host_tests.rs"] +mod tests; diff --git a/codex-rs/code-mode-host/src/main.rs b/codex-rs/code-mode-host/src/main.rs index f328e4d9d0..b215869c8a 100644 --- a/codex-rs/code-mode-host/src/main.rs +++ b/codex-rs/code-mode-host/src/main.rs @@ -1 +1,4 @@ -fn main() {} +#[tokio::main(flavor = "current_thread")] +async fn main() -> anyhow::Result<()> { + codex_code_mode_host::run_stdio().await +} diff --git a/codex-rs/code-mode-host/src/peer.rs b/codex-rs/code-mode-host/src/peer.rs new file mode 100644 index 0000000000..14ff068e7b --- /dev/null +++ b/codex-rs/code-mode-host/src/peer.rs @@ -0,0 +1,541 @@ +use std::collections::HashMap; +use std::collections::VecDeque; +use std::sync::Arc; +use std::sync::Mutex as StdMutex; +use std::sync::PoisonError; +use std::sync::atomic::AtomicI64; +use std::sync::atomic::Ordering; + +use codex_code_mode_protocol::CellId; +use codex_code_mode_protocol::StartedCell; +use codex_code_mode_protocol::host::DelegateRequest; +use codex_code_mode_protocol::host::DelegateRequestId; +use codex_code_mode_protocol::host::DelegateResponse; +use codex_code_mode_protocol::host::EncodedFrame; +use codex_code_mode_protocol::host::HostToClient; +use codex_code_mode_protocol::host::RequestId; +use codex_code_mode_protocol::host::SessionId; +use codex_code_mode_protocol::host::WireResult; +use tokio::sync::Mutex; +use tokio::sync::Notify; +use tokio::sync::OwnedSemaphorePermit; +use tokio::sync::Semaphore; +use tokio::sync::mpsc; +use tokio::sync::oneshot; +use tokio_util::sync::CancellationToken; + +const CELL_MESSAGE_CAPACITY: usize = 128; +const MAX_PENDING_DELEGATE_CALLS: usize = 256; + +pub(super) struct HostPeer { + outgoing_tx: mpsc::Sender, + pending: Mutex>, + delegate_permits: Arc, + cell_routes: StdMutex>, + cell_routes_changed: Notify, + next_request_id: AtomicI64, + disconnected: CancellationToken, + failure: StdMutex>, +} + +struct PendingDelegate { + response_tx: oneshot::Sender>, + dispatched: bool, + _permit: OwnedSemaphorePermit, +} + +enum CellRoute { + Pending(VecDeque), + Active(mpsc::Sender), +} + +enum CellMessage { + Delegate { + id: DelegateRequestId, + request: DelegateRequest, + dispatched_tx: oneshot::Sender>, + }, + Closed, +} + +impl HostPeer { + pub(super) fn new(outgoing_tx: mpsc::Sender) -> Self { + Self { + outgoing_tx, + pending: Mutex::new(HashMap::new()), + delegate_permits: Arc::new(Semaphore::new(MAX_PENDING_DELEGATE_CALLS)), + cell_routes: StdMutex::new(HashMap::new()), + cell_routes_changed: Notify::new(), + next_request_id: AtomicI64::new(1), + disconnected: CancellationToken::new(), + failure: StdMutex::new(None), + } + } + + pub(super) fn send(&self, message: HostToClient) -> Result<(), PeerSendError> { + let frame = EncodedFrame::encode(&message) + .map_err(|err| PeerSendError::Payload(err.to_string()))?; + self.send_frame(frame) + } + + pub(super) fn respond( + &self, + id: RequestId, + result: Result, + ) { + let message = HostToClient::Response { + id, + result: WireResult::from_result(result), + }; + if let Err(PeerSendError::Payload(err)) = self.send(message) { + let _ = self.send(HostToClient::Response { + id, + result: WireResult::Err { + message: format!("code-mode host response exceeds the IPC frame limit: {err}"), + }, + }); + } + } + + fn initial_response( + &self, + id: RequestId, + result: Result, + ) { + let message = HostToClient::InitialResponse { + id, + result: WireResult::from_result(result), + }; + if let Err(PeerSendError::Payload(err)) = self.send(message) { + let _ = self.send(HostToClient::InitialResponse { + id, + result: WireResult::Err { + message: format!( + "code-mode initial response exceeds the IPC frame limit: {err}" + ), + }, + }); + } + } + + pub(super) async fn call( + self: &Arc, + session_id: SessionId, + request: DelegateRequest, + cancellation_token: CancellationToken, + ) -> Result { + if self.disconnected.is_cancelled() { + return Err("code-mode client connection closed".to_string()); + } + let Ok(permit) = Arc::clone(&self.delegate_permits).try_acquire_owned() else { + return Err("code-mode host has too many pending delegate calls".to_string()); + }; + let id = DelegateRequestId::new(self.next_request_id.fetch_add(1, Ordering::Relaxed)); + let (response_tx, response_rx) = oneshot::channel(); + self.pending.lock().await.insert( + id, + PendingDelegate { + response_tx, + dispatched: false, + _permit: permit, + }, + ); + let mut pending = PendingDelegateRequest::new(Arc::clone(self), id); + let cell_id = match &request { + DelegateRequest::InvokeTool { invocation } => invocation.cell_id.clone().into(), + DelegateRequest::Notify { cell_id, .. } => cell_id.clone().into(), + }; + let (dispatched_tx, dispatched_rx) = oneshot::channel(); + if let Err(err) = self.route_cell_message( + (session_id, cell_id), + CellMessage::Delegate { + id, + request, + dispatched_tx, + }, + ) { + self.pending.lock().await.remove(&id); + pending.disarm(); + return Err(err); + } + + let dispatched = tokio::select! { + dispatched = dispatched_rx => dispatched.map_err(|_| { + "code-mode cell route closed before dispatching delegate request".to_string() + })?, + _ = self.disconnected.cancelled() => { + self.pending.lock().await.remove(&id); + pending.disarm(); + return Err("code-mode client connection closed".to_string()); + } + }; + if let Err(err) = dispatched { + self.pending.lock().await.remove(&id); + pending.disarm(); + return Err(err); + } + + tokio::select! { + response = response_rx => { + pending.disarm(); + response.map_err(|_| { + "code-mode client closed before returning delegate output".to_string() + })? + } + _ = cancellation_token.cancelled() => { + if self.remove_pending(id).await.is_some() { + let _ = self.send(HostToClient::CancelDelegateRequest { id }); + } + pending.disarm(); + Err("code mode delegate request cancelled".to_string()) + } + _ = self.disconnected.cancelled() => { + self.pending.lock().await.remove(&id); + pending.disarm(); + Err("code-mode client connection closed".to_string()) + } + } + } + + pub(super) async fn complete( + &self, + id: DelegateRequestId, + response: Result, + ) { + if let Some(pending) = self.remove_pending(id).await { + let _ = pending.response_tx.send(response); + } + } + + pub(super) fn start_cell( + self: &Arc, + session_id: SessionId, + request_id: RequestId, + started: StartedCell, + active_cell_permit: OwnedSemaphorePermit, + ) -> oneshot::Receiver<()> { + let (initial_response_sent_tx, initial_response_sent_rx) = oneshot::channel(); + let key = (session_id, started.cell_id.clone()); + let (messages_tx, messages_rx) = mpsc::channel(CELL_MESSAGE_CAPACITY); + let previous = self + .cell_routes + .lock() + .unwrap_or_else(PoisonError::into_inner) + .insert(key.clone(), CellRoute::Active(messages_tx.clone())); + match previous { + Some(CellRoute::Pending(messages)) => { + for message in messages { + if messages_tx.try_send(message).is_err() { + self.disconnect(); + return initial_response_sent_rx; + } + } + } + Some(CellRoute::Active(_)) => { + self.disconnect(); + return initial_response_sent_rx; + } + None => {} + } + let peer = Arc::clone(self); + self.spawn_critical("cell forwarding", async move { + drive_cell( + peer, + key, + request_id, + started, + messages_rx, + initial_response_sent_tx, + active_cell_permit, + ) + .await; + }); + initial_response_sent_rx + } + + pub(super) fn close_cell(&self, session_id: SessionId, cell_id: CellId) { + let _ = self.route_cell_message((session_id, cell_id), CellMessage::Closed); + } + + pub(super) fn disconnect(&self) { + self.disconnected.cancel(); + } + + pub(super) fn fail(&self, reason: String) { + let mut failure = self.failure.lock().unwrap_or_else(PoisonError::into_inner); + if failure.is_none() { + *failure = Some(reason); + } + drop(failure); + self.disconnect(); + } + + pub(super) fn failure(&self) -> Option { + self.failure + .lock() + .unwrap_or_else(PoisonError::into_inner) + .clone() + } + + pub(super) fn is_disconnected(&self) -> bool { + self.disconnected.is_cancelled() + } + + pub(super) async fn disconnected(&self) { + self.disconnected.cancelled().await; + } + + pub(super) fn disconnection_token(&self) -> CancellationToken { + self.disconnected.clone() + } + + pub(super) async fn wait_for_session_cells(&self, session_id: &SessionId) { + loop { + let changed = self.cell_routes_changed.notified(); + if !self + .cell_routes + .lock() + .unwrap_or_else(PoisonError::into_inner) + .keys() + .any(|(route_session_id, _)| route_session_id == session_id) + { + return; + } + tokio::select! { + _ = changed => {} + _ = self.disconnected.cancelled() => return, + } + } + } + + async fn send_delegate_if_pending( + &self, + id: DelegateRequestId, + session_id: SessionId, + request: DelegateRequest, + dispatched_tx: oneshot::Sender>, + ) { + let result = { + let mut pending = self.pending.lock().await; + let Some(pending) = pending.get_mut(&id) else { + let _ = dispatched_tx.send(Err( + "code-mode delegate request was cancelled before dispatch".to_string(), + )); + return; + }; + match self.send(HostToClient::DelegateRequest { + id, + session_id, + request, + }) { + Ok(()) => { + pending.dispatched = true; + Ok(()) + } + Err(err) => Err(err.to_string()), + } + }; + let _ = dispatched_tx.send(result); + } + + fn route_cell_message( + &self, + key: (SessionId, CellId), + message: CellMessage, + ) -> Result<(), String> { + use std::collections::hash_map::Entry; + + let result = match self + .cell_routes + .lock() + .unwrap_or_else(PoisonError::into_inner) + .entry(key) + { + Entry::Occupied(mut entry) => match entry.get_mut() { + CellRoute::Pending(messages) if messages.len() < CELL_MESSAGE_CAPACITY => { + messages.push_back(message); + Ok(()) + } + CellRoute::Pending(_) => Err("code-mode cell message queue is full".to_string()), + CellRoute::Active(sender) => sender + .try_send(message) + .map_err(|_| "code-mode cell message queue is unavailable".to_string()), + }, + Entry::Vacant(entry) => { + entry.insert(CellRoute::Pending(VecDeque::from([message]))); + Ok(()) + } + }; + if result.is_err() { + self.disconnect(); + } + result + } + + async fn remove_pending(&self, id: DelegateRequestId) -> Option { + self.pending.lock().await.remove(&id) + } + + pub(super) fn spawn_critical(self: &Arc, task_name: &'static str, future: F) + where + F: std::future::Future + Send + 'static, + { + let task = tokio::spawn(future); + let peer = Arc::clone(self); + tokio::spawn(async move { + if let Err(err) = task.await { + peer.fail(format!("code-mode {task_name} task failed: {err}")); + } + }); + } + + fn send_frame(&self, frame: EncodedFrame) -> Result<(), PeerSendError> { + match self.outgoing_tx.try_send(frame) { + Ok(()) => Ok(()), + Err(mpsc::error::TrySendError::Full(_)) => { + self.disconnect(); + Err(PeerSendError::Unavailable( + "code-mode host outgoing queue is full".to_string(), + )) + } + Err(mpsc::error::TrySendError::Closed(_)) => { + self.disconnect(); + Err(PeerSendError::Unavailable( + "code-mode client connection closed".to_string(), + )) + } + } + } +} + +async fn drive_cell( + peer: Arc, + key: (SessionId, CellId), + request_id: RequestId, + started: StartedCell, + mut messages_rx: mpsc::Receiver, + initial_response_sent_tx: oneshot::Sender<()>, + _active_cell_permit: OwnedSemaphorePermit, +) { + let mut initial_response_sent_tx = Some(initial_response_sent_tx); + let initial_response = started.initial_response(); + tokio::pin!(initial_response); + let closed = loop { + tokio::select! { + biased; + result = &mut initial_response => { + peer.initial_response(request_id, result.map(Into::into)); + if let Some(initial_response_sent_tx) = initial_response_sent_tx.take() { + let _ = initial_response_sent_tx.send(()); + } + break false; + } + message = messages_rx.recv() => match message { + Some(CellMessage::Delegate { + id, + request, + dispatched_tx, + }) => { + peer.send_delegate_if_pending(id, key.0.clone(), request, dispatched_tx).await; + } + Some(CellMessage::Closed) | None => break true, + }, + _ = peer.disconnected.cancelled() => { + peer.remove_cell_route(&key); + return; + } + } + }; + + if closed { + peer.initial_response(request_id, initial_response.await.map(Into::into)); + if let Some(initial_response_sent_tx) = initial_response_sent_tx.take() { + let _ = initial_response_sent_tx.send(()); + } + } else { + loop { + tokio::select! { + message = messages_rx.recv() => match message { + Some(CellMessage::Delegate { + id, + request, + dispatched_tx, + }) => { + peer.send_delegate_if_pending(id, key.0.clone(), request, dispatched_tx).await; + } + Some(CellMessage::Closed) | None => break, + }, + _ = peer.disconnected.cancelled() => { + peer.remove_cell_route(&key); + return; + } + } + } + } + let _ = peer.send(HostToClient::CellClosed { + session_id: key.0.clone(), + cell_id: (&key.1).into(), + }); + peer.remove_cell_route(&key); +} + +impl HostPeer { + fn remove_cell_route(&self, key: &(SessionId, CellId)) { + let removed = self + .cell_routes + .lock() + .unwrap_or_else(PoisonError::into_inner) + .remove(key); + if removed.is_some() { + self.cell_routes_changed.notify_waiters(); + } + } +} + +pub(super) enum PeerSendError { + Payload(String), + Unavailable(String), +} + +impl std::fmt::Display for PeerSendError { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Payload(message) | Self::Unavailable(message) => formatter.write_str(message), + } + } +} + +struct PendingDelegateRequest { + peer: Arc, + id: Option, +} + +impl PendingDelegateRequest { + fn new(peer: Arc, id: DelegateRequestId) -> Self { + Self { peer, id: Some(id) } + } + + fn disarm(&mut self) { + self.id = None; + } +} + +impl Drop for PendingDelegateRequest { + fn drop(&mut self) { + let Some(id) = self.id.take() else { + return; + }; + let peer = Arc::clone(&self.peer); + tokio::spawn(async move { + if let Some(pending) = peer.remove_pending(id).await + && pending.dispatched + { + let _ = peer.send(HostToClient::CancelDelegateRequest { id }); + } + }); + } +} + +#[cfg(test)] +#[path = "peer_tests.rs"] +mod tests; diff --git a/codex-rs/code-mode-host/src/peer_tests.rs b/codex-rs/code-mode-host/src/peer_tests.rs new file mode 100644 index 0000000000..a926027ce9 --- /dev/null +++ b/codex-rs/code-mode-host/src/peer_tests.rs @@ -0,0 +1,95 @@ +use std::sync::Arc; +use std::time::Duration; + +use codex_code_mode_protocol::CellId; +use codex_code_mode_protocol::RuntimeResponse; +use codex_code_mode_protocol::StartedCell; +use codex_code_mode_protocol::host::DelegateRequest; +use codex_code_mode_protocol::host::RequestId; +use codex_code_mode_protocol::host::SessionId; +use pretty_assertions::assert_eq; +use tokio::sync::Semaphore; +use tokio::sync::mpsc; +use tokio::sync::oneshot; +use tokio::sync::oneshot::error::TryRecvError; +use tokio_util::sync::CancellationToken; + +use super::HostPeer; +use super::MAX_PENDING_DELEGATE_CALLS; + +fn session_id(value: &str) -> SessionId { + SessionId::new(value).expect("session ID") +} + +#[tokio::test] +async fn start_cell_reports_when_initial_response_is_enqueued() { + let (outgoing_tx, mut outgoing_rx) = mpsc::channel(/*max_capacity*/ 4); + let peer = Arc::new(HostPeer::new(outgoing_tx)); + let cell_id = CellId::new("cell-1".to_string()); + let (response_tx, response_rx) = oneshot::channel(); + let started = StartedCell::new(cell_id.clone(), response_rx); + let active_cell_permits = Arc::new(Semaphore::new(/*permits*/ 1)); + let active_cell_permit = Arc::clone(&active_cell_permits) + .try_acquire_owned() + .expect("active cell permit"); + + let mut initial_response_sent = peer.start_cell( + session_id("session-1"), + RequestId::new(/*value*/ 1), + started, + active_cell_permit, + ); + assert_eq!(initial_response_sent.try_recv(), Err(TryRecvError::Empty)); + + response_tx + .send(RuntimeResponse::Result { + cell_id: cell_id.clone(), + content_items: Vec::new(), + error_text: None, + }) + .expect("initial response receiver"); + initial_response_sent + .await + .expect("initial response completion"); + outgoing_rx.recv().await.expect("initial response frame"); + assert_eq!(active_cell_permits.available_permits(), 0); + + peer.close_cell(session_id("session-1"), cell_id); + let permit = tokio::time::timeout( + Duration::from_secs(1), + Arc::clone(&active_cell_permits).acquire_owned(), + ) + .await + .expect("cell permit should be released") + .expect("cell permit semaphore should remain open"); + drop(permit); +} + +#[tokio::test] +async fn pending_delegate_limit_rejects_call_without_disconnecting() { + let (outgoing_tx, _outgoing_rx) = mpsc::channel(/*max_capacity*/ 1); + let peer = Arc::new(HostPeer::new(outgoing_tx)); + let permits = Arc::clone(&peer.delegate_permits) + .acquire_many_owned(MAX_PENDING_DELEGATE_CALLS as u32) + .await + .expect("delegate permits"); + + let result = peer + .call( + session_id("session-1"), + DelegateRequest::Notify { + call_id: "call-1".to_string(), + cell_id: CellId::new("cell-1".to_string()).into(), + text: "hello".to_string(), + }, + CancellationToken::new(), + ) + .await; + + assert_eq!( + result, + Err("code-mode host has too many pending delegate calls".to_string()) + ); + assert!(!peer.is_disconnected()); + drop(permits); +} diff --git a/codex-rs/code-mode-host/tests/stdio.rs b/codex-rs/code-mode-host/tests/stdio.rs new file mode 100644 index 0000000000..8edcbcaa8d --- /dev/null +++ b/codex-rs/code-mode-host/tests/stdio.rs @@ -0,0 +1,873 @@ +#![allow(clippy::expect_used)] + +use std::sync::Arc; +use std::sync::Mutex; +use std::sync::atomic::AtomicBool; +use std::sync::atomic::Ordering; +use std::time::Duration; + +#[cfg(unix)] +use std::os::unix::fs::PermissionsExt; + +use codex_code_mode::CellId; +use codex_code_mode::CodeModeNestedToolCall; +use codex_code_mode::CodeModeSession; +use codex_code_mode::CodeModeSessionDelegate; +use codex_code_mode::CodeModeSessionProvider; +use codex_code_mode::CodeModeToolKind; +use codex_code_mode::ExecuteRequest; +use codex_code_mode::FunctionCallOutputContentItem; +use codex_code_mode::NotificationFuture; +use codex_code_mode::ProcessOwnedCodeModeSessionProvider; +use codex_code_mode::RuntimeResponse; +use codex_code_mode::ToolDefinition; +use codex_code_mode::ToolInvocationFuture; +use codex_code_mode::WaitOutcome; +use codex_code_mode::WaitRequest; +use codex_code_mode::host::MAX_FRAME_BYTES; +use codex_protocol::ToolName; +use pretty_assertions::assert_eq; +use serde_json::json; +use tokio::sync::Semaphore; +use tokio::sync::mpsc; +use tokio_util::sync::CancellationToken; + +#[derive(Default)] +struct RecordingDelegate { + invocations: Mutex>, + notifications: Mutex>, + closed_cells: Mutex>, +} + +#[derive(Debug, Eq, PartialEq)] +enum CallbackEvent { + Started(String), + Cancelled(String), + CellClosed(CellId), +} + +struct CancellationDelegate { + events_tx: mpsc::UnboundedSender, + fast_tool_release: Semaphore, + slow_tool_started: Semaphore, + hold_slow_cleanup: AtomicBool, + slow_cleanup_release: Semaphore, +} + +struct OversizedResultDelegate; + +impl CodeModeSessionDelegate for OversizedResultDelegate { + fn invoke_tool<'a>( + &'a self, + _invocation: CodeModeNestedToolCall, + _cancellation_token: CancellationToken, + ) -> ToolInvocationFuture<'a> { + Box::pin(async { Ok(json!("x".repeat(MAX_FRAME_BYTES))) }) + } + + fn notify<'a>( + &'a self, + _call_id: String, + _cell_id: CellId, + _text: String, + _cancellation_token: CancellationToken, + ) -> NotificationFuture<'a> { + Box::pin(async { Ok(()) }) + } + + fn cell_closed(&self, _cell_id: &CellId) {} +} + +impl CancellationDelegate { + fn new() -> (Arc, mpsc::UnboundedReceiver) { + let (events_tx, events_rx) = mpsc::unbounded_channel(); + ( + Arc::new(Self { + events_tx, + fast_tool_release: Semaphore::new(/*permits*/ 0), + slow_tool_started: Semaphore::new(/*permits*/ 0), + hold_slow_cleanup: AtomicBool::new(false), + slow_cleanup_release: Semaphore::new(/*permits*/ 0), + }), + events_rx, + ) + } + + #[cfg(unix)] + fn hold_slow_cleanup(&self) { + self.hold_slow_cleanup.store(true, Ordering::Release); + } + + #[cfg(unix)] + fn release_slow_cleanup(&self) { + self.slow_cleanup_release.add_permits(1); + } +} + +impl CodeModeSessionDelegate for CancellationDelegate { + fn invoke_tool<'a>( + &'a self, + invocation: CodeModeNestedToolCall, + cancellation_token: CancellationToken, + ) -> ToolInvocationFuture<'a> { + Box::pin(async move { + let tool_name = invocation.tool_name.name.clone(); + if tool_name == "tool_call_barrier" { + let permit = self + .slow_tool_started + .acquire() + .await + .map_err(|_| "slow tool barrier closed".to_string())?; + permit.forget(); + return Ok(json!({ "tool": tool_name })); + } + let _ = self + .events_tx + .send(CallbackEvent::Started(tool_name.clone())); + if tool_name == "tool_call_slow" { + self.slow_tool_started.add_permits(1); + cancellation_token.cancelled().await; + let _ = self.events_tx.send(CallbackEvent::Cancelled(tool_name)); + if self.hold_slow_cleanup.load(Ordering::Acquire) { + let permit = self + .slow_cleanup_release + .acquire() + .await + .map_err(|_| "slow tool cleanup release closed".to_string())?; + permit.forget(); + } + return Err("slow tool cancelled".to_string()); + } + let permit = self + .fast_tool_release + .acquire() + .await + .map_err(|_| "fast tool release closed".to_string())?; + permit.forget(); + Ok(json!({ "tool": tool_name })) + }) + } + + fn notify<'a>( + &'a self, + _call_id: String, + _cell_id: CellId, + _text: String, + _cancellation_token: CancellationToken, + ) -> NotificationFuture<'a> { + Box::pin(async { Ok(()) }) + } + + fn cell_closed(&self, cell_id: &CellId) { + let _ = self + .events_tx + .send(CallbackEvent::CellClosed(cell_id.clone())); + } +} + +impl CodeModeSessionDelegate for RecordingDelegate { + fn invoke_tool<'a>( + &'a self, + invocation: CodeModeNestedToolCall, + _cancellation_token: CancellationToken, + ) -> ToolInvocationFuture<'a> { + self.invocations + .lock() + .expect("invocations lock") + .push(invocation); + Box::pin(async { Ok(json!({ "value": "output" })) }) + } + + fn notify<'a>( + &'a self, + call_id: String, + cell_id: CellId, + text: String, + _cancellation_token: CancellationToken, + ) -> NotificationFuture<'a> { + self.notifications + .lock() + .expect("notifications lock") + .push((call_id, cell_id, text)); + Box::pin(async { Ok(()) }) + } + + fn cell_closed(&self, cell_id: &CellId) { + self.closed_cells + .lock() + .expect("closed cells lock") + .push(cell_id.clone()); + } +} + +fn cell_id(value: &str) -> CellId { + CellId::new(value.to_string()) +} + +fn execute_request(source: &str) -> ExecuteRequest { + ExecuteRequest { + tool_call_id: "call-1".to_string(), + enabled_tools: Vec::new(), + source: source.to_string(), + yield_time_ms: None, + max_output_tokens: None, + } +} + +async fn execute(session: &Arc, request: ExecuteRequest) -> RuntimeResponse { + session + .execute(request) + .await + .expect("start execution") + .initial_response() + .await + .expect("initial response") +} + +async fn execute_to_terminal( + session: &Arc, + request: ExecuteRequest, +) -> RuntimeResponse { + let started = session.execute(request).await.expect("start execution"); + let mut response = started.initial_response().await.expect("initial response"); + loop { + match response { + RuntimeResponse::Yielded { cell_id, .. } => { + response = match session + .wait(WaitRequest { + cell_id, + yield_time_ms: 60_000, + }) + .await + .expect("wait for terminal response") + { + WaitOutcome::LiveCell(response) | WaitOutcome::MissingCell(response) => { + response + } + }; + } + response => return response, + } + } +} + +async fn next_callback_event( + events_rx: &mut mpsc::UnboundedReceiver, +) -> CallbackEvent { + tokio::time::timeout(Duration::from_secs(5), events_rx.recv()) + .await + .expect("callback event timeout") + .expect("callback event stream closed") +} + +#[tokio::test] +async fn remote_session_persists_values_forwards_delegates_and_controls_cells() { + let provider = ProcessOwnedCodeModeSessionProvider::with_host_program( + codex_utils_cargo_bin::cargo_bin("codex-code-mode-host").expect("host binary"), + ); + let delegate = Arc::new(RecordingDelegate::default()); + let session = provider + .create_session(delegate.clone()) + .await + .expect("create remote session"); + + assert_eq!( + execute(&session, execute_request(r#"store("key", "persisted");"#),).await, + RuntimeResponse::Result { + cell_id: cell_id("1"), + content_items: Vec::new(), + error_text: None, + } + ); + + let mut callback_request = execute_request( + r#" +const result = await tools.echo({ value: String(load("key")) }); +notify("notice"); +text(result.value); +"#, + ); + callback_request.tool_call_id = "call-2".to_string(); + callback_request.enabled_tools = vec![ToolDefinition { + name: "echo".to_string(), + tool_name: ToolName::plain("echo"), + description: String::new(), + kind: CodeModeToolKind::Function, + input_schema: None, + output_schema: None, + }]; + assert_eq!( + execute(&session, callback_request).await, + RuntimeResponse::Result { + cell_id: cell_id("2"), + content_items: vec![FunctionCallOutputContentItem::InputText { + text: "output".to_string(), + }], + error_text: None, + } + ); + assert_eq!( + *delegate.invocations.lock().expect("invocations lock"), + vec![CodeModeNestedToolCall { + cell_id: cell_id("2"), + runtime_tool_call_id: "tool-1".to_string(), + tool_name: ToolName::plain("echo"), + tool_kind: CodeModeToolKind::Function, + input: Some(json!({ "value": "persisted" })), + }] + ); + assert_eq!( + *delegate.notifications.lock().expect("notifications lock"), + vec![("call-2".to_string(), cell_id("2"), "notice".to_string())] + ); + + let mut pending_request = execute_request("await new Promise(() => {});"); + pending_request.tool_call_id = "call-3".to_string(); + pending_request.yield_time_ms = Some(1); + assert_eq!( + execute(&session, pending_request).await, + RuntimeResponse::Yielded { + cell_id: cell_id("3"), + content_items: Vec::new(), + } + ); + assert_eq!( + session + .wait(WaitRequest { + cell_id: cell_id("3"), + yield_time_ms: 1, + }) + .await + .expect("wait for cell"), + WaitOutcome::LiveCell(RuntimeResponse::Yielded { + cell_id: cell_id("3"), + content_items: Vec::new(), + }) + ); + assert_eq!( + session + .terminate(cell_id("3")) + .await + .expect("terminate cell"), + WaitOutcome::LiveCell(RuntimeResponse::Terminated { + cell_id: cell_id("3"), + content_items: Vec::new(), + }) + ); + + session.shutdown().await.expect("shutdown remote session"); + assert_eq!( + *delegate.closed_cells.lock().expect("closed cells lock"), + vec![cell_id("1"), cell_id("2"), cell_id("3")] + ); +} + +#[tokio::test] +async fn dropping_long_wait_releases_observer_before_next_wait() { + let provider = ProcessOwnedCodeModeSessionProvider::with_host_program( + codex_utils_cargo_bin::cargo_bin("codex-code-mode-host").expect("host binary"), + ); + let session = provider + .create_session(Arc::new(RecordingDelegate::default())) + .await + .expect("create remote session"); + let mut request = execute_request("await new Promise(() => {});"); + request.yield_time_ms = Some(1); + let started = session.execute(request).await.expect("start execution"); + let running_cell_id = started.cell_id.clone(); + assert_eq!( + started.initial_response().await.expect("initial response"), + RuntimeResponse::Yielded { + cell_id: running_cell_id.clone(), + content_items: Vec::new(), + } + ); + + let wait_session = Arc::clone(&session); + let wait_cell_id = running_cell_id.clone(); + let first_wait = tokio::spawn(async move { + wait_session + .wait(WaitRequest { + cell_id: wait_cell_id, + yield_time_ms: 60_000, + }) + .await + }); + tokio::time::sleep(Duration::from_millis(100)).await; + first_wait.abort(); + let _ = first_wait.await; + + assert_eq!( + tokio::time::timeout( + Duration::from_secs(2), + session.wait(WaitRequest { + cell_id: running_cell_id.clone(), + yield_time_ms: 1, + }) + ) + .await + .expect("second wait timeout") + .expect("second wait"), + WaitOutcome::LiveCell(RuntimeResponse::Yielded { + cell_id: running_cell_id.clone(), + content_items: Vec::new(), + }) + ); + session + .terminate(running_cell_id) + .await + .expect("terminate cell"); + session.shutdown().await.expect("shutdown remote session"); +} + +#[tokio::test] +async fn unawaited_slow_tool_is_cancelled_after_parallel_tools_complete() { + let provider = ProcessOwnedCodeModeSessionProvider::with_host_program( + codex_utils_cargo_bin::cargo_bin("codex-code-mode-host").expect("host binary"), + ); + let (delegate, mut events_rx) = CancellationDelegate::new(); + let session = provider + .create_session(delegate.clone()) + .await + .expect("create remote session"); + let mut request = execute_request( + r#" +await (async () => { +text("hello world"); +yield_control(); +await Promise.all([ + tools.tool_call_a({}), + tools.tool_call_b({}), +]); +text("hello"); +tools.tool_call_slow({}); +await tools.tool_call_barrier({}); +return; +})(); +"#, + ); + request.enabled_tools = [ + "tool_call_a", + "tool_call_b", + "tool_call_slow", + "tool_call_barrier", + ] + .into_iter() + .map(|name| ToolDefinition { + name: name.to_string(), + tool_name: ToolName::plain(name), + description: String::new(), + kind: CodeModeToolKind::Function, + input_schema: None, + output_schema: None, + }) + .collect(); + + let started = session.execute(request).await.expect("start execution"); + let running_cell_id = started.cell_id.clone(); + assert_eq!( + started.initial_response().await.expect("initial response"), + RuntimeResponse::Yielded { + cell_id: running_cell_id.clone(), + content_items: vec![FunctionCallOutputContentItem::InputText { + text: "hello world".to_string(), + }], + } + ); + + let wait_session = Arc::clone(&session); + let wait_cell_id = running_cell_id.clone(); + let wait_task = tokio::spawn(async move { + wait_session + .wait(WaitRequest { + cell_id: wait_cell_id, + yield_time_ms: 60_000, + }) + .await + }); + + let mut parallel_tools = vec![ + next_callback_event(&mut events_rx).await, + next_callback_event(&mut events_rx).await, + ]; + parallel_tools.sort_by(|left, right| format!("{left:?}").cmp(&format!("{right:?}"))); + assert_eq!( + parallel_tools, + vec![ + CallbackEvent::Started("tool_call_a".to_string()), + CallbackEvent::Started("tool_call_b".to_string()), + ] + ); + delegate.fast_tool_release.add_permits(2); + + assert_eq!( + next_callback_event(&mut events_rx).await, + CallbackEvent::Started("tool_call_slow".to_string()) + ); + let mut closure_events = vec![ + next_callback_event(&mut events_rx).await, + next_callback_event(&mut events_rx).await, + ]; + closure_events.sort_by(|left, right| format!("{left:?}").cmp(&format!("{right:?}"))); + assert_eq!( + closure_events, + vec![ + CallbackEvent::Cancelled("tool_call_slow".to_string()), + CallbackEvent::CellClosed(running_cell_id.clone()), + ] + ); + assert_eq!( + wait_task + .await + .expect("wait task") + .expect("wait for terminal response"), + WaitOutcome::LiveCell(RuntimeResponse::Result { + cell_id: running_cell_id, + content_items: vec![FunctionCallOutputContentItem::InputText { + text: "hello".to_string(), + }], + error_text: None, + }) + ); + session.shutdown().await.expect("shutdown remote session"); +} + +#[tokio::test] +async fn oversized_execute_request_does_not_close_the_shared_host() { + let provider = ProcessOwnedCodeModeSessionProvider::with_host_program( + codex_utils_cargo_bin::cargo_bin("codex-code-mode-host").expect("host binary"), + ); + let session = provider + .create_session(Arc::new(RecordingDelegate::default())) + .await + .expect("create remote session"); + let error = session + .execute(execute_request(&"x".repeat(MAX_FRAME_BYTES))) + .await + .err() + .expect("oversized execute should fail"); + assert!( + error.contains("IPC frame limit"), + "unexpected error: {error}" + ); + + assert_eq!( + execute(&session, execute_request(r#"text("still alive");"#)).await, + RuntimeResponse::Result { + cell_id: cell_id("1"), + content_items: vec![FunctionCallOutputContentItem::InputText { + text: "still alive".to_string(), + }], + error_text: None, + } + ); + session.shutdown().await.expect("shutdown remote session"); +} + +#[tokio::test] +async fn oversized_delegate_payloads_fail_only_the_tool_call() { + let provider = ProcessOwnedCodeModeSessionProvider::with_host_program( + codex_utils_cargo_bin::cargo_bin("codex-code-mode-host").expect("host binary"), + ); + let session = provider + .create_session(Arc::new(OversizedResultDelegate)) + .await + .expect("create remote session"); + let tool = |name: &str| ToolDefinition { + name: name.to_string(), + tool_name: ToolName::plain(name), + description: String::new(), + kind: CodeModeToolKind::Function, + input_schema: None, + output_schema: None, + }; + + let mut oversized_argument = execute_request(&format!( + r#" +try {{ + await tools.big_argument({{ value: "x".repeat({MAX_FRAME_BYTES}) }}); +}} catch (_) {{ + text("argument rejected"); +}} +"# + )); + oversized_argument.enabled_tools = vec![tool("big_argument")]; + oversized_argument.yield_time_ms = Some(60_000); + assert_eq!( + execute_to_terminal(&session, oversized_argument).await, + RuntimeResponse::Result { + cell_id: cell_id("1"), + content_items: vec![FunctionCallOutputContentItem::InputText { + text: "argument rejected".to_string(), + }], + error_text: None, + } + ); + + let mut oversized_result = execute_request( + r#" +try { + await tools.big_result({}); +} catch (_) { + text("result rejected"); +} +"#, + ); + oversized_result.enabled_tools = vec![tool("big_result")]; + oversized_result.yield_time_ms = Some(60_000); + assert_eq!( + execute_to_terminal(&session, oversized_result).await, + RuntimeResponse::Result { + cell_id: cell_id("2"), + content_items: vec![FunctionCallOutputContentItem::InputText { + text: "result rejected".to_string(), + }], + error_text: None, + } + ); + + assert_eq!( + execute(&session, execute_request(r#"text("still alive");"#)).await, + RuntimeResponse::Result { + cell_id: cell_id("3"), + content_items: vec![FunctionCallOutputContentItem::InputText { + text: "still alive".to_string(), + }], + error_text: None, + } + ); + session.shutdown().await.expect("shutdown remote session"); +} + +#[tokio::test] +async fn oversized_initial_response_does_not_close_the_shared_host() { + let provider = ProcessOwnedCodeModeSessionProvider::with_host_program( + codex_utils_cargo_bin::cargo_bin("codex-code-mode-host").expect("host binary"), + ); + let session = provider + .create_session(Arc::new(RecordingDelegate::default())) + .await + .expect("create remote session"); + let started = session + .execute(execute_request(&format!( + r#"text("x".repeat({MAX_FRAME_BYTES}));"# + ))) + .await + .expect("start oversized response"); + let error = started + .initial_response() + .await + .expect_err("oversized initial response should fail"); + assert!( + error.contains("IPC frame limit"), + "unexpected error: {error}" + ); + + assert_eq!( + execute(&session, execute_request(r#"text("still alive");"#)).await, + RuntimeResponse::Result { + cell_id: cell_id("2"), + content_items: vec![FunctionCallOutputContentItem::InputText { + text: "still alive".to_string(), + }], + error_text: None, + } + ); + session.shutdown().await.expect("shutdown remote session"); +} + +#[cfg(unix)] +#[tokio::test] +async fn child_process_loss_cleans_up_and_rebuilds_the_shared_host() { + let host_program = + codex_utils_cargo_bin::cargo_bin("codex-code-mode-host").expect("host binary"); + let proxy_dir = tempfile::tempdir().expect("create host proxy directory"); + let proxy_program = proxy_dir.path().join("host-proxy.sh"); + let pid_path = proxy_dir.path().join("host.pid"); + std::fs::write( + &proxy_program, + format!( + "#!/bin/sh\nprintf '%s\\n' \"$$\" > '{}'\nexec '{}'\n", + pid_path.display(), + host_program.display() + ), + ) + .expect("write host proxy"); + let mut permissions = std::fs::metadata(&proxy_program) + .expect("host proxy metadata") + .permissions(); + permissions.set_mode(/*mode*/ 0o700); + std::fs::set_permissions(&proxy_program, permissions).expect("make host proxy executable"); + + let provider = ProcessOwnedCodeModeSessionProvider::with_host_program(proxy_program); + let (delegate_a, mut events_a) = CancellationDelegate::new(); + delegate_a.hold_slow_cleanup(); + let delegate_b = Arc::new(RecordingDelegate::default()); + let session_a = provider + .create_session(delegate_a.clone()) + .await + .expect("create first remote session"); + let session_b = provider + .create_session(delegate_b.clone()) + .await + .expect("create second remote session"); + + let mut request_a = execute_request("await tools.tool_call_slow({});"); + request_a.yield_time_ms = Some(1); + request_a.enabled_tools = vec![ToolDefinition { + name: "tool_call_slow".to_string(), + tool_name: ToolName::plain("tool_call_slow"), + description: String::new(), + kind: CodeModeToolKind::Function, + input_schema: None, + output_schema: None, + }]; + let started_a = session_a + .execute(request_a) + .await + .expect("start first cell"); + let cell_a = started_a.cell_id.clone(); + assert_eq!( + started_a + .initial_response() + .await + .expect("first initial response"), + RuntimeResponse::Yielded { + cell_id: cell_a.clone(), + content_items: Vec::new(), + } + ); + assert_eq!( + next_callback_event(&mut events_a).await, + CallbackEvent::Started("tool_call_slow".to_string()) + ); + + let mut request_b = execute_request("await new Promise(() => {});"); + request_b.yield_time_ms = Some(1); + let started_b = session_b + .execute(request_b) + .await + .expect("start second cell"); + let cell_b = started_b.cell_id.clone(); + assert_eq!( + started_b + .initial_response() + .await + .expect("second initial response"), + RuntimeResponse::Yielded { + cell_id: cell_b.clone(), + content_items: Vec::new(), + } + ); + + let wait_a_session = Arc::clone(&session_a); + let wait_a_cell = cell_a.clone(); + let wait_a = tokio::spawn(async move { + wait_a_session + .wait(WaitRequest { + cell_id: wait_a_cell, + yield_time_ms: 60_000, + }) + .await + }); + let wait_b_session = Arc::clone(&session_b); + let wait_b_cell = cell_b.clone(); + let wait_b = tokio::spawn(async move { + wait_b_session + .wait(WaitRequest { + cell_id: wait_b_cell, + yield_time_ms: 60_000, + }) + .await + }); + tokio::time::sleep(Duration::from_millis(100)).await; + + let pid = tokio::time::timeout(Duration::from_secs(5), async { + loop { + if let Ok(pid) = std::fs::read_to_string(&pid_path) + && let Ok(pid) = pid.trim().parse::() + { + break pid; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("host pid timeout"); + let kill_status = std::process::Command::new("kill") + .args(["-KILL", &pid.to_string()]) + .status() + .expect("kill host process"); + assert!(kill_status.success()); + + assert!( + tokio::time::timeout(Duration::from_secs(5), wait_a) + .await + .expect("first wait failure timeout") + .expect("first wait task") + .is_err() + ); + assert!( + tokio::time::timeout(Duration::from_secs(5), wait_b) + .await + .expect("second wait failure timeout") + .expect("second wait task") + .is_err() + ); + let closure_events = [ + next_callback_event(&mut events_a).await, + next_callback_event(&mut events_a).await, + ]; + assert!(closure_events.contains(&CallbackEvent::Cancelled("tool_call_slow".to_string()))); + assert!(closure_events.contains(&CallbackEvent::CellClosed(cell_a.clone()))); + tokio::time::timeout(Duration::from_secs(5), async { + loop { + if delegate_b + .closed_cells + .lock() + .expect("closed cells lock") + .contains(&cell_b) + { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("unrelated session cleanup timeout"); + + assert_eq!( + execute(&session_b, execute_request(r#"text("replacement");"#)).await, + RuntimeResponse::Result { + cell_id: cell_id("g2:1"), + content_items: vec![FunctionCallOutputContentItem::InputText { + text: "replacement".to_string(), + }], + error_text: None, + } + ); + let stale_error = session_b + .wait(WaitRequest { + cell_id: cell_b.clone(), + yield_time_ms: 1, + }) + .await + .expect_err("stale cell should be rejected"); + assert!(stale_error.contains("stale code-mode host generation")); + + tokio::time::timeout(Duration::from_secs(5), session_a.shutdown()) + .await + .expect("failed session shutdown timeout") + .expect("shutdown failed session"); + tokio::time::timeout(Duration::from_secs(5), session_b.shutdown()) + .await + .expect("unrelated session shutdown timeout") + .expect("shutdown replacement session"); + + delegate_a.release_slow_cleanup(); + tokio::task::yield_now().await; + assert!(matches!( + events_a.try_recv(), + Err(mpsc::error::TryRecvError::Empty) + )); +} diff --git a/codex-rs/code-mode-protocol/src/host/codec.rs b/codex-rs/code-mode-protocol/src/host/codec.rs index 6bf83d367d..ec07b92d41 100644 --- a/codex-rs/code-mode-protocol/src/host/codec.rs +++ b/codex-rs/code-mode-protocol/src/host/codec.rs @@ -11,6 +11,36 @@ use tokio::io::AsyncWriteExt; /// Maximum JSON payload size accepted for one IPC frame. pub const MAX_FRAME_BYTES: usize = 64 * 1024 * 1024; +/// A serialized IPC frame that has already passed the payload size limit. +#[derive(Clone, Debug)] +pub struct EncodedFrame { + payload: Vec, +} + +impl EncodedFrame { + pub fn encode(message: &T) -> io::Result + where + T: Serialize, + { + let payload = serde_json::to_vec(message).map_err(|err| { + io::Error::new( + io::ErrorKind::InvalidData, + format!("failed to encode code-mode IPC frame: {err}"), + ) + })?; + if payload.len() > MAX_FRAME_BYTES { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!( + "code-mode IPC frame length {} exceeds {MAX_FRAME_BYTES} bytes", + payload.len() + ), + )); + } + Ok(Self { payload }) + } +} + /// Decodes JSON messages prefixed by a four-byte little-endian payload length. pub struct FramedReader { reader: R, @@ -72,22 +102,12 @@ where where T: Serialize, { - let payload = serde_json::to_vec(message).map_err(|err| { - io::Error::new( - io::ErrorKind::InvalidData, - format!("failed to encode code-mode IPC frame: {err}"), - ) - })?; - if payload.len() > MAX_FRAME_BYTES { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - format!( - "code-mode IPC frame length {} exceeds {MAX_FRAME_BYTES} bytes", - payload.len() - ), - )); - } - let length = u32::try_from(payload.len()).map_err(|_| { + self.write_frame(&EncodedFrame::encode(message)?).await + } + + /// Writes and flushes a frame encoded before it entered an I/O queue. + pub async fn write_frame(&mut self, frame: &EncodedFrame) -> io::Result<()> { + let length = u32::try_from(frame.payload.len()).map_err(|_| { io::Error::new( io::ErrorKind::InvalidData, "code-mode IPC frame length exceeds u32", @@ -95,7 +115,7 @@ where })?; self.writer.write_all(&length.to_le_bytes()).await?; - self.writer.write_all(&payload).await?; + self.writer.write_all(&frame.payload).await?; self.writer.flush().await } } diff --git a/codex-rs/code-mode-protocol/src/host/host_tests.rs b/codex-rs/code-mode-protocol/src/host/host_tests.rs index 3aa2a18307..d56e56e1fa 100644 --- a/codex-rs/code-mode-protocol/src/host/host_tests.rs +++ b/codex-rs/code-mode-protocol/src/host/host_tests.rs @@ -365,6 +365,16 @@ fn client_to_host_v1_variants_are_pinned() { }), ); } + + assert_wire_round_trip( + ClientToHost::CancelRequest { + id: request_id(/*value*/ 9), + }, + json!({ + "type": "operation/cancel", + "id": 9, + }), + ); } #[test] diff --git a/codex-rs/code-mode-protocol/src/host/message.rs b/codex-rs/code-mode-protocol/src/host/message.rs index 78f0961d51..0e83c866e4 100644 --- a/codex-rs/code-mode-protocol/src/host/message.rs +++ b/codex-rs/code-mode-protocol/src/host/message.rs @@ -132,6 +132,8 @@ pub enum ClientToHost { ClientHello(ClientHello), #[serde(rename = "operation/request")] Request { id: RequestId, request: HostRequest }, + #[serde(rename = "operation/cancel")] + CancelRequest { id: RequestId }, #[serde(rename = "delegate/response")] DelegateResponse { id: DelegateRequestId, diff --git a/codex-rs/code-mode-protocol/src/host/mod.rs b/codex-rs/code-mode-protocol/src/host/mod.rs index 7e78768ab6..3583ccb3e9 100644 --- a/codex-rs/code-mode-protocol/src/host/mod.rs +++ b/codex-rs/code-mode-protocol/src/host/mod.rs @@ -11,6 +11,7 @@ mod message; mod payload; mod types; +pub use codec::EncodedFrame; pub use codec::FramedReader; pub use codec::FramedWriter; pub use codec::MAX_FRAME_BYTES; diff --git a/codex-rs/code-mode/Cargo.toml b/codex-rs/code-mode/Cargo.toml index 09f1d7380d..011fee6ebc 100644 --- a/codex-rs/code-mode/Cargo.toml +++ b/codex-rs/code-mode/Cargo.toml @@ -19,8 +19,9 @@ workspace = true codex-code-mode-protocol = { workspace = true } codex-protocol = { workspace = true } deno_core_icudata = { workspace = true } +futures = { workspace = true } serde_json = { workspace = true } -tokio = { workspace = true, features = ["macros", "rt", "sync", "time"] } +tokio = { workspace = true, features = ["io-util", "macros", "process", "rt", "sync", "time"] } tokio-util = { workspace = true, features = ["rt"] } tracing = { workspace = true } v8 = { workspace = true } diff --git a/codex-rs/code-mode/src/cell_actor/callbacks.rs b/codex-rs/code-mode/src/cell_actor/callbacks.rs index 9db66ed3a7..08f7cbac99 100644 --- a/codex-rs/code-mode/src/cell_actor/callbacks.rs +++ b/codex-rs/code-mode/src/cell_actor/callbacks.rs @@ -1,11 +1,14 @@ +use std::panic::AssertUnwindSafe; use std::sync::Arc; +use futures::FutureExt; use tokio::task::JoinSet; use tokio_util::sync::CancellationToken; use tracing::warn; use super::CellHost; use super::CellToolCall; +use crate::TaskFailureHandler; use crate::runtime::RuntimeCommand; #[derive(Clone, Copy)] @@ -20,10 +23,20 @@ pub(super) fn spawn_notification( call_id: String, text: String, cancellation_token: CancellationToken, + task_failure_handler: Option, ) { tasks.spawn(async move { - if let Err(err) = host.notify(call_id, text, cancellation_token).await { - warn!("failed to deliver code mode notification: {err}"); + let callback = + AssertUnwindSafe(async move { host.notify(call_id, text, cancellation_token).await }) + .catch_unwind() + .await; + match callback { + Ok(Ok(())) => {} + Ok(Err(err)) => warn!("failed to deliver code mode notification: {err}"), + Err(_) => report_task_failure( + task_failure_handler.as_ref(), + "code mode notification task panicked".to_string(), + ), } }); } @@ -34,14 +47,32 @@ pub(super) fn spawn_tool( invocation: CellToolCall, runtime_tx: std::sync::mpsc::Sender, cancellation_token: CancellationToken, + task_failure_handler: Option, ) { tasks.spawn(async move { let id = invocation.id.clone(); - let command = match host.invoke_tool(invocation, cancellation_token).await { - Ok(result) => RuntimeCommand::ToolResponse { id, result }, - Err(error_text) => RuntimeCommand::ToolError { id, error_text }, + let callback = + AssertUnwindSafe(async move { host.invoke_tool(invocation, cancellation_token).await }) + .catch_unwind() + .await; + let (command, failure_reason) = match callback { + Ok(Ok(result)) => (RuntimeCommand::ToolResponse { id, result }, None), + Ok(Err(error_text)) => (RuntimeCommand::ToolError { id, error_text }, None), + Err(_) => { + let failure_reason = "code mode tool task panicked".to_string(); + ( + RuntimeCommand::ToolError { + id, + error_text: failure_reason.clone(), + }, + Some(failure_reason), + ) + } }; let _ = runtime_tx.send(command); + if let Some(failure_reason) = failure_reason { + report_task_failure(task_failure_handler.as_ref(), failure_reason); + } }); } @@ -50,28 +81,48 @@ pub(super) async fn finish_callbacks( notification_tasks: &mut JoinSet<()>, tool_tasks: &mut JoinSet<()>, completion: CallbackCompletion, + task_failure_handler: Option<&TaskFailureHandler>, ) { if matches!(completion, CallbackCompletion::Cancel) { cancellation_token.cancel(); } - drain_tasks(notification_tasks, "notification").await; + drain_tasks(notification_tasks, "notification", task_failure_handler).await; cancellation_token.cancel(); - drain_tasks(tool_tasks, "tool").await; + drain_tasks(tool_tasks, "tool", task_failure_handler).await; } -pub(super) fn log_task_result( +pub(super) fn report_task_result( task_result: Option>, description: &str, + task_failure_handler: Option<&TaskFailureHandler>, ) { if let Some(Err(err)) = task_result && !err.is_cancelled() { - warn!("code mode {description} task failed: {err}"); + report_task_failure( + task_failure_handler, + format!("code mode {description} task failed: {err}"), + ); } } -async fn drain_tasks(tasks: &mut JoinSet<()>, description: &str) { - while let Some(result) = tasks.join_next().await { - log_task_result(Some(result), description); +fn report_task_failure(task_failure_handler: Option<&TaskFailureHandler>, failure_reason: String) { + warn!("{failure_reason}"); + if let Some(task_failure_handler) = task_failure_handler { + task_failure_handler(failure_reason); } } + +async fn drain_tasks( + tasks: &mut JoinSet<()>, + description: &str, + task_failure_handler: Option<&TaskFailureHandler>, +) { + while let Some(result) = tasks.join_next().await { + report_task_result(Some(result), description, task_failure_handler); + } +} + +#[cfg(test)] +#[path = "callbacks_tests.rs"] +mod tests; diff --git a/codex-rs/code-mode/src/cell_actor/callbacks_tests.rs b/codex-rs/code-mode/src/cell_actor/callbacks_tests.rs new file mode 100644 index 0000000000..585f7808a8 --- /dev/null +++ b/codex-rs/code-mode/src/cell_actor/callbacks_tests.rs @@ -0,0 +1,132 @@ +use std::collections::HashMap; +use std::sync::Arc; +use std::sync::mpsc as std_mpsc; +use std::time::Duration; + +use pretty_assertions::assert_eq; +use serde_json::Value as JsonValue; +use tokio::sync::mpsc; +use tokio::task::JoinSet; +use tokio_util::sync::CancellationToken; + +use super::*; +use crate::cell_actor::CellState; +use crate::cell_actor::CompletionCommit; +use crate::runtime::RuntimeCommand; +use crate::session_runtime::CellEvent; +use crate::session_runtime::ToolKind; +use crate::session_runtime::ToolName; + +struct PanickingCallbackHost; + +impl CellHost for PanickingCallbackHost { + async fn invoke_tool( + &self, + _invocation: CellToolCall, + _cancellation_token: CancellationToken, + ) -> Result { + panic!("tool callback panic probe"); + } + + async fn notify( + &self, + _call_id: String, + _text: String, + _cancellation_token: CancellationToken, + ) -> Result<(), String> { + panic!("notification callback panic probe"); + } + + async fn commit_completion( + &self, + _stored_value_writes: HashMap, + _event: CellEvent, + _pending_initial_yield_items: Option>, + _cell_state: Arc, + ) -> CompletionCommit { + panic!("unexpected completion commit"); + } + + async fn closed(&self) {} +} + +#[tokio::test] +async fn tool_callback_panic_rejects_the_js_promise_and_reports_failure() { + let mut tasks = JoinSet::new(); + let (runtime_tx, runtime_rx) = std_mpsc::channel(); + let (failure_tx, mut failure_rx) = mpsc::unbounded_channel(); + spawn_tool( + &mut tasks, + Arc::new(PanickingCallbackHost), + CellToolCall { + id: "tool-1".to_string(), + name: ToolName { + name: "panic".to_string(), + namespace: None, + }, + kind: ToolKind::Function, + input: None, + }, + runtime_tx, + CancellationToken::new(), + Some(Arc::new(move |reason| { + let _ = failure_tx.send(reason); + })), + ); + + tasks + .join_next() + .await + .expect("tool callback task") + .expect("tool callback wrapper"); + let command = runtime_rx + .recv_timeout(Duration::from_secs(1)) + .expect("tool error command"); + let RuntimeCommand::ToolError { id, error_text } = command else { + panic!("expected a tool error command"); + }; + assert_eq!(id, "tool-1"); + assert_eq!(error_text, "code mode tool task panicked"); + assert_eq!(failure_rx.recv().await, Some(error_text)); +} + +#[tokio::test] +async fn notification_callback_panic_reports_failure() { + let mut tasks = JoinSet::new(); + let (failure_tx, mut failure_rx) = mpsc::unbounded_channel(); + spawn_notification( + &mut tasks, + Arc::new(PanickingCallbackHost), + "notify-1".to_string(), + "hello".to_string(), + CancellationToken::new(), + Some(Arc::new(move |reason| { + let _ = failure_tx.send(reason); + })), + ); + + tasks + .join_next() + .await + .expect("notification callback task") + .expect("notification callback wrapper"); + let failure_reason = failure_rx.recv().await.expect("notification failure"); + assert_eq!(failure_reason, "code mode notification task panicked"); +} + +#[tokio::test] +async fn callback_wrapper_join_error_reports_failure() { + let task_result = tokio::spawn(async { + panic!("callback wrapper panic probe"); + }) + .await; + let (failure_tx, mut failure_rx) = mpsc::unbounded_channel(); + let task_failure_handler: TaskFailureHandler = Arc::new(move |reason| { + let _ = failure_tx.send(reason); + }); + + report_task_result(Some(task_result), "tool", Some(&task_failure_handler)); + + let failure_reason = failure_rx.recv().await.expect("wrapper failure"); + assert!(failure_reason.contains("code mode tool task failed")); +} diff --git a/codex-rs/code-mode/src/cell_actor/mod.rs b/codex-rs/code-mode/src/cell_actor/mod.rs index 24eca22517..7533f4b0cc 100644 --- a/codex-rs/code-mode/src/cell_actor/mod.rs +++ b/codex-rs/code-mode/src/cell_actor/mod.rs @@ -14,7 +14,7 @@ use tokio_util::sync::CancellationToken; use self::callbacks::CallbackCompletion; use self::callbacks::finish_callbacks; -use self::callbacks::log_task_result; +use self::callbacks::report_task_result; use self::callbacks::spawn_notification; use self::callbacks::spawn_tool; use self::conversions::cell_tool_kind; @@ -30,6 +30,7 @@ pub(crate) use self::types::CellToolCall; pub(crate) use self::types::CompletionCommit; use self::types::CompletionDelivery; use self::types::ObservationDelivery; +use crate::TaskFailureHandler; use crate::runtime::PendingRuntimeMode; use crate::runtime::RuntimeCommand; use crate::runtime::RuntimeControlCommand; @@ -50,6 +51,7 @@ impl CellActor { host: Arc, initial_observe_mode: ObserveMode, cell_state: Arc, + task_failure_handler: Option, ) -> Result< ( CellHandle, @@ -66,6 +68,7 @@ impl CellActor { runtime_request(request), event_tx, PendingRuntimeMode::PauseUntilResumed, + task_failure_handler.clone(), )?; let handle = CellHandle::new(command_tx, Arc::clone(&cell_state)); let task = run_cell( @@ -82,6 +85,7 @@ impl CellActor { mode: initial_observe_mode, response_tx: initial_response_tx, }, + task_failure_handler, ); let initial_response = Box::pin(async move { initial_response_rx.await.unwrap_or(Err(CellError::Closed)) }); @@ -107,6 +111,7 @@ async fn run_cell( mut event_rx: mpsc::UnboundedReceiver, command_rx: mpsc::UnboundedReceiver, initial_observer: Observer, + task_failure_handler: Option, ) { let CellContext { runtime_tx, @@ -123,6 +128,7 @@ async fn run_cell( let mut termination = false; let mut runtime_closed = false; let mut runtime_paused = false; + let mut runtime_failure_reported = false; let mut yield_timer: Option>> = None; let mut notification_tasks = JoinSet::new(); let mut tool_tasks = JoinSet::new(); @@ -149,6 +155,7 @@ async fn run_cell( &mut notification_tasks, &mut tool_tasks, CallbackCompletion::Cancel, + task_failure_handler.as_ref(), ).await; finish_termination( &cell_state, @@ -259,6 +266,7 @@ async fn run_cell( &mut notification_tasks, &mut tool_tasks, CallbackCompletion::Cancel, + task_failure_handler.as_ref(), ).await; finish_termination( &cell_state, @@ -269,11 +277,20 @@ async fn run_cell( ); break; } + if !runtime_failure_reported + && let Some(task_failure_handler) = &task_failure_handler + { + runtime_failure_reported = true; + task_failure_handler( + "code-mode V8 runtime thread ended unexpectedly".to_string(), + ); + } finish_callbacks( &callback_cancellation_token, &mut notification_tasks, &mut tool_tasks, CallbackCompletion::DrainNotifications, + task_failure_handler.as_ref(), ) .await; let event = CellEvent::Completed { @@ -376,6 +393,7 @@ async fn run_cell( call_id, text, callback_cancellation_token.child_token(), + task_failure_handler.clone(), ); } RuntimeEvent::ToolCall { id, name, kind, input } => { @@ -394,6 +412,7 @@ async fn run_cell( }, runtime_tx.clone(), callback_cancellation_token.child_token(), + task_failure_handler.clone(), ); } RuntimeEvent::Result { stored_value_writes, error_text } => { @@ -405,6 +424,7 @@ async fn run_cell( &mut notification_tasks, &mut tool_tasks, CallbackCompletion::Cancel, + task_failure_handler.as_ref(), ).await; finish_termination( &cell_state, @@ -420,6 +440,7 @@ async fn run_cell( &mut notification_tasks, &mut tool_tasks, CallbackCompletion::DrainNotifications, + task_failure_handler.as_ref(), ) .await; let event = CellEvent::Completed { @@ -455,13 +476,20 @@ async fn run_cell( } } } + RuntimeEvent::ThreadPanicked => { + runtime_failure_reported = true; + } } } task_result = notification_tasks.join_next(), if !notification_tasks.is_empty() => { - log_task_result(task_result, "notification"); + report_task_result( + task_result, + "notification", + task_failure_handler.as_ref(), + ); } task_result = tool_tasks.join_next(), if !tool_tasks.is_empty() => { - log_task_result(task_result, "tool"); + report_task_result(task_result, "tool", task_failure_handler.as_ref()); } } } @@ -479,6 +507,7 @@ async fn run_cell( &mut notification_tasks, &mut tool_tasks, CallbackCompletion::Cancel, + task_failure_handler.as_ref(), ) .await; host.closed().await; diff --git a/codex-rs/code-mode/src/cell_actor/tests.rs b/codex-rs/code-mode/src/cell_actor/tests.rs index a4d93fc3e6..3612daff91 100644 --- a/codex-rs/code-mode/src/cell_actor/tests.rs +++ b/codex-rs/code-mode/src/cell_actor/tests.rs @@ -102,6 +102,18 @@ fn spawn_cell_actor_harness(initial_observe_mode: ObserveMode) -> CellActorHarne fn spawn_cell_actor_harness_with_host( initial_observe_mode: ObserveMode, host: Arc, +) -> CellActorHarness { + spawn_cell_actor_harness_with_host_and_failure_handler( + initial_observe_mode, + host, + /*task_failure_handler*/ None, + ) +} + +fn spawn_cell_actor_harness_with_host_and_failure_handler( + initial_observe_mode: ObserveMode, + host: Arc, + task_failure_handler: Option, ) -> CellActorHarness { let (event_tx, event_rx) = mpsc::unbounded_channel(); let (command_tx, command_rx) = mpsc::unbounded_channel(); @@ -118,6 +130,7 @@ fn spawn_cell_actor_harness_with_host( }, runtime_event_tx, PendingRuntimeMode::PauseUntilResumed, + /*task_failure_handler*/ None, ) .unwrap(); let (runtime_control_tx, runtime_control_rx) = std_mpsc::channel(); @@ -137,6 +150,7 @@ fn spawn_cell_actor_harness_with_host( mode: initial_observe_mode, response_tx: initial_event_tx, }, + task_failure_handler, )); CellActorHarness { @@ -149,6 +163,54 @@ fn spawn_cell_actor_harness_with_host( } } +#[tokio::test] +async fn unexpected_runtime_thread_exit_is_reported_to_the_session_owner() { + let (failure_tx, mut failure_rx) = mpsc::unbounded_channel(); + let harness = spawn_cell_actor_harness_with_host_and_failure_handler( + ObserveMode::YieldAfter(Duration::from_secs(60)), + Arc::new(TestHost), + Some(Arc::new(move |reason| { + let _ = failure_tx.send(reason); + })), + ); + drop(harness.event_tx); + + assert_eq!( + tokio::time::timeout(Duration::from_secs(1), failure_rx.recv()) + .await + .expect("runtime failure timeout") + .expect("runtime failure"), + "code-mode V8 runtime thread ended unexpectedly" + ); + assert!( + harness + .initial_event_rx + .await + .expect("initial event") + .is_ok() + ); + harness.task.await.expect("cell task"); +} + +#[tokio::test] +async fn runtime_thread_panic_remains_a_cell_error_without_owner_supervision() { + let harness = spawn_cell_actor_harness(ObserveMode::YieldAfter(Duration::from_secs(60))); + harness + .event_tx + .send(RuntimeEvent::ThreadPanicked) + .expect("runtime panic event"); + drop(harness.event_tx); + + assert_eq!( + harness.initial_event_rx.await.expect("initial event"), + Ok(CellEvent::Completed { + content_items: Vec::new(), + error_text: Some("exec runtime ended unexpectedly".to_string()), + }) + ); + harness.task.await.expect("cell task"); +} + async fn wait_for_notification(host: &RecordingHost) { tokio::time::timeout(Duration::from_secs(1), async { while !host.notified.load(Ordering::Acquire) { diff --git a/codex-rs/code-mode/src/lib.rs b/codex-rs/code-mode/src/lib.rs index 089bb6f78d..6d332bf0c4 100644 --- a/codex-rs/code-mode/src/lib.rs +++ b/codex-rs/code-mode/src/lib.rs @@ -1,9 +1,17 @@ mod cell_actor; +mod remote_session; mod runtime; mod service; mod session_runtime; +mod v8_init; + +pub(crate) type TaskFailureHandler = std::sync::Arc; pub use codex_code_mode_protocol::*; +pub use remote_session::ProcessOwnedCodeModeSession; +pub use remote_session::ProcessOwnedCodeModeSessionProvider; pub use service::InProcessCodeModeSession; pub use service::InProcessCodeModeSessionProvider; pub use service::NoopCodeModeSessionDelegate; +pub use v8_init::V8JitMode; +pub use v8_init::initialize_v8; diff --git a/codex-rs/code-mode/src/remote_session.rs b/codex-rs/code-mode/src/remote_session.rs new file mode 100644 index 0000000000..af7d27845b --- /dev/null +++ b/codex-rs/code-mode/src/remote_session.rs @@ -0,0 +1,505 @@ +use std::ffi::OsString; +use std::io; +use std::path::PathBuf; +use std::sync::Arc; +use std::sync::Mutex as StdMutex; +use std::sync::atomic::AtomicBool; +use std::sync::atomic::AtomicU64; +use std::sync::atomic::Ordering; + +use codex_code_mode_protocol::CellId; +use codex_code_mode_protocol::CodeModeSession; +use codex_code_mode_protocol::CodeModeSessionDelegate; +use codex_code_mode_protocol::CodeModeSessionProvider; +use codex_code_mode_protocol::CodeModeSessionProviderFuture; +use codex_code_mode_protocol::CodeModeSessionResultFuture; +use codex_code_mode_protocol::ExecuteRequest; +use codex_code_mode_protocol::StartedCell; +use codex_code_mode_protocol::WaitOutcome; +use codex_code_mode_protocol::WaitRequest; +use codex_code_mode_protocol::host::SessionId; +use tokio::sync::Semaphore; +use tokio::sync::watch; + +use self::connection::Connection; +use self::connection::RemoteSession; +use self::connection::SessionCleanup; +use crate::NoopCodeModeSessionDelegate; + +mod connection; + +const CODE_MODE_HOST_PATH_ENV: &str = "CODEX_CODE_MODE_HOST_PATH"; + +type ShutdownResultReceiver = watch::Receiver>>; + +/// Creates code-mode sessions backed by one lazily spawned process host. +pub struct ProcessOwnedCodeModeSessionProvider { + host_program: PathBuf, + process_host: StdMutex>>, +} + +impl ProcessOwnedCodeModeSessionProvider { + pub fn with_host_program(host_program: PathBuf) -> Self { + Self { + host_program, + process_host: StdMutex::new(None), + } + } + + fn process_host(&self) -> Arc { + let mut process_host = self + .process_host + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if let Some(process_host) = process_host.as_ref() { + return Arc::clone(process_host); + } + + let new_process_host = Arc::new(OwnedProcessHost::new(self.host_program.clone())); + *process_host = Some(Arc::clone(&new_process_host)); + new_process_host + } +} + +impl Default for ProcessOwnedCodeModeSessionProvider { + fn default() -> Self { + Self::with_host_program(default_host_program()) + } +} + +impl CodeModeSessionProvider for ProcessOwnedCodeModeSessionProvider { + fn create_session<'a>( + &'a self, + delegate: Arc, + ) -> CodeModeSessionProviderFuture<'a> { + let session = ProcessOwnedCodeModeSession::with_process_host(delegate, self.process_host()); + Box::pin(async move { + session.connection().await?; + let session: Arc = Arc::new(session); + Ok(session) + }) + } +} + +struct OwnedProcessHost { + host_program: PathBuf, + connection: StdMutex>>, + spawn_permit: Semaphore, + next_session_id: AtomicU64, +} + +impl OwnedProcessHost { + fn new(host_program: PathBuf) -> Self { + Self { + host_program, + connection: StdMutex::new(None), + spawn_permit: Semaphore::new(/*permits*/ 1), + next_session_id: AtomicU64::new(1), + } + } + + async fn connection(&self) -> Result, String> { + if let Some(connection) = self.live_connection() { + return Ok(connection); + } + + let _spawn_permit = self + .spawn_permit + .acquire() + .await + .map_err(|_| "code-mode host spawn coordinator closed".to_string())?; + if let Some(connection) = self.live_connection() { + return Ok(connection); + } + let new_connection = Arc::new(Connection::spawn(&self.host_program).await?); + *self + .connection + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(Arc::clone(&new_connection)); + Ok(new_connection) + } + + fn live_connection(&self) -> Option> { + self.connection + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .as_ref() + .filter(|connection| connection.is_alive()) + .cloned() + } + + fn allocate_session_id(&self) -> SessionId { + let value = self.next_session_id.fetch_add(1, Ordering::Relaxed); + match SessionId::new(format!("session-{value}")) { + Ok(session_id) => session_id, + Err(_) => unreachable!("a generated code-mode session ID is nonempty"), + } + } +} + +enum SessionState { + New, + Opening { + remote: RemoteSession, + result_rx: watch::Receiver>>, + }, + Open(SessionBinding), + Closing, + Closed, +} + +#[derive(Clone)] +struct SessionBinding { + connection: Arc, + remote: RemoteSession, + cleanup: SessionCleanup, +} + +struct SessionInner { + process_host: Arc, + delegate: Arc, + state: StdMutex, + next_generation: AtomicU64, + shutdown_requested: AtomicBool, + shutdown_result: StdMutex>, + retired_cleanups: StdMutex>, +} + +/// A logical code-mode session assigned to a process-owned host. +pub struct ProcessOwnedCodeModeSession { + inner: Arc, +} + +impl ProcessOwnedCodeModeSession { + pub fn new() -> Self { + Self::with_process_host( + Arc::new(NoopCodeModeSessionDelegate), + Arc::new(OwnedProcessHost::new(default_host_program())), + ) + } + + fn with_process_host( + delegate: Arc, + process_host: Arc, + ) -> Self { + Self { + inner: Arc::new(SessionInner { + process_host, + delegate, + state: StdMutex::new(SessionState::New), + next_generation: AtomicU64::new(1), + shutdown_requested: AtomicBool::new(false), + shutdown_result: StdMutex::new(None), + retired_cleanups: StdMutex::new(Vec::new()), + }), + } + } + + async fn connection(&self) -> Result { + self.inner.connection().await + } + + pub async fn execute(&self, request: ExecuteRequest) -> Result { + let binding = self.connection().await?; + binding.connection.execute(binding.remote, request).await + } + + pub async fn wait(&self, request: WaitRequest) -> Result { + let binding = self.connection().await?; + binding.connection.wait(binding.remote, request).await + } + + pub async fn terminate(&self, cell_id: CellId) -> Result { + let binding = self.connection().await?; + binding.connection.terminate(binding.remote, cell_id).await + } + + pub async fn shutdown(&self) -> Result<(), String> { + wait_for_watch(self.inner.request_shutdown()).await + } +} + +impl SessionInner { + async fn connection(self: &Arc) -> Result { + loop { + if self.shutdown_requested.load(Ordering::Acquire) { + return Err("code mode session is shutting down".to_string()); + } + let (result_rx, start) = { + let mut state = self + .state + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + match &*state { + SessionState::New => { + let generation = self.next_generation.fetch_add(1, Ordering::Relaxed); + let remote = RemoteSession { + id: self.process_host.allocate_session_id(), + generation, + }; + let (result_tx, result_rx) = watch::channel(None); + *state = SessionState::Opening { + remote: remote.clone(), + result_rx: result_rx.clone(), + }; + (result_rx, Some((remote, result_tx))) + } + SessionState::Opening { result_rx, .. } => (result_rx.clone(), None), + SessionState::Open(binding) if binding.connection.is_alive() => { + return Ok(binding.clone()); + } + SessionState::Open(binding) => { + self.retain_cleanup(binding.cleanup.clone()); + *state = SessionState::New; + continue; + } + SessionState::Closing | SessionState::Closed => { + return Err("code mode session is shutting down".to_string()); + } + } + }; + if let Some((remote, result_tx)) = start { + let inner = Arc::clone(self); + tokio::spawn(async move { + inner.open(remote, result_tx).await; + }); + } + return wait_for_watch(result_rx).await; + } + } + + async fn open( + self: Arc, + remote: RemoteSession, + result_tx: watch::Sender>>, + ) { + let result = match self.process_host.connection().await { + Ok(connection) => { + let cleanup = connection + .open_session(remote.clone(), Arc::clone(&self.delegate)) + .await; + cleanup.map(|cleanup| SessionBinding { + connection, + remote: remote.clone(), + cleanup, + }) + } + Err(err) => Err(err), + }; + { + let mut state = self + .state + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if matches!( + &*state, + SessionState::Opening { + remote: opening_remote, + .. + } if opening_remote == &remote + ) { + *state = match &result { + Ok(binding) => SessionState::Open(binding.clone()), + Err(_) => SessionState::New, + }; + } + } + result_tx.send_replace(Some(result)); + } + + fn request_shutdown(self: &Arc) -> ShutdownResultReceiver { + self.shutdown_requested.store(true, Ordering::Release); + let mut shutdown_result = self + .shutdown_result + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if let Some(result_rx) = shutdown_result.as_ref() { + return result_rx.clone(); + } + let (result_tx, result_rx) = watch::channel(None); + *shutdown_result = Some(result_rx.clone()); + let inner = Arc::clone(self); + tokio::spawn(async move { + let result = inner.drive_shutdown().await; + result_tx.send_replace(Some(result)); + }); + result_rx + } + + async fn drive_shutdown(self: &Arc) -> Result<(), String> { + loop { + let action = { + let mut state = self + .state + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + match &*state { + SessionState::New => { + *state = SessionState::Closed; + ShutdownAction::Finish + } + SessionState::Opening { result_rx, .. } => { + ShutdownAction::WaitForOpen(result_rx.clone()) + } + SessionState::Open(binding) if !binding.connection.is_alive() => { + let cleanup = binding.cleanup.clone(); + *state = SessionState::Closing; + ShutdownAction::WaitForSessionCleanup(cleanup) + } + SessionState::Open(binding) => { + let binding = binding.clone(); + *state = SessionState::Closing; + ShutdownAction::Close(binding) + } + SessionState::Closing => { + return Err("code-mode session shutdown driver entered twice".to_string()); + } + SessionState::Closed => return Ok(()), + } + }; + match action { + ShutdownAction::WaitForOpen(result_rx) => { + let _ = wait_for_watch(result_rx).await; + } + ShutdownAction::Finish => { + self.wait_for_retired_cleanups().await; + return Ok(()); + } + ShutdownAction::WaitForSessionCleanup(cleanup) => { + cleanup.wait().await; + self.wait_for_retired_cleanups().await; + *self + .state + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = SessionState::Closed; + return Ok(()); + } + ShutdownAction::Close(binding) => { + let result = binding.connection.shutdown_session(binding.remote).await; + if result.is_err() && !binding.connection.is_alive() { + binding.cleanup.wait().await; + } + self.wait_for_retired_cleanups().await; + *self + .state + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = SessionState::Closed; + return result; + } + } + } + } + + fn retain_cleanup(&self, cleanup: SessionCleanup) { + let mut retired = self + .retired_cleanups + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + retired.retain(|cleanup| !cleanup.is_complete()); + if !cleanup.is_complete() { + retired.push(cleanup); + } + } + + async fn wait_for_retired_cleanups(&self) { + let retired = std::mem::take( + &mut *self + .retired_cleanups + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner), + ); + for cleanup in retired { + cleanup.wait().await; + } + } +} + +enum ShutdownAction { + WaitForOpen(watch::Receiver>>), + Finish, + WaitForSessionCleanup(SessionCleanup), + Close(SessionBinding), +} + +async fn wait_for_watch( + mut result_rx: watch::Receiver>>, +) -> Result +where + T: Clone, +{ + loop { + if let Some(result) = result_rx.borrow().clone() { + return result; + } + result_rx + .changed() + .await + .map_err(|_| "code-mode session transition stopped".to_string())?; + } +} + +impl Drop for ProcessOwnedCodeModeSession { + fn drop(&mut self) { + if tokio::runtime::Handle::try_current().is_ok() { + self.inner.request_shutdown(); + } + } +} + +impl Default for ProcessOwnedCodeModeSession { + fn default() -> Self { + Self::new() + } +} + +impl CodeModeSession for ProcessOwnedCodeModeSession { + fn execute<'a>( + &'a self, + request: ExecuteRequest, + ) -> CodeModeSessionResultFuture<'a, StartedCell> { + Box::pin(ProcessOwnedCodeModeSession::execute(self, request)) + } + + fn wait<'a>(&'a self, request: WaitRequest) -> CodeModeSessionResultFuture<'a, WaitOutcome> { + Box::pin(ProcessOwnedCodeModeSession::wait(self, request)) + } + + fn terminate<'a>(&'a self, cell_id: CellId) -> CodeModeSessionResultFuture<'a, WaitOutcome> { + Box::pin(ProcessOwnedCodeModeSession::terminate(self, cell_id)) + } + + fn shutdown<'a>(&'a self) -> CodeModeSessionResultFuture<'a, ()> { + Box::pin(ProcessOwnedCodeModeSession::shutdown(self)) + } +} + +fn default_host_program() -> PathBuf { + resolve_host_program( + std::env::var_os(CODE_MODE_HOST_PATH_ENV), + std::env::current_exe(), + ) +} + +fn resolve_host_program( + override_path: Option, + current_exe: io::Result, +) -> PathBuf { + if let Some(path) = override_path { + return PathBuf::from(path); + } + let executable_name = if cfg!(windows) { + "codex-code-mode-host.exe" + } else { + "codex-code-mode-host" + }; + if let Ok(current_exe) = current_exe + && let Some(parent) = current_exe.parent() + { + return parent.join(executable_name); + } + PathBuf::from(executable_name) +} + +#[cfg(test)] +#[path = "remote_session_tests.rs"] +mod tests; diff --git a/codex-rs/code-mode/src/remote_session/connection.rs b/codex-rs/code-mode/src/remote_session/connection.rs new file mode 100644 index 0000000000..7154784def --- /dev/null +++ b/codex-rs/code-mode/src/remote_session/connection.rs @@ -0,0 +1,458 @@ +use std::path::Path; +use std::process::Stdio; +use std::sync::Arc; +use std::sync::atomic::AtomicBool; +use std::sync::atomic::Ordering; +use std::time::Duration; + +use codex_code_mode_protocol::CellId; +use codex_code_mode_protocol::CodeModeSessionDelegate; +use codex_code_mode_protocol::ExecuteRequest; +use codex_code_mode_protocol::StartedCell; +use codex_code_mode_protocol::WaitOutcome; +use codex_code_mode_protocol::WaitRequest; +use codex_code_mode_protocol::host::CapabilitySet; +use codex_code_mode_protocol::host::ClientHello; +use codex_code_mode_protocol::host::ClientToHost; +use codex_code_mode_protocol::host::EncodedFrame; +use codex_code_mode_protocol::host::FramedReader; +use codex_code_mode_protocol::host::FramedWriter; +use codex_code_mode_protocol::host::HostToClient; +use codex_code_mode_protocol::host::ProtocolVersion; +use codex_code_mode_protocol::host::RequestId; +use codex_code_mode_protocol::host::SupportedProtocolVersions; +use tokio::io::AsyncBufReadExt; +use tokio::io::BufReader; +use tokio::process::Child; +use tokio::process::Command; +use tokio::sync::mpsc; +use tokio::sync::oneshot; +use tokio::task::JoinHandle; +use tokio_util::sync::CancellationToken; +use tracing::debug; +use tracing::warn; + +use self::driver::ConnectionDriver; +use self::driver::DriverCommand; +use self::driver::DriverEvent; +use self::driver::DriverLifecycle; +pub(super) use self::driver::RemoteSession; +pub(super) use self::driver::SessionCleanup; +use self::reader::drive_reader; + +mod driver; +mod reader; + +const IPC_CHANNEL_CAPACITY: usize = 128; +const HOST_HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(10); + +pub(super) struct Connection { + command_tx: mpsc::Sender, + execute_claim_tx: mpsc::UnboundedSender, + alive: Arc, + failure: Arc>>, + cancellation: CancellationToken, +} + +struct CallerCancellation { + token: CancellationToken, + armed: bool, +} + +struct ConnectionSupervisor { + child: Child, + event_tx: mpsc::Sender, + cancellation: CancellationToken, + alive: Arc, + failure: Arc>>, + driver_task: JoinHandle<()>, + reader_task: JoinHandle>, + writer_task: JoinHandle>, +} + +impl CallerCancellation { + fn new() -> Self { + Self { + token: CancellationToken::new(), + armed: true, + } + } + + fn token(&self) -> CancellationToken { + self.token.clone() + } + + fn disarm(mut self) { + self.armed = false; + } +} + +impl Drop for CallerCancellation { + fn drop(&mut self) { + if self.armed { + self.token.cancel(); + } + } +} + +impl Connection { + pub(super) async fn spawn(host_program: &Path) -> Result { + let mut command = Command::new(host_program); + #[cfg(unix)] + command.process_group(0); + let mut child = command + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .kill_on_drop(true) + .spawn() + .map_err(|err| { + format!( + "failed to spawn code-mode host {}: {err}", + host_program.display() + ) + })?; + + if let Some(stderr) = child.stderr.take() { + tokio::spawn(async move { + let mut lines = BufReader::new(stderr).lines(); + loop { + match lines.next_line().await { + Ok(Some(line)) => debug!("code-mode host stderr: {line}"), + Ok(None) => break, + Err(err) => { + warn!("failed to read code-mode host stderr: {err}"); + break; + } + } + } + }); + } + + let stdin = child + .stdin + .take() + .ok_or_else(|| "spawned code-mode host has no stdin".to_string())?; + let stdout = child + .stdout + .take() + .ok_or_else(|| "spawned code-mode host has no stdout".to_string())?; + let mut reader = FramedReader::new(stdout); + let mut writer = FramedWriter::new(stdin); + let handshake = async { + let hello = ClientHello::new( + SupportedProtocolVersions::try_new([ProtocolVersion::V1]) + .map_err(|err| err.to_string())?, + CapabilitySet::empty(), + CapabilitySet::empty(), + ) + .map_err(|err| err.to_string())?; + writer + .write(&ClientToHost::ClientHello(hello)) + .await + .map_err(|err| format!("failed to write code-mode host hello: {err}"))?; + match reader + .read::() + .await + .map_err(|err| format!("failed to read code-mode host hello: {err}"))? + { + Some(HostToClient::HostHello(hello)) + if hello.selected_version() == ProtocolVersion::V1 => + { + Ok(()) + } + Some(HostToClient::HandshakeRejected { reason }) => { + Err(format!("code-mode host rejected the handshake: {reason:?}")) + } + Some(message) => Err(format!( + "code-mode host returned an invalid handshake response: {message:?}" + )), + None => Err("code-mode host exited during handshake".to_string()), + } + }; + let handshake_result = match tokio::time::timeout(HOST_HANDSHAKE_TIMEOUT, handshake).await { + Ok(result) => result, + Err(_) => { + kill_and_reap(&mut child).await; + return Err("timed out negotiating with the code-mode host".to_string()); + } + }; + if let Err(err) = handshake_result { + kill_and_reap(&mut child).await; + return Err(err); + } + + let (command_tx, command_rx) = mpsc::channel(IPC_CHANNEL_CAPACITY); + let (event_tx, event_rx) = mpsc::channel(IPC_CHANNEL_CAPACITY); + let (outgoing_tx, mut outgoing_rx) = mpsc::channel::(IPC_CHANNEL_CAPACITY); + let cancellation = CancellationToken::new(); + let alive = Arc::new(AtomicBool::new(true)); + let failure = Arc::new(std::sync::Mutex::new(None)); + + let writer_cancellation = cancellation.clone(); + let writer_task = tokio::spawn(async move { + loop { + tokio::select! { + _ = writer_cancellation.cancelled() => return Ok(()), + frame = outgoing_rx.recv() => { + let Some(frame) = frame else { + return Err("code-mode host outgoing stream closed".to_string()); + }; + if let Err(err) = writer.write_frame(&frame).await { + return Err(format!("failed to write code-mode host message: {err}")); + } + } + } + } + }); + + let reader_events = event_tx.clone(); + let reader_cancellation = cancellation.clone(); + let reader_task = + tokio::spawn( + async move { drive_reader(reader, reader_events, reader_cancellation).await }, + ); + + let (driver, execute_claim_tx) = ConnectionDriver::new( + command_rx, + event_rx, + event_tx.clone(), + outgoing_tx, + DriverLifecycle { + alive: Arc::clone(&alive), + failure: Arc::clone(&failure), + cancellation: cancellation.clone(), + }, + ); + let driver_task = tokio::spawn(driver.run()); + tokio::spawn( + ConnectionSupervisor { + child, + event_tx, + cancellation: cancellation.clone(), + alive: Arc::clone(&alive), + failure: Arc::clone(&failure), + driver_task, + reader_task, + writer_task, + } + .run(), + ); + + Ok(Self { + command_tx, + execute_claim_tx, + alive, + failure, + cancellation, + }) + } + + pub(super) fn is_alive(&self) -> bool { + if self.command_tx.is_closed() { + mark_connection_dead( + &self.alive, + &self.failure, + "code-mode connection driver closed".to_string(), + ); + } + self.alive.load(Ordering::Acquire) + } + + pub(super) async fn open_session( + &self, + session: RemoteSession, + delegate: Arc, + ) -> Result { + let cleanup = SessionCleanup::new(); + let cancellation = CallerCancellation::new(); + let (response_tx, response_rx) = oneshot::channel(); + self.send(DriverCommand::OpenSession { + session, + delegate, + cleanup: cleanup.clone(), + caller_cancellation: cancellation.token(), + response_tx, + }) + .await?; + let result = self.receive(response_rx).await; + cancellation.disarm(); + result?; + Ok(cleanup) + } + + pub(super) async fn execute( + &self, + session: RemoteSession, + request: ExecuteRequest, + ) -> Result { + let cancellation = CallerCancellation::new(); + let (response_tx, response_rx) = oneshot::channel(); + self.send(DriverCommand::Execute { + session, + request, + caller_cancellation: cancellation.token(), + response_tx, + }) + .await?; + let delivered = match self.receive(response_rx).await { + Ok(delivered) => delivered, + Err(err) => { + cancellation.disarm(); + return Err(err); + } + }; + self.execute_claim_tx + .send(delivered.request_id) + .map_err(|_| self.failure_message())?; + cancellation.disarm(); + Ok(delivered.started) + } + + pub(super) async fn wait( + &self, + session: RemoteSession, + request: WaitRequest, + ) -> Result { + let cancellation = CallerCancellation::new(); + let (response_tx, response_rx) = oneshot::channel(); + self.send(DriverCommand::Wait { + session, + request, + caller_cancellation: cancellation.token(), + response_tx, + }) + .await?; + let result = self.receive(response_rx).await; + cancellation.disarm(); + result + } + + pub(super) async fn terminate( + &self, + session: RemoteSession, + cell_id: CellId, + ) -> Result { + let (response_tx, response_rx) = oneshot::channel(); + self.send(DriverCommand::Terminate { + session, + cell_id, + response_tx, + }) + .await?; + self.receive(response_rx).await + } + + pub(super) async fn shutdown_session(&self, session: RemoteSession) -> Result<(), String> { + let (response_tx, response_rx) = oneshot::channel(); + self.send(DriverCommand::ShutdownSession { + session, + response_tx, + }) + .await?; + self.receive(response_rx).await + } + + async fn send(&self, command: DriverCommand) -> Result<(), String> { + if !self.is_alive() { + return Err(self.failure_message()); + } + self.command_tx + .send(command) + .await + .map_err(|_| self.failure_message()) + } + + async fn receive( + &self, + response_rx: oneshot::Receiver>, + ) -> Result { + response_rx.await.map_err(|_| self.failure_message())? + } + + fn failure_message(&self) -> String { + self.failure + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clone() + .unwrap_or_else(|| "code-mode host connection closed".to_string()) + } +} + +impl Drop for Connection { + fn drop(&mut self) { + mark_connection_dead( + &self.alive, + &self.failure, + "code-mode host connection closed".to_string(), + ); + self.cancellation.cancel(); + } +} + +impl ConnectionSupervisor { + async fn run(mut self) { + let mut child_exited = false; + let reason = tokio::select! { + biased; + _ = self.cancellation.cancelled() => failure_message(&self.failure), + result = &mut self.driver_task => match result { + Ok(()) => "code-mode connection driver exited unexpectedly".to_string(), + Err(err) => format!("code-mode connection driver task failed: {err}"), + }, + result = &mut self.reader_task => task_failure("reader", result), + result = &mut self.writer_task => task_failure("writer", result), + result = self.child.wait() => { + child_exited = true; + match result { + Ok(status) => format!("code-mode host exited with status {status}"), + Err(err) => format!("failed waiting for code-mode host: {err}"), + } + } + }; + mark_connection_dead(&self.alive, &self.failure, reason.clone()); + let _ = self.event_tx.try_send(DriverEvent::Failed(reason)); + self.cancellation.cancel(); + if !child_exited { + kill_and_reap(&mut self.child).await; + } + } +} + +fn task_failure( + task_name: &str, + result: Result, tokio::task::JoinError>, +) -> String { + match result { + Ok(Ok(())) => format!("code-mode connection {task_name} exited unexpectedly"), + Ok(Err(err)) => err, + Err(err) => format!("code-mode connection {task_name} task failed: {err}"), + } +} + +fn mark_connection_dead( + alive: &AtomicBool, + failure: &std::sync::Mutex>, + reason: String, +) { + alive.store(false, Ordering::Release); + let mut failure = failure + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if failure.is_none() { + *failure = Some(reason); + } +} + +fn failure_message(failure: &std::sync::Mutex>) -> String { + failure + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clone() + .unwrap_or_else(|| "code-mode host connection closed".to_string()) +} + +async fn kill_and_reap(child: &mut Child) { + let _ = child.start_kill(); + let _ = child.wait().await; +} diff --git a/codex-rs/code-mode/src/remote_session/connection/driver.rs b/codex-rs/code-mode/src/remote_session/connection/driver.rs new file mode 100644 index 0000000000..102ad47581 --- /dev/null +++ b/codex-rs/code-mode/src/remote_session/connection/driver.rs @@ -0,0 +1,179 @@ +use std::panic::AssertUnwindSafe; +use std::sync::Arc; +use std::sync::atomic::AtomicBool; +use std::sync::atomic::Ordering; + +use codex_code_mode_protocol::CellId; +use codex_code_mode_protocol::CodeModeSessionDelegate; +use codex_code_mode_protocol::host::EncodedFrame; +use codex_code_mode_protocol::host::RequestId; +use tokio::sync::mpsc; +use tokio_util::sync::CancellationToken; + +pub(in crate::remote_session) use self::cleanup::SessionCleanup; +use self::delegate_runtime::DelegateRuntime; +use self::request_tracker::RequestTracker; +use self::session_registry::SessionRegistry; +pub(super) use self::types::DriverCommand; +pub(super) use self::types::DriverEvent; +pub(in crate::remote_session) use self::types::RemoteSession; + +mod cell_ids; +mod cleanup; +mod commands; +mod delegate_runtime; +mod request_tracker; +mod responses; +mod session_registry; +mod types; + +pub(super) struct DriverLifecycle { + pub(super) alive: Arc, + pub(super) failure: Arc>>, + pub(super) cancellation: CancellationToken, +} + +pub(super) struct ConnectionDriver { + command_rx: mpsc::Receiver, + event_rx: mpsc::Receiver, + event_tx: mpsc::Sender, + execute_claim_rx: mpsc::UnboundedReceiver, + outgoing_tx: mpsc::Sender, + requests: RequestTracker, + sessions: SessionRegistry, + delegates: DelegateRuntime, + alive: Arc, + failure: Arc>>, + cancellation: CancellationToken, + failed: bool, +} + +impl ConnectionDriver { + pub(super) fn new( + command_rx: mpsc::Receiver, + event_rx: mpsc::Receiver, + event_tx: mpsc::Sender, + outgoing_tx: mpsc::Sender, + lifecycle: DriverLifecycle, + ) -> (Self, mpsc::UnboundedSender) { + let (execute_claim_tx, execute_claim_rx) = mpsc::unbounded_channel(); + ( + Self { + command_rx, + event_rx, + event_tx: event_tx.clone(), + execute_claim_rx, + outgoing_tx, + requests: RequestTracker::new(), + sessions: SessionRegistry::new(), + delegates: DelegateRuntime::new(event_tx), + alive: lifecycle.alive, + failure: lifecycle.failure, + cancellation: lifecycle.cancellation, + failed: false, + }, + execute_claim_tx, + ) + } + + pub(super) async fn run(mut self) { + loop { + tokio::select! { + biased; + _ = self.cancellation.cancelled() => { + self.fail("code-mode host connection closed".to_string()); + return; + } + event = self.event_rx.recv() => { + let Some(event) = event else { + self.fail("code-mode host event stream closed".to_string()); + return; + }; + if !self.cancel_dropped_callers() || !self.handle_event(event) { + return; + } + } + claim = self.execute_claim_rx.recv() => { + let Some(request_id) = claim else { + self.fail("code-mode execute claim stream closed".to_string()); + return; + }; + self.requests.claim_execute(request_id); + } + command = self.command_rx.recv() => { + let Some(command) = command else { + self.fail("code-mode host command stream closed".to_string()); + return; + }; + if !self.cancel_dropped_callers() || !self.handle_command(command) { + return; + } + } + } + } + } + + fn handle_event(&mut self, event: DriverEvent) -> bool { + let keep_running = match event { + DriverEvent::HostMessage(message) => self.handle_host_message(message), + DriverEvent::DelegateCompleted { id, result } => self.complete_delegate(id, result), + DriverEvent::RequestCancelled(id) => self.cancel_request(id), + DriverEvent::Failed(reason) => { + self.fail(reason); + false + } + }; + if keep_running { + self.flush_deferred_waits() + } else { + false + } + } + + fn queue_frame(&mut self, frame: EncodedFrame) -> bool { + match self.outgoing_tx.try_send(frame) { + Ok(()) => true, + Err(mpsc::error::TrySendError::Full(_)) => { + self.fail("code-mode host outgoing queue is full".to_string()); + false + } + Err(mpsc::error::TrySendError::Closed(_)) => { + self.fail("code-mode host writer closed".to_string()); + false + } + } + } + + fn fail(&mut self, reason: String) { + if self.failed { + return; + } + self.failed = true; + self.alive.store(false, Ordering::Release); + let reason = { + let mut failure = self + .failure + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + failure.get_or_insert(reason).clone() + }; + self.requests.fail_all(&reason); + let failed_sessions = self.sessions.drain(); + self.delegates.fail_all(failed_sessions); + self.cancellation.cancel(); + } +} + +impl Drop for ConnectionDriver { + fn drop(&mut self) { + self.fail("code-mode connection driver stopped unexpectedly".to_string()); + } +} + +fn notify_cell_closed(delegate: &Arc, cell_id: &CellId) { + let _ = std::panic::catch_unwind(AssertUnwindSafe(|| delegate.cell_closed(cell_id))); +} + +#[cfg(test)] +#[path = "driver_tests.rs"] +mod tests; diff --git a/codex-rs/code-mode/src/remote_session/connection/driver/cell_ids.rs b/codex-rs/code-mode/src/remote_session/connection/driver/cell_ids.rs new file mode 100644 index 0000000000..666f6edda6 --- /dev/null +++ b/codex-rs/code-mode/src/remote_session/connection/driver/cell_ids.rs @@ -0,0 +1,111 @@ +use codex_code_mode_protocol::CellId; +use codex_code_mode_protocol::RuntimeResponse; +use codex_code_mode_protocol::WaitOutcome; +use codex_code_mode_protocol::WaitRequest; +use codex_code_mode_protocol::host::WireCellId; +use codex_code_mode_protocol::host::WireRuntimeResponse; +use codex_code_mode_protocol::host::WireWaitOutcome; +use codex_code_mode_protocol::host::WireWaitRequest; + +use super::RemoteSession; + +pub(super) fn public_cell_id(generation: u64, cell_id: &WireCellId) -> CellId { + if generation == 1 { + CellId::new(cell_id.as_str().to_string()) + } else { + CellId::new(format!("g{generation}:{}", cell_id.as_str())) + } +} + +pub(super) fn public_cell_id_from_protocol(generation: u64, cell_id: &CellId) -> CellId { + public_cell_id(generation, &WireCellId::new(cell_id.as_str())) +} + +pub(super) fn remote_cell_id( + session: &RemoteSession, + cell_id: &CellId, +) -> Result { + if session.generation == 1 { + if cell_id.as_str().starts_with('g') && cell_id.as_str().contains(':') { + return Err(format!( + "cell {cell_id} belongs to a stale code-mode host generation" + )); + } + return Ok(WireCellId::new(cell_id.as_str())); + } + let prefix = format!("g{}:", session.generation); + let Some(remote_id) = cell_id.as_str().strip_prefix(&prefix) else { + return Err(format!( + "cell {cell_id} belongs to a stale code-mode host generation" + )); + }; + Ok(WireCellId::new(remote_id)) +} + +pub(super) fn remote_wait_request( + session: &RemoteSession, + request: WaitRequest, +) -> Result { + Ok(WireWaitRequest { + cell_id: remote_cell_id(session, &request.cell_id)?, + yield_time_ms: request.yield_time_ms, + }) +} + +pub(super) fn public_runtime_response( + generation: u64, + response: RuntimeResponse, +) -> RuntimeResponse { + match response { + RuntimeResponse::Yielded { + cell_id, + content_items, + } => RuntimeResponse::Yielded { + cell_id: public_cell_id_from_protocol(generation, &cell_id), + content_items, + }, + RuntimeResponse::Terminated { + cell_id, + content_items, + } => RuntimeResponse::Terminated { + cell_id: public_cell_id_from_protocol(generation, &cell_id), + content_items, + }, + RuntimeResponse::Result { + cell_id, + content_items, + error_text, + } => RuntimeResponse::Result { + cell_id: public_cell_id_from_protocol(generation, &cell_id), + content_items, + error_text, + }, + } +} + +pub(super) fn public_wait_outcome(generation: u64, outcome: WaitOutcome) -> WaitOutcome { + match outcome { + WaitOutcome::LiveCell(response) => { + WaitOutcome::LiveCell(public_runtime_response(generation, response)) + } + WaitOutcome::MissingCell(response) => { + WaitOutcome::MissingCell(public_runtime_response(generation, response)) + } + } +} + +pub(super) fn runtime_response_cell_id(response: &WireRuntimeResponse) -> &WireCellId { + match response { + WireRuntimeResponse::Yielded { cell_id, .. } + | WireRuntimeResponse::Terminated { cell_id, .. } + | WireRuntimeResponse::Result { cell_id, .. } => cell_id, + } +} + +pub(super) fn wait_outcome_cell_id(outcome: &WireWaitOutcome) -> &WireCellId { + match outcome { + WireWaitOutcome::LiveCell(response) | WireWaitOutcome::MissingCell(response) => { + runtime_response_cell_id(response) + } + } +} diff --git a/codex-rs/code-mode/src/remote_session/connection/driver/cleanup.rs b/codex-rs/code-mode/src/remote_session/connection/driver/cleanup.rs new file mode 100644 index 0000000000..b994b64887 --- /dev/null +++ b/codex-rs/code-mode/src/remote_session/connection/driver/cleanup.rs @@ -0,0 +1,40 @@ +use std::sync::Arc; + +use tokio_util::sync::CancellationToken; + +use super::notify_cell_closed; +use super::session_registry::CellOwner; + +struct CleanupInner { + complete: CancellationToken, +} + +#[derive(Clone)] +pub(in crate::remote_session) struct SessionCleanup { + inner: Arc, +} + +impl SessionCleanup { + pub(in crate::remote_session) fn new() -> Self { + Self { + inner: Arc::new(CleanupInner { + complete: CancellationToken::new(), + }), + } + } + + pub(super) fn fail(&self, cells: Vec) { + for owner in cells { + notify_cell_closed(&owner.delegate, &owner.cell_id); + } + self.inner.complete.cancel(); + } + + pub(in crate::remote_session) async fn wait(&self) { + self.inner.complete.cancelled().await; + } + + pub(in crate::remote_session) fn is_complete(&self) -> bool { + self.inner.complete.is_cancelled() + } +} diff --git a/codex-rs/code-mode/src/remote_session/connection/driver/commands.rs b/codex-rs/code-mode/src/remote_session/connection/driver/commands.rs new file mode 100644 index 0000000000..c9cba9c32d --- /dev/null +++ b/codex-rs/code-mode/src/remote_session/connection/driver/commands.rs @@ -0,0 +1,298 @@ +use std::sync::Arc; + +use codex_code_mode_protocol::CellId; +use codex_code_mode_protocol::CodeModeSessionDelegate; +use codex_code_mode_protocol::ExecuteRequest; +use codex_code_mode_protocol::WaitOutcome; +use codex_code_mode_protocol::WaitRequest; +use codex_code_mode_protocol::host::ClientToHost; +use codex_code_mode_protocol::host::EncodedFrame; +use codex_code_mode_protocol::host::HostRequest; +use codex_code_mode_protocol::host::WireWaitRequest; +use tokio::sync::oneshot; +use tokio_util::sync::CancellationToken; + +use super::ConnectionDriver; +use super::cell_ids::remote_cell_id; +use super::cell_ids::remote_wait_request; +use super::types::CancellableRequest; +use super::types::DeferredWait; +use super::types::DeliveredExecute; +use super::types::DriverCommand; +use super::types::PendingRequest; +use super::types::RemoteSession; + +impl ConnectionDriver { + pub(super) fn handle_command(&mut self, command: DriverCommand) -> bool { + match command { + DriverCommand::OpenSession { + session, + delegate, + cleanup, + caller_cancellation, + response_tx, + } => self.open_session(session, delegate, cleanup, caller_cancellation, response_tx), + DriverCommand::Execute { + session, + request, + caller_cancellation, + response_tx, + } => self.execute(session, request, caller_cancellation, response_tx), + DriverCommand::Wait { + session, + request, + caller_cancellation, + response_tx, + } => self.wait(session, request, caller_cancellation, response_tx), + DriverCommand::Terminate { + session, + cell_id, + response_tx, + } => self.terminate(session, cell_id, response_tx), + DriverCommand::ShutdownSession { + session, + response_tx, + } => self.shutdown_session(session, response_tx), + } + } + + fn open_session( + &mut self, + session: RemoteSession, + delegate: Arc, + cleanup: super::cleanup::SessionCleanup, + caller_cancellation: CancellationToken, + response_tx: oneshot::Sender>, + ) -> bool { + if self.sessions.contains(&session.id) || self.requests.contains_pending_open(&session) { + let _ = response_tx.send(Err(format!( + "code-mode session {} is already open", + session.id + ))); + return true; + } + let request_id = match self.requests.allocate_id() { + Ok(id) => id, + Err(err) => { + let _ = response_tx.send(Err(err)); + return false; + } + }; + let message = ClientToHost::Request { + id: request_id, + request: HostRequest::OpenSession { + session_id: session.id.clone(), + }, + }; + let frame = match EncodedFrame::encode(&message) { + Ok(frame) => frame, + Err(err) => { + let _ = response_tx.send(Err(format!( + "failed to encode code-mode open-session request: {err}" + ))); + return true; + } + }; + let cancellation = CancellableRequest::new(caller_cancellation); + self.requests.insert_pending( + request_id, + PendingRequest::OpenSession { + session, + delegate, + cleanup, + cancellation, + response_tx, + }, + &self.event_tx, + ); + self.queue_frame(frame) + } + + fn execute( + &mut self, + session: RemoteSession, + request: ExecuteRequest, + caller_cancellation: CancellationToken, + response_tx: oneshot::Sender>, + ) -> bool { + if let Err(err) = self.sessions.require_ready(&session) { + let _ = response_tx.send(Err(err)); + return true; + } + let request = match request.try_into() { + Ok(request) => request, + Err(err) => { + let _ = response_tx.send(Err(format!( + "failed to encode code-mode execute request: {err}" + ))); + return true; + } + }; + let request_id = match self.requests.allocate_id() { + Ok(id) => id, + Err(err) => { + let _ = response_tx.send(Err(err)); + return false; + } + }; + let message = ClientToHost::Request { + id: request_id, + request: HostRequest::Execute { + session_id: session.id.clone(), + request, + }, + }; + let frame = match EncodedFrame::encode(&message) { + Ok(frame) => frame, + Err(err) => { + let _ = response_tx.send(Err(format!( + "code-mode execute request exceeds the IPC frame limit: {err}" + ))); + return true; + } + }; + let (initial_response_tx, initial_response_rx) = oneshot::channel(); + let cancellation = CancellableRequest::new(caller_cancellation); + self.requests.insert_pending( + request_id, + PendingRequest::Execute { + session, + response_tx, + initial_response_tx, + initial_response_rx, + cancellation, + }, + &self.event_tx, + ); + self.queue_frame(frame) + } + + fn wait( + &mut self, + session: RemoteSession, + request: WaitRequest, + caller_cancellation: CancellationToken, + response_tx: oneshot::Sender>, + ) -> bool { + if let Err(err) = self.sessions.require_ready(&session) { + let _ = response_tx.send(Err(err)); + return true; + } + let request = match remote_wait_request(&session, request) { + Ok(request) => request, + Err(err) => { + let _ = response_tx.send(Err(err)); + return true; + } + }; + if self.requests.has_cancelled_wait(&session, &request.cell_id) { + self.requests.push_deferred_wait(DeferredWait { + session, + request, + caller_cancellation, + response_tx, + }); + return true; + } + self.start_wait(session, request, caller_cancellation, response_tx) + } + + pub(super) fn start_wait( + &mut self, + session: RemoteSession, + request: WireWaitRequest, + caller_cancellation: CancellationToken, + response_tx: oneshot::Sender>, + ) -> bool { + let cell_id = request.cell_id.clone(); + self.send_request( + HostRequest::Wait { + session_id: session.id.clone(), + request, + }, + PendingRequest::Wait { + session, + cell_id, + cancellation: CancellableRequest::new(caller_cancellation), + response_tx, + }, + ) + } + + fn terminate( + &mut self, + session: RemoteSession, + cell_id: CellId, + response_tx: oneshot::Sender>, + ) -> bool { + if let Err(err) = self.sessions.require_ready(&session) { + let _ = response_tx.send(Err(err)); + return true; + } + let cell_id = match remote_cell_id(&session, &cell_id) { + Ok(cell_id) => cell_id, + Err(err) => { + let _ = response_tx.send(Err(err)); + return true; + } + }; + let pending_cell_id = cell_id.clone(); + self.send_request( + HostRequest::Terminate { + session_id: session.id.clone(), + cell_id, + }, + PendingRequest::Terminate { + session, + cell_id: pending_cell_id, + response_tx, + }, + ) + } + + fn shutdown_session( + &mut self, + session: RemoteSession, + response_tx: oneshot::Sender>, + ) -> bool { + if let Err(err) = self.sessions.begin_shutdown(&session) { + let _ = response_tx.send(Err(err)); + return true; + } + self.send_request( + HostRequest::ShutdownSession { + session_id: session.id.clone(), + }, + PendingRequest::ShutdownSession { + session, + response_tx, + }, + ) + } + + pub(super) fn send_request(&mut self, request: HostRequest, pending: PendingRequest) -> bool { + let request_id = match self.requests.allocate_id() { + Ok(id) => id, + Err(err) => { + pending.fail(err); + return false; + } + }; + let message = ClientToHost::Request { + id: request_id, + request, + }; + let frame = match EncodedFrame::encode(&message) { + Ok(frame) => frame, + Err(err) => { + pending.fail(format!( + "code-mode request exceeds the IPC frame limit: {err}" + )); + return true; + } + }; + self.requests + .insert_pending(request_id, pending, &self.event_tx); + self.queue_frame(frame) + } +} diff --git a/codex-rs/code-mode/src/remote_session/connection/driver/delegate_runtime.rs b/codex-rs/code-mode/src/remote_session/connection/driver/delegate_runtime.rs new file mode 100644 index 0000000000..c79270661d --- /dev/null +++ b/codex-rs/code-mode/src/remote_session/connection/driver/delegate_runtime.rs @@ -0,0 +1,339 @@ +//! Client-side delegate task and closure lifecycle. +//! +//! Cancellation revokes the task's completion path before removing its active-call state. The +//! delegate future may finish later, but it can no longer send a response or affect cell closure. + +use std::collections::HashMap; +use std::collections::HashSet; +use std::collections::VecDeque; + +use codex_code_mode_protocol::CodeModeNestedToolCall; +use codex_code_mode_protocol::host::ClientToHost; +use codex_code_mode_protocol::host::DelegateRequest; +use codex_code_mode_protocol::host::DelegateRequestId; +use codex_code_mode_protocol::host::DelegateResponse; +use codex_code_mode_protocol::host::EncodedFrame; +use codex_code_mode_protocol::host::SessionId; +use codex_code_mode_protocol::host::WireCellId; +use codex_code_mode_protocol::host::WireResult; +use tokio::sync::mpsc; +use tokio_util::sync::CancellationToken; + +use super::ConnectionDriver; +use super::notify_cell_closed; +use super::session_registry::CellOwner; +use super::session_registry::DelegateTarget; +use super::session_registry::FailedSession; +use super::types::DriverEvent; + +const MAX_RECENT_DELEGATE_REQUEST_IDS: usize = 4096; + +#[derive(Clone, Eq, Hash, PartialEq)] +struct CellKey { + session_id: codex_code_mode_protocol::host::SessionId, + cell_id: codex_code_mode_protocol::CellId, +} + +impl CellKey { + fn for_owner(owner: &CellOwner) -> Self { + Self { + session_id: owner.session_id.clone(), + cell_id: owner.cell_id.clone(), + } + } +} + +struct DelegateCall { + cell: CellKey, + cancellation: CancellationToken, + completion_stop: CancellationToken, +} + +impl DelegateCall { + fn revoke(&self) { + self.cancellation.cancel(); + self.completion_stop.cancel(); + } +} + +enum DelegateTask { + InvokeTool(CodeModeNestedToolCall), + Notify { + call_id: String, + cell_id: codex_code_mode_protocol::CellId, + text: String, + }, +} + +pub(super) struct DelegateEffects { + pub(super) response: Option<(DelegateRequestId, Result)>, + pub(super) closed_cells: Vec, +} + +impl DelegateEffects { + fn empty() -> Self { + Self { + response: None, + closed_cells: Vec::new(), + } + } + + fn append(&mut self, mut other: Self) { + debug_assert!(self.response.is_none()); + self.response = other.response.take(); + self.closed_cells.append(&mut other.closed_cells); + } +} + +pub(super) struct DelegateRuntime { + calls: HashMap, + seen_requests: HashSet, + request_order: VecDeque, + event_tx: mpsc::Sender, +} + +impl DelegateRuntime { + pub(super) fn new(event_tx: mpsc::Sender) -> Self { + Self { + calls: HashMap::new(), + seen_requests: HashSet::new(), + request_order: VecDeque::new(), + event_tx, + } + } + + pub(super) fn start( + &mut self, + id: DelegateRequestId, + target: DelegateTarget, + request: DelegateRequest, + ) -> Result<(), String> { + if self.calls.contains_key(&id) || self.seen_requests.contains(&id) { + return Err(format!("duplicate code-mode delegate request ID {id:?}")); + } + self.remember_request(id); + let cancellation = CancellationToken::new(); + let task_request = match request { + DelegateRequest::InvokeTool { invocation } => { + let mut invocation: CodeModeNestedToolCall = invocation.into(); + invocation.cell_id = target.cell_id.clone(); + DelegateTask::InvokeTool(invocation) + } + DelegateRequest::Notify { + call_id, + cell_id: _, + text, + } => DelegateTask::Notify { + call_id, + cell_id: target.cell_id.clone(), + text, + }, + }; + let delegate = target.delegate; + let task_cancellation = cancellation.clone(); + let delegate_task = tokio::spawn(async move { + match task_request { + DelegateTask::InvokeTool(invocation) => delegate + .invoke_tool(invocation, task_cancellation) + .await + .map(|result| DelegateResponse::ToolResult { result }), + DelegateTask::Notify { + call_id, + cell_id, + text, + } => delegate + .notify(call_id, cell_id, text, task_cancellation) + .await + .map(|()| DelegateResponse::NotificationDelivered), + } + }); + let completion_stop = CancellationToken::new(); + self.calls.insert( + id, + DelegateCall { + cell: CellKey { + session_id: target.session_id, + cell_id: target.cell_id, + }, + cancellation, + completion_stop: completion_stop.clone(), + }, + ); + let event_tx = self.event_tx.clone(); + tokio::spawn(async move { + let result = tokio::select! { + biased; + _ = completion_stop.cancelled() => return, + result = delegate_task => match result { + Ok(result) => result, + Err(err) => Err(format!("code-mode delegate task failed: {err}")), + }, + }; + tokio::select! { + biased; + _ = completion_stop.cancelled() => {} + _ = event_tx.send(DriverEvent::DelegateCompleted { id, result }) => {} + } + }); + Ok(()) + } + + pub(super) fn cancel(&mut self, id: DelegateRequestId) { + if let Some(call) = self.calls.remove(&id) { + call.revoke(); + } + } + + pub(super) fn complete( + &mut self, + id: DelegateRequestId, + result: Result, + ) -> DelegateEffects { + if self.calls.remove(&id).is_none() { + return DelegateEffects::empty(); + } + let mut effects = DelegateEffects::empty(); + effects.response = Some((id, result)); + effects + } + + pub(super) fn close_cell(&mut self, owner: CellOwner) -> DelegateEffects { + let key = CellKey::for_owner(&owner); + self.calls.retain(|_, call| { + if call.cell != key { + return true; + } + call.revoke(); + false + }); + let mut effects = DelegateEffects::empty(); + effects.closed_cells.push(owner); + effects + } + + pub(super) fn close_cells(&mut self, owners: Vec) -> DelegateEffects { + let mut effects = DelegateEffects::empty(); + for owner in owners { + effects.append(self.close_cell(owner)); + } + effects + } + + pub(super) fn fail_all(&mut self, failed_sessions: Vec) { + for (_, call) in self.calls.drain() { + call.revoke(); + } + for session in failed_sessions { + session.cleanup.fail(session.cells); + } + } + + fn remember_request(&mut self, id: DelegateRequestId) { + self.seen_requests.insert(id); + self.request_order.push_back(id); + while self.request_order.len() > MAX_RECENT_DELEGATE_REQUEST_IDS { + if let Some(expired) = self.request_order.pop_front() { + self.seen_requests.remove(&expired); + } + } + } +} + +impl ConnectionDriver { + pub(super) fn start_delegate( + &mut self, + id: DelegateRequestId, + session_id: SessionId, + request: DelegateRequest, + ) -> bool { + let wire_cell_id = match &request { + DelegateRequest::InvokeTool { invocation } => &invocation.cell_id, + DelegateRequest::Notify { cell_id, .. } => cell_id, + }; + let target = match self.sessions.delegate_target(&session_id, wire_cell_id) { + Ok(target) => target, + Err(err) => { + self.fail(err); + return false; + } + }; + if let Err(err) = self.delegates.start(id, target, request) { + self.fail(err); + return false; + } + true + } + + pub(super) fn complete_delegate( + &mut self, + id: DelegateRequestId, + result: Result, + ) -> bool { + let effects = self.delegates.complete(id, result); + self.apply_delegate_effects(effects) + } + + fn send_delegate_response( + &mut self, + id: DelegateRequestId, + result: Result, + ) -> bool { + let message = ClientToHost::DelegateResponse { + id, + result: WireResult::from_result(result), + }; + let frame = match EncodedFrame::encode(&message) { + Ok(frame) => frame, + Err(err) => { + let fallback = ClientToHost::DelegateResponse { + id, + result: WireResult::Err { + message: format!( + "code-mode delegate response exceeds the IPC frame limit: {err}" + ), + }, + }; + match EncodedFrame::encode(&fallback) { + Ok(frame) => frame, + Err(fallback_err) => { + self.fail(format!( + "failed to encode code-mode delegate error response: {fallback_err}" + )); + return false; + } + } + } + }; + self.queue_frame(frame) + } + + pub(super) fn close_cell(&mut self, session_id: SessionId, cell_id: WireCellId) -> bool { + let owner = match self.sessions.remove_cell(&session_id, &cell_id) { + Ok(owner) => owner, + Err(err) => { + self.fail(err); + return false; + } + }; + let effects = self.delegates.close_cell(owner); + self.apply_delegate_effects(effects) + } + + pub(super) fn close_session_locally(&mut self, session_id: &SessionId) -> DelegateEffects { + self.requests.remove_unclaimed_for_session(session_id); + let owners = self.sessions.remove_session(session_id); + self.delegates.close_cells(owners) + } + + pub(super) fn apply_delegate_effects(&mut self, effects: DelegateEffects) -> bool { + if let Some((id, result)) = effects.response + && !self.send_delegate_response(id, result) + { + return false; + } + for closed in effects.closed_cells { + notify_cell_closed(&closed.delegate, &closed.cell_id); + } + true + } +} diff --git a/codex-rs/code-mode/src/remote_session/connection/driver/request_tracker.rs b/codex-rs/code-mode/src/remote_session/connection/driver/request_tracker.rs new file mode 100644 index 0000000000..0e7fc0881d --- /dev/null +++ b/codex-rs/code-mode/src/remote_session/connection/driver/request_tracker.rs @@ -0,0 +1,186 @@ +use std::collections::HashMap; +use std::collections::VecDeque; + +use codex_code_mode_protocol::host::RequestId; +use codex_code_mode_protocol::host::SessionId; +use codex_code_mode_protocol::host::WireCellId; +use tokio::sync::mpsc; + +use super::types::DeferredWait; +use super::types::DriverEvent; +use super::types::InitialResponse; +use super::types::PendingRequest; +use super::types::RemoteSession; +use super::types::UnclaimedExecute; + +pub(super) enum CancellationAction { + Send(RequestId), + Terminate { + request_id: RequestId, + execute: UnclaimedExecute, + }, +} + +pub(super) struct RequestTracker { + pending: HashMap, + unclaimed_executes: HashMap, + initial_responses: HashMap, + deferred_waits: VecDeque, + next_request_id: i64, +} + +impl RequestTracker { + pub(super) fn new() -> Self { + Self { + pending: HashMap::new(), + unclaimed_executes: HashMap::new(), + initial_responses: HashMap::new(), + deferred_waits: VecDeque::new(), + next_request_id: 1, + } + } + + pub(super) fn contains_pending_open(&self, session: &RemoteSession) -> bool { + self.pending.values().any(|pending| { + matches!( + pending, + PendingRequest::OpenSession { + session: pending_session, + .. + } if pending_session.id == session.id + ) + }) + } + + pub(super) fn allocate_id(&mut self) -> Result { + let id = self.next_request_id; + self.next_request_id = self + .next_request_id + .checked_add(1) + .ok_or_else(|| "code-mode host request ID space exhausted".to_string())?; + Ok(RequestId::new(id)) + } + + pub(super) fn insert_pending( + &mut self, + id: RequestId, + pending: PendingRequest, + event_tx: &mpsc::Sender, + ) { + self.pending.insert(id, pending); + if let Some(cancellation) = self + .pending + .get_mut(&id) + .and_then(PendingRequest::cancellation_mut) + { + cancellation.spawn_watcher(id, event_tx.clone()); + } + } + + pub(super) fn remove_pending(&mut self, id: RequestId) -> Option { + self.pending.remove(&id) + } + + pub(super) fn insert_initial_response(&mut self, id: RequestId, response: InitialResponse) { + self.initial_responses.insert(id, response); + } + + pub(super) fn remove_initial_response(&mut self, id: RequestId) -> Option { + self.initial_responses.remove(&id) + } + + pub(super) fn insert_unclaimed_execute(&mut self, id: RequestId, execute: UnclaimedExecute) { + self.unclaimed_executes.insert(id, execute); + } + + pub(super) fn claim_execute(&mut self, id: RequestId) { + self.unclaimed_executes.remove(&id); + } + + pub(super) fn collect_cancellations(&mut self) -> Vec { + let mut actions = self + .pending + .iter_mut() + .filter_map(|(id, pending)| { + let cancellation = pending.cancellation_mut()?; + (cancellation.is_cancelled() && cancellation.mark_reported()) + .then_some(CancellationAction::Send(*id)) + }) + .collect::>(); + actions.extend( + self.unclaimed_executes + .extract_if(|_, execute| { + execute.cancellation.is_cancelled() && execute.cancellation.mark_reported() + }) + .map(|(request_id, execute)| CancellationAction::Terminate { + request_id, + execute, + }), + ); + actions + } + + pub(super) fn mark_cancelled(&mut self, id: RequestId) -> Option { + if let Some(cancellation) = self + .pending + .get_mut(&id) + .and_then(PendingRequest::cancellation_mut) + { + return cancellation + .mark_reported() + .then_some(CancellationAction::Send(id)); + } + let execute = self.unclaimed_executes.get_mut(&id)?; + if !execute.cancellation.mark_reported() { + return None; + } + self.unclaimed_executes + .remove(&id) + .map(|execute| CancellationAction::Terminate { + request_id: id, + execute, + }) + } + + pub(super) fn has_cancelled_wait(&self, session: &RemoteSession, cell_id: &WireCellId) -> bool { + self.pending.values().any(|pending| { + matches!( + pending, + PendingRequest::Wait { + session: pending_session, + cell_id: pending_cell_id, + cancellation, + .. + } if pending_session == session + && pending_cell_id == cell_id + && cancellation.is_cancelled() + ) + }) + } + + pub(super) fn push_deferred_wait(&mut self, wait: DeferredWait) { + self.deferred_waits.push_back(wait); + } + + pub(super) fn take_deferred_waits(&mut self) -> VecDeque { + std::mem::take(&mut self.deferred_waits) + } + + pub(super) fn remove_unclaimed_for_session(&mut self, session_id: &SessionId) { + self.unclaimed_executes + .retain(|_, execute| &execute.session.id != session_id); + } + + pub(super) fn fail_all(&mut self, reason: &str) { + for (_, pending) in self.pending.drain() { + pending.fail(reason.to_string()); + } + self.unclaimed_executes.clear(); + for (_, initial) in self.initial_responses.drain() { + let _ = initial.response_tx.send(Err(reason.to_string())); + } + for wait in self.deferred_waits.drain(..) { + let _ = wait.response_tx.send(Err(reason.to_string())); + } + } +} diff --git a/codex-rs/code-mode/src/remote_session/connection/driver/responses.rs b/codex-rs/code-mode/src/remote_session/connection/driver/responses.rs new file mode 100644 index 0000000000..19c78ec000 --- /dev/null +++ b/codex-rs/code-mode/src/remote_session/connection/driver/responses.rs @@ -0,0 +1,397 @@ +use codex_code_mode_protocol::StartedCell; +use codex_code_mode_protocol::host::ClientToHost; +use codex_code_mode_protocol::host::EncodedFrame; +use codex_code_mode_protocol::host::HostRequest; +use codex_code_mode_protocol::host::HostResponse; +use codex_code_mode_protocol::host::HostToClient; +use codex_code_mode_protocol::host::RequestId; +use codex_code_mode_protocol::host::WireCellId; +use tokio::sync::oneshot; + +use super::ConnectionDriver; +use super::cell_ids::public_runtime_response; +use super::cell_ids::public_wait_outcome; +use super::cell_ids::runtime_response_cell_id; +use super::cell_ids::wait_outcome_cell_id; +use super::request_tracker::CancellationAction; +use super::session_registry::CellAdmissionError; +use super::types::DeliveredExecute; +use super::types::InitialResponse; +use super::types::PendingRequest; +use super::types::RemoteSession; +use super::types::UnclaimedExecute; + +impl ConnectionDriver { + pub(super) fn flush_deferred_waits(&mut self) -> bool { + let mut deferred = self.requests.take_deferred_waits(); + while let Some(wait) = deferred.pop_front() { + if wait.caller_cancellation.is_cancelled() { + let _ = wait + .response_tx + .send(Err("code-mode request cancelled".to_string())); + continue; + } + if self + .requests + .has_cancelled_wait(&wait.session, &wait.request.cell_id) + { + self.requests.push_deferred_wait(wait); + continue; + } + if !self.start_wait( + wait.session, + wait.request, + wait.caller_cancellation, + wait.response_tx, + ) { + for wait in deferred { + let _ = wait + .response_tx + .send(Err("code-mode host connection closed".to_string())); + } + return false; + } + } + true + } + + pub(super) fn handle_host_message(&mut self, message: HostToClient) -> bool { + match message { + HostToClient::Response { id, result } => { + self.complete_request(id, result.into_result()) + } + HostToClient::InitialResponse { id, result } => { + self.complete_initial_response(id, result.into_result()) + } + HostToClient::DelegateRequest { + id, + session_id, + request, + } => self.start_delegate(id, session_id, request), + HostToClient::CancelDelegateRequest { id } => { + self.delegates.cancel(id); + true + } + HostToClient::CellClosed { + session_id, + cell_id, + } => self.close_cell(session_id, cell_id), + HostToClient::HostHello(_) | HostToClient::HandshakeRejected { .. } => { + self.fail("code-mode host sent a second handshake response".to_string()); + false + } + } + } + + fn complete_request(&mut self, id: RequestId, result: Result) -> bool { + let Some(pending) = self.requests.remove_pending(id) else { + self.fail(format!("code-mode host returned unknown request ID {id:?}")); + return false; + }; + match pending { + PendingRequest::OpenSession { + session, + delegate, + cleanup, + cancellation, + response_tx, + } => match result { + Ok(HostResponse::SessionReady { session_id }) if session_id == session.id => { + let abandoned = cancellation.is_cancelled() || response_tx.is_closed(); + self.sessions + .insert_ready(session.clone(), delegate, cleanup); + if abandoned || response_tx.send(Ok(())).is_err() { + return self.shutdown_abandoned_session(session); + } + } + Ok(_) => { + let reason = + "code-mode host returned an invalid open-session response".to_string(); + let _ = response_tx.send(Err(reason.clone())); + self.fail(reason); + return false; + } + Err(err) => { + let _ = response_tx.send(Err(err)); + } + }, + PendingRequest::Execute { + session, + response_tx, + initial_response_tx, + initial_response_rx, + cancellation, + } => match result { + Ok(HostResponse::ExecutionStarted { cell_id }) => { + // The host owns a checked, never-reused ID sequence. Retain only live + // IDs so client memory scales with concurrency, not session lifetime. + let remote_cell_id = cell_id.clone(); + let public_id = match self.sessions.admit_cell(&session, cell_id) { + Ok(public_id) => public_id, + Err(CellAdmissionError::MissingSession) => { + let _ = response_tx + .send(Err("code-mode session closed during execute".to_string())); + return true; + } + Err(CellAdmissionError::DuplicateCell) => { + let reason = format!( + "code-mode host reused live cell {} in session {}", + remote_cell_id.as_str(), + session.id + ); + let _ = response_tx.send(Err(reason.clone())); + self.fail(reason); + return false; + } + }; + self.requests.insert_initial_response( + id, + InitialResponse { + generation: session.generation, + cell_id: remote_cell_id.clone(), + response_tx: initial_response_tx, + }, + ); + let started = StartedCell::from_result_receiver(public_id, initial_response_rx); + if cancellation.is_cancelled() || response_tx.is_closed() { + return self.terminate_abandoned_cell(session, remote_cell_id); + } + let delivered = DeliveredExecute { + request_id: id, + started, + }; + if response_tx.send(Ok(delivered)).is_err() { + return self.terminate_abandoned_cell(session, remote_cell_id); + } + self.requests.insert_unclaimed_execute( + id, + UnclaimedExecute { + session, + cell_id: remote_cell_id, + cancellation, + }, + ); + } + Ok(_) => { + let reason = "code-mode host returned an invalid execute response".to_string(); + let _ = response_tx.send(Err(reason.clone())); + self.fail(reason); + return false; + } + Err(err) => { + let _ = response_tx.send(Err(err)); + } + }, + PendingRequest::Wait { + session, + cell_id, + cancellation: _, + response_tx, + } => { + let result = match result { + Ok(HostResponse::WaitCompleted { outcome }) => { + if wait_outcome_cell_id(&outcome) != &cell_id { + let reason = format!( + "code-mode host returned cell {} for request targeting {}", + wait_outcome_cell_id(&outcome).as_str(), + cell_id.as_str() + ); + let _ = response_tx.send(Err(reason.clone())); + self.fail(reason); + return false; + } + Ok(public_wait_outcome(session.generation, outcome.into())) + } + Ok(_) => { + let reason = "code-mode host returned an invalid cell response".to_string(); + let _ = response_tx.send(Err(reason.clone())); + self.fail(reason); + return false; + } + Err(err) => Err(err), + }; + let _ = response_tx.send(result); + } + PendingRequest::Terminate { + session, + cell_id, + response_tx, + } => { + let result = match result { + Ok(HostResponse::WaitCompleted { outcome }) => { + if wait_outcome_cell_id(&outcome) != &cell_id { + let reason = format!( + "code-mode host returned cell {} for request targeting {}", + wait_outcome_cell_id(&outcome).as_str(), + cell_id.as_str() + ); + let _ = response_tx.send(Err(reason.clone())); + self.fail(reason); + return false; + } + public_wait_outcome(session.generation, outcome.into()) + } + Ok(_) => { + let reason = "code-mode host returned an invalid cell response".to_string(); + let _ = response_tx.send(Err(reason.clone())); + self.fail(reason); + return false; + } + Err(err) => { + let _ = response_tx.send(Err(err)); + return true; + } + }; + let _ = response_tx.send(Ok(result)); + } + PendingRequest::ShutdownSession { + session, + response_tx, + } => match result { + Ok(HostResponse::SessionClosed { session_id }) if session_id == session.id => { + let effects = self.close_session_locally(&session.id); + if !self.apply_delegate_effects(effects) { + return false; + } + let _ = response_tx.send(Ok(())); + } + Ok(_) => { + let err = "code-mode host returned an invalid shutdown response".to_string(); + let _ = response_tx.send(Err(err.clone())); + self.fail(err); + return false; + } + Err(err) => { + let _ = response_tx.send(Err(err.clone())); + self.fail(err); + return false; + } + }, + } + true + } + + pub(super) fn cancel_dropped_callers(&mut self) -> bool { + for action in self.requests.collect_cancellations() { + if !self.apply_cancellation(action) { + return false; + } + } + true + } + + pub(super) fn cancel_request(&mut self, id: RequestId) -> bool { + self.requests + .mark_cancelled(id) + .is_none_or(|action| self.apply_cancellation(action)) + } + + fn apply_cancellation(&mut self, action: CancellationAction) -> bool { + match action { + CancellationAction::Send(id) => self.send_cancel_request(id), + CancellationAction::Terminate { + request_id, + execute, + } => { + if !self.send_cancel_request(request_id) { + return false; + } + self.terminate_abandoned_cell(execute.session, execute.cell_id) + } + } + } + + fn send_cancel_request(&mut self, id: RequestId) -> bool { + let frame = match EncodedFrame::encode(&ClientToHost::CancelRequest { id }) { + Ok(frame) => frame, + Err(err) => { + self.fail(format!( + "failed to encode code-mode cancellation request: {err}" + )); + return false; + } + }; + self.queue_frame(frame) + } + + fn shutdown_abandoned_session(&mut self, session: RemoteSession) -> bool { + let Some(should_shutdown) = self.sessions.begin_abandoned_shutdown(&session.id) else { + self.fail(format!( + "code-mode host committed abandoned session {} without local state", + session.id + )); + return false; + }; + if !should_shutdown { + return true; + } + let (response_tx, response_rx) = oneshot::channel(); + drop(response_rx); + self.send_request( + HostRequest::ShutdownSession { + session_id: session.id.clone(), + }, + PendingRequest::ShutdownSession { + session, + response_tx, + }, + ) + } + + fn terminate_abandoned_cell(&mut self, session: RemoteSession, cell_id: WireCellId) -> bool { + let Some(is_closing) = self.sessions.is_closing(&session.id) else { + self.fail(format!( + "code-mode host admitted an abandoned cell in unknown session {}", + session.id + )); + return false; + }; + if is_closing { + return true; + } + let (response_tx, response_rx) = oneshot::channel(); + drop(response_rx); + self.send_request( + HostRequest::Terminate { + session_id: session.id.clone(), + cell_id: cell_id.clone(), + }, + PendingRequest::Terminate { + session, + cell_id, + response_tx, + }, + ) + } + + fn complete_initial_response( + &mut self, + id: RequestId, + result: Result, + ) -> bool { + let Some(initial) = self.requests.remove_initial_response(id) else { + self.fail(format!( + "code-mode host returned initial response for unknown request ID {id:?}" + )); + return false; + }; + let response = match result { + Ok(response) if runtime_response_cell_id(&response) == &initial.cell_id => { + Ok(public_runtime_response(initial.generation, response.into())) + } + Ok(response) => { + let reason = format!( + "code-mode host returned initial response for cell {} instead of {}", + runtime_response_cell_id(&response).as_str(), + initial.cell_id.as_str() + ); + let _ = initial.response_tx.send(Err(reason.clone())); + self.fail(reason); + return false; + } + Err(err) => Err(err), + }; + let _ = initial.response_tx.send(response); + true + } +} diff --git a/codex-rs/code-mode/src/remote_session/connection/driver/session_registry.rs b/codex-rs/code-mode/src/remote_session/connection/driver/session_registry.rs new file mode 100644 index 0000000000..a3d1c82ae6 --- /dev/null +++ b/codex-rs/code-mode/src/remote_session/connection/driver/session_registry.rs @@ -0,0 +1,222 @@ +use std::collections::HashMap; +use std::sync::Arc; + +use codex_code_mode_protocol::CellId; +use codex_code_mode_protocol::CodeModeSessionDelegate; +use codex_code_mode_protocol::host::SessionId; +use codex_code_mode_protocol::host::WireCellId; + +use super::cell_ids::public_cell_id; +use super::cleanup::SessionCleanup; +use super::types::RemoteSession; + +pub(super) struct CellOwner { + pub(super) session_id: SessionId, + pub(super) cell_id: CellId, + pub(super) delegate: Arc, +} + +pub(super) struct DelegateTarget { + pub(super) session_id: SessionId, + pub(super) cell_id: CellId, + pub(super) delegate: Arc, +} + +pub(super) struct FailedSession { + pub(super) cleanup: SessionCleanup, + pub(super) cells: Vec, +} + +pub(super) enum CellAdmissionError { + MissingSession, + DuplicateCell, +} + +struct SessionRecord { + remote: RemoteSession, + delegate: Arc, + cleanup: SessionCleanup, + phase: SessionPhase, + cells: HashMap, +} + +#[derive(Clone, Copy, Eq, PartialEq)] +enum SessionPhase { + Ready, + Closing, +} + +pub(super) struct SessionRegistry { + records: HashMap, +} + +impl SessionRegistry { + pub(super) fn new() -> Self { + Self { + records: HashMap::new(), + } + } + + pub(super) fn contains(&self, session_id: &SessionId) -> bool { + self.records.contains_key(session_id) + } + + pub(super) fn insert_ready( + &mut self, + session: RemoteSession, + delegate: Arc, + cleanup: SessionCleanup, + ) { + self.records.insert( + session.id.clone(), + SessionRecord { + remote: session, + delegate, + cleanup, + phase: SessionPhase::Ready, + cells: HashMap::new(), + }, + ); + } + + pub(super) fn require_ready(&self, session: &RemoteSession) -> Result<(), String> { + let record = self + .records + .get(&session.id) + .ok_or_else(|| format!("unknown code-mode session {}", session.id))?; + if record.remote != *session { + return Err("stale code-mode session generation".to_string()); + } + if record.phase != SessionPhase::Ready { + return Err("code-mode session is shutting down".to_string()); + } + Ok(()) + } + + pub(super) fn begin_shutdown(&mut self, session: &RemoteSession) -> Result<(), String> { + let record = self + .records + .get_mut(&session.id) + .ok_or_else(|| format!("unknown code-mode session {}", session.id))?; + if record.remote != *session { + return Err("stale code-mode session generation".to_string()); + } + if record.phase == SessionPhase::Closing { + return Err("code-mode session is already closing".to_string()); + } + record.phase = SessionPhase::Closing; + Ok(()) + } + + pub(super) fn begin_abandoned_shutdown(&mut self, session_id: &SessionId) -> Option { + let record = self.records.get_mut(session_id)?; + if record.phase == SessionPhase::Closing { + return Some(false); + } + record.phase = SessionPhase::Closing; + Some(true) + } + + pub(super) fn is_closing(&self, session_id: &SessionId) -> Option { + self.records + .get(session_id) + .map(|record| record.phase == SessionPhase::Closing) + } + + pub(super) fn admit_cell( + &mut self, + session: &RemoteSession, + cell_id: WireCellId, + ) -> Result { + let Some(record) = self.records.get_mut(&session.id) else { + return Err(CellAdmissionError::MissingSession); + }; + if record.cells.contains_key(&cell_id) { + return Err(CellAdmissionError::DuplicateCell); + } + let public_id = public_cell_id(session.generation, &cell_id); + record.cells.insert(cell_id, public_id.clone()); + Ok(public_id) + } + + pub(super) fn delegate_target( + &self, + session_id: &SessionId, + cell_id: &WireCellId, + ) -> Result { + let session = self + .records + .get(session_id) + .ok_or_else(|| format!("code-mode host delegated for unknown session {session_id}"))?; + let public_id = session.cells.get(cell_id).cloned().ok_or_else(|| { + format!( + "code-mode host delegated for unknown cell {} in session {session_id}", + cell_id.as_str() + ) + })?; + Ok(DelegateTarget { + session_id: session_id.clone(), + cell_id: public_id, + delegate: Arc::clone(&session.delegate), + }) + } + + pub(super) fn remove_cell( + &mut self, + session_id: &SessionId, + cell_id: &WireCellId, + ) -> Result { + let session = self.records.get_mut(session_id).ok_or_else(|| { + format!( + "code-mode host closed cell {} in unknown session {session_id}", + cell_id.as_str() + ) + })?; + let public_id = session + .cells + .remove(cell_id) + .ok_or_else(|| format!("code-mode host closed unknown cell in session {session_id}"))?; + Ok(CellOwner { + session_id: session_id.clone(), + cell_id: public_id, + delegate: Arc::clone(&session.delegate), + }) + } + + pub(super) fn remove_session(&mut self, session_id: &SessionId) -> Vec { + let Some(session) = self.records.remove(session_id) else { + return Vec::new(); + }; + session + .cells + .into_values() + .map(|cell_id| CellOwner { + session_id: session_id.clone(), + cell_id, + delegate: Arc::clone(&session.delegate), + }) + .collect() + } + + pub(super) fn drain(&mut self) -> Vec { + let sessions = std::mem::take(&mut self.records); + sessions + .into_iter() + .map(|(session_id, session)| { + let cells = session + .cells + .into_values() + .map(|cell_id| CellOwner { + session_id: session_id.clone(), + cell_id, + delegate: Arc::clone(&session.delegate), + }) + .collect(); + FailedSession { + cleanup: session.cleanup, + cells, + } + }) + .collect() + } +} diff --git a/codex-rs/code-mode/src/remote_session/connection/driver/types.rs b/codex-rs/code-mode/src/remote_session/connection/driver/types.rs new file mode 100644 index 0000000000..d4f6a102f3 --- /dev/null +++ b/codex-rs/code-mode/src/remote_session/connection/driver/types.rs @@ -0,0 +1,196 @@ +use std::sync::Arc; + +use codex_code_mode_protocol::CellId; +use codex_code_mode_protocol::CodeModeSessionDelegate; +use codex_code_mode_protocol::ExecuteRequest; +use codex_code_mode_protocol::RuntimeResponse; +use codex_code_mode_protocol::StartedCell; +use codex_code_mode_protocol::WaitOutcome; +use codex_code_mode_protocol::WaitRequest; +use codex_code_mode_protocol::host::DelegateRequestId; +use codex_code_mode_protocol::host::DelegateResponse; +use codex_code_mode_protocol::host::HostToClient; +use codex_code_mode_protocol::host::RequestId; +use codex_code_mode_protocol::host::SessionId; +use codex_code_mode_protocol::host::WireCellId; +use codex_code_mode_protocol::host::WireWaitRequest; +use tokio::sync::mpsc; +use tokio::sync::oneshot; +use tokio_util::sync::CancellationToken; + +use super::cleanup::SessionCleanup; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(in crate::remote_session) struct RemoteSession { + pub(in crate::remote_session) id: SessionId, + pub(in crate::remote_session) generation: u64, +} + +pub(in crate::remote_session::connection) enum DriverCommand { + OpenSession { + session: RemoteSession, + delegate: Arc, + cleanup: SessionCleanup, + caller_cancellation: CancellationToken, + response_tx: oneshot::Sender>, + }, + Execute { + session: RemoteSession, + request: ExecuteRequest, + caller_cancellation: CancellationToken, + response_tx: oneshot::Sender>, + }, + Wait { + session: RemoteSession, + request: WaitRequest, + caller_cancellation: CancellationToken, + response_tx: oneshot::Sender>, + }, + Terminate { + session: RemoteSession, + cell_id: CellId, + response_tx: oneshot::Sender>, + }, + ShutdownSession { + session: RemoteSession, + response_tx: oneshot::Sender>, + }, +} + +pub(in crate::remote_session::connection) enum DriverEvent { + HostMessage(HostToClient), + DelegateCompleted { + id: DelegateRequestId, + result: Result, + }, + RequestCancelled(RequestId), + Failed(String), +} + +pub(super) struct CancellableRequest { + caller_cancellation: CancellationToken, + watcher_stop: CancellationToken, + reported: bool, +} + +impl CancellableRequest { + pub(super) fn new(caller_cancellation: CancellationToken) -> Self { + Self { + caller_cancellation, + watcher_stop: CancellationToken::new(), + reported: false, + } + } + + pub(super) fn is_cancelled(&self) -> bool { + self.caller_cancellation.is_cancelled() + } + + pub(super) fn mark_reported(&mut self) -> bool { + if self.reported { + return false; + } + self.reported = true; + true + } + + pub(super) fn spawn_watcher(&self, id: RequestId, event_tx: mpsc::Sender) { + let caller_cancellation = self.caller_cancellation.clone(); + let watcher_stop = self.watcher_stop.clone(); + tokio::spawn(async move { + tokio::select! { + _ = caller_cancellation.cancelled() => { + let _ = event_tx.send(DriverEvent::RequestCancelled(id)).await; + } + _ = watcher_stop.cancelled() => {} + } + }); + } +} + +impl Drop for CancellableRequest { + fn drop(&mut self) { + self.watcher_stop.cancel(); + } +} + +pub(super) struct InitialResponse { + pub(super) generation: u64, + pub(super) cell_id: WireCellId, + pub(super) response_tx: oneshot::Sender>, +} + +pub(in crate::remote_session::connection) struct DeliveredExecute { + pub(in crate::remote_session::connection) request_id: RequestId, + pub(in crate::remote_session::connection) started: StartedCell, +} + +pub(super) struct UnclaimedExecute { + pub(super) session: RemoteSession, + pub(super) cell_id: WireCellId, + pub(super) cancellation: CancellableRequest, +} + +pub(super) enum PendingRequest { + OpenSession { + session: RemoteSession, + delegate: Arc, + cleanup: SessionCleanup, + cancellation: CancellableRequest, + response_tx: oneshot::Sender>, + }, + Execute { + session: RemoteSession, + response_tx: oneshot::Sender>, + initial_response_tx: oneshot::Sender>, + initial_response_rx: oneshot::Receiver>, + cancellation: CancellableRequest, + }, + Wait { + session: RemoteSession, + cell_id: WireCellId, + cancellation: CancellableRequest, + response_tx: oneshot::Sender>, + }, + Terminate { + session: RemoteSession, + cell_id: WireCellId, + response_tx: oneshot::Sender>, + }, + ShutdownSession { + session: RemoteSession, + response_tx: oneshot::Sender>, + }, +} + +pub(super) struct DeferredWait { + pub(super) session: RemoteSession, + pub(super) request: WireWaitRequest, + pub(super) caller_cancellation: CancellationToken, + pub(super) response_tx: oneshot::Sender>, +} + +impl PendingRequest { + pub(super) fn cancellation_mut(&mut self) -> Option<&mut CancellableRequest> { + match self { + Self::OpenSession { cancellation, .. } + | Self::Execute { cancellation, .. } + | Self::Wait { cancellation, .. } => Some(cancellation), + Self::Terminate { .. } | Self::ShutdownSession { .. } => None, + } + } + + pub(super) fn fail(self, reason: String) { + match self { + Self::OpenSession { response_tx, .. } | Self::ShutdownSession { response_tx, .. } => { + let _ = response_tx.send(Err(reason)); + } + Self::Execute { response_tx, .. } => { + let _ = response_tx.send(Err(reason)); + } + Self::Wait { response_tx, .. } | Self::Terminate { response_tx, .. } => { + let _ = response_tx.send(Err(reason)); + } + } + } +} diff --git a/codex-rs/code-mode/src/remote_session/connection/driver_tests.rs b/codex-rs/code-mode/src/remote_session/connection/driver_tests.rs new file mode 100644 index 0000000000..97ed9b553e --- /dev/null +++ b/codex-rs/code-mode/src/remote_session/connection/driver_tests.rs @@ -0,0 +1,1514 @@ +use std::sync::Arc; +use std::sync::Mutex as StdMutex; +use std::sync::atomic::AtomicBool; +use std::sync::atomic::AtomicUsize; +use std::sync::atomic::Ordering; +use std::time::Duration; + +use codex_code_mode_protocol::CellId; +use codex_code_mode_protocol::CodeModeNestedToolCall; +use codex_code_mode_protocol::CodeModeSessionDelegate; +use codex_code_mode_protocol::ExecuteRequest; +use codex_code_mode_protocol::NotificationFuture; +use codex_code_mode_protocol::ToolInvocationFuture; +use codex_code_mode_protocol::WaitRequest; +use codex_code_mode_protocol::host::DelegateRequest; +use codex_code_mode_protocol::host::DelegateRequestId; +use codex_code_mode_protocol::host::HostResponse; +use codex_code_mode_protocol::host::HostToClient; +use codex_code_mode_protocol::host::RequestId; +use codex_code_mode_protocol::host::SessionId; +use codex_code_mode_protocol::host::WireNestedToolCall; +use codex_code_mode_protocol::host::WireResult; +use codex_code_mode_protocol::host::WireRuntimeResponse; +use codex_code_mode_protocol::host::WireWaitOutcome; +use codex_protocol::ToolName; +use pretty_assertions::assert_eq; +use tokio::sync::mpsc; +use tokio::sync::oneshot; +use tokio_util::sync::CancellationToken; + +use super::ConnectionDriver; +use super::DriverCommand; +use super::DriverEvent; +use super::DriverLifecycle; +use super::RemoteSession; +use super::SessionCleanup; + +struct DriverHarness { + command_tx: mpsc::Sender, + event_tx: mpsc::Sender, + execute_claim_tx: mpsc::UnboundedSender, + outgoing_rx: mpsc::Receiver, + cancellation: CancellationToken, + alive: Arc, + driver_task: tokio::task::JoinHandle<()>, +} + +impl DriverHarness { + fn start() -> Self { + let (command_tx, command_rx) = mpsc::channel(/*max_capacity*/ 16); + let (event_tx, event_rx) = mpsc::channel(/*max_capacity*/ 16); + let (outgoing_tx, outgoing_rx) = mpsc::channel(/*max_capacity*/ 16); + let cancellation = CancellationToken::new(); + let alive = Arc::new(AtomicBool::new(true)); + let (driver, execute_claim_tx) = ConnectionDriver::new( + command_rx, + event_rx, + event_tx.clone(), + outgoing_tx, + DriverLifecycle { + alive: Arc::clone(&alive), + failure: Arc::new(StdMutex::new(None)), + cancellation: cancellation.clone(), + }, + ); + let driver_task = tokio::spawn(driver.run()); + Self { + command_tx, + event_tx, + execute_claim_tx, + outgoing_rx, + cancellation, + alive, + driver_task, + } + } + + async fn open( + &mut self, + session: RemoteSession, + delegate: Arc, + ) -> SessionCleanup { + let cleanup = SessionCleanup::new(); + let (response_tx, response_rx) = oneshot::channel(); + self.command_tx + .send(DriverCommand::OpenSession { + session: session.clone(), + delegate, + cleanup: cleanup.clone(), + caller_cancellation: CancellationToken::new(), + response_tx, + }) + .await + .expect("open command"); + self.outgoing_rx.recv().await.expect("open frame"); + self.event_tx + .send(DriverEvent::HostMessage(HostToClient::Response { + id: RequestId::new(/*value*/ 1), + result: WireResult::Ok { + value: HostResponse::SessionReady { + session_id: session.id, + }, + }, + })) + .await + .expect("open response"); + response_rx + .await + .expect("open reply") + .expect("open session"); + cleanup + } + + async fn start_cell( + &mut self, + session: RemoteSession, + request_id: i64, + cell_id: &str, + ) -> codex_code_mode_protocol::StartedCell { + let (response_tx, response_rx) = oneshot::channel(); + self.command_tx + .send(DriverCommand::Execute { + session, + request: ExecuteRequest { + tool_call_id: format!("call-{request_id}"), + enabled_tools: Vec::new(), + source: "await new Promise(() => {})".to_string(), + yield_time_ms: Some(1), + max_output_tokens: None, + }, + caller_cancellation: CancellationToken::new(), + response_tx, + }) + .await + .expect("execute command"); + self.outgoing_rx.recv().await.expect("execute frame"); + self.event_tx + .send(DriverEvent::HostMessage(HostToClient::Response { + id: RequestId::new(request_id), + result: WireResult::Ok { + value: HostResponse::ExecutionStarted { + cell_id: CellId::new(cell_id.to_string()).into(), + }, + }, + })) + .await + .expect("execute response"); + let delivered = response_rx + .await + .expect("execute reply") + .expect("execute session"); + self.execute_claim_tx + .send(delivered.request_id) + .expect("claim execute"); + delivered.started + } + + async fn start_tool_delegate(&self, session: &RemoteSession, id: DelegateRequestId) { + self.event_tx + .send(DriverEvent::HostMessage(HostToClient::DelegateRequest { + id, + session_id: session.id.clone(), + request: DelegateRequest::InvokeTool { + invocation: WireNestedToolCall { + cell_id: CellId::new("1".to_string()).into(), + runtime_tool_call_id: "tool-1".to_string(), + tool_name: ToolName::plain("slow").into(), + tool_kind: codex_code_mode_protocol::CodeModeToolKind::Function.into(), + input: None, + }, + }, + })) + .await + .expect("delegate request"); + } +} + +impl Drop for DriverHarness { + fn drop(&mut self) { + self.cancellation.cancel(); + } +} + +#[derive(Default)] +struct RecordingDelegate { + closed_cells: StdMutex>, + invocations: AtomicUsize, + notifications: AtomicUsize, +} + +struct PanickingDelegate; + +#[derive(Debug, Eq, PartialEq)] +enum HeldDelegateEvent { + Started, + Cancelled, + Finished, + CellClosed(CellId), +} + +struct HeldDelegate { + events_tx: mpsc::UnboundedSender, + release: CancellationToken, +} + +impl HeldDelegate { + fn new() -> ( + Arc, + mpsc::UnboundedReceiver, + CancellationToken, + ) { + let (events_tx, events_rx) = mpsc::unbounded_channel(); + let release = CancellationToken::new(); + ( + Arc::new(Self { + events_tx, + release: release.clone(), + }), + events_rx, + release, + ) + } +} + +impl CodeModeSessionDelegate for HeldDelegate { + fn invoke_tool<'a>( + &'a self, + _invocation: CodeModeNestedToolCall, + cancellation_token: CancellationToken, + ) -> ToolInvocationFuture<'a> { + let events_tx = self.events_tx.clone(); + let release = self.release.clone(); + Box::pin(async move { + let _ = events_tx.send(HeldDelegateEvent::Started); + cancellation_token.cancelled().await; + let _ = events_tx.send(HeldDelegateEvent::Cancelled); + release.cancelled().await; + let _ = events_tx.send(HeldDelegateEvent::Finished); + Err("cancelled".to_string()) + }) + } + + fn notify<'a>( + &'a self, + _call_id: String, + _cell_id: CellId, + _text: String, + _cancellation_token: CancellationToken, + ) -> NotificationFuture<'a> { + Box::pin(async { Ok(()) }) + } + + fn cell_closed(&self, cell_id: &CellId) { + let _ = self + .events_tx + .send(HeldDelegateEvent::CellClosed(cell_id.clone())); + } +} + +impl CodeModeSessionDelegate for PanickingDelegate { + fn invoke_tool<'a>( + &'a self, + _invocation: CodeModeNestedToolCall, + _cancellation_token: CancellationToken, + ) -> ToolInvocationFuture<'a> { + Box::pin(async { panic!("delegate panic probe") }) + } + + fn notify<'a>( + &'a self, + _call_id: String, + _cell_id: CellId, + _text: String, + _cancellation_token: CancellationToken, + ) -> NotificationFuture<'a> { + Box::pin(async { Ok(()) }) + } + + fn cell_closed(&self, _cell_id: &CellId) {} +} + +impl CodeModeSessionDelegate for RecordingDelegate { + fn invoke_tool<'a>( + &'a self, + _invocation: CodeModeNestedToolCall, + cancellation_token: CancellationToken, + ) -> ToolInvocationFuture<'a> { + self.invocations.fetch_add(1, Ordering::Relaxed); + Box::pin(async move { + cancellation_token.cancelled().await; + Err("cancelled".to_string()) + }) + } + + fn notify<'a>( + &'a self, + _call_id: String, + _cell_id: CellId, + _text: String, + _cancellation_token: CancellationToken, + ) -> NotificationFuture<'a> { + self.notifications.fetch_add(1, Ordering::Relaxed); + Box::pin(async { Ok(()) }) + } + + fn cell_closed(&self, cell_id: &CellId) { + self.closed_cells + .lock() + .expect("closed cells lock") + .push(cell_id.clone()); + } +} + +fn remote_session() -> RemoteSession { + RemoteSession { + id: SessionId::new("session-1").expect("session ID"), + generation: 1, + } +} + +async fn next_held_delegate_event( + events_rx: &mut mpsc::UnboundedReceiver, +) -> HeldDelegateEvent { + tokio::time::timeout(Duration::from_secs(1), events_rx.recv()) + .await + .expect("delegate event timeout") + .expect("delegate event stream") +} + +#[tokio::test] +async fn dropped_open_waiter_shuts_down_committed_session() { + let mut harness = DriverHarness::start(); + let session = remote_session(); + let (open_tx, open_rx) = oneshot::channel(); + let cleanup = SessionCleanup::new(); + harness + .command_tx + .send(DriverCommand::OpenSession { + session: session.clone(), + delegate: Arc::new(RecordingDelegate::default()), + cleanup, + caller_cancellation: CancellationToken::new(), + response_tx: open_tx, + }) + .await + .expect("open command"); + drop(open_rx); + harness.outgoing_rx.recv().await.expect("open frame"); + harness + .event_tx + .send(DriverEvent::HostMessage(HostToClient::Response { + id: RequestId::new(/*value*/ 1), + result: WireResult::Ok { + value: HostResponse::SessionReady { + session_id: session.id.clone(), + }, + }, + })) + .await + .expect("open response"); + harness + .outgoing_rx + .recv() + .await + .expect("abandoned session shutdown frame"); + harness + .event_tx + .send(DriverEvent::HostMessage(HostToClient::Response { + id: RequestId::new(/*value*/ 2), + result: WireResult::Ok { + value: HostResponse::SessionClosed { + session_id: session.id.clone(), + }, + }, + })) + .await + .expect("shutdown response"); + + let (execute_tx, execute_rx) = oneshot::channel(); + harness + .command_tx + .send(DriverCommand::Execute { + session: session.clone(), + request: ExecuteRequest { + tool_call_id: "call-1".to_string(), + enabled_tools: Vec::new(), + source: "text('ok')".to_string(), + yield_time_ms: None, + max_output_tokens: None, + }, + caller_cancellation: CancellationToken::new(), + response_tx: execute_tx, + }) + .await + .expect("execute command"); + assert_eq!( + execute_rx + .await + .expect("execute reply") + .err() + .expect("closed session should reject execute"), + "unknown code-mode session session-1" + ); +} + +#[tokio::test] +async fn delegate_cancel_is_best_effort_and_sends_no_late_response() { + let mut harness = DriverHarness::start(); + let session = remote_session(); + let delegate = Arc::new(RecordingDelegate::default()); + harness.open(session.clone(), delegate.clone()).await; + let _started = harness + .start_cell(session.clone(), /*request_id*/ 2, "1") + .await; + let request_id = DelegateRequestId::new(/*value*/ 7); + harness + .event_tx + .send(DriverEvent::HostMessage(HostToClient::DelegateRequest { + id: request_id, + session_id: session.id.clone(), + request: DelegateRequest::InvokeTool { + invocation: WireNestedToolCall { + cell_id: CellId::new("1".to_string()).into(), + runtime_tool_call_id: "tool-1".to_string(), + tool_name: ToolName::plain("slow").into(), + tool_kind: codex_code_mode_protocol::CodeModeToolKind::Function.into(), + input: None, + }, + }, + })) + .await + .expect("delegate request"); + harness + .event_tx + .send(DriverEvent::HostMessage( + HostToClient::CancelDelegateRequest { id: request_id }, + )) + .await + .expect("delegate cancel"); + tokio::task::yield_now().await; + assert!(matches!( + harness.outgoing_rx.try_recv(), + Err(mpsc::error::TryRecvError::Empty) + )); + harness + .event_tx + .send(DriverEvent::HostMessage(HostToClient::DelegateRequest { + id: request_id, + session_id: session.id, + request: DelegateRequest::Notify { + call_id: "notify-reused".to_string(), + cell_id: CellId::new("1".to_string()).into(), + text: "duplicate".to_string(), + }, + })) + .await + .expect("reused delegate request"); + tokio::task::yield_now().await; + + assert!(!harness.alive.load(Ordering::Acquire)); + assert_eq!(delegate.invocations.load(Ordering::Relaxed), 1); + assert_eq!(delegate.notifications.load(Ordering::Relaxed), 0); +} + +#[tokio::test] +async fn terminate_closes_cell_without_waiting_for_delegate_cleanup() { + let mut harness = DriverHarness::start(); + let session = remote_session(); + let (delegate, mut events_rx, release) = HeldDelegate::new(); + harness.open(session.clone(), delegate).await; + let _started = harness + .start_cell(session.clone(), /*request_id*/ 2, "1") + .await; + let delegate_id = DelegateRequestId::new(/*value*/ 7); + harness.start_tool_delegate(&session, delegate_id).await; + assert_eq!( + next_held_delegate_event(&mut events_rx).await, + HeldDelegateEvent::Started + ); + + let (response_tx, response_rx) = oneshot::channel(); + harness + .command_tx + .send(DriverCommand::Terminate { + session: session.clone(), + cell_id: CellId::new("1".to_string()), + response_tx, + }) + .await + .expect("terminate command"); + harness.outgoing_rx.recv().await.expect("terminate frame"); + harness + .event_tx + .send(DriverEvent::HostMessage( + HostToClient::CancelDelegateRequest { id: delegate_id }, + )) + .await + .expect("delegate cancel"); + harness + .event_tx + .send(DriverEvent::HostMessage(HostToClient::CellClosed { + session_id: session.id, + cell_id: CellId::new("1".to_string()).into(), + })) + .await + .expect("cell close"); + harness + .event_tx + .send(DriverEvent::HostMessage(HostToClient::Response { + id: RequestId::new(/*value*/ 3), + result: WireResult::Ok { + value: HostResponse::WaitCompleted { + outcome: WireWaitOutcome::LiveCell(WireRuntimeResponse::Terminated { + cell_id: CellId::new("1".to_string()).into(), + content_items: Vec::new(), + }), + }, + }, + })) + .await + .expect("terminate response"); + + let closure_events = [ + next_held_delegate_event(&mut events_rx).await, + next_held_delegate_event(&mut events_rx).await, + ]; + assert!(closure_events.contains(&HeldDelegateEvent::Cancelled)); + assert!(closure_events.contains(&HeldDelegateEvent::CellClosed(CellId::new("1".to_string())))); + assert_eq!( + response_rx.await.expect("terminate reply"), + Ok(codex_code_mode_protocol::WaitOutcome::LiveCell( + codex_code_mode_protocol::RuntimeResponse::Terminated { + cell_id: CellId::new("1".to_string()), + content_items: Vec::new(), + } + )) + ); + assert!(matches!( + events_rx.try_recv(), + Err(mpsc::error::TryRecvError::Empty | mpsc::error::TryRecvError::Disconnected) + )); + + release.cancel(); + assert_eq!( + next_held_delegate_event(&mut events_rx).await, + HeldDelegateEvent::Finished + ); + assert!(matches!( + events_rx.try_recv(), + Err(mpsc::error::TryRecvError::Empty | mpsc::error::TryRecvError::Disconnected) + )); + assert!(matches!( + harness.outgoing_rx.try_recv(), + Err(mpsc::error::TryRecvError::Empty) + )); + assert!(harness.alive.load(Ordering::Acquire)); +} + +#[tokio::test] +async fn shutdown_closes_cell_without_waiting_for_delegate_cleanup() { + let mut harness = DriverHarness::start(); + let session = remote_session(); + let (delegate, mut events_rx, release) = HeldDelegate::new(); + harness.open(session.clone(), delegate).await; + let _started = harness + .start_cell(session.clone(), /*request_id*/ 2, "1") + .await; + let delegate_id = DelegateRequestId::new(/*value*/ 7); + harness.start_tool_delegate(&session, delegate_id).await; + assert_eq!( + next_held_delegate_event(&mut events_rx).await, + HeldDelegateEvent::Started + ); + + let (response_tx, response_rx) = oneshot::channel(); + harness + .command_tx + .send(DriverCommand::ShutdownSession { + session: session.clone(), + response_tx, + }) + .await + .expect("shutdown command"); + harness.outgoing_rx.recv().await.expect("shutdown frame"); + harness + .event_tx + .send(DriverEvent::HostMessage( + HostToClient::CancelDelegateRequest { id: delegate_id }, + )) + .await + .expect("delegate cancel"); + harness + .event_tx + .send(DriverEvent::HostMessage(HostToClient::CellClosed { + session_id: session.id.clone(), + cell_id: CellId::new("1".to_string()).into(), + })) + .await + .expect("cell close"); + harness + .event_tx + .send(DriverEvent::HostMessage(HostToClient::Response { + id: RequestId::new(/*value*/ 3), + result: WireResult::Ok { + value: HostResponse::SessionClosed { + session_id: session.id, + }, + }, + })) + .await + .expect("shutdown response"); + + let closure_events = [ + next_held_delegate_event(&mut events_rx).await, + next_held_delegate_event(&mut events_rx).await, + ]; + assert!(closure_events.contains(&HeldDelegateEvent::Cancelled)); + assert!(closure_events.contains(&HeldDelegateEvent::CellClosed(CellId::new("1".to_string())))); + assert_eq!(response_rx.await.expect("shutdown reply"), Ok(())); + assert!(matches!( + events_rx.try_recv(), + Err(mpsc::error::TryRecvError::Empty) + )); + + release.cancel(); + assert_eq!( + next_held_delegate_event(&mut events_rx).await, + HeldDelegateEvent::Finished + ); + assert!(matches!( + events_rx.try_recv(), + Err(mpsc::error::TryRecvError::Empty | mpsc::error::TryRecvError::Disconnected) + )); + assert!(matches!( + harness.outgoing_rx.try_recv(), + Err(mpsc::error::TryRecvError::Empty) + )); + assert!(harness.alive.load(Ordering::Acquire)); +} + +#[tokio::test] +async fn completed_delegate_request_id_cannot_be_reused() { + let mut harness = DriverHarness::start(); + let session = remote_session(); + let delegate = Arc::new(RecordingDelegate::default()); + harness.open(session.clone(), delegate.clone()).await; + let _started = harness + .start_cell(session.clone(), /*request_id*/ 2, "1") + .await; + let request_id = DelegateRequestId::new(/*value*/ 7); + let request = || DelegateRequest::Notify { + call_id: "notify-1".to_string(), + cell_id: CellId::new("1".to_string()).into(), + text: "once".to_string(), + }; + harness + .event_tx + .send(DriverEvent::HostMessage(HostToClient::DelegateRequest { + id: request_id, + session_id: session.id.clone(), + request: request(), + })) + .await + .expect("delegate request"); + harness + .outgoing_rx + .recv() + .await + .expect("delegate response frame"); + harness + .event_tx + .send(DriverEvent::HostMessage(HostToClient::DelegateRequest { + id: request_id, + session_id: session.id, + request: request(), + })) + .await + .expect("reused delegate request"); + tokio::task::yield_now().await; + + assert!(!harness.alive.load(Ordering::Acquire)); + assert_eq!(delegate.notifications.load(Ordering::Relaxed), 1); +} + +#[tokio::test] +async fn delegate_task_panic_becomes_tool_error_without_killing_connection() { + let mut harness = DriverHarness::start(); + let session = remote_session(); + harness + .open(session.clone(), Arc::new(PanickingDelegate)) + .await; + let _started = harness + .start_cell(session.clone(), /*request_id*/ 2, "1") + .await; + harness + .event_tx + .send(DriverEvent::HostMessage(HostToClient::DelegateRequest { + id: DelegateRequestId::new(/*value*/ 7), + session_id: session.id.clone(), + request: DelegateRequest::InvokeTool { + invocation: WireNestedToolCall { + cell_id: CellId::new("1".to_string()).into(), + runtime_tool_call_id: "tool-1".to_string(), + tool_name: ToolName::plain("panic").into(), + tool_kind: codex_code_mode_protocol::CodeModeToolKind::Function.into(), + input: None, + }, + }, + })) + .await + .expect("delegate request"); + tokio::time::timeout(Duration::from_secs(1), harness.outgoing_rx.recv()) + .await + .expect("delegate response timeout") + .expect("delegate response frame"); + + assert!(harness.alive.load(Ordering::Acquire)); + harness + .event_tx + .send(DriverEvent::HostMessage(HostToClient::CellClosed { + session_id: session.id, + cell_id: CellId::new("1".to_string()).into(), + })) + .await + .expect("cell close"); +} + +#[tokio::test] +async fn delegate_for_unknown_cell_fails_connection_without_invocation() { + let mut harness = DriverHarness::start(); + let session = remote_session(); + let delegate = Arc::new(RecordingDelegate::default()); + harness.open(session.clone(), delegate.clone()).await; + + harness + .event_tx + .send(DriverEvent::HostMessage(HostToClient::DelegateRequest { + id: DelegateRequestId::new(/*value*/ 7), + session_id: session.id, + request: DelegateRequest::InvokeTool { + invocation: WireNestedToolCall { + cell_id: CellId::new("missing".to_string()).into(), + runtime_tool_call_id: "tool-1".to_string(), + tool_name: ToolName::plain("slow").into(), + tool_kind: codex_code_mode_protocol::CodeModeToolKind::Function.into(), + input: None, + }, + }, + })) + .await + .expect("delegate request"); + tokio::task::yield_now().await; + + assert!(!harness.alive.load(Ordering::Acquire)); + assert_eq!(delegate.invocations.load(Ordering::Relaxed), 0); +} + +#[tokio::test] +async fn delegate_after_cell_close_fails_connection_without_invocation() { + let mut harness = DriverHarness::start(); + let session = remote_session(); + let delegate = Arc::new(RecordingDelegate::default()); + harness.open(session.clone(), delegate.clone()).await; + let _started = harness + .start_cell(session.clone(), /*request_id*/ 2, "1") + .await; + harness + .event_tx + .send(DriverEvent::HostMessage(HostToClient::CellClosed { + session_id: session.id.clone(), + cell_id: CellId::new("1".to_string()).into(), + })) + .await + .expect("cell close"); + harness + .event_tx + .send(DriverEvent::HostMessage(HostToClient::DelegateRequest { + id: DelegateRequestId::new(/*value*/ 7), + session_id: session.id, + request: DelegateRequest::Notify { + call_id: "notify-1".to_string(), + cell_id: CellId::new("1".to_string()).into(), + text: "late".to_string(), + }, + })) + .await + .expect("delegate request"); + tokio::task::yield_now().await; + + assert!(!harness.alive.load(Ordering::Acquire)); + assert_eq!(delegate.invocations.load(Ordering::Relaxed), 0); +} + +#[tokio::test] +async fn mismatched_initial_response_fails_connection_and_closes_cell_once() { + let mut harness = DriverHarness::start(); + let session = remote_session(); + let delegate = Arc::new(RecordingDelegate::default()); + harness.open(session.clone(), delegate.clone()).await; + let started = harness.start_cell(session, /*request_id*/ 2, "1").await; + harness + .event_tx + .send(DriverEvent::HostMessage(HostToClient::InitialResponse { + id: RequestId::new(/*value*/ 2), + result: WireResult::Ok { + value: WireRuntimeResponse::Yielded { + cell_id: CellId::new("2".to_string()).into(), + content_items: Vec::new(), + }, + }, + })) + .await + .expect("initial response"); + + assert!(started.initial_response().await.is_err()); + assert!(!harness.alive.load(Ordering::Acquire)); + assert_eq!( + *delegate.closed_cells.lock().expect("closed cells lock"), + vec![CellId::new("1".to_string())] + ); +} + +#[tokio::test] +async fn mismatched_wait_response_fails_connection() { + let mut harness = DriverHarness::start(); + let session = remote_session(); + let delegate = Arc::new(RecordingDelegate::default()); + harness.open(session.clone(), delegate.clone()).await; + let _started = harness + .start_cell(session.clone(), /*request_id*/ 2, "1") + .await; + let (response_tx, response_rx) = oneshot::channel(); + harness + .command_tx + .send(DriverCommand::Wait { + session, + request: WaitRequest { + cell_id: CellId::new("1".to_string()), + yield_time_ms: 1, + }, + caller_cancellation: CancellationToken::new(), + response_tx, + }) + .await + .expect("wait command"); + harness.outgoing_rx.recv().await.expect("wait frame"); + harness + .event_tx + .send(DriverEvent::HostMessage(HostToClient::Response { + id: RequestId::new(/*value*/ 3), + result: WireResult::Ok { + value: HostResponse::WaitCompleted { + outcome: WireWaitOutcome::LiveCell(WireRuntimeResponse::Yielded { + cell_id: CellId::new("2".to_string()).into(), + content_items: Vec::new(), + }), + }, + }, + })) + .await + .expect("wait response"); + + assert!(response_rx.await.expect("wait reply").is_err()); + assert!(!harness.alive.load(Ordering::Acquire)); + assert_eq!( + *delegate.closed_cells.lock().expect("closed cells lock"), + vec![CellId::new("1".to_string())] + ); +} + +#[tokio::test] +async fn mismatched_terminate_response_fails_connection() { + let mut harness = DriverHarness::start(); + let session = remote_session(); + let delegate = Arc::new(RecordingDelegate::default()); + harness.open(session.clone(), delegate.clone()).await; + let _started = harness + .start_cell(session.clone(), /*request_id*/ 2, "1") + .await; + let (response_tx, response_rx) = oneshot::channel(); + harness + .command_tx + .send(DriverCommand::Terminate { + session, + cell_id: CellId::new("1".to_string()), + response_tx, + }) + .await + .expect("terminate command"); + harness.outgoing_rx.recv().await.expect("terminate frame"); + harness + .event_tx + .send(DriverEvent::HostMessage(HostToClient::Response { + id: RequestId::new(/*value*/ 3), + result: WireResult::Ok { + value: HostResponse::WaitCompleted { + outcome: WireWaitOutcome::MissingCell(WireRuntimeResponse::Terminated { + cell_id: CellId::new("2".to_string()).into(), + content_items: Vec::new(), + }), + }, + }, + })) + .await + .expect("terminate response"); + + assert!(response_rx.await.expect("terminate reply").is_err()); + assert!(!harness.alive.load(Ordering::Acquire)); + assert_eq!( + *delegate.closed_cells.lock().expect("closed cells lock"), + vec![CellId::new("1".to_string())] + ); +} + +#[tokio::test] +async fn remote_wait_accepts_durations_longer_than_five_minutes() { + let mut harness = DriverHarness::start(); + let session = remote_session(); + harness + .open(session.clone(), Arc::new(RecordingDelegate::default())) + .await; + let _started = harness + .start_cell(session.clone(), /*request_id*/ 2, "1") + .await; + let (response_tx, response_rx) = oneshot::channel(); + harness + .command_tx + .send(DriverCommand::Wait { + session, + request: WaitRequest { + cell_id: CellId::new("1".to_string()), + yield_time_ms: 300_001, + }, + caller_cancellation: CancellationToken::new(), + response_tx, + }) + .await + .expect("wait command"); + tokio::time::timeout(Duration::from_secs(1), harness.outgoing_rx.recv()) + .await + .expect("wait frame timeout") + .expect("wait frame"); + harness + .event_tx + .send(DriverEvent::HostMessage(HostToClient::Response { + id: RequestId::new(/*value*/ 3), + result: WireResult::Ok { + value: HostResponse::WaitCompleted { + outcome: WireWaitOutcome::LiveCell(WireRuntimeResponse::Yielded { + cell_id: CellId::new("1".to_string()).into(), + content_items: Vec::new(), + }), + }, + }, + })) + .await + .expect("wait response"); + + assert_eq!( + response_rx.await.expect("wait reply"), + Ok(codex_code_mode_protocol::WaitOutcome::LiveCell( + codex_code_mode_protocol::RuntimeResponse::Yielded { + cell_id: CellId::new("1".to_string()), + content_items: Vec::new(), + } + )) + ); +} + +#[tokio::test] +async fn cancelled_wait_is_retired_before_next_wait_is_sent() { + let mut harness = DriverHarness::start(); + let session = remote_session(); + harness + .open(session.clone(), Arc::new(RecordingDelegate::default())) + .await; + let _started = harness + .start_cell(session.clone(), /*request_id*/ 2, "1") + .await; + let first_cancellation = CancellationToken::new(); + let (first_tx, first_rx) = oneshot::channel(); + harness + .command_tx + .send(DriverCommand::Wait { + session: session.clone(), + request: WaitRequest { + cell_id: CellId::new("1".to_string()), + yield_time_ms: 60_000, + }, + caller_cancellation: first_cancellation.clone(), + response_tx: first_tx, + }) + .await + .expect("first wait command"); + harness.outgoing_rx.recv().await.expect("first wait frame"); + first_cancellation.cancel(); + drop(first_rx); + + let (second_tx, second_rx) = oneshot::channel(); + harness + .command_tx + .send(DriverCommand::Wait { + session, + request: WaitRequest { + cell_id: CellId::new("1".to_string()), + yield_time_ms: 1, + }, + caller_cancellation: CancellationToken::new(), + response_tx: second_tx, + }) + .await + .expect("second wait command"); + harness + .outgoing_rx + .recv() + .await + .expect("cancel request frame"); + assert!(matches!( + harness.outgoing_rx.try_recv(), + Err(mpsc::error::TryRecvError::Empty) + )); + + harness + .event_tx + .send(DriverEvent::HostMessage(HostToClient::Response { + id: RequestId::new(/*value*/ 3), + result: WireResult::Err { + message: "code-mode request cancelled".to_string(), + }, + })) + .await + .expect("cancelled wait response"); + harness.outgoing_rx.recv().await.expect("second wait frame"); + harness + .event_tx + .send(DriverEvent::HostMessage(HostToClient::Response { + id: RequestId::new(/*value*/ 4), + result: WireResult::Ok { + value: HostResponse::WaitCompleted { + outcome: WireWaitOutcome::LiveCell(WireRuntimeResponse::Yielded { + cell_id: CellId::new("1".to_string()).into(), + content_items: Vec::new(), + }), + }, + }, + })) + .await + .expect("second wait response"); + + assert_eq!( + second_rx.await.expect("second wait reply"), + Ok(codex_code_mode_protocol::WaitOutcome::LiveCell( + codex_code_mode_protocol::RuntimeResponse::Yielded { + cell_id: CellId::new("1".to_string()), + content_items: Vec::new(), + } + )) + ); +} + +#[tokio::test] +async fn abandoned_execute_is_tracked_and_terminated_after_admission() { + let mut harness = DriverHarness::start(); + let session = remote_session(); + let delegate = Arc::new(RecordingDelegate::default()); + harness.open(session.clone(), delegate.clone()).await; + let cancellation = CancellationToken::new(); + let (execute_tx, execute_rx) = oneshot::channel(); + harness + .command_tx + .send(DriverCommand::Execute { + session: session.clone(), + request: ExecuteRequest { + tool_call_id: "call-1".to_string(), + enabled_tools: Vec::new(), + source: "await new Promise(() => {})".to_string(), + yield_time_ms: Some(1), + max_output_tokens: None, + }, + caller_cancellation: cancellation.clone(), + response_tx: execute_tx, + }) + .await + .expect("execute command"); + harness.outgoing_rx.recv().await.expect("execute frame"); + cancellation.cancel(); + drop(execute_rx); + harness + .event_tx + .send(DriverEvent::HostMessage(HostToClient::Response { + id: RequestId::new(/*value*/ 2), + result: WireResult::Ok { + value: HostResponse::ExecutionStarted { + cell_id: CellId::new("1".to_string()).into(), + }, + }, + })) + .await + .expect("execute response"); + + harness + .outgoing_rx + .recv() + .await + .expect("execute cancellation frame"); + harness + .outgoing_rx + .recv() + .await + .expect("abandoned cell termination frame"); + harness + .event_tx + .send(DriverEvent::HostMessage(HostToClient::InitialResponse { + id: RequestId::new(/*value*/ 2), + result: WireResult::Ok { + value: WireRuntimeResponse::Terminated { + cell_id: CellId::new("1".to_string()).into(), + content_items: Vec::new(), + }, + }, + })) + .await + .expect("initial response"); + harness + .event_tx + .send(DriverEvent::HostMessage(HostToClient::Response { + id: RequestId::new(/*value*/ 3), + result: WireResult::Ok { + value: HostResponse::WaitCompleted { + outcome: WireWaitOutcome::LiveCell(WireRuntimeResponse::Terminated { + cell_id: CellId::new("1".to_string()).into(), + content_items: Vec::new(), + }), + }, + }, + })) + .await + .expect("terminate response"); + harness + .event_tx + .send(DriverEvent::HostMessage(HostToClient::CellClosed { + session_id: session.id, + cell_id: CellId::new("1".to_string()).into(), + })) + .await + .expect("cell close"); + tokio::task::yield_now().await; + + assert!(harness.alive.load(Ordering::Acquire)); + assert_eq!( + *delegate.closed_cells.lock().expect("closed cells lock"), + vec![CellId::new("1".to_string())] + ); +} + +#[tokio::test] +async fn delivered_but_unclaimed_execute_is_terminated_when_the_caller_is_cancelled() { + let mut harness = DriverHarness::start(); + let session = remote_session(); + let delegate = Arc::new(RecordingDelegate::default()); + harness.open(session.clone(), delegate.clone()).await; + let cancellation = CancellationToken::new(); + let (execute_tx, execute_rx) = oneshot::channel(); + harness + .command_tx + .send(DriverCommand::Execute { + session: session.clone(), + request: ExecuteRequest { + tool_call_id: "call-1".to_string(), + enabled_tools: Vec::new(), + source: "await new Promise(() => {})".to_string(), + yield_time_ms: Some(1), + max_output_tokens: None, + }, + caller_cancellation: cancellation.clone(), + response_tx: execute_tx, + }) + .await + .expect("execute command"); + harness.outgoing_rx.recv().await.expect("execute frame"); + harness + .event_tx + .send(DriverEvent::HostMessage(HostToClient::Response { + id: RequestId::new(/*value*/ 2), + result: WireResult::Ok { + value: HostResponse::ExecutionStarted { + cell_id: CellId::new("1".to_string()).into(), + }, + }, + })) + .await + .expect("execute response"); + let delivered = execute_rx + .await + .expect("execute reply") + .expect("delivered execute"); + assert_eq!(delivered.request_id, RequestId::new(/*value*/ 2)); + cancellation.cancel(); + + harness + .outgoing_rx + .recv() + .await + .expect("execute cancellation frame"); + harness + .outgoing_rx + .recv() + .await + .expect("unclaimed cell termination frame"); + harness + .event_tx + .send(DriverEvent::HostMessage(HostToClient::InitialResponse { + id: RequestId::new(/*value*/ 2), + result: WireResult::Ok { + value: WireRuntimeResponse::Terminated { + cell_id: CellId::new("1".to_string()).into(), + content_items: Vec::new(), + }, + }, + })) + .await + .expect("initial response"); + assert!(delivered.started.initial_response().await.is_ok()); + harness + .event_tx + .send(DriverEvent::HostMessage(HostToClient::Response { + id: RequestId::new(/*value*/ 3), + result: WireResult::Ok { + value: HostResponse::WaitCompleted { + outcome: WireWaitOutcome::LiveCell(WireRuntimeResponse::Terminated { + cell_id: CellId::new("1".to_string()).into(), + content_items: Vec::new(), + }), + }, + }, + })) + .await + .expect("terminate response"); + harness + .event_tx + .send(DriverEvent::HostMessage(HostToClient::CellClosed { + session_id: session.id, + cell_id: CellId::new("1".to_string()).into(), + })) + .await + .expect("cell close"); + tokio::task::yield_now().await; + + assert!(harness.alive.load(Ordering::Acquire)); + assert_eq!( + *delegate.closed_cells.lock().expect("closed cells lock"), + vec![CellId::new("1".to_string())] + ); +} + +#[tokio::test] +async fn session_accepts_more_than_4096_cells_without_growing_a_tombstone_set() { + const CELL_COUNT: usize = 4097; + + let mut harness = DriverHarness::start(); + let session = remote_session(); + let delegate = Arc::new(RecordingDelegate::default()); + harness.open(session.clone(), delegate.clone()).await; + + for sequence in 1..=CELL_COUNT { + let request_id = i64::try_from(sequence).expect("cell sequence fits in i64") + 1; + let cell_id = sequence.to_string(); + let started = harness + .start_cell(session.clone(), request_id, &cell_id) + .await; + harness + .event_tx + .send(DriverEvent::HostMessage(HostToClient::InitialResponse { + id: RequestId::new(request_id), + result: WireResult::Ok { + value: WireRuntimeResponse::Yielded { + cell_id: CellId::new(cell_id.clone()).into(), + content_items: Vec::new(), + }, + }, + })) + .await + .expect("initial response"); + assert!(started.initial_response().await.is_ok()); + harness + .event_tx + .send(DriverEvent::HostMessage(HostToClient::CellClosed { + session_id: session.id.clone(), + cell_id: CellId::new(cell_id).into(), + })) + .await + .expect("cell close"); + } + + tokio::time::timeout(Duration::from_secs(1), async { + while delegate + .closed_cells + .lock() + .expect("closed cells lock") + .len() + != CELL_COUNT + { + tokio::task::yield_now().await; + } + }) + .await + .expect("cell close callbacks timeout"); + assert!(harness.alive.load(Ordering::Acquire)); +} + +#[tokio::test] +async fn connection_failure_closes_every_live_cell_once() { + let mut harness = DriverHarness::start(); + let session = remote_session(); + let delegate = Arc::new(RecordingDelegate::default()); + let cleanup = harness.open(session.clone(), delegate.clone()).await; + let (execute_tx, execute_rx) = oneshot::channel(); + harness + .command_tx + .send(DriverCommand::Execute { + session, + request: ExecuteRequest { + tool_call_id: "call-1".to_string(), + enabled_tools: Vec::new(), + source: "await new Promise(() => {})".to_string(), + yield_time_ms: Some(1), + max_output_tokens: None, + }, + caller_cancellation: CancellationToken::new(), + response_tx: execute_tx, + }) + .await + .expect("execute command"); + harness.outgoing_rx.recv().await.expect("execute frame"); + harness + .event_tx + .send(DriverEvent::HostMessage(HostToClient::Response { + id: RequestId::new(/*value*/ 2), + result: WireResult::Ok { + value: HostResponse::ExecutionStarted { + cell_id: CellId::new("1".to_string()).into(), + }, + }, + })) + .await + .expect("execute response"); + let _started = execute_rx + .await + .expect("execute reply") + .expect("execute session"); + harness + .event_tx + .send(DriverEvent::Failed("host crashed".to_string())) + .await + .expect("failure event"); + tokio::time::timeout(Duration::from_secs(1), cleanup.wait()) + .await + .expect("session cleanup timeout"); + assert_eq!( + *delegate.closed_cells.lock().expect("closed cells lock"), + vec![CellId::new("1".to_string())] + ); +} + +#[tokio::test] +async fn session_cleanup_does_not_wait_for_delegate_completion() { + let mut harness = DriverHarness::start(); + let session = remote_session(); + let (delegate, mut events_rx, release) = HeldDelegate::new(); + let cleanup = harness.open(session.clone(), delegate).await; + let _started = harness + .start_cell(session.clone(), /*request_id*/ 2, "1") + .await; + harness + .start_tool_delegate(&session, DelegateRequestId::new(/*value*/ 7)) + .await; + assert_eq!( + next_held_delegate_event(&mut events_rx).await, + HeldDelegateEvent::Started + ); + + harness + .event_tx + .send(DriverEvent::Failed("host crashed".to_string())) + .await + .expect("failure event"); + let closure_events = [ + next_held_delegate_event(&mut events_rx).await, + next_held_delegate_event(&mut events_rx).await, + ]; + assert!(closure_events.contains(&HeldDelegateEvent::Cancelled)); + assert!(closure_events.contains(&HeldDelegateEvent::CellClosed(CellId::new("1".to_string())))); + tokio::time::timeout(Duration::from_secs(1), cleanup.wait()) + .await + .expect("session cleanup timeout"); + assert!(matches!( + events_rx.try_recv(), + Err(mpsc::error::TryRecvError::Empty) + )); + + release.cancel(); + assert_eq!( + next_held_delegate_event(&mut events_rx).await, + HeldDelegateEvent::Finished + ); + assert!(matches!( + events_rx.try_recv(), + Err(mpsc::error::TryRecvError::Empty | mpsc::error::TryRecvError::Disconnected) + )); +} + +#[tokio::test] +async fn aborting_driver_marks_connection_dead_and_closes_cells() { + let mut harness = DriverHarness::start(); + let session = remote_session(); + let delegate = Arc::new(RecordingDelegate::default()); + harness.open(session.clone(), delegate.clone()).await; + let _started = harness + .start_cell(session.clone(), /*request_id*/ 2, "1") + .await; + let (wait_tx, wait_rx) = oneshot::channel(); + harness + .command_tx + .send(DriverCommand::Wait { + session, + request: WaitRequest { + cell_id: CellId::new("1".to_string()), + yield_time_ms: 60_000, + }, + caller_cancellation: CancellationToken::new(), + response_tx: wait_tx, + }) + .await + .expect("wait command"); + harness.outgoing_rx.recv().await.expect("wait frame"); + + harness.driver_task.abort(); + for _ in 0..10 { + if !harness.alive.load(Ordering::Acquire) { + break; + } + tokio::task::yield_now().await; + } + + assert!(!harness.alive.load(Ordering::Acquire)); + assert!(harness.cancellation.is_cancelled()); + assert!(wait_rx.await.expect("wait failure").is_err()); + assert_eq!( + *delegate.closed_cells.lock().expect("closed cells lock"), + vec![CellId::new("1".to_string())] + ); +} + +#[tokio::test] +async fn dropped_shutdown_waiter_does_not_abort_remote_cleanup() { + let mut harness = DriverHarness::start(); + let session = remote_session(); + harness + .open(session.clone(), Arc::new(RecordingDelegate::default())) + .await; + let (shutdown_tx, shutdown_rx) = oneshot::channel(); + harness + .command_tx + .send(DriverCommand::ShutdownSession { + session: session.clone(), + response_tx: shutdown_tx, + }) + .await + .expect("shutdown command"); + drop(shutdown_rx); + harness.outgoing_rx.recv().await.expect("shutdown frame"); + harness + .event_tx + .send(DriverEvent::HostMessage(HostToClient::Response { + id: RequestId::new(/*value*/ 2), + result: WireResult::Ok { + value: HostResponse::SessionClosed { + session_id: session.id.clone(), + }, + }, + })) + .await + .expect("shutdown response"); + + let (execute_tx, execute_rx) = oneshot::channel(); + harness + .command_tx + .send(DriverCommand::Execute { + session, + request: ExecuteRequest { + tool_call_id: "call-2".to_string(), + enabled_tools: Vec::new(), + source: "text('unreachable')".to_string(), + yield_time_ms: None, + max_output_tokens: None, + }, + caller_cancellation: CancellationToken::new(), + response_tx: execute_tx, + }) + .await + .expect("execute command"); + assert_eq!( + execute_rx + .await + .expect("execute reply") + .err() + .expect("closed session should reject execute"), + "unknown code-mode session session-1" + ); + assert!(matches!( + harness.outgoing_rx.try_recv(), + Err(mpsc::error::TryRecvError::Empty) + )); +} diff --git a/codex-rs/code-mode/src/remote_session/connection/reader.rs b/codex-rs/code-mode/src/remote_session/connection/reader.rs new file mode 100644 index 0000000000..fbfc94de42 --- /dev/null +++ b/codex-rs/code-mode/src/remote_session/connection/reader.rs @@ -0,0 +1,29 @@ +use codex_code_mode_protocol::host::FramedReader; +use codex_code_mode_protocol::host::HostToClient; +use tokio::process::ChildStdout; +use tokio::sync::mpsc; +use tokio_util::sync::CancellationToken; + +use super::driver::DriverEvent; + +pub(super) async fn drive_reader( + mut reader: FramedReader, + events: mpsc::Sender, + cancellation: CancellationToken, +) -> Result<(), String> { + loop { + let message = tokio::select! { + _ = cancellation.cancelled() => return Ok(()), + result = reader.read::() => result, + }; + let message = match message { + Ok(Some(message)) => message, + Ok(None) => return Err("code-mode host closed its stdout".to_string()), + Err(err) => return Err(format!("failed to read code-mode host message: {err}")), + }; + events + .send(DriverEvent::HostMessage(message)) + .await + .map_err(|_| "code-mode connection driver closed".to_string())?; + } +} diff --git a/codex-rs/code-mode/src/remote_session_tests.rs b/codex-rs/code-mode/src/remote_session_tests.rs new file mode 100644 index 0000000000..03d75e5239 --- /dev/null +++ b/codex-rs/code-mode/src/remote_session_tests.rs @@ -0,0 +1,103 @@ +use std::io; +use std::path::PathBuf; +use std::sync::Arc; + +use codex_code_mode_protocol::CodeModeSessionProvider; + +use super::ProcessOwnedCodeModeSession; +use super::ProcessOwnedCodeModeSessionProvider; +use super::resolve_host_program; +use crate::NoopCodeModeSessionDelegate; + +#[test] +fn provider_reuses_its_live_process_host() { + let provider = ProcessOwnedCodeModeSessionProvider::default(); + + let first = provider.process_host(); + let second = provider.process_host(); + + assert!(Arc::ptr_eq(&first, &second)); +} + +#[test] +fn host_program_override_takes_precedence() { + assert_eq!( + resolve_host_program( + Some("custom-code-mode-host".into()), + Ok(PathBuf::from("/opt/codex/bin/codex")), + ), + PathBuf::from("custom-code-mode-host") + ); +} + +#[test] +fn host_program_is_next_to_the_main_executable_even_when_missing() { + let executable_name = if cfg!(windows) { + "codex-code-mode-host.exe" + } else { + "codex-code-mode-host" + }; + + assert_eq!( + resolve_host_program( + /*override_path*/ None, + Ok(PathBuf::from("/opt/codex/bin/codex")), + ), + PathBuf::from("/opt/codex/bin").join(executable_name) + ); +} + +#[test] +fn host_program_falls_back_to_its_name_when_main_executable_is_unknown() { + let executable_name = if cfg!(windows) { + "codex-code-mode-host.exe" + } else { + "codex-code-mode-host" + }; + + assert_eq!( + resolve_host_program( + /*override_path*/ None, + Err(io::Error::new( + io::ErrorKind::NotFound, + "missing executable" + )), + ), + PathBuf::from(executable_name) + ); +} + +#[tokio::test] +async fn provider_reports_host_spawn_failure() { + let provider = ProcessOwnedCodeModeSessionProvider::with_host_program( + "codex-code-mode-host-does-not-exist".into(), + ); + + let error = provider + .create_session(Arc::new(NoopCodeModeSessionDelegate)) + .await + .err() + .expect("session creation should fail"); + + assert!(error.contains("failed to spawn code-mode host")); +} + +#[tokio::test] +async fn shutdown_before_open_does_not_spawn_the_host() { + let session = ProcessOwnedCodeModeSession::new(); + + session.shutdown().await.expect("shutdown session"); + let error = session + .execute(codex_code_mode_protocol::ExecuteRequest { + tool_call_id: "call-1".to_string(), + enabled_tools: Vec::new(), + source: "text('unreachable')".to_string(), + yield_time_ms: None, + max_output_tokens: None, + }) + .await + .err() + .expect("shutdown session should reject execution"); + + assert_eq!(error, "code mode session is shutting down"); +} diff --git a/codex-rs/code-mode/src/runtime/mod.rs b/codex-rs/code-mode/src/runtime/mod.rs index 42af5c0b95..0d32858724 100644 --- a/codex-rs/code-mode/src/runtime/mod.rs +++ b/codex-rs/code-mode/src/runtime/mod.rs @@ -5,7 +5,8 @@ mod timers; mod value; use std::collections::HashMap; -use std::sync::OnceLock; +use std::panic::AssertUnwindSafe; +use std::panic::catch_unwind; use std::sync::mpsc as std_mpsc; use std::thread; @@ -18,6 +19,9 @@ use codex_protocol::ToolName; use serde_json::Value as JsonValue; use tokio::sync::mpsc; +use crate::TaskFailureHandler; +use crate::v8_init::ensure_v8_initialized; + const EXIT_SENTINEL: &str = "__codex_code_mode_exit__"; #[derive(Debug)] @@ -63,6 +67,7 @@ pub(crate) enum RuntimeEvent { stored_value_writes: HashMap, error_text: Option, }, + ThreadPanicked, } pub(crate) fn spawn_runtime( @@ -70,6 +75,7 @@ pub(crate) fn spawn_runtime( request: ExecuteRequest, event_tx: mpsc::UnboundedSender, pending_mode: PendingRuntimeMode, + task_failure_handler: Option, ) -> Result< ( std_mpsc::Sender, @@ -78,7 +84,7 @@ pub(crate) fn spawn_runtime( ), String, > { - initialize_v8()?; + ensure_v8_initialized()?; let (command_tx, command_rx) = std_mpsc::channel(); let (control_tx, control_rx) = std_mpsc::channel(); @@ -96,7 +102,7 @@ pub(crate) fn spawn_runtime( stored_values, }; - thread::spawn(move || { + spawn_supervised_runtime_thread(event_tx.clone(), task_failure_handler, move || { run_runtime( config, event_tx, @@ -114,6 +120,21 @@ pub(crate) fn spawn_runtime( Ok((command_tx, control_tx, isolate_handle)) } +fn spawn_supervised_runtime_thread( + event_tx: mpsc::UnboundedSender, + task_failure_handler: Option, + runtime: impl FnOnce() + Send + 'static, +) { + thread::spawn(move || { + if catch_unwind(AssertUnwindSafe(runtime)).is_err() { + if let Some(task_failure_handler) = task_failure_handler { + task_failure_handler("code-mode V8 runtime thread panicked".to_string()); + } + let _ = event_tx.send(RuntimeEvent::ThreadPanicked); + } + }); +} + #[derive(Clone)] struct RuntimeConfig { tool_call_id: String, @@ -144,22 +165,6 @@ pub(super) enum CompletionState { }, } -fn initialize_v8() -> Result<(), String> { - static PLATFORM: OnceLock, String>> = OnceLock::new(); - - match PLATFORM.get_or_init(|| { - v8::icu::set_common_data_77(deno_core_icudata::ICU_DATA) - .map_err(|error_code| format!("failed to initialize ICU data: {error_code}"))?; - let platform = v8::new_default_platform(0, false).make_shared(); - v8::V8::initialize_platform(platform.clone()); - v8::V8::initialize(); - Ok(platform) - }) { - Ok(_) => Ok(()), - Err(error_text) => Err(error_text.clone()), - } -} - fn run_runtime( config: RuntimeConfig, event_tx: mpsc::UnboundedSender, @@ -336,6 +341,7 @@ mod tests { use super::RuntimeControlCommand; use super::RuntimeEvent; use super::spawn_runtime; + use super::spawn_supervised_runtime_thread; use crate::FunctionCallOutputContentItem; fn execute_request(source: &str) -> ExecuteRequest { @@ -348,6 +354,45 @@ mod tests { } } + #[tokio::test] + async fn runtime_thread_panic_before_initialization_is_reported_directly() { + let (event_tx, event_rx) = mpsc::unbounded_channel(); + drop(event_rx); + let (failure_tx, mut failure_rx) = mpsc::unbounded_channel(); + spawn_supervised_runtime_thread( + event_tx, + Some(std::sync::Arc::new(move |reason| { + let _ = failure_tx.send(reason); + })), + || panic!("runtime thread panic probe"), + ); + + assert_eq!( + tokio::time::timeout(Duration::from_secs(1), failure_rx.recv()) + .await + .expect("runtime failure timeout") + .expect("runtime failure"), + "code-mode V8 runtime thread panicked" + ); + } + + #[tokio::test] + async fn runtime_thread_panic_is_forwarded_without_owner_supervision() { + let (event_tx, mut event_rx) = mpsc::unbounded_channel(); + spawn_supervised_runtime_thread( + event_tx, + /*task_failure_handler*/ None, + || panic!("runtime thread panic probe"), + ); + + assert!(matches!( + tokio::time::timeout(Duration::from_secs(1), event_rx.recv()) + .await + .expect("runtime panic event timeout"), + Some(RuntimeEvent::ThreadPanicked) + )); + } + #[tokio::test] async fn terminate_execution_stops_cpu_bound_module() { let (event_tx, mut event_rx) = mpsc::unbounded_channel(); @@ -356,6 +401,7 @@ mod tests { execute_request("while (true) {}"), event_tx, PendingRuntimeMode::Continue, + /*task_failure_handler*/ None, ) .unwrap(); @@ -398,6 +444,7 @@ await new Promise(() => {}); ), event_tx, PendingRuntimeMode::PauseUntilResumed, + /*task_failure_handler*/ None, ) .unwrap(); diff --git a/codex-rs/code-mode/src/service.rs b/codex-rs/code-mode/src/service.rs index 57400001f0..8c5b94bb75 100644 --- a/codex-rs/code-mode/src/service.rs +++ b/codex-rs/code-mode/src/service.rs @@ -87,6 +87,18 @@ impl InProcessCodeModeSession { } } + pub fn with_delegate_and_task_failure_handler( + delegate: Arc, + task_failure_handler: Arc, + ) -> Self { + Self { + runtime: SessionRuntime::new_with_task_failure_handler( + Arc::new(ProtocolDelegate { delegate }), + Some(task_failure_handler), + ), + } + } + pub async fn execute(&self, request: ExecuteRequest) -> Result { let yield_time_ms = request.yield_time_ms.unwrap_or(DEFAULT_EXEC_YIELD_TIME_MS); let started = self diff --git a/codex-rs/code-mode/src/session_runtime/mod.rs b/codex-rs/code-mode/src/session_runtime/mod.rs index 5688c04303..cc014230f9 100644 --- a/codex-rs/code-mode/src/session_runtime/mod.rs +++ b/codex-rs/code-mode/src/session_runtime/mod.rs @@ -24,6 +24,7 @@ pub(crate) use self::types::SessionRuntimeDelegate; pub(crate) use self::types::ToolDefinition; pub(crate) use self::types::ToolKind; pub(crate) use self::types::ToolName; +use crate::TaskFailureHandler; use crate::cell_actor::CellActor; use crate::cell_actor::CellError; use crate::cell_actor::CellEventFuture; @@ -46,11 +47,19 @@ struct Inner { cell_tasks: TaskTracker, shutdown_token: CancellationToken, delegate: Arc, + task_failure_handler: Option, next_cell_id: AtomicU64, } impl SessionRuntime { pub(crate) fn new(delegate: Arc) -> Self { + Self::new_with_task_failure_handler(delegate, /*task_failure_handler*/ None) + } + + pub(crate) fn new_with_task_failure_handler( + delegate: Arc, + task_failure_handler: Option, + ) -> Self { Self { inner: Arc::new(Inner { stored_values: Mutex::new(HashMap::new()), @@ -58,6 +67,7 @@ impl SessionRuntime { cell_tasks: TaskTracker::new(), shutdown_token: CancellationToken::new(), delegate, + task_failure_handler, next_cell_id: AtomicU64::new(1), }), } @@ -71,7 +81,7 @@ impl SessionRuntime { if self.inner.shutdown_token.is_cancelled() { return Err(Error::ShuttingDown); } - let cell_id = self.allocate_cell_id(); + let cell_id = self.allocate_cell_id()?; let initial_event = self .start_cell(cell_id.clone(), request, initial_observe_mode) .await?; @@ -133,13 +143,14 @@ impl SessionRuntime { Ok(()) } - fn allocate_cell_id(&self) -> CellId { - CellId::new( - self.inner - .next_cell_id - .fetch_add(1, Ordering::Relaxed) - .to_string(), - ) + fn allocate_cell_id(&self) -> Result { + self.inner + .next_cell_id + .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |next_cell_id| { + next_cell_id.checked_add(1) + }) + .map(|cell_id| CellId::new(cell_id.to_string())) + .map_err(|_| Error::CellIdSpaceExhausted) } async fn start_cell( @@ -167,10 +178,21 @@ impl SessionRuntime { host, initial_observe_mode, cell_state, + self.inner.task_failure_handler.clone(), ) .map_err(Error::Runtime)?; cells.insert(cell_id.clone(), handle); - self.inner.cell_tasks.spawn(task); + let task = self.inner.cell_tasks.spawn(task); + if let Some(task_failure_handler) = self.inner.task_failure_handler.clone() { + let failed_cell_id = cell_id.clone(); + let _failure_watcher = self.inner.cell_tasks.spawn(async move { + if let Err(err) = task.await { + task_failure_handler(format!( + "code-mode cell {failed_cell_id} task failed: {err}" + )); + } + }); + } drop(cells); Ok(map_actor_event(cell_id, initial_event)) } diff --git a/codex-rs/code-mode/src/session_runtime/tests.rs b/codex-rs/code-mode/src/session_runtime/tests.rs index 969f06a22d..1d90179bb3 100644 --- a/codex-rs/code-mode/src/session_runtime/tests.rs +++ b/codex-rs/code-mode/src/session_runtime/tests.rs @@ -15,6 +15,8 @@ use crate::cell_actor::CompletionCommit; struct RecordingDelegate; +struct PanickingClosedDelegate; + impl SessionRuntimeDelegate for RecordingDelegate { async fn invoke_tool( &self, @@ -37,6 +39,62 @@ impl SessionRuntimeDelegate for RecordingDelegate { fn cell_closed(&self, _cell_id: &CellId) {} } +impl SessionRuntimeDelegate for PanickingClosedDelegate { + async fn invoke_tool( + &self, + _invocation: NestedToolCall, + _cancellation_token: CancellationToken, + ) -> Result { + Ok(JsonValue::Null) + } + + async fn notify( + &self, + _call_id: String, + _cell_id: CellId, + _text: String, + _cancellation_token: CancellationToken, + ) -> Result<(), String> { + Ok(()) + } + + fn cell_closed(&self, _cell_id: &CellId) { + panic!("cell close panic probe"); + } +} + +#[tokio::test] +async fn reports_cell_actor_panics_to_the_owner() { + let (failure_tx, mut failure_rx) = tokio::sync::mpsc::unbounded_channel(); + let runtime = SessionRuntime::new_with_task_failure_handler( + Arc::new(PanickingClosedDelegate), + Some(Arc::new(move |reason| { + let _ = failure_tx.send(reason); + })), + ); + let started = runtime + .execute( + execute_request(r#"text("done");"#), + ObserveMode::YieldAfter(Duration::from_secs(1)), + ) + .await + .expect("start cell"); + assert_eq!( + started.initial_event().await, + Ok(CellEvent::Completed { + content_items: vec![OutputItem::Text { + text: "done".to_string(), + }], + error_text: None, + }) + ); + runtime.shutdown().await.expect("shutdown runtime"); + let failure = failure_rx + .try_recv() + .expect("shutdown should wait for the cell failure watcher"); + assert!(failure.contains("code-mode cell 1 task failed")); +} + #[tokio::test] async fn termination_rejects_a_waiting_store_commit_before_the_next_cell_can_load_it() { let runtime = SessionRuntime::new(Arc::new(RecordingDelegate)); @@ -118,6 +176,26 @@ fn execute_request(source: &str) -> CreateCellRequest { } } +#[tokio::test] +async fn cell_id_allocation_fails_before_wrapping() { + let runtime = SessionRuntime::new(Arc::new(RecordingDelegate)); + runtime + .inner + .next_cell_id + .store(u64::MAX, Ordering::Relaxed); + + assert_eq!( + runtime + .execute( + execute_request(r#"text("unreachable");"#), + ObserveMode::YieldAfter(Duration::from_secs(1)), + ) + .await + .err(), + Some(Error::CellIdSpaceExhausted) + ); +} + #[tokio::test] #[expect( clippy::await_holding_invalid_type, diff --git a/codex-rs/code-mode/src/session_runtime/types.rs b/codex-rs/code-mode/src/session_runtime/types.rs index b04bd0f236..a33ffa1f1f 100644 --- a/codex-rs/code-mode/src/session_runtime/types.rs +++ b/codex-rs/code-mode/src/session_runtime/types.rs @@ -138,6 +138,7 @@ pub(crate) trait SessionRuntimeDelegate: Send + Sync + 'static { #[derive(Clone, Debug, Eq, PartialEq)] pub(crate) enum Error { ShuttingDown, + CellIdSpaceExhausted, DuplicateCell(CellId), MissingCell(CellId), BusyObserver(CellId), @@ -150,6 +151,9 @@ impl fmt::Display for Error { fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { match self { Self::ShuttingDown => formatter.write_str("code mode session is shutting down"), + Self::CellIdSpaceExhausted => { + formatter.write_str("code mode session exhausted its cell ID space") + } Self::DuplicateCell(cell_id) => write!(formatter, "exec cell {cell_id} already exists"), Self::MissingCell(cell_id) => write!(formatter, "exec cell {cell_id} not found"), Self::BusyObserver(cell_id) => { diff --git a/codex-rs/code-mode/src/v8_init.rs b/codex-rs/code-mode/src/v8_init.rs new file mode 100644 index 0000000000..0d00d9fad5 --- /dev/null +++ b/codex-rs/code-mode/src/v8_init.rs @@ -0,0 +1,65 @@ +use std::sync::OnceLock; + +/// Controls whether V8 may generate executable code at runtime. +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub enum V8JitMode { + #[default] + Enabled, + Disabled, +} + +struct V8Initialization { + _platform: v8::SharedRef, + jit_mode: V8JitMode, +} + +static V8_INITIALIZATION: OnceLock> = OnceLock::new(); + +/// Initializes the process-wide V8 platform with the requested JIT mode. +/// +/// Call this before executing any code-mode cells when JIT must be disabled. +/// V8 cannot change JIT mode after initialization, so a later call requesting +/// a different mode returns an error. Code mode initializes V8 with JIT enabled +/// by default when this function has not been called explicitly. +pub fn initialize_v8(jit_mode: V8JitMode) -> Result<(), String> { + match V8_INITIALIZATION.get_or_init(|| initialize_v8_with_mode(jit_mode)) { + Ok(initialization) if initialization.jit_mode == jit_mode => Ok(()), + Ok(initialization) => Err(format!( + "V8 was already initialized with JIT {}", + initialization.jit_mode.description() + )), + Err(error_text) => Err(error_text.clone()), + } +} + +pub(crate) fn ensure_v8_initialized() -> Result<(), String> { + match V8_INITIALIZATION.get_or_init(|| initialize_v8_with_mode(V8JitMode::Enabled)) { + Ok(_) => Ok(()), + Err(error_text) => Err(error_text.clone()), + } +} + +fn initialize_v8_with_mode(jit_mode: V8JitMode) -> Result { + v8::icu::set_common_data_77(deno_core_icudata::ICU_DATA) + .map_err(|error_code| format!("failed to initialize ICU data: {error_code}"))?; + match jit_mode { + V8JitMode::Enabled => {} + V8JitMode::Disabled => v8::V8::set_flags_from_string("--jitless"), + } + let platform = v8::new_default_platform(0, false).make_shared(); + v8::V8::initialize_platform(platform.clone()); + v8::V8::initialize(); + Ok(V8Initialization { + _platform: platform, + jit_mode, + }) +} + +impl V8JitMode { + fn description(self) -> &'static str { + match self { + Self::Enabled => "enabled", + Self::Disabled => "disabled", + } + } +} diff --git a/codex-rs/code-mode/tests/jit.rs b/codex-rs/code-mode/tests/jit.rs new file mode 100644 index 0000000000..0f8e866c47 --- /dev/null +++ b/codex-rs/code-mode/tests/jit.rs @@ -0,0 +1,41 @@ +use codex_code_mode::ExecuteRequest; +use codex_code_mode::InProcessCodeModeSession; +use codex_code_mode::RuntimeResponse; +use codex_code_mode::V8JitMode; +use codex_code_mode::initialize_v8; +use pretty_assertions::assert_eq; + +#[tokio::test] +async fn code_mode_runs_with_jit_disabled() { + initialize_v8(V8JitMode::Disabled).expect("initialize V8 without JIT"); + + let service = InProcessCodeModeSession::new(); + let started = service + .execute(ExecuteRequest { + tool_call_id: "call_1".to_string(), + enabled_tools: Vec::new(), + source: "21 * 2;".to_string(), + yield_time_ms: None, + max_output_tokens: None, + }) + .await + .expect("start code-mode cell"); + let cell_id = started.cell_id.clone(); + let response = started + .initial_response() + .await + .expect("execute code-mode cell"); + + assert_eq!( + response, + RuntimeResponse::Result { + cell_id, + content_items: Vec::new(), + error_text: None, + } + ); + assert_eq!( + initialize_v8(V8JitMode::Enabled), + Err("V8 was already initialized with JIT disabled".to_string()) + ); +} diff --git a/codex-rs/codex-api/src/common.rs b/codex-rs/codex-api/src/common.rs index bd037deb58..254a5cb38f 100644 --- a/codex-rs/codex-api/src/common.rs +++ b/codex-rs/codex-api/src/common.rs @@ -121,21 +121,12 @@ pub struct SafetyBuffering { pub reasons: Vec, #[serde(skip)] pub show_buffering_ui: bool, - #[serde(skip)] + #[serde(rename = "retry_model")] pub faster_model: Option, } -impl SafetyBuffering { - pub(crate) fn with_treatment(mut self, treatment: &SafetyBufferingTreatment) -> Self { - self.show_buffering_ui = treatment.show_buffering_ui; - self.faster_model.clone_from(&treatment.faster_model); - self - } -} - #[derive(Debug, Clone, Default, PartialEq, Eq)] pub(crate) struct SafetyBufferingTreatment { - pub show_buffering_ui: bool, pub faster_model: Option, } diff --git a/codex-rs/codex-api/src/endpoint/realtime_websocket/methods.rs b/codex-rs/codex-api/src/endpoint/realtime_websocket/methods.rs index 499635579f..0ae443d00a 100644 --- a/codex-rs/codex-api/src/endpoint/realtime_websocket/methods.rs +++ b/codex-rs/codex-api/src/endpoint/realtime_websocket/methods.rs @@ -391,6 +391,13 @@ impl RealtimeWebsocketWriter { } impl RealtimeWebsocketEvents { + pub async fn take_transcript_tail(&self) -> Vec { + let mut active_transcript = self.active_transcript.lock().await; + let tail = active_transcript.entries[active_transcript.last_handoff_entry_count..].to_vec(); + active_transcript.last_handoff_entry_count = active_transcript.entries.len(); + tail + } + pub async fn next_event(&self) -> Result, ApiError> { if self.is_closed.load(Ordering::SeqCst) { return Ok(None); @@ -962,6 +969,56 @@ mod tests { ); } + #[tokio::test] + async fn takes_only_transcript_after_last_handoff_once() { + let (_tx_message, rx_message) = async_channel::unbounded(); + let events = RealtimeWebsocketEvents { + rx_message, + active_transcript: Arc::new(Mutex::new(ActiveTranscriptState::default())), + event_parser: RealtimeEventParser::V1, + is_closed: Arc::new(AtomicBool::new(false)), + }; + + assert_eq!(events.take_transcript_tail().await, vec![]); + + let mut covered = RealtimeEvent::InputTranscriptDelta(RealtimeTranscriptDelta { + delta: "already handed off".to_string(), + }); + events.update_active_transcript(&mut covered).await; + let mut handoff = RealtimeEvent::HandoffRequested(RealtimeHandoffRequested { + handoff_id: "handoff_1".to_string(), + item_id: "item_1".to_string(), + input_transcript: "already handed off".to_string(), + active_transcript: vec![], + }); + events.update_active_transcript(&mut handoff).await; + assert_eq!( + handoff, + RealtimeEvent::HandoffRequested(RealtimeHandoffRequested { + handoff_id: "handoff_1".to_string(), + item_id: "item_1".to_string(), + input_transcript: "already handed off".to_string(), + active_transcript: vec![RealtimeTranscriptEntry { + role: "user".to_string(), + text: "already handed off".to_string(), + }], + }) + ); + + let mut tail = RealtimeEvent::OutputTranscriptDelta(RealtimeTranscriptDelta { + delta: "tail".to_string(), + }); + events.update_active_transcript(&mut tail).await; + assert_eq!( + events.take_transcript_tail().await, + vec![RealtimeTranscriptEntry { + role: "assistant".to_string(), + text: "tail".to_string(), + }] + ); + assert_eq!(events.take_transcript_tail().await, vec![]); + } + #[test] fn parse_input_transcript_delta_event() { let payload = json!({ diff --git a/codex-rs/codex-api/src/endpoint/responses_websocket.rs b/codex-rs/codex-api/src/endpoint/responses_websocket.rs index 755228a9f5..7f261bba36 100644 --- a/codex-rs/codex-api/src/endpoint/responses_websocket.rs +++ b/codex-rs/codex-api/src/endpoint/responses_websocket.rs @@ -43,7 +43,6 @@ use tracing::debug; use tracing::error; use tracing::info; use tracing::instrument; -use tracing::trace; use tungstenite::extensions::ExtensionsConfig; use tungstenite::extensions::compression::deflate::DeflateConfig; use tungstenite::protocol::WebSocketConfig; @@ -690,17 +689,10 @@ async fn run_websocket_response_stream( { let _ = turn_state.set(response_turn_state); } - if let Some(headers) = event.headers.as_ref().and_then(Value::as_object) - && let Some(treatment) = - treatment_from_headers(&json_headers_to_http_headers(headers)) - { - safety_buffering_treatment = treatment; - } let model_verifications = event.model_verifications(); let turn_moderation_metadata = event.turn_moderation_metadata(); - let safety_buffering = event - .safety_buffering() - .map(|buffering| buffering.with_treatment(&safety_buffering_treatment)); + let safety_buffering = + safety_buffering_for_event(&event, &mut safety_buffering_treatment); if event.kind() == "codex.rate_limits" { if let Some(snapshot) = parse_rate_limit_event(&text) { let _ = tx_event.send(Ok(ResponseEvent::RateLimits(snapshot))).await; @@ -775,6 +767,19 @@ async fn run_websocket_response_stream( Ok(()) } +fn safety_buffering_for_event( + event: &ResponsesStreamEvent, + treatment: &mut SafetyBufferingTreatment, +) -> Option { + if let Some(headers) = event.headers.as_ref().and_then(Value::as_object) + && let Some(updated_treatment) = + treatment_from_headers(&json_headers_to_http_headers(headers)) + { + *treatment = updated_treatment; + } + event.safety_buffering(treatment) +} + async fn send_websocket_request( ws_stream: &WsStream, request_text: String, @@ -782,8 +787,6 @@ async fn send_websocket_request( telemetry: Option<&Arc>, connection_reused: bool, ) -> Result<(), ApiError> { - trace!("websocket request: {request_text}"); - let request_start = Instant::now(); let result = tokio::time::timeout( idle_timeout, @@ -1042,4 +1045,75 @@ mod tests { Some(&HeaderValue::from_static("default-only")) ); } + + #[test] + fn websocket_safety_buffering_uses_event_before_header_fallback() { + let metadata: ResponsesStreamEvent = serde_json::from_value(json!({ + "type": "codex.response.metadata", + "headers": { + "x-codex-safety-buffering-enabled": "true", + "x-codex-safety-buffering-faster-model": "gpt-fast-header" + } + })) + .expect("deserialize treatment metadata"); + let event: ResponsesStreamEvent = serde_json::from_value(json!({ + "type": "response.output_text.delta", + "safety_buffering": { + "use_cases": ["cyber"], + "reasons": ["user_risk"], + "retry_model": "gpt-fast-wire" + } + })) + .expect("deserialize safety buffering event"); + let mut treatment = SafetyBufferingTreatment::default(); + + assert!(safety_buffering_for_event(&metadata, &mut treatment).is_none()); + let buffering = safety_buffering_for_event(&event, &mut treatment) + .expect("expected safety buffering payload"); + + assert_eq!( + buffering, + crate::common::SafetyBuffering { + use_cases: vec!["cyber".to_string()], + reasons: vec!["user_risk".to_string()], + show_buffering_ui: true, + faster_model: Some("gpt-fast-wire".to_string()), + } + ); + } + + #[test] + fn websocket_safety_buffering_event_controls_visibility_when_header_disables_it() { + let metadata: ResponsesStreamEvent = serde_json::from_value(json!({ + "type": "codex.response.metadata", + "headers": { + "x-codex-safety-buffering-enabled": "false", + "x-codex-safety-buffering-faster-model": "gpt-fast-header" + } + })) + .expect("deserialize treatment metadata"); + let event: ResponsesStreamEvent = serde_json::from_value(json!({ + "type": "response.output_text.delta", + "safety_buffering": { + "use_cases": ["cyber"], + "reasons": ["user_risk"] + } + })) + .expect("deserialize safety buffering event"); + let mut treatment = SafetyBufferingTreatment::default(); + + assert!(safety_buffering_for_event(&metadata, &mut treatment).is_none()); + let buffering = safety_buffering_for_event(&event, &mut treatment) + .expect("expected safety buffering payload"); + + assert_eq!( + buffering, + crate::common::SafetyBuffering { + use_cases: vec!["cyber".to_string()], + reasons: vec!["user_risk".to_string()], + show_buffering_ui: true, + faster_model: Some("gpt-fast-header".to_string()), + } + ); + } } diff --git a/codex-rs/codex-api/src/safety_buffering.rs b/codex-rs/codex-api/src/safety_buffering.rs index e5f3dcd79e..aaf09c8eb4 100644 --- a/codex-rs/codex-api/src/safety_buffering.rs +++ b/codex-rs/codex-api/src/safety_buffering.rs @@ -6,23 +6,17 @@ pub(crate) const X_CODEX_SAFETY_BUFFERING_FASTER_MODEL_HEADER: &str = "x-codex-safety-buffering-faster-model"; pub(crate) fn treatment_from_headers(headers: &HeaderMap) -> Option { - let show_buffering_ui = headers - .get(X_CODEX_SAFETY_BUFFERING_ENABLED_HEADER) - .and_then(|value| value.to_str().ok())? - .eq_ignore_ascii_case("true"); - let faster_model = if show_buffering_ui { - headers - .get(X_CODEX_SAFETY_BUFFERING_FASTER_MODEL_HEADER) - .and_then(|value| value.to_str().ok()) - .map(str::to_string) - } else { - None - }; + if !headers.contains_key(X_CODEX_SAFETY_BUFFERING_ENABLED_HEADER) + && !headers.contains_key(X_CODEX_SAFETY_BUFFERING_FASTER_MODEL_HEADER) + { + return None; + } + let faster_model = headers + .get(X_CODEX_SAFETY_BUFFERING_FASTER_MODEL_HEADER) + .and_then(|value| value.to_str().ok()) + .map(str::to_string); - Some(SafetyBufferingTreatment { - show_buffering_ui, - faster_model, - }) + Some(SafetyBufferingTreatment { faster_model }) } #[cfg(test)] @@ -46,7 +40,26 @@ mod tests { assert_eq!( treatment_from_headers(&headers), Some(SafetyBufferingTreatment { - show_buffering_ui: true, + faster_model: Some("faster-model".to_string()), + }) + ); + } + + #[test] + fn buffering_enabled_header_does_not_gate_the_faster_model_fallback() { + let mut headers = HeaderMap::new(); + headers.insert( + X_CODEX_SAFETY_BUFFERING_ENABLED_HEADER, + HeaderValue::from_static("false"), + ); + headers.insert( + X_CODEX_SAFETY_BUFFERING_FASTER_MODEL_HEADER, + HeaderValue::from_static("faster-model"), + ); + + assert_eq!( + treatment_from_headers(&headers), + Some(SafetyBufferingTreatment { faster_model: Some("faster-model".to_string()), }) ); diff --git a/codex-rs/codex-api/src/sse/responses.rs b/codex-rs/codex-api/src/sse/responses.rs index 71f73c0232..17d31a02ee 100644 --- a/codex-rs/codex-api/src/sse/responses.rs +++ b/codex-rs/codex-api/src/sse/responses.rs @@ -231,8 +231,18 @@ impl ResponsesStreamEvent { .map(|metadata| TurnModerationMetadataEvent { metadata }) } - pub(crate) fn safety_buffering(&self) -> Option { - serde_json::from_value(self.safety_buffering.as_ref()?.clone()).ok() + pub(crate) fn safety_buffering( + &self, + treatment: &SafetyBufferingTreatment, + ) -> Option { + let value = self.safety_buffering.as_ref()?; + let retry_model_present = value.as_object()?.contains_key("retry_model"); + let mut buffering: SafetyBuffering = serde_json::from_value(value.clone()).ok()?; + buffering.show_buffering_ui = true; + if !retry_model_present { + buffering.faster_model.clone_from(&treatment.faster_model); + } + Some(buffering) } } @@ -515,9 +525,7 @@ async fn process_sse_with_treatment( }; let model_verifications = event.model_verifications(); let turn_moderation_metadata = event.turn_moderation_metadata(); - let safety_buffering = event - .safety_buffering() - .map(|buffering| buffering.with_treatment(&safety_buffering_treatment)); + let safety_buffering = event.safety_buffering(&safety_buffering_treatment); if let Some(model) = event.response_model() && last_server_model.as_deref() != Some(model.as_str()) @@ -1354,7 +1362,8 @@ mod tests { "delta": "hello", "safety_buffering": { "use_cases": ["cyber"], - "reasons": ["user_risk"] + "reasons": ["user_risk"], + "retry_model": "gpt-fast-wire" } }), json!({ @@ -1381,7 +1390,10 @@ mod tests { assert_matches!( &events[1], ResponseEvent::SafetyBuffering(buffering) - if buffering.use_cases == ["cyber"] && buffering.reasons == ["user_risk"] + if buffering.use_cases == ["cyber"] + && buffering.reasons == ["user_risk"] + && buffering.show_buffering_ui + && buffering.faster_model.as_deref() == Some("gpt-fast-wire") ); assert_matches!(&events[2], ResponseEvent::OutputTextDelta(delta) if delta == "hello"); assert_matches!( @@ -1398,6 +1410,46 @@ mod tests { assert_matches!(&events[6], ResponseEvent::Completed { response_id, .. } if response_id == "resp-1"); } + #[test] + fn safety_buffering_prefers_wire_retry_model_and_only_falls_back_when_omitted() { + let treatment = SafetyBufferingTreatment { + faster_model: Some("gpt-fast-header".to_string()), + }; + + for (retry_model, expected_faster_model) in [ + (None, Some("gpt-fast-header")), + (Some(Value::Null), None), + (Some(json!("gpt-fast-wire")), Some("gpt-fast-wire")), + ] { + let mut event = json!({ + "type": "response.output_text.delta", + "safety_buffering": { + "use_cases": ["cyber"], + "reasons": ["user_risk"] + } + }); + if let Some(retry_model) = retry_model { + event["safety_buffering"]["retry_model"] = retry_model; + } + let event: ResponsesStreamEvent = + serde_json::from_value(event).expect("deserialize safety buffering event"); + + let buffering = event + .safety_buffering(&treatment) + .expect("expected safety buffering payload"); + + assert_eq!( + buffering, + SafetyBuffering { + use_cases: vec!["cyber".to_string()], + reasons: vec!["user_risk".to_string()], + show_buffering_ui: true, + faster_model: expected_faster_model.map(str::to_string), + } + ); + } + } + #[test] fn responses_stream_event_response_model_reads_top_level_headers() { let ev: ResponsesStreamEvent = serde_json::from_value(json!({ diff --git a/codex-rs/codex-api/tests/models_integration.rs b/codex-rs/codex-api/tests/models_integration.rs index 5f421dc1ad..b528604329 100644 --- a/codex-rs/codex-api/tests/models_integration.rs +++ b/codex-rs/codex-api/tests/models_integration.rs @@ -80,6 +80,7 @@ async fn models_client_hits_models_endpoint() { upgrade: None, base_instructions: "base instructions".to_string(), model_messages: None, + include_skills_usage_instructions: false, supports_reasoning_summaries: false, default_reasoning_summary: ReasoningSummary::Auto, support_verbosity: false, diff --git a/codex-rs/codex-client/src/chatgpt_cloudflare_cookies.rs b/codex-rs/codex-client/src/chatgpt_cloudflare_cookies.rs index c5f4bbd4eb..7e5d22057c 100644 --- a/codex-rs/codex-client/src/chatgpt_cloudflare_cookies.rs +++ b/codex-rs/codex-client/src/chatgpt_cloudflare_cookies.rs @@ -128,11 +128,12 @@ mod tests { fn stores_and_returns_cloudflare_cookies_for_chatgpt_hosts() { let store = ChatGptCloudflareCookieStore::default(); let url = reqwest::Url::parse("https://chatgpt.com/backend-api/codex/responses").unwrap(); + let load_balancer = HeaderValue::from_static("__cflb=west; Path=/; Secure; HttpOnly"); let cfuvid = HeaderValue::from_static("_cfuvid=visitor; Path=/; Secure; HttpOnly"); let clearance = HeaderValue::from_static("cf_clearance=clearance; Path=/; Secure; HttpOnly"); - store.set_cookies(&mut [&cfuvid, &clearance].into_iter(), &url); + store.set_cookies(&mut [&load_balancer, &cfuvid, &clearance].into_iter(), &url); let mut cookies = store .cookies(&url) @@ -148,6 +149,7 @@ mod tests { assert_eq!( cookies, vec![ + "__cflb=west".to_string(), "_cfuvid=visitor".to_string(), "cf_clearance=clearance".to_string() ] diff --git a/codex-rs/codex-mcp/src/catalog.rs b/codex-rs/codex-mcp/src/catalog.rs index 3a752c82ba..d60ea843df 100644 --- a/codex-rs/codex-mcp/src/catalog.rs +++ b/codex-rs/codex-mcp/src/catalog.rs @@ -371,6 +371,35 @@ impl ResolvedMcpCatalog { .collect() } + /// Returns whether both catalogs resolve to the same winning servers and sources. + pub fn has_same_servers(&self, other: &Self) -> bool { + self.servers == other.servers + } + + /// Replaces the resolved server set while preserving known server sources. + /// + /// Names not present in the existing catalog are treated as config-owned. + pub fn with_materialized_servers(&self, servers: HashMap) -> Self { + let mut builder = Self::builder(); + for (name, config) in servers { + let source = self + .server(&name) + .map(|server| server.source.clone()) + .unwrap_or(McpServerSource::Config); + let precedence = match &source { + McpServerSource::Plugin(_) => RegistrationPrecedence::Plugin(Reverse(0)), + McpServerSource::SelectedPlugin(_) => { + RegistrationPrecedence::SelectedPlugin(Reverse(0)) + } + McpServerSource::Config => RegistrationPrecedence::Config, + McpServerSource::Compatibility { .. } => RegistrationPrecedence::Compatibility, + McpServerSource::Extension { .. } => RegistrationPrecedence::Extension(0), + }; + builder.register(McpServerRegistration::new(name, source, config, precedence)); + } + builder.build() + } + /// Returns package attribution for each winning plugin-owned server. pub fn plugin_attributions_by_server_name(&self) -> HashMap { self.servers diff --git a/codex-rs/codex-mcp/src/catalog_tests.rs b/codex-rs/codex-mcp/src/catalog_tests.rs index 0b2ffef50d..6a0515db39 100644 --- a/codex-rs/codex-mcp/src/catalog_tests.rs +++ b/codex-rs/codex-mcp/src/catalog_tests.rs @@ -304,6 +304,17 @@ fn selected_plugins_override_discovered_plugins_but_not_config() { }] ); + let refreshed = server("https://refreshed.example/mcp"); + let catalog = + catalog.with_materialized_servers(HashMap::from([("docs".to_string(), refreshed.clone())])); + assert_eq!( + catalog.server("docs"), + Some(&super::ResolvedMcpServer { + source: selected_plugin_source("selected-alpha"), + config: refreshed, + }) + ); + let mut builder = catalog.to_builder(); let configured = server("https://config.example/mcp"); builder.register(McpServerRegistration::from_config( diff --git a/codex-rs/codex-mcp/src/codex_apps.rs b/codex-rs/codex-mcp/src/codex_apps.rs index c1b5fd5fa5..a33f735366 100644 --- a/codex-rs/codex-mcp/src/codex_apps.rs +++ b/codex-rs/codex-mcp/src/codex_apps.rs @@ -1,70 +1,9 @@ //! Codex Apps support for the host-owned apps MCP server. //! -//! This module owns the pieces that are unique to ChatGPT-hosted app -//! connectors: cache scoping by authenticated user, disk cache reads/writes, -//! connector allow-list filtering, and the normalization that turns app +//! This module owns the normalization that turns ChatGPT-hosted app //! connector/tool metadata into model-visible MCP callable names. -use std::path::PathBuf; -use std::time::Instant; - -use crate::runtime::emit_duration; -use crate::tools::MCP_TOOLS_CACHE_WRITE_DURATION_METRIC; -use crate::tools::ToolInfo; -use anyhow::Context; -use codex_login::CodexAuth; -use codex_protocol::mcp::McpServerInfo; use codex_utils_plugins::mcp_connector::sanitize_name; -use serde::Deserialize; -use serde::Serialize; -use sha1::Digest; -use sha1::Sha1; -use tracing::instrument; - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct CodexAppsToolsCacheKey { - pub(crate) account_id: Option, - pub(crate) chatgpt_user_id: Option, - pub(crate) is_workspace_account: bool, -} - -pub fn codex_apps_tools_cache_key(auth: Option<&CodexAuth>) -> CodexAppsToolsCacheKey { - CodexAppsToolsCacheKey { - account_id: auth.and_then(CodexAuth::get_account_id), - chatgpt_user_id: auth.and_then(CodexAuth::get_chatgpt_user_id), - is_workspace_account: auth.is_some_and(CodexAuth::is_workspace_account), - } -} - -#[derive(Clone)] -pub(crate) struct CodexAppsToolsCacheContext { - pub(crate) codex_home: PathBuf, - pub(crate) user_key: CodexAppsToolsCacheKey, -} - -impl CodexAppsToolsCacheContext { - pub(crate) fn tools_cache_path(&self) -> PathBuf { - self.cache_path_in(CODEX_APPS_TOOLS_CACHE_DIR) - } - - pub(crate) fn server_info_cache_path(&self) -> PathBuf { - self.cache_path_in(CODEX_APPS_SERVER_INFO_CACHE_DIR) - } - - fn cache_path_in(&self, cache_dir: &str) -> PathBuf { - let user_key_json = serde_json::to_string(&self.user_key).unwrap_or_default(); - let user_key_hash = sha1_hex(&user_key_json); - self.codex_home - .join(cache_dir) - .join(format!("{user_key_hash}.json")) - } -} - -pub(crate) enum CachedCodexAppsToolsLoad { - Hit(Vec), - Missing, - Invalid, -} pub(crate) fn normalize_codex_apps_tool_title(connector_name: Option<&str>, value: &str) -> String { let Some(connector_name) = connector_name @@ -124,152 +63,3 @@ pub(crate) fn normalize_codex_apps_callable_namespace( server_name.to_string() } } - -pub(crate) fn write_codex_apps_tools_cache( - cache_context: Option<&CodexAppsToolsCacheContext>, - server_info: &McpServerInfo, - tools: &[ToolInfo], -) { - if let Some(cache_context) = cache_context { - let cache_write_start = Instant::now(); - write_cached_codex_apps_tools(cache_context, tools); - if let Err(err) = write_cached_codex_apps_server_info(cache_context, server_info) { - tracing::warn!("failed to write Codex Apps server info cache: {err:#}"); - } - emit_duration( - MCP_TOOLS_CACHE_WRITE_DURATION_METRIC, - cache_write_start.elapsed(), - &[], - ); - } -} - -pub(crate) fn load_startup_cached_codex_apps_tools_snapshot( - cache_context: Option<&CodexAppsToolsCacheContext>, -) -> Option> { - let cache_context = cache_context?; - - match load_cached_codex_apps_tools(cache_context) { - CachedCodexAppsToolsLoad::Hit(tools) => Some(tools), - CachedCodexAppsToolsLoad::Missing | CachedCodexAppsToolsLoad::Invalid => None, - } -} - -pub(crate) fn load_startup_cached_codex_apps_server_info( - cache_context: Option<&CodexAppsToolsCacheContext>, -) -> Option { - load_cached_codex_apps_server_info(cache_context?) -} - -#[cfg(test)] -pub(crate) fn read_cached_codex_apps_tools( - cache_context: &CodexAppsToolsCacheContext, -) -> Option> { - match load_cached_codex_apps_tools(cache_context) { - CachedCodexAppsToolsLoad::Hit(tools) => Some(tools), - CachedCodexAppsToolsLoad::Missing | CachedCodexAppsToolsLoad::Invalid => None, - } -} - -#[instrument(level = "trace", skip_all)] -pub(crate) fn load_cached_codex_apps_tools( - cache_context: &CodexAppsToolsCacheContext, -) -> CachedCodexAppsToolsLoad { - let cache_path = cache_context.tools_cache_path(); - let bytes = match std::fs::read(cache_path) { - Ok(bytes) => bytes, - Err(err) if err.kind() == std::io::ErrorKind::NotFound => { - return CachedCodexAppsToolsLoad::Missing; - } - Err(_) => return CachedCodexAppsToolsLoad::Invalid, - }; - let cache: CodexAppsToolsDiskCache = match serde_json::from_slice(&bytes) { - Ok(cache) => cache, - Err(_) => return CachedCodexAppsToolsLoad::Invalid, - }; - if cache.schema_version != CODEX_APPS_TOOLS_CACHE_SCHEMA_VERSION { - return CachedCodexAppsToolsLoad::Invalid; - } - CachedCodexAppsToolsLoad::Hit(cache.tools) -} - -pub(crate) fn write_cached_codex_apps_tools( - cache_context: &CodexAppsToolsCacheContext, - tools: &[ToolInfo], -) { - let cache_path = cache_context.tools_cache_path(); - if let Some(parent) = cache_path.parent() - && std::fs::create_dir_all(parent).is_err() - { - return; - } - let Ok(bytes) = serde_json::to_vec_pretty(&CodexAppsToolsDiskCache { - schema_version: CODEX_APPS_TOOLS_CACHE_SCHEMA_VERSION, - tools: tools.to_vec(), - }) else { - return; - }; - let _ = std::fs::write(cache_path, bytes); -} - -#[instrument(level = "trace", skip_all)] -pub(crate) fn load_cached_codex_apps_server_info( - cache_context: &CodexAppsToolsCacheContext, -) -> Option { - let bytes = std::fs::read(cache_context.server_info_cache_path()).ok()?; - let cache: CodexAppsServerInfoDiskCache = serde_json::from_slice(&bytes).ok()?; - (cache.schema_version == CODEX_APPS_SERVER_INFO_CACHE_SCHEMA_VERSION) - .then_some(cache.server_info) -} - -fn write_cached_codex_apps_server_info( - cache_context: &CodexAppsToolsCacheContext, - server_info: &McpServerInfo, -) -> anyhow::Result<()> { - let cache_path = cache_context.server_info_cache_path(); - if let Some(parent) = cache_path.parent() { - std::fs::create_dir_all(parent).with_context(|| { - format!( - "failed to create Codex Apps server info cache directory `{}`", - parent.display() - ) - })?; - } - let bytes = serde_json::to_vec_pretty(&CodexAppsServerInfoDiskCache { - schema_version: CODEX_APPS_SERVER_INFO_CACHE_SCHEMA_VERSION, - server_info: server_info.clone(), - }) - .context("failed to serialize Codex Apps server info cache")?; - std::fs::write(&cache_path, bytes).with_context(|| { - format!( - "failed to write Codex Apps server info cache `{}`", - cache_path.display() - ) - })?; - Ok(()) -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -struct CodexAppsToolsDiskCache { - schema_version: u8, - tools: Vec, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -struct CodexAppsServerInfoDiskCache { - schema_version: u8, - server_info: McpServerInfo, -} - -const CODEX_APPS_TOOLS_CACHE_DIR: &str = "cache/codex_apps_tools"; -pub(crate) const CODEX_APPS_TOOLS_CACHE_SCHEMA_VERSION: u8 = 4; - -const CODEX_APPS_SERVER_INFO_CACHE_DIR: &str = "cache/codex_apps_server_info"; -const CODEX_APPS_SERVER_INFO_CACHE_SCHEMA_VERSION: u8 = 1; - -fn sha1_hex(s: &str) -> String { - let mut hasher = Sha1::new(); - hasher.update(s.as_bytes()); - let sha1 = hasher.finalize(); - format!("{sha1:x}") -} diff --git a/codex-rs/codex-mcp/src/codex_apps_cache.rs b/codex-rs/codex-mcp/src/codex_apps_cache.rs new file mode 100644 index 0000000000..ea39444628 --- /dev/null +++ b/codex-rs/codex-mcp/src/codex_apps_cache.rs @@ -0,0 +1,378 @@ +//! Shared raw tool cache for the host-owned Codex Apps MCP server. +//! +//! Cache entries are process-local live state scoped by the active Codex auth +//! key. Disk is best-effort cold-start persistence; entries do not reread disk +//! after creation. + +use std::collections::HashMap; +use std::path::Path; +use std::path::PathBuf; +use std::sync::Arc; +use std::sync::Mutex; +use std::sync::atomic::AtomicU64; +use std::sync::atomic::Ordering; +use std::time::Instant; + +use anyhow::Context; +use arc_swap::ArcSwapOption; +use codex_login::CodexAuth; +use codex_protocol::mcp::McpServerInfo; +use serde::Deserialize; +use serde::Serialize; +use sha1::Digest; +use sha1::Sha1; +use tracing::instrument; + +use crate::runtime::emit_duration; +use crate::tools::MCP_TOOLS_CACHE_WRITE_DURATION_METRIC; +use crate::tools::ToolInfo; + +const MCP_TOOLS_CACHE_PUBLISH_DURATION_METRIC: &str = "codex.mcp.tools.cache_publish.duration_ms"; + +/// The CodexAuth bits that identify a Codex Apps catalog. +/// +/// Debug bearer-token overrides bypass the shared cache, so shared entries only +/// need the CodexAuth-backed identity. +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct CodexAppsToolsCacheKey { + pub(crate) account_id: Option, + pub(crate) chatgpt_user_id: Option, + pub(crate) is_workspace_account: bool, +} + +/// Builds the CodexAuth-backed Codex Apps cache key. +pub fn codex_apps_tools_cache_key(auth: Option<&CodexAuth>) -> CodexAppsToolsCacheKey { + CodexAppsToolsCacheKey { + account_id: auth.and_then(CodexAuth::get_account_id), + chatgpt_user_id: auth.and_then(CodexAuth::get_chatgpt_user_id), + is_workspace_account: auth.is_some_and(CodexAuth::is_workspace_account), + } +} + +/// Process-scoped registry for shared Codex Apps raw tool snapshots. +/// +/// Two clients share an entry only when they would read the same Codex Apps +/// catalog. New entries may seed from disk; live entries read from memory only. +#[derive(Clone, Default)] +pub struct CodexAppsToolsCache { + entries: Arc>>>, +} + +/// Handle to one shared Codex Apps tools cache entry. +/// +/// The connection manager creates this from the auth key, then tool +/// reads and refreshes for that managed client use the same entry. +#[derive(Clone)] +pub(crate) struct CodexAppsToolsCacheContext { + entry: Arc, +} + +impl CodexAppsToolsCacheContext { + pub(crate) fn tools_cache_path(&self) -> PathBuf { + self.entry + .identity + .cache_path_in(CODEX_APPS_TOOLS_CACHE_DIR) + } + + pub(crate) fn server_info_cache_path(&self) -> PathBuf { + self.entry + .identity + .cache_path_in(CODEX_APPS_SERVER_INFO_CACHE_DIR) + } + + pub(crate) fn current_tools(&self) -> Option> { + self.entry + .current_tools + .load_full() + .map(|tools| tools.as_ref().clone()) + } + + pub(crate) fn has_current_tools(&self) -> bool { + self.entry.current_tools.load_full().is_some() + } + + pub(crate) fn begin_fetch( + &self, + source: CodexAppsToolsFetchSource, + ) -> CodexAppsToolsFetchTicket { + CodexAppsToolsFetchTicket { + generation: self + .entry + .next_fetch_generation + .fetch_add(1, Ordering::Relaxed) + + 1, + source, + } + } + + pub(crate) fn publish_if_newest_accepted( + &self, + ticket: CodexAppsToolsFetchTicket, + server_info: &McpServerInfo, + tools: Vec, + ) -> Vec { + let publish_start = Instant::now(); + let mut last_accepted_generation = lock_unpoisoned(&self.entry.last_accepted_generation); + if ticket.generation <= *last_accepted_generation { + emit_duration( + MCP_TOOLS_CACHE_PUBLISH_DURATION_METRIC, + publish_start.elapsed(), + &[("source", ticket.source.as_str()), ("result", "stale")], + ); + return self.current_tools().unwrap_or(tools); + } + + *last_accepted_generation = ticket.generation; + self.entry + .current_tools + .store(Some(Arc::new(tools.clone()))); + persist_codex_apps_cache(self, server_info, &tools); + emit_duration( + MCP_TOOLS_CACHE_PUBLISH_DURATION_METRIC, + publish_start.elapsed(), + &[("source", ticket.source.as_str()), ("result", "published")], + ); + tools + } + + #[cfg(test)] + pub(crate) fn store_current_tools_for_test(&self, tools: Vec) { + self.entry.current_tools.store(Some(Arc::new(tools))); + } +} + +impl CodexAppsToolsCache { + pub(crate) fn context( + &self, + codex_home: PathBuf, + auth_key: CodexAppsToolsCacheKey, + ) -> CodexAppsToolsCacheContext { + let identity = CodexAppsToolsCacheIdentity { + codex_home, + auth_key, + }; + let mut entries = lock_unpoisoned(&self.entries); + let entry = entries + .entry(identity.clone()) + .or_insert_with(|| Arc::new(CodexAppsToolsCacheEntry::new(identity))) + .clone(); + CodexAppsToolsCacheContext { entry } + } +} + +#[derive(Debug, Clone, Copy)] +pub(crate) enum CodexAppsToolsFetchSource { + Startup, + HardRefresh, +} + +impl CodexAppsToolsFetchSource { + fn as_str(self) -> &'static str { + match self { + Self::Startup => "startup", + Self::HardRefresh => "hard_refresh", + } + } +} + +pub(crate) struct CodexAppsToolsFetchTicket { + generation: u64, + source: CodexAppsToolsFetchSource, +} + +struct CodexAppsToolsCacheEntry { + identity: CodexAppsToolsCacheIdentity, + current_tools: ArcSwapOption>, + next_fetch_generation: AtomicU64, + last_accepted_generation: Mutex, +} + +impl CodexAppsToolsCacheEntry { + fn new(identity: CodexAppsToolsCacheIdentity) -> Self { + let current_tools = load_cached_codex_apps_tools_for_identity(&identity).map(Arc::new); + Self { + identity, + current_tools: ArcSwapOption::from(current_tools), + next_fetch_generation: AtomicU64::new(0), + last_accepted_generation: Mutex::new(0), + } + } +} + +/// Everything that decides whether two Codex Apps clients can share tools. +/// +/// The auth key says whose catalog we are reading. `codex_home` keeps the +/// persisted cache under the right home directory. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +struct CodexAppsToolsCacheIdentity { + codex_home: PathBuf, + auth_key: CodexAppsToolsCacheKey, +} + +impl CodexAppsToolsCacheIdentity { + fn cache_path_in(&self, cache_dir: &str) -> PathBuf { + // `codex_home` is already the parent directory. Keep it out of the + // filename hash so non-UTF-8 Unix paths cannot collapse distinct auth + // keys onto the same disk cache file. + let identity_json = serde_json::to_string(&self.auth_key).unwrap_or_default(); + let identity_hash = sha1_hex(&identity_json); + self.codex_home + .join(cache_dir) + .join(format!("{identity_hash}.json")) + } +} + +#[cfg(test)] +fn write_cached_codex_apps_tools_for_test( + cache_context: &CodexAppsToolsCacheContext, + server_info: &McpServerInfo, + tools: &[ToolInfo], +) { + cache_context + .entry + .current_tools + .store(Some(Arc::new(tools.to_vec()))); + persist_codex_apps_cache(cache_context, server_info, tools); +} + +pub(crate) fn load_startup_cached_codex_apps_server_info( + cache_context: &CodexAppsToolsCacheContext, +) -> Option { + load_cached_codex_apps_server_info(cache_context) +} + +#[cfg(test)] +fn read_cached_codex_apps_tools( + cache_context: &CodexAppsToolsCacheContext, +) -> Option> { + load_cached_codex_apps_tools_for_identity(&cache_context.entry.identity) +} + +#[instrument(level = "trace", skip_all)] +fn load_cached_codex_apps_tools_for_identity( + identity: &CodexAppsToolsCacheIdentity, +) -> Option> { + let cache_path = identity.cache_path_in(CODEX_APPS_TOOLS_CACHE_DIR); + let bytes = std::fs::read(cache_path).ok()?; + let cache: CodexAppsToolsDiskCache = serde_json::from_slice(&bytes).ok()?; + (cache.schema_version == CODEX_APPS_TOOLS_CACHE_SCHEMA_VERSION).then_some(cache.tools) +} + +fn write_cached_codex_apps_tools( + cache_context: &CodexAppsToolsCacheContext, + tools: &[ToolInfo], +) -> anyhow::Result<()> { + let cache_path = cache_context.tools_cache_path(); + let bytes = serde_json::to_vec_pretty(&CodexAppsToolsDiskCache { + schema_version: CODEX_APPS_TOOLS_CACHE_SCHEMA_VERSION, + tools: tools.to_vec(), + }) + .context("failed to serialize Codex Apps tools cache")?; + write_codex_apps_cache_file(&cache_path, "tools", bytes) +} + +#[instrument(level = "trace", skip_all)] +fn load_cached_codex_apps_server_info( + cache_context: &CodexAppsToolsCacheContext, +) -> Option { + let bytes = std::fs::read(cache_context.server_info_cache_path()).ok()?; + let cache: CodexAppsServerInfoDiskCache = serde_json::from_slice(&bytes).ok()?; + (cache.schema_version == CODEX_APPS_SERVER_INFO_CACHE_SCHEMA_VERSION) + .then_some(cache.server_info) +} + +fn write_cached_codex_apps_server_info( + cache_context: &CodexAppsToolsCacheContext, + server_info: &McpServerInfo, +) -> anyhow::Result<()> { + let cache_path = cache_context.server_info_cache_path(); + let bytes = serde_json::to_vec_pretty(&CodexAppsServerInfoDiskCache { + schema_version: CODEX_APPS_SERVER_INFO_CACHE_SCHEMA_VERSION, + server_info: server_info.clone(), + }) + .context("failed to serialize Codex Apps server info cache")?; + write_codex_apps_cache_file(&cache_path, "server info", bytes) +} + +fn write_codex_apps_cache_file( + cache_path: &Path, + cache_name: &str, + bytes: Vec, +) -> anyhow::Result<()> { + if let Some(parent) = cache_path.parent() { + std::fs::create_dir_all(parent).with_context(|| { + format!( + "failed to create Codex Apps {cache_name} cache directory `{}`", + parent.display() + ) + })?; + } + std::fs::write(cache_path, bytes).with_context(|| { + format!( + "failed to write Codex Apps {cache_name} cache `{}`", + cache_path.display() + ) + })?; + Ok(()) +} + +fn persist_codex_apps_cache( + cache_context: &CodexAppsToolsCacheContext, + server_info: &McpServerInfo, + tools: &[ToolInfo], +) { + let cache_write_start = Instant::now(); + let tools_result = write_cached_codex_apps_tools(cache_context, tools); + if let Err(err) = &tools_result { + tracing::warn!("failed to write Codex Apps tools cache: {err:#}"); + } + let server_info_result = write_cached_codex_apps_server_info(cache_context, server_info); + if let Err(err) = &server_info_result { + tracing::warn!("failed to write Codex Apps server info cache: {err:#}"); + } + let status = if tools_result.is_ok() && server_info_result.is_ok() { + "success" + } else { + "failure" + }; + emit_duration( + MCP_TOOLS_CACHE_WRITE_DURATION_METRIC, + cache_write_start.elapsed(), + &[("status", status)], + ); +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +struct CodexAppsToolsDiskCache { + schema_version: u8, + tools: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +struct CodexAppsServerInfoDiskCache { + schema_version: u8, + server_info: McpServerInfo, +} + +const CODEX_APPS_TOOLS_CACHE_DIR: &str = "cache/codex_apps_tools"; +const CODEX_APPS_TOOLS_CACHE_SCHEMA_VERSION: u8 = 4; + +const CODEX_APPS_SERVER_INFO_CACHE_DIR: &str = "cache/codex_apps_server_info"; +const CODEX_APPS_SERVER_INFO_CACHE_SCHEMA_VERSION: u8 = 1; + +fn sha1_hex(s: &str) -> String { + let mut hasher = Sha1::new(); + hasher.update(s.as_bytes()); + let sha1 = hasher.finalize(); + format!("{sha1:x}") +} + +fn lock_unpoisoned(mutex: &Mutex) -> std::sync::MutexGuard<'_, T> { + mutex + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) +} + +#[cfg(test)] +#[path = "codex_apps_cache_tests.rs"] +mod tests; diff --git a/codex-rs/codex-mcp/src/codex_apps_cache_tests.rs b/codex-rs/codex-mcp/src/codex_apps_cache_tests.rs new file mode 100644 index 0000000000..46ad311a1b --- /dev/null +++ b/codex-rs/codex-mcp/src/codex_apps_cache_tests.rs @@ -0,0 +1,464 @@ +use super::*; +use crate::mcp::CODEX_APPS_MCP_SERVER_NAME; +use crate::tools::ToolInfo; +use codex_protocol::ToolName; +use codex_protocol::mcp::McpServerInfo; +use pretty_assertions::assert_eq; +use rmcp::model::JsonObject; +use rmcp::model::Tool; +use std::collections::HashSet; +#[cfg(unix)] +use std::os::unix::ffi::OsStringExt; +use std::path::PathBuf; +use std::sync::Arc; +use tempfile::tempdir; + +fn create_test_tool(server_name: &str, tool_name: &str) -> ToolInfo { + ToolInfo { + server_name: server_name.to_string(), + supports_parallel_tool_calls: false, + server_origin: None, + callable_name: tool_name.to_string(), + callable_namespace: server_name.to_string(), + namespace_description: None, + tool: Tool::new( + tool_name.to_string(), + format!("Test tool: {tool_name}"), + Arc::new(JsonObject::default()), + ), + connector_id: None, + connector_name: None, + plugin_display_names: Vec::new(), + } +} + +fn create_test_tool_with_connector( + server_name: &str, + tool_name: &str, + connector_id: &str, + connector_name: Option<&str>, +) -> ToolInfo { + let mut tool = create_test_tool(server_name, tool_name); + tool.connector_id = Some(connector_id.to_string()); + tool.connector_name = connector_name.map(ToOwned::to_owned); + tool +} + +fn create_codex_apps_tools_cache_context( + codex_home: PathBuf, + account_id: Option<&str>, + chatgpt_user_id: Option<&str>, +) -> CodexAppsToolsCacheContext { + CodexAppsToolsCache::default().context( + codex_home, + CodexAppsToolsCacheKey { + account_id: account_id.map(ToOwned::to_owned), + chatgpt_user_id: chatgpt_user_id.map(ToOwned::to_owned), + is_workspace_account: false, + }, + ) +} + +fn create_test_server_info(title: &str) -> McpServerInfo { + McpServerInfo { + name: "codex-apps".to_string(), + title: Some(title.to_string()), + version: "1.0.0".to_string(), + description: None, + icons: None, + website_url: None, + } +} + +fn model_tool_names(tools: &[ToolInfo]) -> HashSet { + tools + .iter() + .map(ToolInfo::canonical_tool_name) + .collect::>() +} + +#[test] +fn codex_apps_tools_cache_is_overwritten_by_last_write() { + let codex_home = tempdir().expect("tempdir"); + let cache_context = create_codex_apps_tools_cache_context( + codex_home.path().to_path_buf(), + Some("account-one"), + Some("user-one"), + ); + let tools_gateway_1 = vec![create_test_tool(CODEX_APPS_MCP_SERVER_NAME, "one")]; + let tools_gateway_2 = vec![create_test_tool(CODEX_APPS_MCP_SERVER_NAME, "two")]; + + write_cached_codex_apps_tools(&cache_context, &tools_gateway_1).expect("write first cache"); + let cached_gateway_1 = + read_cached_codex_apps_tools(&cache_context).expect("cache entry exists for first write"); + assert_eq!(cached_gateway_1[0].callable_name, "one"); + + write_cached_codex_apps_tools(&cache_context, &tools_gateway_2).expect("write second cache"); + let cached_gateway_2 = + read_cached_codex_apps_tools(&cache_context).expect("cache entry exists for second write"); + assert_eq!(cached_gateway_2[0].callable_name, "two"); +} + +#[test] +fn codex_apps_tools_cache_is_scoped_per_user() { + let codex_home = tempdir().expect("tempdir"); + let cache_context_user_1 = create_codex_apps_tools_cache_context( + codex_home.path().to_path_buf(), + Some("account-one"), + Some("user-one"), + ); + let cache_context_user_2 = create_codex_apps_tools_cache_context( + codex_home.path().to_path_buf(), + Some("account-two"), + Some("user-two"), + ); + let tools_user_1 = vec![create_test_tool(CODEX_APPS_MCP_SERVER_NAME, "one")]; + let tools_user_2 = vec![create_test_tool(CODEX_APPS_MCP_SERVER_NAME, "two")]; + + write_cached_codex_apps_tools(&cache_context_user_1, &tools_user_1) + .expect("write user one cache"); + write_cached_codex_apps_tools(&cache_context_user_2, &tools_user_2) + .expect("write user two cache"); + + let read_user_1 = + read_cached_codex_apps_tools(&cache_context_user_1).expect("cache entry for user one"); + let read_user_2 = + read_cached_codex_apps_tools(&cache_context_user_2).expect("cache entry for user two"); + + assert_eq!(read_user_1[0].callable_name, "one"); + assert_eq!(read_user_2[0].callable_name, "two"); + assert_ne!( + cache_context_user_1.tools_cache_path(), + cache_context_user_2.tools_cache_path(), + "each user should get an isolated cache file" + ); +} + +#[test] +fn codex_apps_tools_cache_preserves_formerly_disallowed_connectors() { + let codex_home = tempdir().expect("tempdir"); + let cache_context = create_codex_apps_tools_cache_context( + codex_home.path().to_path_buf(), + Some("account-one"), + Some("user-one"), + ); + let tools = vec![ + create_test_tool_with_connector( + CODEX_APPS_MCP_SERVER_NAME, + "formerly_blocked_tool", + "connector_2b0a9009c9c64bf9933a3dae3f2b1254", + Some("Formerly Blocked"), + ), + create_test_tool_with_connector( + CODEX_APPS_MCP_SERVER_NAME, + "calendar_tool", + "calendar", + Some("Calendar"), + ), + ]; + + write_cached_codex_apps_tools(&cache_context, &tools).expect("write cache"); + let cached = read_cached_codex_apps_tools(&cache_context).expect("cache entry exists for user"); + + assert_eq!( + cached + .iter() + .map(|tool| (tool.callable_name.as_str(), tool.connector_id.as_deref())) + .collect::>(), + vec![ + ( + "formerly_blocked_tool", + Some("connector_2b0a9009c9c64bf9933a3dae3f2b1254") + ), + ("calendar_tool", Some("calendar")), + ] + ); +} + +#[test] +fn codex_apps_tools_cache_is_ignored_when_schema_version_mismatches() { + let codex_home = tempdir().expect("tempdir"); + let cache_context = create_codex_apps_tools_cache_context( + codex_home.path().to_path_buf(), + Some("account-one"), + Some("user-one"), + ); + let cache_path = cache_context.tools_cache_path(); + if let Some(parent) = cache_path.parent() { + std::fs::create_dir_all(parent).expect("create parent"); + } + let bytes = serde_json::to_vec_pretty(&serde_json::json!({ + "schema_version": CODEX_APPS_TOOLS_CACHE_SCHEMA_VERSION + 1, + "tools": [create_test_tool(CODEX_APPS_MCP_SERVER_NAME, "one")], + })) + .expect("serialize"); + std::fs::write(cache_path, bytes).expect("write"); + + assert!(read_cached_codex_apps_tools(&cache_context).is_none()); +} + +#[test] +fn codex_apps_tools_cache_is_ignored_when_json_is_invalid() { + let codex_home = tempdir().expect("tempdir"); + let cache_context = create_codex_apps_tools_cache_context( + codex_home.path().to_path_buf(), + Some("account-one"), + Some("user-one"), + ); + let cache_path = cache_context.tools_cache_path(); + if let Some(parent) = cache_path.parent() { + std::fs::create_dir_all(parent).expect("create parent"); + } + std::fs::write(cache_path, b"{not json").expect("write"); + + assert!(read_cached_codex_apps_tools(&cache_context).is_none()); +} + +#[test] +fn startup_cached_codex_apps_tools_loads_from_disk_cache() { + let codex_home = tempdir().expect("tempdir"); + let writer_cache_context = create_codex_apps_tools_cache_context( + codex_home.path().to_path_buf(), + Some("account-one"), + Some("user-one"), + ); + let cached_tools = vec![create_test_tool( + CODEX_APPS_MCP_SERVER_NAME, + "calendar_search", + )]; + let server_info = create_test_server_info("Codex Apps"); + write_cached_codex_apps_tools_for_test(&writer_cache_context, &server_info, &cached_tools); + let cache_context = create_codex_apps_tools_cache_context( + codex_home.path().to_path_buf(), + Some("account-one"), + Some("user-one"), + ); + + let startup_tools = cache_context + .current_tools() + .expect("expected startup snapshot to load from cache"); + let cached_server_info = load_startup_cached_codex_apps_server_info(&cache_context); + + assert_eq!(startup_tools.len(), 1); + assert_eq!(startup_tools[0].server_name, CODEX_APPS_MCP_SERVER_NAME); + assert_eq!(startup_tools[0].callable_name, "calendar_search"); + assert_eq!(cached_server_info, Some(server_info)); +} + +#[test] +fn startup_cached_codex_apps_tools_loads_without_server_info_cache() { + let codex_home = tempdir().expect("tempdir"); + let writer_cache_context = create_codex_apps_tools_cache_context( + codex_home.path().to_path_buf(), + Some("account-one"), + Some("user-one"), + ); + let cache_path = writer_cache_context.tools_cache_path(); + if let Some(parent) = cache_path.parent() { + std::fs::create_dir_all(parent).expect("create parent"); + } + let bytes = serde_json::to_vec_pretty(&serde_json::json!({ + "schema_version": CODEX_APPS_TOOLS_CACHE_SCHEMA_VERSION, + "tools": [create_test_tool(CODEX_APPS_MCP_SERVER_NAME, "calendar_search")], + })) + .expect("serialize"); + std::fs::write(cache_path, bytes).expect("write"); + let cache_context = create_codex_apps_tools_cache_context( + codex_home.path().to_path_buf(), + Some("account-one"), + Some("user-one"), + ); + + let startup_tools = cache_context + .current_tools() + .expect("legacy startup snapshot should remain available"); + let cached_server_info = load_startup_cached_codex_apps_server_info(&cache_context); + + assert_eq!(startup_tools.len(), 1); + assert_eq!(startup_tools[0].callable_name, "calendar_search"); + assert_eq!(cached_server_info, None); +} + +#[test] +fn codex_apps_server_info_cache_survives_legacy_tools_cache_write() { + let codex_home = tempdir().expect("tempdir"); + let cache_context = create_codex_apps_tools_cache_context( + codex_home.path().to_path_buf(), + Some("account-one"), + Some("user-one"), + ); + let server_info = create_test_server_info("Codex Apps"); + write_cached_codex_apps_tools_for_test( + &cache_context, + &server_info, + &[create_test_tool( + CODEX_APPS_MCP_SERVER_NAME, + "calendar_search", + )], + ); + + let cache_path = cache_context.tools_cache_path(); + if let Some(parent) = cache_path.parent() { + std::fs::create_dir_all(parent).expect("create parent"); + } + let bytes = serde_json::to_vec_pretty(&serde_json::json!({ + "schema_version": CODEX_APPS_TOOLS_CACHE_SCHEMA_VERSION - 1, + "tools": [create_test_tool(CODEX_APPS_MCP_SERVER_NAME, "calendar_search")], + })) + .expect("serialize"); + std::fs::write(cache_path, bytes).expect("write legacy tools cache"); + let startup_cache_context = create_codex_apps_tools_cache_context( + codex_home.path().to_path_buf(), + Some("account-one"), + Some("user-one"), + ); + + assert_eq!( + load_startup_cached_codex_apps_server_info(&startup_cache_context), + Some(server_info) + ); + assert!(startup_cache_context.current_tools().is_none()); +} + +#[test] +fn codex_apps_tools_cache_context_does_not_reread_disk_after_creation() { + let codex_home = tempdir().expect("tempdir"); + let writer_cache_context = create_codex_apps_tools_cache_context( + codex_home.path().to_path_buf(), + Some("account-one"), + Some("user-one"), + ); + let cached_tools = vec![create_test_tool(CODEX_APPS_MCP_SERVER_NAME, "cached")]; + write_cached_codex_apps_tools(&writer_cache_context, &cached_tools).expect("write cache"); + let reader_cache_context = create_codex_apps_tools_cache_context( + codex_home.path().to_path_buf(), + Some("account-one"), + Some("user-one"), + ); + let updated_tools = vec![create_test_tool(CODEX_APPS_MCP_SERVER_NAME, "updated")]; + write_cached_codex_apps_tools(&writer_cache_context, &updated_tools).expect("rewrite cache"); + + assert_eq!( + reader_cache_context + .current_tools() + .expect("in-memory tools")[0] + .callable_name, + "cached" + ); + assert_eq!( + read_cached_codex_apps_tools(&writer_cache_context).expect("disk tools")[0].callable_name, + "updated" + ); +} + +#[test] +fn codex_apps_tools_cache_publishes_newest_shared_snapshot() { + let codex_home = tempdir().expect("tempdir"); + let cache = CodexAppsToolsCache::default(); + let cache_context_1 = cache.context( + codex_home.path().to_path_buf(), + CodexAppsToolsCacheKey { + account_id: Some("account-one".to_string()), + chatgpt_user_id: Some("user-one".to_string()), + is_workspace_account: false, + }, + ); + let cache_context_2 = cache.context( + codex_home.path().to_path_buf(), + CodexAppsToolsCacheKey { + account_id: Some("account-one".to_string()), + chatgpt_user_id: Some("user-one".to_string()), + is_workspace_account: false, + }, + ); + let older_ticket = cache_context_1.begin_fetch(CodexAppsToolsFetchSource::Startup); + let newer_ticket = cache_context_2.begin_fetch(CodexAppsToolsFetchSource::HardRefresh); + let server_info = create_test_server_info("Codex Apps"); + let newer_tools = vec![create_test_tool(CODEX_APPS_MCP_SERVER_NAME, "newer")]; + let older_tools = vec![create_test_tool(CODEX_APPS_MCP_SERVER_NAME, "older")]; + + let published_tools = + cache_context_2.publish_if_newest_accepted(newer_ticket, &server_info, newer_tools); + assert_eq!( + model_tool_names(&published_tools), + model_tool_names( + &cache_context_1 + .current_tools() + .expect("new snapshot should publish") + ) + ); + let current_tools = + cache_context_1.publish_if_newest_accepted(older_ticket, &server_info, older_tools); + + assert_eq!(current_tools[0].callable_name, "newer"); + assert_eq!( + cache_context_2.current_tools().expect("shared snapshot")[0].callable_name, + "newer" + ); + assert_eq!( + read_cached_codex_apps_tools(&cache_context_1).expect("persisted snapshot")[0] + .callable_name, + "newer" + ); +} + +#[test] +fn codex_apps_tools_cache_keeps_live_publish_when_disk_persistence_fails() { + let codex_home = tempdir().expect("tempdir"); + let codex_home_file = codex_home.path().join("not-a-directory"); + std::fs::write(&codex_home_file, b"occupied").expect("create codex home file"); + let cache_context = CodexAppsToolsCache::default().context( + codex_home_file, + CodexAppsToolsCacheKey { + account_id: Some("account-one".to_string()), + chatgpt_user_id: Some("user-one".to_string()), + is_workspace_account: false, + }, + ); + let tools = vec![create_test_tool(CODEX_APPS_MCP_SERVER_NAME, "live")]; + let published_tools = cache_context.publish_if_newest_accepted( + cache_context.begin_fetch(CodexAppsToolsFetchSource::HardRefresh), + &create_test_server_info("Codex Apps"), + tools.clone(), + ); + + assert_eq!(model_tool_names(&published_tools), model_tool_names(&tools)); + assert_eq!( + model_tool_names(&cache_context.current_tools().expect("live snapshot")), + model_tool_names(&tools) + ); +} + +#[cfg(unix)] +#[test] +fn codex_apps_tools_cache_scopes_non_utf8_home_disk_paths() { + let codex_home = PathBuf::from(std::ffi::OsString::from_vec( + b"/tmp/codex-home-\xff".to_vec(), + )); + let cache = CodexAppsToolsCache::default(); + let user_one_context = cache.context( + codex_home.clone(), + CodexAppsToolsCacheKey { + account_id: Some("account-one".to_string()), + chatgpt_user_id: Some("user-one".to_string()), + is_workspace_account: false, + }, + ); + let user_two_context = cache.context( + codex_home, + CodexAppsToolsCacheKey { + account_id: Some("account-two".to_string()), + chatgpt_user_id: Some("user-two".to_string()), + is_workspace_account: false, + }, + ); + let cache_paths = [ + user_one_context.tools_cache_path(), + user_two_context.tools_cache_path(), + ]; + + assert_eq!( + cache_paths.iter().collect::>().len(), + cache_paths.len() + ); +} diff --git a/codex-rs/codex-mcp/src/connection_manager.rs b/codex-rs/codex-mcp/src/connection_manager.rs index a6575e0c8f..4cc91ca208 100644 --- a/codex-rs/codex-mcp/src/connection_manager.rs +++ b/codex-rs/codex-mcp/src/connection_manager.rs @@ -7,7 +7,6 @@ //! `codex-core`. use std::collections::HashMap; -use std::path::Path; use std::path::PathBuf; use std::sync::Arc; use std::sync::atomic::AtomicBool; @@ -16,10 +15,11 @@ use std::time::Duration; use std::time::Instant; use crate::McpAuthStatusEntry; -use crate::codex_apps::CodexAppsToolsCacheContext; -use crate::codex_apps::CodexAppsToolsCacheKey; -use crate::codex_apps::write_codex_apps_tools_cache; +use crate::codex_apps_cache::CodexAppsToolsCache; +use crate::codex_apps_cache::CodexAppsToolsCacheKey; +use crate::codex_apps_cache::CodexAppsToolsFetchSource; use crate::elicitation::ElicitationRequestManager; +use crate::elicitation::ElicitationRequestRouter; use crate::elicitation::ElicitationReviewerHandle; use crate::mcp::CODEX_APPS_MCP_SERVER_NAME; use crate::mcp::ToolPluginProvenance; @@ -57,9 +57,12 @@ use codex_protocol::protocol::Event; use codex_protocol::protocol::EventMsg; use codex_protocol::protocol::McpStartupCompleteEvent; use codex_protocol::protocol::McpStartupFailure; +use codex_protocol::protocol::McpStartupFailureReason; use codex_protocol::protocol::McpStartupStatus; use codex_protocol::protocol::McpStartupUpdateEvent; use codex_rmcp_client::ElicitationResponse; +use codex_rmcp_client::McpAuthState; +use codex_rmcp_client::McpLoginRequirement; use rmcp::model::ElicitationCapability; use rmcp::model::ListResourceTemplatesResult; use rmcp::model::ListResourcesResult; @@ -115,6 +118,7 @@ struct McpClientRuntime { tx_event: Sender, runtime_context: McpRuntimeContext, codex_home: PathBuf, + codex_apps_tools_cache: CodexAppsToolsCache, codex_apps_tools_cache_key: CodexAppsToolsCacheKey, client_elicitation_capability: ElicitationCapability, supports_openai_form_elicitation: bool, @@ -125,27 +129,44 @@ impl McpClientRuntime { fn create_client( &self, server_name: String, + startup_submit_id: String, server: EffectiveMcpServer, cancel_token: CancellationToken, elicitation_requests: ElicitationRequestManager, tool_plugin_provenance: Arc, ) -> AsyncManagedClient { - let tools_cache_context = if server_name == CODEX_APPS_MCP_SERVER_NAME { - codex_apps_tools_cache_context(&self.codex_home, &self.codex_apps_tools_cache_key) - } else { - regular_mcp_tools_cache_context() - }; + let uses_env_bearer_token = server + .configured_config() + .is_some_and(|config| match &config.transport { + McpServerTransportConfig::StreamableHttp { + bearer_token_env_var, + .. + } => bearer_token_env_var.is_some(), + McpServerTransportConfig::Stdio { .. } => false, + }); + let codex_apps_tools_cache_context = + should_share_codex_apps_tools_cache(&server_name, uses_env_bearer_token).then(|| { + self.codex_apps_tools_cache.context( + self.codex_home.clone(), + self.codex_apps_tools_cache_key.clone(), + ) + }); let runtime_auth_provider = - chatgpt_auth_provider_for_server(&server, self.chatgpt_auth_provider.clone()); + if server_name == CODEX_APPS_MCP_SERVER_NAME && uses_env_bearer_token { + None + } else { + chatgpt_auth_provider_for_server(&server, self.chatgpt_auth_provider.clone()) + }; AsyncManagedClient::new( server_name, + startup_submit_id, server, self.store_mode, self.keyring_backend_kind, cancel_token, self.tx_event.clone(), elicitation_requests, - tools_cache_context, + codex_apps_tools_cache_context, tool_plugin_provenance, self.runtime_context.clone(), runtime_auth_provider, @@ -184,6 +205,7 @@ impl McpConnectionManager { initial_permission_profile: PermissionProfile, runtime_context: McpRuntimeContext, codex_home: PathBuf, + codex_apps_tools_cache: CodexAppsToolsCache, codex_apps_tools_cache_key: CodexAppsToolsCacheKey, prefix_mcp_tool_names: bool, client_elicitation_capability: ElicitationCapability, @@ -191,6 +213,8 @@ impl McpConnectionManager { tool_plugin_provenance: ToolPluginProvenance, auth: Option<&CodexAuth>, elicitation_reviewer: Option, + elicitation_lifecycle: Option, + elicitation_router: ElicitationRequestRouter, ) -> Self { let mut required_servers = mcp_servers .iter() @@ -205,6 +229,8 @@ impl McpConnectionManager { approval_policy.value(), initial_permission_profile, elicitation_reviewer, + elicitation_lifecycle, + elicitation_router, ); let tool_plugin_provenance = Arc::new(tool_plugin_provenance); let startup_submit_id = submit_id.clone(); @@ -217,6 +243,7 @@ impl McpConnectionManager { tx_event: tx_event.clone(), runtime_context, codex_home, + codex_apps_tools_cache, codex_apps_tools_cache_key, client_elicitation_capability, supports_openai_form_elicitation, @@ -240,6 +267,7 @@ impl McpConnectionManager { .await; let async_managed_client = client_runtime.create_client( server_name.clone(), + startup_submit_id.clone(), server, cancel_token.clone(), elicitation_requests.clone(), @@ -348,6 +376,8 @@ impl McpConnectionManager { approval_policy.value(), permission_profile.clone(), /*reviewer*/ None, + /*lifecycle*/ None, + ElicitationRequestRouter::default(), ), startup_cancellation_token: CancellationToken::new(), client_runtime: None, @@ -405,6 +435,7 @@ impl McpConnectionManager { .await; let replacement_client = runtime.create_client( server_name.clone(), + submit_id.clone(), server, cancel_token.clone(), self.elicitation_requests.clone(), @@ -558,6 +589,10 @@ impl McpConnectionManager { self.elicitation_requests.set_auto_deny(auto_deny); } + pub fn elicitation_router(&self) -> ElicitationRequestRouter { + self.elicitation_requests.router() + } + pub async fn resolve_elicitation( &self, server_name: String, @@ -585,13 +620,14 @@ impl McpConnectionManager { pub async fn list_all_tools(&self) -> Vec { let mut tools = Vec::new(); for (server_name, managed_client) in &self.clients { - let has_cached_tool_info_snapshot = managed_client.cached_tool_info_snapshot.is_some(); + managed_client.reconnect_failed_startup().await; + let has_cached_tools = managed_client.has_cached_tools(); let startup_complete = managed_client .startup_complete .load(std::sync::atomic::Ordering::Acquire); trace!( server_name = %server_name, - has_cached_tool_info_snapshot, + has_cached_tools, startup_complete, "waiting for MCP server tools while building tool list" ); @@ -600,7 +636,7 @@ impl McpConnectionManager { .instrument(trace_span!( "list_tools_for_server", server_name = %server_name, - has_cached_tool_info_snapshot, + has_cached_tools, startup_complete )) .await @@ -623,9 +659,9 @@ impl McpConnectionManager { /// Force-refresh codex apps tools by bypassing the in-process cache. /// - /// On success, the refreshed tools replace the cache contents and the - /// latest filtered tools are returned directly to the caller. On - /// failure, the existing cache remains unchanged. + /// On success, the refreshed tools replace shared cache contents when the + /// cache is enabled and the latest filtered tools are returned directly to + /// the caller. On failure, existing shared cache contents remain unchanged. pub async fn hard_refresh_codex_apps_tools_cache(&self) -> Result> { let managed_client = self .clients @@ -637,6 +673,10 @@ impl McpConnectionManager { let list_start = Instant::now(); let fetch_start = Instant::now(); + let fetch_ticket = managed_client + .codex_apps_tools_cache_context + .as_ref() + .map(|cache_context| cache_context.begin_fetch(CodexAppsToolsFetchSource::HardRefresh)); let tools = list_tools_for_client_uncached( CODEX_APPS_MCP_SERVER_NAME, /*is_codex_apps_mcp_server*/ true, @@ -654,11 +694,16 @@ impl McpConnectionManager { &[], ); - write_codex_apps_tools_cache( - managed_client.codex_apps_tools_cache_context.as_ref(), - &managed_client.server_info, - &tools, - ); + let tools = + match ( + managed_client.codex_apps_tools_cache_context.as_ref(), + fetch_ticket, + ) { + (Some(cache_context), Some(fetch_ticket)) => cache_context + .publish_if_newest_accepted(fetch_ticket, &managed_client.server_info, tools), + (None, None) => tools, + _ => unreachable!("Codex Apps fetch ticket requires cache context"), + }; emit_duration( MCP_TOOLS_LIST_DURATION_METRIC, list_start.elapsed(), @@ -998,22 +1043,6 @@ impl Drop for McpConnectionManager { } } -/// Creates the per-user tools cache context used only by the Codex Apps server. -fn codex_apps_tools_cache_context( - codex_home: &Path, - codex_apps_tools_cache_key: &CodexAppsToolsCacheKey, -) -> Option { - Some(CodexAppsToolsCacheContext { - codex_home: codex_home.to_path_buf(), - user_key: codex_apps_tools_cache_key.clone(), - }) -} - -/// Keeps regular MCP servers isolated from the Codex Apps tools cache. -fn regular_mcp_tools_cache_context() -> Option { - None -} - /// Makes ChatGPT authentication available to servers that explicitly opt in. /// The HTTP transport applies it only when no configured authorization resolves. fn chatgpt_auth_provider_for_server( @@ -1046,6 +1075,7 @@ async fn observe_startup( Err(StartupOutcomeError::Cancelled) => McpStartupStatus::Cancelled, Err(error) => McpStartupStatus::Failed { error: mcp_init_error_display(&server_name, auth_entry.as_ref(), error), + reason: mcp_startup_failure_reason(auth_entry.as_ref(), error), }, }; let _ = emit_update( @@ -1057,6 +1087,9 @@ async fn observe_startup( }, ) .await; + if matches!(&outcome, Err(StartupOutcomeError::Failed { .. })) { + client.reconnect_failed_startup().await; + } (server_name, outcome) } @@ -1068,13 +1101,17 @@ fn record_startup_outcome( match outcome { Ok(_) => summary.ready.push(server_name), Err(StartupOutcomeError::Cancelled) => summary.cancelled.push(server_name), - Err(StartupOutcomeError::Failed { error }) => summary.failed.push(McpStartupFailure { + Err(StartupOutcomeError::Failed { error, .. }) => summary.failed.push(McpStartupFailure { server: server_name, error, }), } } +fn should_share_codex_apps_tools_cache(server_name: &str, uses_env_bearer_token: bool) -> bool { + server_name == CODEX_APPS_MCP_SERVER_NAME && !uses_env_bearer_token +} + async fn emit_update( submit_id: &str, tx_event: &Sender, @@ -1088,6 +1125,28 @@ async fn emit_update( .await } +fn mcp_startup_failure_reason( + entry: Option<&McpAuthStatusEntry>, + error: &StartupOutcomeError, +) -> Option { + if !error.is_authentication_required() { + return None; + } + + match entry.map(|entry| entry.auth_state) { + Some(McpAuthState::LoggedOut(McpLoginRequirement::Reauthentication)) => { + Some(McpStartupFailureReason::ReauthenticationRequired) + } + Some( + McpAuthState::Unsupported + | McpAuthState::LoggedOut(McpLoginRequirement::Login) + | McpAuthState::BearerToken + | McpAuthState::OAuth, + ) + | None => None, + } +} + fn mcp_init_error_display( server_name: &str, entry: Option<&McpAuthStatusEntry>, @@ -1134,20 +1193,20 @@ fn mcp_init_error_display( fn startup_outcome_error_message(error: StartupOutcomeError) -> String { match error { StartupOutcomeError::Cancelled => "MCP startup cancelled".to_string(), - StartupOutcomeError::Failed { error } => error, + StartupOutcomeError::Failed { error, .. } => error, } } fn is_mcp_client_auth_required_error(error: &StartupOutcomeError) -> bool { match error { - StartupOutcomeError::Failed { error } => error.contains("Auth required"), + StartupOutcomeError::Failed { error, .. } => error.contains("Auth required"), _ => false, } } fn is_mcp_client_startup_timeout_error(error: &StartupOutcomeError) -> bool { match error { - StartupOutcomeError::Failed { error } => { + StartupOutcomeError::Failed { error, .. } => { error.contains("request timed out") || error.contains("timed out handshaking with MCP server") } diff --git a/codex-rs/codex-mcp/src/connection_manager_tests.rs b/codex-rs/codex-mcp/src/connection_manager_tests.rs index d66c755f12..8263e900ef 100644 --- a/codex-rs/codex-mcp/src/connection_manager_tests.rs +++ b/codex-rs/codex-mcp/src/connection_manager_tests.rs @@ -1,16 +1,16 @@ use super::*; -use crate::codex_apps::CODEX_APPS_TOOLS_CACHE_SCHEMA_VERSION; -use crate::codex_apps::CodexAppsToolsCacheContext; -use crate::codex_apps::load_startup_cached_codex_apps_server_info; -use crate::codex_apps::load_startup_cached_codex_apps_tools_snapshot; -use crate::codex_apps::read_cached_codex_apps_tools; -use crate::codex_apps::write_cached_codex_apps_tools; -use crate::codex_apps::write_codex_apps_tools_cache; +use crate::codex_apps_cache::CodexAppsToolsCache; +use crate::codex_apps_cache::CodexAppsToolsCacheContext; use crate::declared_openai_file_input_param_names; +use crate::elicitation::ElicitationLifecycle; use crate::elicitation::ElicitationRequestManager; +use crate::elicitation::ElicitationRequestRouter; use crate::elicitation::elicitation_is_rejected_by_policy; use crate::rmcp_client::AsyncManagedClient; +use crate::rmcp_client::CODEX_APPS_RECONNECT_INITIAL_BACKOFF; +use crate::rmcp_client::CodexAppsStartupReconnect; use crate::rmcp_client::ManagedClient; +use crate::rmcp_client::ManagedClientFuture; use crate::rmcp_client::StartupOutcomeError; use crate::server::EffectiveMcpServer; use crate::server::McpServerMetadata; @@ -25,13 +25,16 @@ use codex_config::Constrained; use codex_config::McpServerConfig; use codex_config::McpServerToolConfig; use codex_config::types::AuthKeyringBackendKind; +use codex_config::types::OAuthCredentialsStoreMode; use codex_exec_server::EnvironmentManager; use codex_protocol::ToolName; use codex_protocol::mcp::McpServerInfo; use codex_protocol::models::PermissionProfile; use codex_protocol::protocol::GranularApprovalConfig; -use codex_protocol::protocol::McpAuthStatus; +use codex_rmcp_client::InProcessTransportFactory; +use codex_rmcp_client::RmcpClient; use futures::FutureExt; +use futures::future::BoxFuture; use pretty_assertions::assert_eq; use rmcp::model::CreateElicitationRequestParams; use rmcp::model::ElicitationAction; @@ -41,8 +44,11 @@ use rmcp::model::Meta; use rmcp::model::NumberOrString; use rmcp::model::Tool; use std::collections::HashSet; +use std::io; use std::sync::Arc; +use std::sync::atomic::AtomicUsize; use tempfile::tempdir; +use tokio::io::DuplexStream; fn create_test_tool(server_name: &str, tool_name: &str) -> ToolInfo { ToolInfo { @@ -63,31 +69,19 @@ fn create_test_tool(server_name: &str, tool_name: &str) -> ToolInfo { } } -fn create_test_tool_with_connector( - server_name: &str, - tool_name: &str, - connector_id: &str, - connector_name: Option<&str>, -) -> ToolInfo { - let mut tool = create_test_tool(server_name, tool_name); - tool.connector_id = Some(connector_id.to_string()); - tool.connector_name = connector_name.map(ToOwned::to_owned); - tool -} - fn create_codex_apps_tools_cache_context( codex_home: PathBuf, account_id: Option<&str>, chatgpt_user_id: Option<&str>, ) -> CodexAppsToolsCacheContext { - CodexAppsToolsCacheContext { + CodexAppsToolsCache::default().context( codex_home, - user_key: CodexAppsToolsCacheKey { + CodexAppsToolsCacheKey { account_id: account_id.map(ToOwned::to_owned), chatgpt_user_id: chatgpt_user_id.map(ToOwned::to_owned), is_workspace_account: false, }, - } + ) } fn create_test_server_info(title: &str) -> McpServerInfo { @@ -128,6 +122,94 @@ fn local_stdio_server(command: &str) -> EffectiveMcpServer { }) } +struct TestInProcessTransportFactory; + +impl InProcessTransportFactory for TestInProcessTransportFactory { + fn open(&self) -> BoxFuture<'static, io::Result> { + async { + let (client_stream, _server_stream) = tokio::io::duplex(1); + Ok(client_stream) + } + .boxed() + } +} + +async fn create_test_managed_client(tools: Vec) -> ManagedClient { + ManagedClient { + client: Arc::new( + RmcpClient::new_in_process_client(Arc::new(TestInProcessTransportFactory)) + .await + .expect("create in-process RMCP client"), + ), + server_info: create_test_server_info("Ready"), + tools, + tool_filter: ToolFilter::default(), + tool_timeout: None, + server_instructions: None, + server_supports_sandbox_state_meta_capability: false, + codex_apps_tools_cache_context: None, + } +} + +async fn create_ready_async_managed_client(tools: Vec) -> AsyncManagedClient { + AsyncManagedClient { + client: futures::future::ready::>(Ok( + create_test_managed_client(tools).await, + )) + .boxed() + .shared(), + is_codex_apps_mcp_server: false, + cached_server_info: None, + codex_apps_tools_cache_context: None, + tool_filter: ToolFilter::default(), + startup_complete: Arc::new(std::sync::atomic::AtomicBool::new(true)), + startup_reconnect: None, + tool_plugin_provenance: Arc::new(ToolPluginProvenance::default()), + cancel_token: CancellationToken::new(), + } +} + +fn create_test_manager_with_failed_apps_startup( + cached_tools: Vec, + reconnect_factory: Arc ManagedClientFuture + Send + Sync>, +) -> McpConnectionManager { + let client: ManagedClientFuture = futures::future::ready(Err(StartupOutcomeError::Failed { + error: "startup failed".to_string(), + is_authentication_required: false, + })) + .boxed() + .shared(); + let codex_home = tempdir().expect("tempdir"); + let cache_context = create_codex_apps_tools_cache_context( + codex_home.path().to_path_buf(), + Some("reconnect-test-account"), + Some("reconnect-test-user"), + ); + cache_context.store_current_tools_for_test(cached_tools); + let approval_policy = Constrained::allow_any(AskForApproval::OnRequest); + let permission_profile = Constrained::allow_any(PermissionProfile::default()); + let mut manager = McpConnectionManager::new_uninitialized( + &approval_policy, + &permission_profile, + /*prefix_mcp_tool_names*/ true, + ); + manager.clients.insert( + CODEX_APPS_MCP_SERVER_NAME.to_string(), + AsyncManagedClient { + client, + is_codex_apps_mcp_server: true, + cached_server_info: None, + codex_apps_tools_cache_context: Some(cache_context), + tool_filter: ToolFilter::default(), + startup_complete: Arc::new(std::sync::atomic::AtomicBool::new(true)), + startup_reconnect: Some(Arc::new(CodexAppsStartupReconnect::new(reconnect_factory))), + tool_plugin_provenance: Arc::new(ToolPluginProvenance::default()), + cancel_token: CancellationToken::new(), + }, + ); + manager +} + fn model_tool_names(tools: &[ToolInfo]) -> HashSet { tools .iter() @@ -269,6 +351,8 @@ async fn disabled_permissions_auto_accept_elicitation_with_empty_form_schema() { AskForApproval::Never, PermissionProfile::Disabled, /*reviewer*/ None, + /*lifecycle*/ None, + ElicitationRequestRouter::default(), ); let (tx_event, _rx_event) = async_channel::bounded(1); let sender = manager.make_sender("server".to_string(), tx_event); @@ -304,6 +388,8 @@ async fn disabled_permissions_do_not_auto_accept_elicitation_with_requested_fiel AskForApproval::Never, PermissionProfile::Disabled, /*reviewer*/ None, + /*lifecycle*/ None, + ElicitationRequestRouter::default(), ); let (tx_event, _rx_event) = async_channel::bounded(1); let sender = manager.make_sender("server".to_string(), tx_event); @@ -337,6 +423,120 @@ async fn disabled_permissions_do_not_auto_accept_elicitation_with_requested_fiel ); } +#[tokio::test] +async fn shared_elicitation_router_targets_the_exact_pending_request() { + struct Registration(Arc); + + impl Drop for Registration { + fn drop(&mut self) { + self.0.fetch_sub(1, std::sync::atomic::Ordering::SeqCst); + } + } + + let router = ElicitationRequestRouter::default(); + let outstanding = Arc::new(AtomicUsize::new(0)); + let lifecycle = ElicitationLifecycle::new({ + let outstanding = outstanding.clone(); + move || { + outstanding.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + Registration(outstanding.clone()) + } + }); + let manager_a = ElicitationRequestManager::new( + AskForApproval::OnRequest, + PermissionProfile::default(), + /*reviewer*/ None, + Some(lifecycle.clone()), + router.clone(), + ); + let manager_b = ElicitationRequestManager::new( + AskForApproval::OnRequest, + PermissionProfile::default(), + /*reviewer*/ None, + Some(lifecycle), + router, + ); + let (tx_event, rx_event) = async_channel::bounded(2); + let sender_a = manager_a.make_sender("server".to_string(), tx_event.clone()); + let sender_b = manager_b.make_sender("server".to_string(), tx_event); + let elicitation = codex_rmcp_client::Elicitation::Mcp( + CreateElicitationRequestParams::FormElicitationParams { + meta: None, + message: "Which runtime?".to_string(), + requested_schema: rmcp::model::ElicitationSchema::builder() + .required_property( + "runtime", + rmcp::model::PrimitiveSchema::String(rmcp::model::StringSchema::new()), + ) + .build() + .expect("schema should build"), + }, + ); + + let pending_a = tokio::spawn(sender_a(NumberOrString::Number(1), elicitation.clone())); + let EventMsg::ElicitationRequest(request_a) = rx_event.recv().await.expect("request A").msg + else { + panic!("expected elicitation request"); + }; + let pending_b = tokio::spawn(sender_b(NumberOrString::Number(1), elicitation)); + let EventMsg::ElicitationRequest(request_b) = rx_event.recv().await.expect("request B").msg + else { + panic!("expected elicitation request"); + }; + assert_eq!(outstanding.load(std::sync::atomic::Ordering::SeqCst), 2); + let ( + codex_protocol::mcp::RequestId::String(request_a_id), + codex_protocol::mcp::RequestId::String(request_b_id), + ) = (request_a.id, request_b.id) + else { + panic!("expected Codex-owned string request IDs"); + }; + assert_ne!(request_a_id, request_b_id); + + let response_a = ElicitationResponse { + action: ElicitationAction::Accept, + content: Some(serde_json::json!({"runtime": "a"})), + meta: None, + }; + manager_b + .resolve( + "server".to_string(), + NumberOrString::String(request_a_id.into()), + response_a.clone(), + ) + .await + .expect("runtime B should route a response to runtime A"); + let response_b = ElicitationResponse { + action: ElicitationAction::Accept, + content: Some(serde_json::json!({"runtime": "b"})), + meta: None, + }; + manager_a + .resolve( + "server".to_string(), + NumberOrString::String(request_b_id.into()), + response_b.clone(), + ) + .await + .expect("runtime A should route a response to runtime B"); + + assert_eq!( + pending_a + .await + .expect("request A task") + .expect("request A response"), + response_a + ); + assert_eq!( + pending_b + .await + .expect("request B task") + .expect("request B response"), + response_b + ); + assert_eq!(outstanding.load(std::sync::atomic::Ordering::SeqCst), 0); +} + #[test] fn test_normalize_tools_short_non_duplicated_names() { let tools = vec![ @@ -581,235 +781,25 @@ fn filter_tools_applies_per_server_filters() { } #[test] -fn codex_apps_tools_cache_is_overwritten_by_last_write() { - let codex_home = tempdir().expect("tempdir"); - let cache_context = create_codex_apps_tools_cache_context( - codex_home.path().to_path_buf(), - Some("account-one"), - Some("user-one"), - ); - let tools_gateway_1 = vec![create_test_tool(CODEX_APPS_MCP_SERVER_NAME, "one")]; - let tools_gateway_2 = vec![create_test_tool(CODEX_APPS_MCP_SERVER_NAME, "two")]; - - write_cached_codex_apps_tools(&cache_context, &tools_gateway_1); - let cached_gateway_1 = - read_cached_codex_apps_tools(&cache_context).expect("cache entry exists for first write"); - assert_eq!(cached_gateway_1[0].callable_name, "one"); - - write_cached_codex_apps_tools(&cache_context, &tools_gateway_2); - let cached_gateway_2 = - read_cached_codex_apps_tools(&cache_context).expect("cache entry exists for second write"); - assert_eq!(cached_gateway_2[0].callable_name, "two"); -} - -#[test] -fn codex_apps_tools_cache_is_scoped_per_user() { - let codex_home = tempdir().expect("tempdir"); - let cache_context_user_1 = create_codex_apps_tools_cache_context( - codex_home.path().to_path_buf(), - Some("account-one"), - Some("user-one"), - ); - let cache_context_user_2 = create_codex_apps_tools_cache_context( - codex_home.path().to_path_buf(), - Some("account-two"), - Some("user-two"), - ); - let tools_user_1 = vec![create_test_tool(CODEX_APPS_MCP_SERVER_NAME, "one")]; - let tools_user_2 = vec![create_test_tool(CODEX_APPS_MCP_SERVER_NAME, "two")]; - - write_cached_codex_apps_tools(&cache_context_user_1, &tools_user_1); - write_cached_codex_apps_tools(&cache_context_user_2, &tools_user_2); - - let read_user_1 = - read_cached_codex_apps_tools(&cache_context_user_1).expect("cache entry for user one"); - let read_user_2 = - read_cached_codex_apps_tools(&cache_context_user_2).expect("cache entry for user two"); - - assert_eq!(read_user_1[0].callable_name, "one"); - assert_eq!(read_user_2[0].callable_name, "two"); - assert_ne!( - cache_context_user_1.tools_cache_path(), - cache_context_user_2.tools_cache_path(), - "each user should get an isolated cache file" - ); -} - -#[test] -fn codex_apps_tools_cache_preserves_formerly_disallowed_connectors() { - let codex_home = tempdir().expect("tempdir"); - let cache_context = create_codex_apps_tools_cache_context( - codex_home.path().to_path_buf(), - Some("account-one"), - Some("user-one"), - ); - let tools = vec![ - create_test_tool_with_connector( - CODEX_APPS_MCP_SERVER_NAME, - "formerly_blocked_tool", - "connector_2b0a9009c9c64bf9933a3dae3f2b1254", - Some("Formerly Blocked"), - ), - create_test_tool_with_connector( - CODEX_APPS_MCP_SERVER_NAME, - "calendar_tool", - "calendar", - Some("Calendar"), - ), - ]; - - write_cached_codex_apps_tools(&cache_context, &tools); - let cached = read_cached_codex_apps_tools(&cache_context).expect("cache entry exists for user"); - - assert_eq!( - cached - .iter() - .map(|tool| (tool.callable_name.as_str(), tool.connector_id.as_deref())) - .collect::>(), - vec![ - ( - "formerly_blocked_tool", - Some("connector_2b0a9009c9c64bf9933a3dae3f2b1254") - ), - ("calendar_tool", Some("calendar")), - ] - ); -} - -#[test] -fn codex_apps_tools_cache_is_ignored_when_schema_version_mismatches() { - let codex_home = tempdir().expect("tempdir"); - let cache_context = create_codex_apps_tools_cache_context( - codex_home.path().to_path_buf(), - Some("account-one"), - Some("user-one"), - ); - let cache_path = cache_context.tools_cache_path(); - if let Some(parent) = cache_path.parent() { - std::fs::create_dir_all(parent).expect("create parent"); - } - let bytes = serde_json::to_vec_pretty(&serde_json::json!({ - "schema_version": CODEX_APPS_TOOLS_CACHE_SCHEMA_VERSION + 1, - "tools": [create_test_tool(CODEX_APPS_MCP_SERVER_NAME, "one")], - })) - .expect("serialize"); - std::fs::write(cache_path, bytes).expect("write"); - - assert!(read_cached_codex_apps_tools(&cache_context).is_none()); -} - -#[test] -fn codex_apps_tools_cache_is_ignored_when_json_is_invalid() { - let codex_home = tempdir().expect("tempdir"); - let cache_context = create_codex_apps_tools_cache_context( - codex_home.path().to_path_buf(), - Some("account-one"), - Some("user-one"), - ); - let cache_path = cache_context.tools_cache_path(); - if let Some(parent) = cache_path.parent() { - std::fs::create_dir_all(parent).expect("create parent"); - } - std::fs::write(cache_path, b"{not json").expect("write"); - - assert!(read_cached_codex_apps_tools(&cache_context).is_none()); -} - -#[test] -fn startup_cached_codex_apps_tools_loads_from_disk_cache() { - let codex_home = tempdir().expect("tempdir"); - let cache_context = create_codex_apps_tools_cache_context( - codex_home.path().to_path_buf(), - Some("account-one"), - Some("user-one"), - ); - let cached_tools = vec![create_test_tool( +fn codex_apps_env_bearer_token_bypasses_shared_tools_cache() { + assert!(!should_share_codex_apps_tools_cache( CODEX_APPS_MCP_SERVER_NAME, - "calendar_search", - )]; - let server_info = create_test_server_info("Codex Apps"); - write_codex_apps_tools_cache(Some(&cache_context), &server_info, &cached_tools); - - let startup_tools = load_startup_cached_codex_apps_tools_snapshot(Some(&cache_context)) - .expect("expected startup snapshot to load from cache"); - let cached_server_info = load_startup_cached_codex_apps_server_info(Some(&cache_context)); - - assert_eq!(startup_tools.len(), 1); - assert_eq!(startup_tools[0].server_name, CODEX_APPS_MCP_SERVER_NAME); - assert_eq!(startup_tools[0].callable_name, "calendar_search"); - assert_eq!(cached_server_info, Some(server_info)); -} - -#[test] -fn startup_cached_codex_apps_tools_loads_without_server_info_cache() { - let codex_home = tempdir().expect("tempdir"); - let cache_context = create_codex_apps_tools_cache_context( - codex_home.path().to_path_buf(), - Some("account-one"), - Some("user-one"), - ); - let cache_path = cache_context.tools_cache_path(); - if let Some(parent) = cache_path.parent() { - std::fs::create_dir_all(parent).expect("create parent"); - } - let bytes = serde_json::to_vec_pretty(&serde_json::json!({ - "schema_version": CODEX_APPS_TOOLS_CACHE_SCHEMA_VERSION, - "tools": [create_test_tool(CODEX_APPS_MCP_SERVER_NAME, "calendar_search")], - })) - .expect("serialize"); - std::fs::write(cache_path, bytes).expect("write"); - - let startup_tools = load_startup_cached_codex_apps_tools_snapshot(Some(&cache_context)) - .expect("legacy startup snapshot should remain available"); - let cached_server_info = load_startup_cached_codex_apps_server_info(Some(&cache_context)); - - assert_eq!(startup_tools.len(), 1); - assert_eq!(startup_tools[0].callable_name, "calendar_search"); - assert_eq!(cached_server_info, None); -} - -#[test] -fn codex_apps_server_info_cache_survives_legacy_tools_cache_write() { - let codex_home = tempdir().expect("tempdir"); - let cache_context = create_codex_apps_tools_cache_context( - codex_home.path().to_path_buf(), - Some("account-one"), - Some("user-one"), - ); - let server_info = create_test_server_info("Codex Apps"); - write_codex_apps_tools_cache( - Some(&cache_context), - &server_info, - &[create_test_tool( - CODEX_APPS_MCP_SERVER_NAME, - "calendar_search", - )], - ); - - let cache_path = cache_context.tools_cache_path(); - if let Some(parent) = cache_path.parent() { - std::fs::create_dir_all(parent).expect("create parent"); - } - let bytes = serde_json::to_vec_pretty(&serde_json::json!({ - "schema_version": CODEX_APPS_TOOLS_CACHE_SCHEMA_VERSION - 1, - "tools": [create_test_tool(CODEX_APPS_MCP_SERVER_NAME, "calendar_search")], - })) - .expect("serialize"); - std::fs::write(cache_path, bytes).expect("write legacy tools cache"); - - assert_eq!( - load_startup_cached_codex_apps_server_info(Some(&cache_context)), - Some(server_info) - ); - assert!(load_startup_cached_codex_apps_tools_snapshot(Some(&cache_context)).is_none()); + /*uses_env_bearer_token*/ true, + )); } #[tokio::test] -async fn list_all_tools_uses_cached_tool_info_snapshot_while_client_is_pending() { - let startup_tools = vec![create_test_tool( +async fn list_all_tools_uses_shared_codex_apps_cache_while_client_is_pending() { + let codex_home = tempdir().expect("tempdir"); + let cache_context = create_codex_apps_tools_cache_context( + codex_home.path().to_path_buf(), + Some("account-one"), + Some("user-one"), + ); + cache_context.store_current_tools_for_test(vec![create_test_tool( CODEX_APPS_MCP_SERVER_NAME, "calendar_create_event", - )]; + )]); let pending_client = futures::future::pending::>() .boxed() .shared(); @@ -825,9 +815,11 @@ async fn list_all_tools_uses_cached_tool_info_snapshot_while_client_is_pending() AsyncManagedClient { client: pending_client, is_codex_apps_mcp_server: true, - cached_tool_info_snapshot: Some(startup_tools), cached_server_info: None, + codex_apps_tools_cache_context: Some(cache_context), + tool_filter: ToolFilter::default(), startup_complete: Arc::new(std::sync::atomic::AtomicBool::new(false)), + startup_reconnect: None, tool_plugin_provenance: Arc::new(ToolPluginProvenance::default()), cancel_token: CancellationToken::new(), }, @@ -840,7 +832,7 @@ async fn list_all_tools_uses_cached_tool_info_snapshot_while_client_is_pending() tool.canonical_tool_name() == ToolName::namespaced("mcp__codex_apps", "calendar_create_event") }) - .expect("tool from startup cache"); + .expect("tool from shared cache"); assert_eq!(tool.server_name, CODEX_APPS_MCP_SERVER_NAME); assert_eq!(tool.callable_name, "calendar_create_event"); } @@ -863,9 +855,11 @@ async fn list_available_server_infos_uses_cache_while_client_is_pending() { AsyncManagedClient { client: pending_client, is_codex_apps_mcp_server: true, - cached_tool_info_snapshot: Some(Vec::new()), cached_server_info: Some(server_info.clone()), + codex_apps_tools_cache_context: None, + tool_filter: ToolFilter::default(), startup_complete: Arc::new(std::sync::atomic::AtomicBool::new(false)), + startup_reconnect: None, tool_plugin_provenance: Arc::new(ToolPluginProvenance::default()), cancel_token: CancellationToken::new(), }, @@ -885,10 +879,8 @@ async fn list_available_server_infos_uses_cache_while_client_is_pending() { #[tokio::test] async fn list_all_tools_accepts_canonical_namespaced_tool_names() { - let startup_tools = vec![create_test_tool("rmcp", "echo")]; - let pending_client = futures::future::pending::>() - .boxed() - .shared(); + let managed_client = + create_ready_async_managed_client(vec![create_test_tool("rmcp", "echo")]).await; let approval_policy = Constrained::allow_any(AskForApproval::OnRequest); let permission_profile = Constrained::allow_any(PermissionProfile::default()); let mut manager = McpConnectionManager::new_uninitialized( @@ -896,18 +888,7 @@ async fn list_all_tools_accepts_canonical_namespaced_tool_names() { &permission_profile, /*prefix_mcp_tool_names*/ false, ); - manager.clients.insert( - "rmcp".to_string(), - AsyncManagedClient { - client: pending_client, - is_codex_apps_mcp_server: false, - cached_tool_info_snapshot: Some(startup_tools), - cached_server_info: None, - startup_complete: Arc::new(std::sync::atomic::AtomicBool::new(false)), - tool_plugin_provenance: Arc::new(ToolPluginProvenance::default()), - cancel_token: CancellationToken::new(), - }, - ); + manager.clients.insert("rmcp".to_string(), managed_client); let tools = manager.list_all_tools().await; let tool = tools @@ -929,10 +910,8 @@ async fn list_all_tools_accepts_canonical_namespaced_tool_names() { #[tokio::test] async fn list_all_tools_applies_legacy_mcp_prefix_by_default() { - let startup_tools = vec![create_test_tool("rmcp", "echo")]; - let pending_client = futures::future::pending::>() - .boxed() - .shared(); + let managed_client = + create_ready_async_managed_client(vec![create_test_tool("rmcp", "echo")]).await; let approval_policy = Constrained::allow_any(AskForApproval::OnRequest); let permission_profile = Constrained::allow_any(PermissionProfile::default()); let mut manager = McpConnectionManager::new_uninitialized( @@ -940,18 +919,7 @@ async fn list_all_tools_applies_legacy_mcp_prefix_by_default() { &permission_profile, /*prefix_mcp_tool_names*/ true, ); - manager.clients.insert( - "rmcp".to_string(), - AsyncManagedClient { - client: pending_client, - is_codex_apps_mcp_server: false, - cached_tool_info_snapshot: Some(startup_tools), - cached_server_info: None, - startup_complete: Arc::new(std::sync::atomic::AtomicBool::new(false)), - tool_plugin_provenance: Arc::new(ToolPluginProvenance::default()), - cancel_token: CancellationToken::new(), - }, - ); + manager.clients.insert("rmcp".to_string(), managed_client); let tools = manager.list_all_tools().await; let tool = tools @@ -972,7 +940,7 @@ async fn list_all_tools_applies_legacy_mcp_prefix_by_default() { } #[tokio::test] -async fn list_all_tools_blocks_while_client_is_pending_without_cached_tool_info_snapshot() { +async fn list_all_tools_blocks_while_client_is_pending_without_cached_tools() { let pending_client = futures::future::pending::>() .boxed() .shared(); @@ -988,9 +956,11 @@ async fn list_all_tools_blocks_while_client_is_pending_without_cached_tool_info_ AsyncManagedClient { client: pending_client, is_codex_apps_mcp_server: true, - cached_tool_info_snapshot: None, cached_server_info: None, + codex_apps_tools_cache_context: None, + tool_filter: ToolFilter::default(), startup_complete: Arc::new(std::sync::atomic::AtomicBool::new(false)), + startup_reconnect: None, tool_plugin_provenance: Arc::new(ToolPluginProvenance::default()), cancel_token: CancellationToken::new(), }, @@ -1025,9 +995,11 @@ async fn shutdown_cancels_pending_tool_listing() { AsyncManagedClient { client: pending_client, is_codex_apps_mcp_server: true, - cached_tool_info_snapshot: None, cached_server_info: None, + codex_apps_tools_cache_context: None, + tool_filter: ToolFilter::default(), startup_complete: Arc::new(std::sync::atomic::AtomicBool::new(false)), + startup_reconnect: None, tool_plugin_provenance: Arc::new(ToolPluginProvenance::default()), cancel_token, }, @@ -1070,9 +1042,11 @@ async fn shutdown_continues_after_caller_is_aborted() { AsyncManagedClient { client: blocking_client, is_codex_apps_mcp_server: true, - cached_tool_info_snapshot: None, cached_server_info: None, + codex_apps_tools_cache_context: None, + tool_filter: ToolFilter::default(), startup_complete: Arc::new(std::sync::atomic::AtomicBool::new(false)), + startup_reconnect: None, tool_plugin_provenance: Arc::new(ToolPluginProvenance::default()), cancel_token: CancellationToken::new(), }, @@ -1098,7 +1072,14 @@ async fn shutdown_continues_after_caller_is_aborted() { } #[tokio::test] -async fn list_all_tools_does_not_block_when_cached_tool_info_snapshot_is_empty() { +async fn list_all_tools_does_not_block_when_shared_codex_apps_cache_is_empty() { + let codex_home = tempdir().expect("tempdir"); + let cache_context = create_codex_apps_tools_cache_context( + codex_home.path().to_path_buf(), + Some("account-one"), + Some("user-one"), + ); + cache_context.store_current_tools_for_test(Vec::new()); let pending_client = futures::future::pending::>() .boxed() .shared(); @@ -1114,9 +1095,11 @@ async fn list_all_tools_does_not_block_when_cached_tool_info_snapshot_is_empty() AsyncManagedClient { client: pending_client, is_codex_apps_mcp_server: true, - cached_tool_info_snapshot: Some(Vec::new()), cached_server_info: None, + codex_apps_tools_cache_context: Some(cache_context), + tool_filter: ToolFilter::default(), startup_complete: Arc::new(std::sync::atomic::AtomicBool::new(false)), + startup_reconnect: None, tool_plugin_provenance: Arc::new(ToolPluginProvenance::default()), cancel_token: CancellationToken::new(), }, @@ -1124,20 +1107,27 @@ async fn list_all_tools_does_not_block_when_cached_tool_info_snapshot_is_empty() let timeout_result = tokio::time::timeout(Duration::from_millis(10), manager.list_all_tools()).await; - let tools = timeout_result.expect("cache-hit startup snapshot should not block"); + let tools = timeout_result.expect("shared empty cache should not block"); assert!(tools.is_empty()); } #[tokio::test] -async fn list_all_tools_uses_cached_tool_info_snapshot_when_client_startup_fails() { - let startup_tools = vec![create_test_tool( +async fn list_all_tools_uses_shared_codex_apps_cache_when_client_startup_fails() { + let codex_home = tempdir().expect("tempdir"); + let cache_context = create_codex_apps_tools_cache_context( + codex_home.path().to_path_buf(), + Some("account-one"), + Some("user-one"), + ); + cache_context.store_current_tools_for_test(vec![create_test_tool( CODEX_APPS_MCP_SERVER_NAME, "calendar_create_event", - )]; + )]); let server_info = create_test_server_info("Codex Apps"); let failed_client = futures::future::ready::>(Err( StartupOutcomeError::Failed { error: "startup failed".to_string(), + is_authentication_required: false, }, )) .boxed() @@ -1155,9 +1145,11 @@ async fn list_all_tools_uses_cached_tool_info_snapshot_when_client_startup_fails AsyncManagedClient { client: failed_client, is_codex_apps_mcp_server: true, - cached_tool_info_snapshot: Some(startup_tools), cached_server_info: Some(server_info.clone()), + codex_apps_tools_cache_context: Some(cache_context), + tool_filter: ToolFilter::default(), startup_complete, + startup_reconnect: None, tool_plugin_provenance: Arc::new(ToolPluginProvenance::default()), cancel_token: CancellationToken::new(), }, @@ -1170,7 +1162,7 @@ async fn list_all_tools_uses_cached_tool_info_snapshot_when_client_startup_fails tool.canonical_tool_name() == ToolName::namespaced("mcp__codex_apps", "calendar_create_event") }) - .expect("tool from startup cache"); + .expect("tool from shared cache"); assert_eq!(tool.server_name, CODEX_APPS_MCP_SERVER_NAME); assert_eq!(tool.callable_name, "calendar_create_event"); assert_eq!( @@ -1183,12 +1175,239 @@ async fn list_all_tools_uses_cached_tool_info_snapshot_when_client_startup_fails } #[tokio::test] -async fn list_all_tools_adds_server_metadata_to_cached_tools() { - let server_name = "docs"; - let startup_tools = vec![create_test_tool(server_name, "search")]; - let pending_client = futures::future::pending::>() +async fn list_all_tools_reconnects_failed_codex_apps_startup_and_reuses_client() { + let recovered_client = create_test_managed_client(vec![create_test_tool( + CODEX_APPS_MCP_SERVER_NAME, + "drive_search", + )]) + .await; + let attempts = Arc::new(AtomicUsize::new(0)); + let attempts_for_reconnect = Arc::clone(&attempts); + let reconnect_finished = Arc::new(tokio::sync::Notify::new()); + let reconnect_finished_for_factory = Arc::clone(&reconnect_finished); + let reconnect_factory = Arc::new(move || { + attempts_for_reconnect.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + let reconnect_finished = Arc::clone(&reconnect_finished_for_factory); + let recovered_client = recovered_client.clone(); + async move { + reconnect_finished.notify_one(); + Ok(recovered_client) + } .boxed() - .shared(); + .shared() + }); + let manager = create_test_manager_with_failed_apps_startup(Vec::new(), reconnect_factory); + + let reconnect_finished_wait = reconnect_finished.notified(); + let tools = manager.list_all_tools().await; + assert!(tools.is_empty()); + reconnect_finished_wait.await; + + let tools = manager.list_all_tools().await; + assert_eq!( + tools + .iter() + .map(|tool| tool.callable_name.as_str()) + .collect::>(), + vec!["drive_search"] + ); + assert_eq!(attempts.load(std::sync::atomic::Ordering::SeqCst), 1); + + let tools = manager.list_all_tools().await; + assert_eq!( + tools + .iter() + .map(|tool| tool.callable_name.as_str()) + .collect::>(), + vec!["drive_search"] + ); + assert_eq!(attempts.load(std::sync::atomic::Ordering::SeqCst), 1); +} + +#[tokio::test(start_paused = true)] +async fn later_tool_list_retries_after_failed_reconnect_and_keeps_cached_tools() { + let recovered_client = create_test_managed_client(vec![create_test_tool( + CODEX_APPS_MCP_SERVER_NAME, + "drive_search", + )]) + .await; + let attempts = Arc::new(AtomicUsize::new(0)); + let attempts_for_reconnect = Arc::clone(&attempts); + let reconnect_finished = Arc::new(tokio::sync::Notify::new()); + let reconnect_finished_for_factory = Arc::clone(&reconnect_finished); + let reconnect_factory = Arc::new(move || { + let attempt = attempts_for_reconnect.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + let reconnect_finished = Arc::clone(&reconnect_finished_for_factory); + let recovered_client = recovered_client.clone(); + async move { + let result = if attempt < 2 { + Err(StartupOutcomeError::Failed { + error: "recreated startup failed".to_string(), + is_authentication_required: false, + }) + } else { + Ok(recovered_client) + }; + reconnect_finished.notify_one(); + result + } + .boxed() + .shared() + }); + let manager = create_test_manager_with_failed_apps_startup( + vec![create_test_tool( + CODEX_APPS_MCP_SERVER_NAME, + "cached_drive_search", + )], + reconnect_factory, + ); + + let first_reconnect_finished = reconnect_finished.notified(); + let tools = manager.list_all_tools().await; + assert_eq!( + tools + .iter() + .map(|tool| tool.callable_name.as_str()) + .collect::>(), + vec!["cached_drive_search"] + ); + first_reconnect_finished.await; + assert_eq!(attempts.load(std::sync::atomic::Ordering::SeqCst), 1); + + let tools = manager.list_all_tools().await; + assert_eq!( + tools + .iter() + .map(|tool| tool.callable_name.as_str()) + .collect::>(), + vec!["cached_drive_search"] + ); + assert_eq!(attempts.load(std::sync::atomic::Ordering::SeqCst), 1); + + tokio::time::advance(CODEX_APPS_RECONNECT_INITIAL_BACKOFF).await; + let second_reconnect_finished = reconnect_finished.notified(); + let tools = manager.list_all_tools().await; + assert_eq!( + tools + .iter() + .map(|tool| tool.callable_name.as_str()) + .collect::>(), + vec!["cached_drive_search"] + ); + second_reconnect_finished.await; + assert_eq!(attempts.load(std::sync::atomic::Ordering::SeqCst), 2); + + tokio::time::advance(CODEX_APPS_RECONNECT_INITIAL_BACKOFF).await; + let tools = manager.list_all_tools().await; + assert_eq!( + tools + .iter() + .map(|tool| tool.callable_name.as_str()) + .collect::>(), + vec!["cached_drive_search"] + ); + assert_eq!(attempts.load(std::sync::atomic::Ordering::SeqCst), 2); + + tokio::time::advance(CODEX_APPS_RECONNECT_INITIAL_BACKOFF).await; + let third_reconnect_finished = reconnect_finished.notified(); + let tools = manager.list_all_tools().await; + assert_eq!( + tools + .iter() + .map(|tool| tool.callable_name.as_str()) + .collect::>(), + vec!["cached_drive_search"] + ); + third_reconnect_finished.await; + assert_eq!(attempts.load(std::sync::atomic::Ordering::SeqCst), 3); + + let tools = manager.list_all_tools().await; + assert_eq!( + tools + .iter() + .map(|tool| tool.callable_name.as_str()) + .collect::>(), + vec!["drive_search"] + ); +} + +#[tokio::test] +async fn tool_lists_do_not_block_and_share_codex_apps_startup_reconnect() { + let recovered_client = create_test_managed_client(vec![create_test_tool( + CODEX_APPS_MCP_SERVER_NAME, + "drive_search", + )]) + .await; + let attempts = Arc::new(AtomicUsize::new(0)); + let attempts_for_reconnect = Arc::clone(&attempts); + let reconnect_started = Arc::new(tokio::sync::Notify::new()); + let reconnect_started_for_factory = Arc::clone(&reconnect_started); + let release_reconnect = Arc::new(tokio::sync::Notify::new()); + let release_reconnect_for_factory = Arc::clone(&release_reconnect); + let reconnect_factory = Arc::new(move || { + let recovered_client = recovered_client.clone(); + let attempts = Arc::clone(&attempts_for_reconnect); + let reconnect_started = Arc::clone(&reconnect_started_for_factory); + let release_reconnect = Arc::clone(&release_reconnect_for_factory); + async move { + attempts.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + reconnect_started.notify_one(); + release_reconnect.notified().await; + Ok(recovered_client) + } + .boxed() + .shared() + }); + let manager = Arc::new(create_test_manager_with_failed_apps_startup( + vec![create_test_tool( + CODEX_APPS_MCP_SERVER_NAME, + "cached_drive_search", + )], + reconnect_factory, + )); + let reconnect_started_wait = reconnect_started.notified(); + let first_tools = tokio::time::timeout(Duration::from_millis(10), manager.list_all_tools()) + .await + .expect("cached tools should not wait for reconnect"); + + reconnect_started_wait.await; + let second_tools = tokio::time::timeout(Duration::from_millis(10), manager.list_all_tools()) + .await + .expect("concurrent cached tools should not wait for reconnect"); + assert_eq!(attempts.load(std::sync::atomic::Ordering::SeqCst), 1); + assert_eq!( + first_tools + .iter() + .map(|tool| tool.callable_name.as_str()) + .collect::>(), + vec!["cached_drive_search"] + ); + assert_eq!( + second_tools + .iter() + .map(|tool| tool.callable_name.as_str()) + .collect::>(), + vec!["cached_drive_search"] + ); + + release_reconnect.notify_one(); + tokio::task::yield_now().await; + let tools = manager.list_all_tools().await; + assert_eq!( + tools + .iter() + .map(|tool| tool.callable_name.as_str()) + .collect::>(), + vec!["drive_search"] + ); + assert_eq!(attempts.load(std::sync::atomic::Ordering::SeqCst), 1); +} + +#[tokio::test] +async fn list_all_tools_adds_server_metadata_to_tools() { + let server_name = "docs"; + let managed_client = + create_ready_async_managed_client(vec![create_test_tool(server_name, "search")]).await; let approval_policy = Constrained::allow_any(AskForApproval::OnRequest); let permission_profile = Constrained::allow_any(PermissionProfile::default()); let mut manager = McpConnectionManager::new_uninitialized( @@ -1209,18 +1428,9 @@ async fn list_all_tools_adds_server_metadata_to_cached_tools() { tool_approval_modes: HashMap::new(), }, ); - manager.clients.insert( - server_name.to_string(), - AsyncManagedClient { - client: pending_client, - is_codex_apps_mcp_server: false, - cached_tool_info_snapshot: Some(startup_tools), - cached_server_info: None, - startup_complete: Arc::new(std::sync::atomic::AtomicBool::new(false)), - tool_plugin_provenance: Arc::new(ToolPluginProvenance::default()), - cancel_token: CancellationToken::new(), - }, - ); + manager + .clients + .insert(server_name.to_string(), managed_client); let tools = manager.list_all_tools().await; assert_eq!(tools.len(), 1); @@ -1341,6 +1551,7 @@ async fn no_local_runtime_fails_local_stdio_but_keeps_local_http_server() { PathBuf::from("/tmp"), ), codex_home.path().to_path_buf(), + CodexAppsToolsCache::default(), CodexAppsToolsCacheKey { account_id: None, chatgpt_user_id: None, @@ -1352,6 +1563,8 @@ async fn no_local_runtime_fails_local_stdio_but_keeps_local_http_server() { ToolPluginProvenance::default(), /*auth*/ None, /*elicitation_reviewer*/ None, + /*elicitation_lifecycle*/ None, + ElicitationRequestRouter::default(), ) .await; @@ -1405,6 +1618,7 @@ async fn server_replacement_preserves_and_does_not_cancel_unrelated_clients() { PathBuf::from("/tmp"), ), codex_home.path().to_path_buf(), + CodexAppsToolsCache::default(), CodexAppsToolsCacheKey { account_id: None, chatgpt_user_id: None, @@ -1416,6 +1630,8 @@ async fn server_replacement_preserves_and_does_not_cancel_unrelated_clients() { ToolPluginProvenance::default(), /*auth*/ None, /*elicitation_reviewer*/ None, + /*elicitation_lifecycle*/ None, + ElicitationRequestRouter::default(), ) .await, ); @@ -1555,7 +1771,7 @@ fn mcp_init_error_display_prompts_for_github_pat() { oauth_resource: None, tools: HashMap::new(), }), - auth_status: McpAuthStatus::Unsupported, + auth_state: McpAuthState::Unsupported, }; let err: StartupOutcomeError = anyhow::anyhow!("OAuth is unsupported").into(); @@ -1582,6 +1798,50 @@ fn mcp_init_error_display_prompts_for_login_when_auth_required() { assert_eq!(expected, display); } +#[test] +fn mcp_startup_failure_reason_requires_existing_oauth_and_auth_failure() { + for (auth_state, is_authentication_required, expected) in [ + ( + Some(McpAuthState::LoggedOut( + McpLoginRequirement::Reauthentication, + )), + true, + Some(McpStartupFailureReason::ReauthenticationRequired), + ), + ( + Some(McpAuthState::LoggedOut( + McpLoginRequirement::Reauthentication, + )), + false, + None, + ), + ( + Some(McpAuthState::LoggedOut(McpLoginRequirement::Login)), + true, + None, + ), + (Some(McpAuthState::Unsupported), true, None), + (Some(McpAuthState::BearerToken), true, None), + (Some(McpAuthState::OAuth), true, None), + (None, true, None), + ] { + let entry = auth_state.map(|auth_state| McpAuthStatusEntry { + config: None, + auth_state, + }); + let error = StartupOutcomeError::Failed { + error: "startup failed".to_string(), + is_authentication_required, + }; + + assert_eq!( + mcp_startup_failure_reason(entry.as_ref(), &error), + expected, + "auth_state={auth_state:?}, is_authentication_required={is_authentication_required}" + ); + } +} + #[test] fn mcp_init_error_display_reports_generic_errors() { let server_name = "custom"; @@ -1609,7 +1869,7 @@ fn mcp_init_error_display_reports_generic_errors() { oauth_resource: None, tools: HashMap::new(), }), - auth_status: McpAuthStatus::Unsupported, + auth_state: McpAuthState::Unsupported, }; let err: StartupOutcomeError = anyhow::anyhow!("boom").into(); diff --git a/codex-rs/codex-mcp/src/elicitation.rs b/codex-rs/codex-mcp/src/elicitation.rs index d6a685b502..b6bb7fa791 100644 --- a/codex-rs/codex-mcp/src/elicitation.rs +++ b/codex-rs/codex-mcp/src/elicitation.rs @@ -8,6 +8,8 @@ use std::collections::HashMap; use std::sync::Arc; use std::sync::Mutex as StdMutex; +use std::sync::atomic::AtomicU64; +use std::sync::atomic::Ordering; use crate::mcp::McpPermissionPromptAutoApproveContext; use crate::mcp::mcp_permission_prompt_is_auto_approved; @@ -32,6 +34,8 @@ use rmcp::model::RequestId; use tokio::sync::Mutex; use tokio::sync::oneshot; +static NEXT_ELICITATION_REQUEST_ID: AtomicU64 = AtomicU64::new(0); + #[derive(Debug, Clone)] pub struct ElicitationReviewRequest { pub server_name: String, @@ -48,13 +52,68 @@ pub trait ElicitationReviewer: Send + Sync { pub type ElicitationReviewerHandle = Arc; +/// Holds an owner-provided registration while an MCP elicitation is waiting for a response. +#[derive(Clone)] +pub struct ElicitationLifecycle { + register: Arc Box + Send + Sync>, +} + +impl ElicitationLifecycle { + pub fn new(register: impl Fn() -> T + Send + Sync + 'static) -> Self + where + T: Send + Sync + 'static, + { + Self { + register: Arc::new(move || Box::new(register())), + } + } + + fn start(&self) -> ActiveElicitation { + ActiveElicitation { + _registration: (self.register)(), + } + } +} + +struct ActiveElicitation { + _registration: Box, +} + +/// Routes model-visible elicitation response tokens to their exact pending responders. +/// +/// One router is shared by every MCP runtime created for a thread. The public response token is +/// generated by Codex rather than copied from the MCP connection, so separate runtimes may reuse +/// the same server request ID without colliding. +#[derive(Clone, Default)] +pub struct ElicitationRequestRouter { + requests: Arc>, +} + +impl ElicitationRequestRouter { + async fn resolve( + &self, + server_name: String, + id: RequestId, + response: ElicitationResponse, + ) -> Result<()> { + self.requests + .lock() + .await + .remove(&(server_name, id)) + .ok_or_else(|| anyhow!("elicitation request not found"))? + .send(response) + .map_err(|e| anyhow!("failed to send elicitation response: {e:?}")) + } +} + #[derive(Clone)] pub(crate) struct ElicitationRequestManager { - requests: Arc>, + router: ElicitationRequestRouter, pub(crate) approval_policy: Arc>, pub(crate) permission_profile: Arc>, auto_deny: Arc>, reviewer: Option, + lifecycle: Option, } impl ElicitationRequestManager { @@ -62,13 +121,16 @@ impl ElicitationRequestManager { approval_policy: AskForApproval, permission_profile: PermissionProfile, reviewer: Option, + lifecycle: Option, + router: ElicitationRequestRouter, ) -> Self { Self { - requests: Arc::new(Mutex::new(HashMap::new())), + router, approval_policy: Arc::new(StdMutex::new(approval_policy)), permission_profile: Arc::new(StdMutex::new(permission_profile)), auto_deny: Arc::new(StdMutex::new(false)), reviewer, + lifecycle, } } @@ -91,13 +153,11 @@ impl ElicitationRequestManager { id: RequestId, response: ElicitationResponse, ) -> Result<()> { - self.requests - .lock() - .await - .remove(&(server_name, id)) - .ok_or_else(|| anyhow!("elicitation request not found"))? - .send(response) - .map_err(|e| anyhow!("failed to send elicitation response: {e:?}")) + self.router.resolve(server_name, id, response).await + } + + pub(crate) fn router(&self) -> ElicitationRequestRouter { + self.router.clone() } pub(crate) fn make_sender( @@ -105,19 +165,21 @@ impl ElicitationRequestManager { server_name: String, tx_event: Sender, ) -> SendElicitation { - let elicitation_requests = self.requests.clone(); + let router = self.router.clone(); let approval_policy = self.approval_policy.clone(); let permission_profile = self.permission_profile.clone(); let auto_deny = self.auto_deny.clone(); let reviewer = self.reviewer.clone(); + let lifecycle = self.lifecycle.clone(); Box::new(move |id, elicitation| { - let elicitation_requests = elicitation_requests.clone(); + let router = router.clone(); let tx_event = tx_event.clone(); let server_name = server_name.clone(); let approval_policy = approval_policy.clone(); let permission_profile = permission_profile.clone(); let auto_deny = auto_deny.clone(); let reviewer = reviewer.clone(); + let lifecycle = lifecycle.clone(); async move { let auto_deny = auto_deny .lock() @@ -171,6 +233,11 @@ impl ElicitationRequestManager { } } + let public_request_id = format!( + "codex-mcp-elicitation-{}", + NEXT_ELICITATION_REQUEST_ID.fetch_add(1, Ordering::Relaxed) + ); + let routed_request_id = RequestId::String(public_request_id.clone().into()); let request = match elicitation { Elicitation::Mcp( rmcp::model::CreateElicitationRequestParams::FormElicitationParams { @@ -214,9 +281,10 @@ impl ElicitationRequestManager { }, }; let (tx, rx) = oneshot::channel(); + let _active_elicitation = lifecycle.as_ref().map(ElicitationLifecycle::start); { - let mut lock = elicitation_requests.lock().await; - lock.insert((server_name.clone(), id.clone()), tx); + let mut lock = router.requests.lock().await; + lock.insert((server_name.clone(), routed_request_id), tx); } let _ = tx_event .send(Event { @@ -224,14 +292,7 @@ impl ElicitationRequestManager { msg: EventMsg::ElicitationRequest(ElicitationRequestEvent { turn_id: None, server_name, - id: match id.clone() { - rmcp::model::NumberOrString::String(value) => { - ProtocolRequestId::String(value.to_string()) - } - rmcp::model::NumberOrString::Number(value) => { - ProtocolRequestId::Integer(value) - } - }, + id: ProtocolRequestId::String(public_request_id), request, }), }) diff --git a/codex-rs/codex-mcp/src/lib.rs b/codex-rs/codex-mcp/src/lib.rs index b4d397eac9..77128d7970 100644 --- a/codex-rs/codex-mcp/src/lib.rs +++ b/codex-rs/codex-mcp/src/lib.rs @@ -1,5 +1,7 @@ pub use connection_manager::McpConnectionManager; pub use connection_manager::tool_is_model_visible; +pub use elicitation::ElicitationLifecycle; +pub use elicitation::ElicitationRequestRouter; pub use elicitation::ElicitationReviewRequest; pub use elicitation::ElicitationReviewer; pub use elicitation::ElicitationReviewerHandle; @@ -35,8 +37,9 @@ pub use auth_elicitation::auth_elicitation_id; pub use auth_elicitation::build_auth_elicitation; pub use auth_elicitation::build_auth_elicitation_plan; pub use auth_elicitation::connector_auth_failure_from_tool_result; -pub use codex_apps::CodexAppsToolsCacheKey; -pub use codex_apps::codex_apps_tools_cache_key; +pub use codex_apps_cache::CodexAppsToolsCache; +pub use codex_apps_cache::CodexAppsToolsCacheKey; +pub use codex_apps_cache::codex_apps_tools_cache_key; pub use mcp::codex_apps_mcp_server_config; pub use mcp::configured_mcp_servers; pub use mcp::effective_mcp_servers; @@ -61,7 +64,9 @@ pub use mcp::McpOAuthScopesSource; pub use mcp::ResolvedMcpOAuthScopes; pub use mcp::compute_auth_statuses; pub use mcp::discover_supported_scopes; +pub use mcp::discover_supported_scopes_with_http_client; pub use mcp::oauth_login_support; +pub use mcp::oauth_login_support_with_http_client; pub use mcp::resolve_oauth_scopes; pub use mcp::should_retry_without_scopes; @@ -73,6 +78,7 @@ pub use tools::declared_openai_file_input_param_names; pub(crate) mod auth_elicitation; mod catalog; pub(crate) mod codex_apps; +pub(crate) mod codex_apps_cache; pub(crate) mod connection_manager; pub(crate) mod elicitation; pub(crate) mod mcp; diff --git a/codex-rs/codex-mcp/src/mcp/auth.rs b/codex-rs/codex-mcp/src/mcp/auth.rs index d19766af1f..fcdaefe3a9 100644 --- a/codex-rs/codex-mcp/src/mcp/auth.rs +++ b/codex-rs/codex-mcp/src/mcp/auth.rs @@ -1,4 +1,5 @@ use std::collections::HashMap; +use std::sync::Arc; use anyhow::Result; use codex_config::McpServerAuth; @@ -6,15 +7,19 @@ use codex_config::McpServerConfig; use codex_config::McpServerTransportConfig; use codex_config::types::AuthKeyringBackendKind; use codex_config::types::OAuthCredentialsStoreMode; +use codex_exec_server::HttpClient; use codex_login::CodexAuth; -use codex_protocol::protocol::McpAuthStatus; +use codex_rmcp_client::McpAuthState; use codex_rmcp_client::OAuthProviderError; use codex_rmcp_client::determine_streamable_http_auth_status; +use codex_rmcp_client::determine_streamable_http_auth_status_with_http_client; use codex_rmcp_client::discover_streamable_http_oauth; +use codex_rmcp_client::discover_streamable_http_oauth_with_http_client; use futures::FutureExt; use futures::future::join_all; use tracing::warn; +use crate::runtime::McpRuntimeContext; use crate::server::EffectiveMcpServer; #[derive(Debug, Clone)] @@ -49,10 +54,54 @@ pub struct ResolvedMcpOAuthScopes { #[derive(Debug, Clone)] pub struct McpAuthStatusEntry { pub config: Option, - pub auth_status: McpAuthStatus, + pub auth_state: McpAuthState, } pub async fn oauth_login_support(transport: &McpServerTransportConfig) -> McpOAuthLoginSupport { + let Some(mut config) = oauth_login_candidate(transport) else { + return McpOAuthLoginSupport::Unsupported; + }; + match discover_streamable_http_oauth( + &config.url, + config.http_headers.clone(), + config.env_http_headers.clone(), + ) + .await + { + Ok(Some(discovery)) => { + config.discovered_scopes = discovery.scopes_supported; + McpOAuthLoginSupport::Supported(config) + } + Ok(None) => McpOAuthLoginSupport::Unsupported, + Err(err) => McpOAuthLoginSupport::Unknown(err), + } +} + +pub async fn oauth_login_support_with_http_client( + transport: &McpServerTransportConfig, + http_client: Arc, +) -> McpOAuthLoginSupport { + let Some(mut config) = oauth_login_candidate(transport) else { + return McpOAuthLoginSupport::Unsupported; + }; + match discover_streamable_http_oauth_with_http_client( + &config.url, + config.http_headers.clone(), + config.env_http_headers.clone(), + http_client, + ) + .await + { + Ok(Some(discovery)) => { + config.discovered_scopes = discovery.scopes_supported; + McpOAuthLoginSupport::Supported(config) + } + Ok(None) => McpOAuthLoginSupport::Unsupported, + Err(err) => McpOAuthLoginSupport::Unknown(err), + } +} + +fn oauth_login_candidate(transport: &McpServerTransportConfig) -> Option { let McpServerTransportConfig::StreamableHttp { url, bearer_token_env_var, @@ -60,24 +109,17 @@ pub async fn oauth_login_support(transport: &McpServerTransportConfig) -> McpOAu env_http_headers, } = transport else { - return McpOAuthLoginSupport::Unsupported; + return None; }; - if bearer_token_env_var.is_some() { - return McpOAuthLoginSupport::Unsupported; - } - - match discover_streamable_http_oauth(url, http_headers.clone(), env_http_headers.clone()).await - { - Ok(Some(discovery)) => McpOAuthLoginSupport::Supported(McpOAuthLoginConfig { - url: url.clone(), - http_headers: http_headers.clone(), - env_http_headers: env_http_headers.clone(), - discovered_scopes: discovery.scopes_supported, - }), - Ok(None) => McpOAuthLoginSupport::Unsupported, - Err(err) => McpOAuthLoginSupport::Unknown(err), + return None; } + Some(McpOAuthLoginConfig { + url: url.clone(), + http_headers: http_headers.clone(), + env_http_headers: env_http_headers.clone(), + discovered_scopes: None, + }) } pub async fn discover_supported_scopes( @@ -89,6 +131,16 @@ pub async fn discover_supported_scopes( } } +pub async fn discover_supported_scopes_with_http_client( + transport: &McpServerTransportConfig, + http_client: Arc, +) -> Option> { + match oauth_login_support_with_http_client(transport, http_client).await { + McpOAuthLoginSupport::Supported(config) => config.discovered_scopes, + McpOAuthLoginSupport::Unsupported | McpOAuthLoginSupport::Unknown(_) => None, + } +} + pub fn resolve_oauth_scopes( explicit_scopes: Option>, configured_scopes: Option>, @@ -133,6 +185,7 @@ pub async fn compute_auth_statuses<'a, I>( store_mode: OAuthCredentialsStoreMode, keyring_backend_kind: AuthKeyringBackendKind, auth: Option<&CodexAuth>, + runtime_context: &McpRuntimeContext, ) -> HashMap where I: IntoIterator, @@ -140,6 +193,7 @@ where let futures = servers.into_iter().map(|(name, server)| { let name = name.clone(); let config = server.configured_config().cloned(); + let runtime_context = runtime_context.clone(); let has_runtime_auth = config .as_ref() .is_some_and(|config| matches!(&config.auth, McpServerAuth::ChatGpt)) @@ -154,7 +208,7 @@ where ) }); async move { - let auth_status = match config.as_ref() { + let auth_state = match config.as_ref() { Some(config) => { match compute_auth_status( &name, @@ -162,6 +216,7 @@ where store_mode, keyring_backend_kind, has_runtime_auth, + &runtime_context, ) .await { @@ -170,16 +225,13 @@ where warn!( "failed to determine auth status for MCP server `{name}`: {error:?}" ); - McpAuthStatus::Unsupported + McpAuthState::Unsupported } } } - None => McpAuthStatus::Unsupported, - }; - let entry = McpAuthStatusEntry { - config, - auth_status, + None => McpAuthState::Unsupported, }; + let entry = McpAuthStatusEntry { config, auth_state }; (name, entry) } }); @@ -193,34 +245,53 @@ async fn compute_auth_status( store_mode: OAuthCredentialsStoreMode, keyring_backend_kind: AuthKeyringBackendKind, has_runtime_auth: bool, -) -> Result { + runtime_context: &McpRuntimeContext, +) -> Result { if !config.enabled { - return Ok(McpAuthStatus::Unsupported); + return Ok(McpAuthState::Unsupported); } if has_runtime_auth { - return Ok(McpAuthStatus::BearerToken); + return Ok(McpAuthState::BearerToken); } match &config.transport { - McpServerTransportConfig::Stdio { .. } => Ok(McpAuthStatus::Unsupported), + McpServerTransportConfig::Stdio { .. } => Ok(McpAuthState::Unsupported), McpServerTransportConfig::StreamableHttp { url, bearer_token_env_var, http_headers, env_http_headers, } => { - determine_streamable_http_auth_status( - server_name, - url, - bearer_token_env_var.as_deref(), - http_headers.clone(), - env_http_headers.clone(), - store_mode, - keyring_backend_kind, - ) - .boxed() - .await + if config.is_local_environment() { + determine_streamable_http_auth_status( + server_name, + url, + bearer_token_env_var.as_deref(), + http_headers.clone(), + env_http_headers.clone(), + store_mode, + keyring_backend_kind, + ) + .boxed() + .await + } else { + let http_client = runtime_context + .resolve_http_client(server_name, config) + .map_err(anyhow::Error::msg)?; + determine_streamable_http_auth_status_with_http_client( + server_name, + url, + bearer_token_env_var.as_deref(), + http_headers.clone(), + env_http_headers.clone(), + store_mode, + keyring_backend_kind, + http_client, + ) + .boxed() + .await + } } } } diff --git a/codex-rs/codex-mcp/src/mcp/mod.rs b/codex-rs/codex-mcp/src/mcp/mod.rs index f2c8f1d020..87ea25dd9f 100644 --- a/codex-rs/codex-mcp/src/mcp/mod.rs +++ b/codex-rs/codex-mcp/src/mcp/mod.rs @@ -5,7 +5,9 @@ pub use auth::McpOAuthScopesSource; pub use auth::ResolvedMcpOAuthScopes; pub use auth::compute_auth_statuses; pub use auth::discover_supported_scopes; +pub use auth::discover_supported_scopes_with_http_client; pub use auth::oauth_login_support; +pub use auth::oauth_login_support_with_http_client; pub use auth::resolve_oauth_scopes; pub use auth::should_retry_without_scopes; @@ -42,7 +44,8 @@ use serde_json::Value; use tokio_util::sync::CancellationToken; use crate::ResolvedMcpCatalog; -use crate::codex_apps::codex_apps_tools_cache_key; +use crate::codex_apps_cache::CodexAppsToolsCache; +use crate::codex_apps_cache::codex_apps_tools_cache_key; use crate::connection_manager::McpConnectionManager; use crate::runtime::McpRuntimeContext; use crate::server::EffectiveMcpServer; @@ -299,6 +302,7 @@ pub async fn read_mcp_resource( config: &McpConfig, auth: Option<&CodexAuth>, runtime_context: McpRuntimeContext, + codex_apps_tools_cache: CodexAppsToolsCache, server: &str, uri: &str, ) -> anyhow::Result { @@ -309,6 +313,7 @@ pub async fn read_mcp_resource( config.mcp_oauth_credentials_store_mode, config.auth_keyring_backend_kind, auth, + &runtime_context, ) .await; let (tx_event, rx_event) = unbounded(); @@ -326,6 +331,7 @@ pub async fn read_mcp_resource( PermissionProfile::default(), runtime_context, config.codex_home.clone(), + codex_apps_tools_cache, codex_apps_tools_cache_key(auth), config.prefix_mcp_tool_names, config.client_elicitation_capability.clone(), @@ -333,6 +339,8 @@ pub async fn read_mcp_resource( tool_plugin_provenance(config), auth, /*elicitation_reviewer*/ None, + /*elicitation_lifecycle*/ None, + crate::elicitation::ElicitationRequestRouter::default(), ) .await; @@ -358,6 +366,7 @@ pub async fn collect_mcp_server_status_snapshot_with_detail( auth: Option<&CodexAuth>, submit_id: String, runtime_context: McpRuntimeContext, + codex_apps_tools_cache: CodexAppsToolsCache, detail: McpSnapshotDetail, ) -> McpServerStatusSnapshot { let mcp_servers = effective_mcp_servers(config, auth); @@ -378,6 +387,7 @@ pub async fn collect_mcp_server_status_snapshot_with_detail( config.mcp_oauth_credentials_store_mode, config.auth_keyring_backend_kind, auth, + &runtime_context, ) .await; @@ -399,6 +409,7 @@ pub async fn collect_mcp_server_status_snapshot_with_detail( PermissionProfile::default(), runtime_context, config.codex_home.clone(), + codex_apps_tools_cache, codex_apps_tools_cache_key(auth), config.prefix_mcp_tool_names, config.client_elicitation_capability.clone(), @@ -406,6 +417,8 @@ pub async fn collect_mcp_server_status_snapshot_with_detail( tool_plugin_provenance, auth, /*elicitation_reviewer*/ None, + /*elicitation_lifecycle*/ None, + crate::elicitation::ElicitationRequestRouter::default(), ) .await; @@ -558,7 +571,7 @@ fn auth_statuses_from_entries( ) -> HashMap { auth_status_entries .iter() - .map(|(name, entry)| (name.clone(), entry.auth_status)) + .map(|(name, entry)| (name.clone(), McpAuthStatus::from(entry.auth_state))) .collect::>() } diff --git a/codex-rs/codex-mcp/src/plugin_config.rs b/codex-rs/codex-mcp/src/plugin_config.rs index 968d6a64df..4f74a97d16 100644 --- a/codex-rs/codex-mcp/src/plugin_config.rs +++ b/codex-rs/codex-mcp/src/plugin_config.rs @@ -173,8 +173,35 @@ fn environment_cwd( fn bind_environment_env_vars(config: &mut McpServerConfig) -> Result<(), String> { let is_local_environment = config.is_local_environment(); - let McpServerTransportConfig::Stdio { env_vars, .. } = &mut config.transport else { - return Ok(()); + let env_vars = match &mut config.transport { + McpServerTransportConfig::Stdio { env_vars, .. } => env_vars, + // Never resolve executor-owned environment references in the host process. + // Remove this rejection once the owning executor resolves these fields. + McpServerTransportConfig::StreamableHttp { + bearer_token_env_var, + env_http_headers, + .. + } => { + if is_local_environment { + return Ok(()); + } + if bearer_token_env_var.is_some() { + return Err( + "`bearer_token_env_var` requires executor-side environment resolution for an executor-owned HTTP MCP" + .to_string(), + ); + } + if env_http_headers + .as_ref() + .is_some_and(|headers| !headers.is_empty()) + { + return Err( + "`env_http_headers` requires executor-side environment resolution for an executor-owned HTTP MCP" + .to_string(), + ); + } + return Ok(()); + } }; for env_var in env_vars { match env_var { diff --git a/codex-rs/codex-mcp/src/plugin_config_tests.rs b/codex-rs/codex-mcp/src/plugin_config_tests.rs index 576bd29618..c2f1f71a54 100644 --- a/codex-rs/codex-mcp/src/plugin_config_tests.rs +++ b/codex-rs/codex-mcp/src/plugin_config_tests.rs @@ -275,6 +275,98 @@ fn environment_placement_rejects_orchestrator_env_vars() { ); } +#[test] +fn remote_environment_placement_rejects_http_env_references() { + let plugin_root = plugin_root(); + let outcome = parse_executor_plugin_mcp_config( + &plugin_root_uri(&plugin_root), + r#"{ + "bearer": { + "url": "https://example.com/bearer", + "bearer_token_env_var": "TOKEN" + }, + "headers": { + "url": "https://example.com/headers", + "env_http_headers": {"Authorization": "TOKEN"} + } + }"#, + "executor-1", + ) + .expect("parse plugin MCP config"); + + assert_eq!( + outcome, + PluginMcpConfigParseOutcome { + servers: BTreeMap::new(), + errors: vec![ + PluginMcpServerParseError { + name: "bearer".to_string(), + message: "`bearer_token_env_var` requires executor-side environment resolution for an executor-owned HTTP MCP" + .to_string(), + }, + PluginMcpServerParseError { + name: "headers".to_string(), + message: "`env_http_headers` requires executor-side environment resolution for an executor-owned HTTP MCP" + .to_string(), + }, + ], + } + ); +} + +#[test] +fn local_environment_placement_preserves_http_env_references() { + let plugin_root = plugin_root(); + let outcome = parse_executor_plugin_mcp_config( + &plugin_root_uri(&plugin_root), + r#"{ + "demo": { + "url": "https://example.com/mcp", + "bearer_token_env_var": "TOKEN", + "env_http_headers": {"X-Account": "ACCOUNT_ID"} + } + }"#, + DEFAULT_MCP_SERVER_ENVIRONMENT_ID, + ) + .expect("parse plugin MCP config"); + + assert_eq!( + outcome, + PluginMcpConfigParseOutcome { + servers: BTreeMap::from([( + "demo".to_string(), + McpServerConfig { + auth: Default::default(), + transport: McpServerTransportConfig::StreamableHttp { + url: "https://example.com/mcp".to_string(), + bearer_token_env_var: Some("TOKEN".to_string()), + http_headers: None, + env_http_headers: Some(HashMap::from([( + "X-Account".to_string(), + "ACCOUNT_ID".to_string(), + )])), + }, + environment_id: DEFAULT_MCP_SERVER_ENVIRONMENT_ID.to_string(), + enabled: true, + required: false, + supports_parallel_tool_calls: false, + disabled_reason: None, + startup_timeout_sec: None, + tool_timeout_sec: None, + default_tools_approval_mode: None, + enabled_tools: None, + disabled_tools: None, + scopes: None, + oauth: None, + oauth_resource: None, + tools: HashMap::new(), + }, + )]), + errors: Vec::new(), + } + ); +} + #[test] fn local_environment_placement_preserves_local_env_vars() { let plugin_root = plugin_root(); diff --git a/codex-rs/codex-mcp/src/rmcp_client.rs b/codex-rs/codex-mcp/src/rmcp_client.rs index 1e8f5a331d..37819203f6 100644 --- a/codex-rs/codex-mcp/src/rmcp_client.rs +++ b/codex-rs/codex-mcp/src/rmcp_client.rs @@ -2,7 +2,7 @@ //! //! This module owns startup of individual RMCP clients: building the transport, //! initializing the server, listing raw tools, applying per-server tool filters, -//! and exposing cached startup snapshots while a client is still connecting. +//! and exposing cached Codex Apps tools while a client is still connecting. //! Higher-level aggregation and resource/tool APIs live in //! [`crate::connection_manager`]. @@ -12,20 +12,18 @@ use std::collections::HashMap; use std::env; use std::ffi::OsString; use std::sync::Arc; +use std::sync::Mutex as StdMutex; use std::sync::atomic::AtomicBool; use std::sync::atomic::Ordering; use std::time::Duration; use std::time::Instant; -use crate::codex_apps::CachedCodexAppsToolsLoad; -use crate::codex_apps::CodexAppsToolsCacheContext; -use crate::codex_apps::load_cached_codex_apps_tools; -use crate::codex_apps::load_startup_cached_codex_apps_server_info; -use crate::codex_apps::load_startup_cached_codex_apps_tools_snapshot; use crate::codex_apps::normalize_codex_apps_callable_name; use crate::codex_apps::normalize_codex_apps_callable_namespace; use crate::codex_apps::normalize_codex_apps_tool_title; -use crate::codex_apps::write_codex_apps_tools_cache; +use crate::codex_apps_cache::CodexAppsToolsCacheContext; +use crate::codex_apps_cache::CodexAppsToolsFetchSource; +use crate::codex_apps_cache::load_startup_cached_codex_apps_server_info; use crate::elicitation::ElicitationRequestManager; use crate::mcp::CODEX_APPS_MCP_SERVER_NAME; use crate::mcp::ToolPluginProvenance; @@ -51,11 +49,15 @@ use codex_exec_server::HttpClient; use codex_exec_server::ReqwestHttpClient; use codex_protocol::mcp::McpServerInfo; use codex_protocol::protocol::Event; +use codex_protocol::protocol::EventMsg; +use codex_protocol::protocol::McpStartupStatus; +use codex_protocol::protocol::McpStartupUpdateEvent; use codex_rmcp_client::ExecutorStdioServerLauncher; use codex_rmcp_client::LocalStdioServerLauncher; use codex_rmcp_client::RmcpClient; use codex_rmcp_client::StdioServerLauncher; use codex_rmcp_client::ToolWithConnectorId; +use codex_rmcp_client::is_authentication_required_error; use futures::future::BoxFuture; use futures::future::FutureExt; use futures::future::Shared; @@ -66,6 +68,7 @@ use rmcp::model::InitializeRequestParams; use rmcp::model::JsonObject; use rmcp::model::ProtocolVersion; use rmcp::model::Tool as RmcpTool; +use tokio::time::Instant as TokioInstant; use tokio_util::sync::CancellationToken; use tracing::Instrument; use tracing::instrument; @@ -82,6 +85,9 @@ pub(crate) const MCP_TOOLS_FETCH_UNCACHED_DURATION_METRIC: &str = pub(crate) const DEFAULT_STARTUP_TIMEOUT: Duration = Duration::from_secs(30); pub(crate) const DEFAULT_TOOL_TIMEOUT: Duration = Duration::from_secs(300); +pub(crate) const CODEX_APPS_RECONNECT_INITIAL_BACKOFF: Duration = Duration::from_secs(1); +const CODEX_APPS_RECONNECT_MAX_BACKOFF: Duration = Duration::from_secs(30); + const UNTRUSTED_CONNECTOR_META_KEYS: &[&str] = &[ "connector_id", "connector_name", @@ -105,9 +111,10 @@ pub(crate) struct ManagedClient { impl ManagedClient { fn listed_tools(&self) -> Vec { let total_start = Instant::now(); - if let Some(cache_context) = self.codex_apps_tools_cache_context.as_ref() - && let CachedCodexAppsToolsLoad::Hit(tools) = - load_cached_codex_apps_tools(cache_context) + if let Some(tools) = self + .codex_apps_tools_cache_context + .as_ref() + .and_then(CodexAppsToolsCacheContext::current_tools) { emit_duration( MCP_TOOLS_LIST_DURATION_METRIC, @@ -129,59 +136,174 @@ impl ManagedClient { } } -#[derive(Clone)] -pub(crate) struct AsyncManagedClient { - pub(crate) client: Shared>>, - pub(crate) is_codex_apps_mcp_server: bool, - pub(crate) cached_tool_info_snapshot: Option>, - pub(crate) cached_server_info: Option, - pub(crate) startup_complete: Arc, - pub(crate) tool_plugin_provenance: Arc, - pub(crate) cancel_token: CancellationToken, +pub(crate) type ManagedClientFuture = + Shared>>; + +#[derive(Default)] +struct CodexAppsStartupReconnectState { + current_client: Option, + reconnect_in_flight: bool, + consecutive_failures: u32, + retry_not_before: Option, } -impl AsyncManagedClient { - // Keep this constructor flat so the startup inputs remain readable at the - // single call site instead of introducing a one-off params wrapper. - #[instrument(level = "trace", skip_all, fields(server_name = %server_name))] - #[allow(clippy::too_many_arguments)] - pub(crate) fn new( +#[derive(Clone)] +struct CodexAppsStartupStatusContext { + submit_id: String, + server_name: String, + tx_event: Sender, +} + +pub(crate) struct CodexAppsStartupReconnect { + factory: Arc ManagedClientFuture + Send + Sync>, + state: StdMutex, + startup_status_context: Option, +} + +impl CodexAppsStartupReconnect { + pub(crate) fn new(factory: Arc ManagedClientFuture + Send + Sync>) -> Self { + Self { + factory, + state: StdMutex::new(CodexAppsStartupReconnectState::default()), + startup_status_context: None, + } + } + + fn with_startup_status_context( + mut self, + submit_id: String, server_name: String, - server: EffectiveMcpServer, - store_mode: OAuthCredentialsStoreMode, - keyring_backend_kind: AuthKeyringBackendKind, - cancel_token: CancellationToken, tx_event: Sender, - elicitation_requests: ElicitationRequestManager, - codex_apps_tools_cache_context: Option, - tool_plugin_provenance: Arc, - runtime_context: McpRuntimeContext, - runtime_auth_provider: Option, - client_elicitation_capability: ElicitationCapability, - supports_openai_form_elicitation: bool, ) -> Self { + self.startup_status_context = Some(CodexAppsStartupStatusContext { + submit_id, + server_name, + tx_event, + }); + self + } + + fn current_client(&self) -> Option { + self.state + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .current_client + .clone() + } + + fn reconnect_in_background(self: &Arc) { + { + let mut state = self + .state + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if state.current_client.is_some() || state.reconnect_in_flight { + return; + } + if state + .retry_not_before + .is_some_and(|retry_not_before| TokioInstant::now() < retry_not_before) + { + return; + } + state.reconnect_in_flight = true; + } + + let reconnect = Arc::clone(self); + tokio::spawn(async move { + let result = (reconnect.factory)().await; + let startup_status_context = reconnect.startup_status_context.clone(); + let recovered = { + let mut state = reconnect + .state + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + state.reconnect_in_flight = false; + match result { + Ok(client) => { + state.current_client = Some(client); + state.consecutive_failures = 0; + state.retry_not_before = None; + true + } + Err(error) => { + state.consecutive_failures = state.consecutive_failures.saturating_add(1); + let retry_after = codex_apps_reconnect_backoff(state.consecutive_failures); + state.retry_not_before = Some(TokioInstant::now() + retry_after); + warn!( + error = %error, + retry_after_ms = retry_after.as_millis(), + "Apps MCP startup reconnect failed; continuing with cached tools" + ); + false + } + } + }; + + if recovered && let Some(context) = startup_status_context { + let _ = context + .tx_event + .send(Event { + id: context.submit_id, + msg: EventMsg::McpStartupUpdate(McpStartupUpdateEvent { + server: context.server_name, + status: McpStartupStatus::Ready, + }), + }) + .await; + } + }); + } +} + +fn codex_apps_reconnect_backoff(consecutive_failures: u32) -> Duration { + let exponent = consecutive_failures.saturating_sub(1).min(5); + CODEX_APPS_RECONNECT_INITIAL_BACKOFF + .saturating_mul(1 << exponent) + .min(CODEX_APPS_RECONNECT_MAX_BACKOFF) +} + +#[derive(Clone)] +struct ManagedClientStartup { + server_name: String, + server: EffectiveMcpServer, + store_mode: OAuthCredentialsStoreMode, + keyring_backend_kind: AuthKeyringBackendKind, + tx_event: Sender, + elicitation_requests: ElicitationRequestManager, + codex_apps_tools_cache_context: Option, + runtime_context: McpRuntimeContext, + runtime_auth_provider: Option, + client_elicitation_capability: ElicitationCapability, + supports_openai_form_elicitation: bool, + cancel_token: CancellationToken, + startup_complete: Arc, +} + +impl ManagedClientStartup { + fn start(&self) -> ManagedClientFuture { + let Self { + server_name, + server, + store_mode, + keyring_backend_kind, + tx_event, + elicitation_requests, + codex_apps_tools_cache_context, + runtime_context, + runtime_auth_provider, + client_elicitation_capability, + supports_openai_form_elicitation, + cancel_token, + startup_complete, + } = self.clone(); let is_codex_apps_mcp_server = server_name == CODEX_APPS_MCP_SERVER_NAME; let tool_filter = server .configured_config() .map(ToolFilter::from_config) .unwrap_or_default(); - let (cached_tool_info_snapshot, cached_server_info) = if is_codex_apps_mcp_server { - ( - load_startup_cached_codex_apps_tools_snapshot( - codex_apps_tools_cache_context.as_ref(), - ), - load_startup_cached_codex_apps_server_info(codex_apps_tools_cache_context.as_ref()), - ) - } else { - (None, None) - }; - let cached_tool_info_snapshot = - cached_tool_info_snapshot.map(|tools| filter_tools(tools, &tool_filter)); - let startup_tool_filter = tool_filter; - let startup_complete = Arc::new(AtomicBool::new(false)); - let startup_complete_for_fut = Arc::clone(&startup_complete); - let cancel_token_for_fut = cancel_token.clone(); - let fut = async move { + let cancel_token_for_fut = cancel_token; + async move { let outcome = match async { if let Err(error) = validate_mcp_server_name(&server_name) { return Err(error.into()); @@ -211,7 +333,7 @@ impl AsyncManagedClient { .configured_config() .and_then(|config| config.tool_timeout_sec) .unwrap_or(DEFAULT_TOOL_TIMEOUT), - tool_filter: startup_tool_filter, + tool_filter, tx_event, elicitation_requests, codex_apps_tools_cache_context, @@ -228,11 +350,95 @@ impl AsyncManagedClient { Err(CancelErr::Cancelled) => Err(StartupOutcomeError::Cancelled), }; - startup_complete_for_fut.store(true, Ordering::Release); + startup_complete.store(true, Ordering::Release); outcome + } + .in_current_span() + .boxed() + .shared() + } +} + +#[derive(Clone)] +pub(crate) struct AsyncManagedClient { + pub(crate) client: ManagedClientFuture, + pub(crate) is_codex_apps_mcp_server: bool, + pub(crate) cached_server_info: Option, + pub(crate) codex_apps_tools_cache_context: Option, + pub(crate) tool_filter: ToolFilter, + pub(crate) startup_complete: Arc, + pub(crate) startup_reconnect: Option>, + pub(crate) tool_plugin_provenance: Arc, + pub(crate) cancel_token: CancellationToken, +} + +impl AsyncManagedClient { + // Keep this constructor flat so the startup inputs remain readable at the + // single call site instead of introducing a one-off params wrapper. + #[instrument(level = "trace", skip_all, fields(server_name = %server_name))] + #[allow(clippy::too_many_arguments)] + pub(crate) fn new( + server_name: String, + startup_submit_id: String, + server: EffectiveMcpServer, + store_mode: OAuthCredentialsStoreMode, + keyring_backend_kind: AuthKeyringBackendKind, + cancel_token: CancellationToken, + tx_event: Sender, + elicitation_requests: ElicitationRequestManager, + codex_apps_tools_cache_context: Option, + tool_plugin_provenance: Arc, + runtime_context: McpRuntimeContext, + runtime_auth_provider: Option, + client_elicitation_capability: ElicitationCapability, + supports_openai_form_elicitation: bool, + ) -> Self { + let is_codex_apps_mcp_server = server_name == CODEX_APPS_MCP_SERVER_NAME; + let reconnect_server_name = server_name.clone(); + let reconnect_tx_event = tx_event.clone(); + let tool_filter = server + .configured_config() + .map(ToolFilter::from_config) + .unwrap_or_default(); + let cached_server_info = if is_codex_apps_mcp_server { + codex_apps_tools_cache_context + .as_ref() + .and_then(load_startup_cached_codex_apps_server_info) + } else { + None }; - let client = fut.in_current_span().boxed().shared(); - if cached_tool_info_snapshot.is_some() { + let startup_complete = Arc::new(AtomicBool::new(false)); + let startup = Arc::new(ManagedClientStartup { + server_name, + server, + store_mode, + keyring_backend_kind, + tx_event, + elicitation_requests, + codex_apps_tools_cache_context: codex_apps_tools_cache_context.clone(), + runtime_context, + runtime_auth_provider, + client_elicitation_capability, + supports_openai_form_elicitation, + cancel_token: cancel_token.clone(), + startup_complete: Arc::clone(&startup_complete), + }); + let client = startup.start(); + let startup_reconnect = is_codex_apps_mcp_server.then(|| { + let startup = Arc::clone(&startup); + Arc::new( + CodexAppsStartupReconnect::new(Arc::new(move || startup.start())) + .with_startup_status_context( + startup_submit_id, + reconnect_server_name, + reconnect_tx_event, + ), + ) + }); + if codex_apps_tools_cache_context + .as_ref() + .is_some_and(CodexAppsToolsCacheContext::has_current_tools) + { let startup_task = client.clone(); tokio::spawn(async move { let _ = startup_task.await; @@ -242,18 +448,39 @@ impl AsyncManagedClient { Self { client, is_codex_apps_mcp_server, - cached_tool_info_snapshot, cached_server_info, + codex_apps_tools_cache_context, + tool_filter, startup_complete, + startup_reconnect, tool_plugin_provenance, cancel_token, } } pub(crate) async fn client(&self) -> Result { + if let Some(client) = self + .startup_reconnect + .as_ref() + .and_then(|reconnect| reconnect.current_client()) + { + return Ok(client); + } self.client.clone().await } + pub(crate) async fn reconnect_failed_startup(&self) { + let Some(startup_reconnect) = self.startup_reconnect.as_ref() else { + return; + }; + if !self.startup_complete.load(Ordering::Acquire) { + return; + } + if matches!(self.client().await, Err(StartupOutcomeError::Failed { .. })) { + startup_reconnect.reconnect_in_background(); + } + } + pub(crate) async fn shutdown(&self) { self.cancel_token.cancel(); match self.client().await { @@ -265,15 +492,29 @@ impl AsyncManagedClient { } } + pub(crate) fn has_cached_tools(&self) -> bool { + self.codex_apps_tools_cache_context + .as_ref() + .is_some_and(CodexAppsToolsCacheContext::has_current_tools) + } + + fn cached_tools(&self) -> Option> { + self.codex_apps_tools_cache_context + .as_ref() + .and_then(CodexAppsToolsCacheContext::current_tools) + .map(|tools| filter_tools(tools, &self.tool_filter)) + } + pub(crate) async fn listed_tools(&self) -> Option> { // Keep cache payloads raw; plugin provenance is resolved per-session at read time. - let tools = if let Some(startup_tools) = self.cached_tool_info_snapshot_while_initializing() + let tools = if !self.startup_complete.load(Ordering::Acquire) + && let Some(startup_tools) = self.cached_tools() { Some(startup_tools) } else { match self.client().await { Ok(client) => Some(client.listed_tools()), - Err(_) => self.cached_tool_info_snapshot.clone(), + Err(_) => self.cached_tools(), } }?; Some(if self.is_codex_apps_mcp_server { @@ -282,13 +523,6 @@ impl AsyncManagedClient { prepare_regular_mcp_tools_for_model(tools, &self.tool_plugin_provenance) }) } - - fn cached_tool_info_snapshot_while_initializing(&self) -> Option> { - if !self.startup_complete.load(Ordering::Acquire) { - return self.cached_tool_info_snapshot.clone(); - } - None - } } #[derive(Debug, Clone, thiserror::Error)] @@ -298,13 +532,30 @@ pub(crate) enum StartupOutcomeError { // We can't store the original error here because anyhow::Error doesn't implement // `Clone`. #[error("MCP startup failed: {error}")] - Failed { error: String }, + Failed { + error: String, + is_authentication_required: bool, + }, +} + +impl StartupOutcomeError { + pub(crate) fn is_authentication_required(&self) -> bool { + match self { + Self::Cancelled => false, + Self::Failed { + is_authentication_required, + .. + } => *is_authentication_required, + } + } } impl From for StartupOutcomeError { fn from(error: anyhow::Error) -> Self { + let is_authentication_required = is_authentication_required_error(&error); Self::Failed { error: error.to_string(), + is_authentication_required, } } } @@ -570,6 +821,9 @@ async fn start_server_task( .is_some(); let list_start = Instant::now(); let fetch_start = Instant::now(); + let fetch_ticket = codex_apps_tools_cache_context + .as_ref() + .map(|cache_context| cache_context.begin_fetch(CodexAppsToolsFetchSource::Startup)); let tools = list_tools_for_client_uncached( &server_name, is_codex_apps_mcp_server, @@ -585,21 +839,20 @@ async fn start_server_task( &[], ); let server_info = mcp_server_info_from_implementation(initialize_result.server_info); - let codex_apps_tools_cache_context = if is_codex_apps_mcp_server { - write_codex_apps_tools_cache( - codex_apps_tools_cache_context.as_ref(), - &server_info, - &tools, - ); + let tools = match (codex_apps_tools_cache_context.as_ref(), fetch_ticket) { + (Some(cache_context), Some(fetch_ticket)) => { + cache_context.publish_if_newest_accepted(fetch_ticket, &server_info, tools) + } + (None, None) => tools, + _ => unreachable!("Codex Apps fetch ticket requires cache context"), + }; + if is_codex_apps_mcp_server { emit_duration( MCP_TOOLS_LIST_DURATION_METRIC, list_start.elapsed(), &[("cache", "miss")], ); - codex_apps_tools_cache_context - } else { - None - }; + } let tools = filter_tools(tools, &tool_filter); let managed = ManagedClient { @@ -757,6 +1010,17 @@ mod tests { use pretty_assertions::assert_eq; use rmcp::model::JsonObject; use rmcp::model::Meta; + use rmcp::transport::auth::AuthError; + + #[test] + fn startup_outcome_error_identifies_authentication_required() { + let error = anyhow::Error::new(AuthError::AuthorizationRequired) + .context("failed to initialize MCP server"); + + let error = StartupOutcomeError::from(error); + + assert!(error.is_authentication_required()); + } #[test] fn mcp_initialize_advertises_openai_form_only_when_supported() { diff --git a/codex-rs/codex-mcp/src/runtime.rs b/codex-rs/codex-mcp/src/runtime.rs index be9d50d3f4..b9b769ffb4 100644 --- a/codex-rs/codex-mcp/src/runtime.rs +++ b/codex-rs/codex-mcp/src/runtime.rs @@ -11,6 +11,8 @@ use std::time::Duration; use codex_exec_server::Environment; use codex_exec_server::EnvironmentManager; +use codex_exec_server::HttpClient; +use codex_exec_server::ReqwestHttpClient; use codex_protocol::models::PermissionProfile; use codex_utils_path_uri::PathUri; use serde::Deserialize; @@ -81,6 +83,20 @@ impl McpRuntimeContext { config.environment_id )) } + + /// Resolves the HTTP capability owned by the server's configured environment. + pub fn resolve_http_client( + &self, + server_name: &str, + config: &codex_config::McpServerConfig, + ) -> Result, String> { + Ok(self + .resolve_server_environment(server_name, config)? + .map_or_else( + || Arc::new(ReqwestHttpClient) as Arc, + |environment| environment.get_http_client(), + )) + } } pub(crate) fn emit_duration(metric: &str, duration: Duration, tags: &[(&str, &str)]) { diff --git a/codex-rs/config/Cargo.toml b/codex-rs/config/Cargo.toml index 509ffd189a..4fe529d743 100644 --- a/codex-rs/config/Cargo.toml +++ b/codex-rs/config/Cargo.toml @@ -30,6 +30,7 @@ gethostname = { workspace = true } indexmap = { workspace = true, features = ["serde"] } multimap = { workspace = true } prost = "0.14.3" +regex-lite = { workspace = true } schemars = { workspace = true } serde = { workspace = true, features = ["derive"] } serde_ignored = { workspace = true } diff --git a/codex-rs/config/src/config_requirements.rs b/codex-rs/config/src/config_requirements.rs index e757c2149c..00af972336 100644 --- a/codex-rs/config/src/config_requirements.rs +++ b/codex-rs/config/src/config_requirements.rs @@ -2,6 +2,7 @@ use codex_protocol::config_types::ApprovalsReviewer; use codex_protocol::config_types::SandboxMode; use codex_protocol::config_types::WebSearchMode; use codex_protocol::models::PermissionProfile; +use codex_protocol::openai_models::ReasoningEffort; use codex_protocol::protocol::AskForApproval; use codex_utils_absolute_path::AbsolutePathBuf; use serde::Deserialize; @@ -19,6 +20,7 @@ use super::requirements_exec_policy::RequirementsExecPolicyToml; use crate::Constrained; use crate::ConstraintError; use crate::ManagedHooksRequirementsToml; +use crate::mcp_requirements::McpServerRequirement; use crate::mcp_types::AppToolApproval; use crate::permissions_toml::PermissionProfileToml; use crate::types::WindowsSandboxModeToml; @@ -217,18 +219,6 @@ impl ConfigRequirements { } } -#[derive(Deserialize, Debug, Clone, PartialEq, Eq)] -#[serde(untagged)] -pub enum McpServerIdentity { - Command { command: String }, - Url { url: String }, -} - -#[derive(Deserialize, Debug, Clone, PartialEq, Eq)] -pub struct McpServerRequirement { - pub identity: McpServerIdentity, -} - #[derive(Deserialize, Debug, Clone, Default, PartialEq, Eq)] pub struct PluginRequirementsToml { pub mcp_servers: Option>, @@ -887,9 +877,36 @@ pub struct ConfigRequirementsToml { #[serde(rename = "experimental_network")] pub network: Option, pub permissions: Option, + pub models: Option, pub guardian_policy_config: Option, } +#[derive(Deserialize, Debug, Clone, Default, PartialEq, Eq)] +pub struct ModelsRequirementsToml { + pub new_thread: Option, +} + +impl ModelsRequirementsToml { + fn is_empty(&self) -> bool { + self.new_thread + .as_ref() + .is_none_or(NewThreadModelDefaultsToml::is_empty) + } +} + +#[derive(Deserialize, Debug, Clone, Default, PartialEq, Eq)] +pub struct NewThreadModelDefaultsToml { + pub model: Option, + pub model_reasoning_effort: Option, + pub service_tier: Option, +} + +impl NewThreadModelDefaultsToml { + fn is_empty(&self) -> bool { + self.model.is_none() && self.model_reasoning_effort.is_none() && self.service_tier.is_none() + } +} + #[derive(Deserialize, Debug, Clone, PartialEq)] pub struct RemoteSandboxConfigToml { pub hostname_patterns: Vec, @@ -941,6 +958,7 @@ pub struct ConfigRequirementsWithSources { pub enforce_residency: Option>, pub network: Option>, pub permissions: Option>, + pub models: Option>, pub guardian_policy_config: Option>, } @@ -985,6 +1003,7 @@ impl ConfigRequirementsWithSources { enforce_residency: _, network: _, permissions: _, + models: _, guardian_policy_config: _, } = &other; @@ -1021,6 +1040,7 @@ impl ConfigRequirementsWithSources { enforce_residency, network, permissions, + models, guardian_policy_config, } ); @@ -1057,6 +1077,7 @@ impl ConfigRequirementsWithSources { enforce_residency, network, permissions, + models, guardian_policy_config, } = self; ConfigRequirementsToml { @@ -1082,6 +1103,7 @@ impl ConfigRequirementsWithSources { enforce_residency: enforce_residency.map(|sourced| sourced.value), network: network.map(|sourced| sourced.value), permissions: permissions.map(|sourced| sourced.value), + models: models.map(|sourced| sourced.value), guardian_policy_config: guardian_policy_config.map(|sourced| sourced.value), } } @@ -1194,6 +1216,10 @@ impl ConfigRequirementsToml { && self.enforce_residency.is_none() && self.network.is_none() && self.permissions.is_none() + && self + .models + .as_ref() + .is_none_or(ModelsRequirementsToml::is_empty) && self .guardian_policy_config .as_deref() @@ -1201,13 +1227,33 @@ impl ConfigRequirementsToml { } } +fn validate_mcp_server_requirements( + requirements: &BTreeMap, + source: &RequirementSource, + plugin_name: Option<&str>, +) -> Result<(), ConstraintError> { + for (server_name, requirement) in requirements { + requirement + .validate() + .map_err(|reason| ConstraintError::McpServerRequirementParse { + server_name: plugin_name + .map(|plugin_name| format!("{plugin_name}/{server_name}")) + .unwrap_or_else(|| server_name.clone()), + requirement_source: source.clone(), + reason, + })?; + } + Ok(()) +} + impl TryFrom for ConfigRequirements { type Error = ConstraintError; fn try_from(toml: ConfigRequirementsWithSources) -> Result { // Profile catalog selection remains on ConfigRequirementsToml for - // config loading and requirements API projection. The normalized - // constraints below only need the compiled PermissionProfile envelope. + // config loading and requirements API projection. Managed new-thread + // defaults also remain there because they are initialization values, + // not runtime constraints. let ConfigRequirementsWithSources { allowed_approval_policies, allowed_approvals_reviewers, @@ -1230,9 +1276,29 @@ impl TryFrom for ConfigRequirements { enforce_residency, network, permissions, + models: _, guardian_policy_config, } = toml; + if let Some(requirements) = &mcp_servers { + validate_mcp_server_requirements( + &requirements.value, + &requirements.source, + /*plugin_name*/ None, + )?; + } + if let Some(plugin_requirements) = &plugins { + for (plugin_name, plugin) in &plugin_requirements.value { + if let Some(requirements) = &plugin.mcp_servers { + validate_mcp_server_requirements( + requirements, + &plugin_requirements.source, + Some(plugin_name), + )?; + } + } + } + let approval_policy = match allowed_approval_policies { Some(Sourced { value: policies, @@ -1542,6 +1608,9 @@ pub fn sandbox_mode_requirement_for_permission_profile( mod tests { use super::*; use crate::HookEventsToml; + use crate::McpServerCommandMatcher; + use crate::McpServerIdentity; + use crate::McpServerValueMatcher; use anyhow::Result; use codex_execpolicy::Decision; use codex_execpolicy::Evaluation; @@ -1605,6 +1674,7 @@ mod tests { enforce_residency, network, permissions, + models, guardian_policy_config, } = toml; ConfigRequirementsWithSources { @@ -1640,6 +1710,7 @@ mod tests { .map(|value| Sourced::new(value, RequirementSource::Unknown)), network: network.map(|value| Sourced::new(value, RequirementSource::Unknown)), permissions: permissions.map(|value| Sourced::new(value, RequirementSource::Unknown)), + models: models.map(|value| Sourced::new(value, RequirementSource::Unknown)), guardian_policy_config: guardian_policy_config .map(|value| Sourced::new(value, RequirementSource::Unknown)), } @@ -1792,6 +1863,31 @@ mod tests { Ok(()) } + #[test] + fn deserialize_new_thread_model_defaults() -> Result<()> { + let requirements: ConfigRequirementsToml = from_str( + r#" + [models.new_thread] + model = "managed-model" + model_reasoning_effort = "medium" + service_tier = "fast" + "#, + )?; + + assert_eq!( + requirements.models, + Some(ModelsRequirementsToml { + new_thread: Some(NewThreadModelDefaultsToml { + model: Some("managed-model".to_string()), + model_reasoning_effort: Some(ReasoningEffort::Medium), + service_tier: Some("fast".to_string()), + }), + }) + ); + assert!(!requirements.is_empty()); + Ok(()) + } + #[test] fn merge_unset_fields_copies_every_field_and_sets_sources() { let mut target = ConfigRequirementsWithSources::default(); @@ -1814,6 +1910,13 @@ mod tests { let computer_use = ComputerUseRequirementsToml { allow_locked_computer_use: Some(false), }; + let models = ModelsRequirementsToml { + new_thread: Some(NewThreadModelDefaultsToml { + model: Some("managed-model".to_string()), + model_reasoning_effort: Some(ReasoningEffort::Medium), + service_tier: Some("fast".to_string()), + }), + }; let enforce_residency = ResidencyRequirement::Us; let enforce_source = source.clone(); let guardian_policy_config = "Use the company-managed guardian policy.".to_string(); @@ -1843,6 +1946,7 @@ mod tests { enforce_residency: Some(enforce_residency), network: None, permissions: None, + models: Some(models.clone()), guardian_policy_config: Some(guardian_policy_config.clone()), }; @@ -1893,6 +1997,7 @@ mod tests { enforce_residency: Some(Sourced::new(enforce_residency, enforce_source)), network: None, permissions: None, + models: Some(Sourced::new(models, source.clone())), guardian_policy_config: Some(Sourced::new(guardian_policy_config, source)), } ); @@ -1940,6 +2045,7 @@ mod tests { enforce_residency: None, network: None, permissions: None, + models: None, guardian_policy_config: None, } ); @@ -1995,6 +2101,7 @@ mod tests { enforce_residency: None, network: None, permissions: None, + models: None, guardian_policy_config: None, } ); @@ -3454,6 +3561,9 @@ command = "python3 /enterprise/hooks/pre.py" #[test] fn deserialize_mcp_server_requirements() -> Result<()> { let toml_str = r#" + [mcp_servers.docs] + description = "ignored legacy field" + [mcp_servers.docs.identity] command = "codex-mcp" @@ -3469,7 +3579,7 @@ command = "python3 /enterprise/hooks/pre.py" BTreeMap::from([ ( "docs".to_string(), - McpServerRequirement { + McpServerRequirement::Identity { identity: McpServerIdentity::Command { command: "codex-mcp".to_string(), }, @@ -3477,7 +3587,7 @@ command = "python3 /enterprise/hooks/pre.py" ), ( "remote".to_string(), - McpServerRequirement { + McpServerRequirement::Identity { identity: McpServerIdentity::Url { url: "https://example.com/mcp".to_string(), }, @@ -3490,6 +3600,74 @@ command = "python3 /enterprise/hooks/pre.py" Ok(()) } + #[test] + fn deserialize_mcp_server_matcher_requirements() -> Result<()> { + let toml_str = r#" + [mcp_servers.internal_mcp_proxy.identity] + command = { executable = "company-cli", args = [ + { match = "exact", value = "mcp" }, + { match = "exact", value = "proxy" }, + { match = "exact", value = "--server" }, + { match = "regex", expression = '^https://[A-Za-z0-9-]+\.mcp\.internal\.example\.com(?::443)?(?:/.*)?$' }, + ] } + "#; + let requirements: ConfigRequirements = + with_unknown_source(from_str(toml_str)?).try_into()?; + + assert_eq!( + requirements.mcp_servers, + Some(Sourced::new( + BTreeMap::from([( + "internal_mcp_proxy".to_string(), + McpServerRequirement::Command(McpServerCommandMatcher { + executable: "company-cli".to_string(), + args: vec![ + McpServerValueMatcher::Exact { + value: "mcp".to_string(), + }, + McpServerValueMatcher::Exact { + value: "proxy".to_string(), + }, + McpServerValueMatcher::Exact { + value: "--server".to_string(), + }, + McpServerValueMatcher::Regex { + expression: r"^https://[A-Za-z0-9-]+\.mcp\.internal\.example\.com(?::443)?(?:/.*)?$" + .to_string(), + }, + ], + }), + )]), + RequirementSource::Unknown, + )) + ); + Ok(()) + } + + #[test] + fn invalid_mcp_server_requirement_regex_reports_the_server_name_and_source() -> Result<()> { + let toml_str = r#" + [mcp_servers.broken_rule.identity] + url = { match = "regex", expression = "[" } + "#; + + let err = ConfigRequirements::try_from(with_unknown_source(from_str(toml_str)?)) + .expect_err("invalid matcher regex should fail requirements normalization"); + let ConstraintError::McpServerRequirementParse { + server_name, + requirement_source, + reason, + } = err + else { + panic!("unexpected error: {err:?}"); + }; + + assert_eq!(server_name, "broken_rule"); + assert_eq!(requirement_source, RequirementSource::Unknown); + assert!(reason.contains("invalid regex `[`"), "{reason}"); + Ok(()) + } + #[test] fn deserialize_plugin_mcp_server_requirements() -> Result<()> { let toml_str = r#" @@ -3511,7 +3689,7 @@ command = "python3 /enterprise/hooks/pre.py" PluginRequirementsToml { mcp_servers: Some(BTreeMap::from([( "remote".to_string(), - McpServerRequirement { + McpServerRequirement::Identity { identity: McpServerIdentity::Url { url: "https://example.com/mcp".to_string(), }, @@ -3524,7 +3702,7 @@ command = "python3 /enterprise/hooks/pre.py" PluginRequirementsToml { mcp_servers: Some(BTreeMap::from([( "sample".to_string(), - McpServerRequirement { + McpServerRequirement::Identity { identity: McpServerIdentity::Command { command: "sample-mcp".to_string(), }, @@ -3539,6 +3717,70 @@ command = "python3 /enterprise/hooks/pre.py" Ok(()) } + #[test] + fn deserialize_plugin_mcp_server_matcher_requirement() -> Result<()> { + let toml_str = r#" + [plugins."sample@test".mcp_servers.internal_proxy.identity] + command = { executable = "company-cli", args = [ + { match = "exact", value = "mcp" }, + { match = "regex", expression = '^https://[a-z]+\.example\.com$' }, + ] } + "#; + let requirements: ConfigRequirements = + with_unknown_source(from_str(toml_str)?).try_into()?; + + assert_eq!( + requirements.plugins, + Some(Sourced::new( + BTreeMap::from([( + "sample@test".to_string(), + PluginRequirementsToml { + mcp_servers: Some(BTreeMap::from([( + "internal_proxy".to_string(), + McpServerRequirement::Command(McpServerCommandMatcher { + executable: "company-cli".to_string(), + args: vec![ + McpServerValueMatcher::Exact { + value: "mcp".to_string(), + }, + McpServerValueMatcher::Regex { + expression: r"^https://[a-z]+\.example\.com$".to_string(), + }, + ], + }), + )])), + }, + )]), + RequirementSource::Unknown, + )) + ); + Ok(()) + } + + #[test] + fn invalid_plugin_mcp_server_regex_reports_plugin_and_server_name() -> Result<()> { + let toml_str = r#" + [plugins."sample@test".mcp_servers.broken_rule.identity] + url = { match = "regex", expression = "[" } + "#; + + let err = ConfigRequirements::try_from(with_unknown_source(from_str(toml_str)?)) + .expect_err("invalid plugin MCP regex should fail requirements normalization"); + let ConstraintError::McpServerRequirementParse { + server_name, + requirement_source, + reason, + } = err + else { + panic!("unexpected error: {err:?}"); + }; + + assert_eq!(server_name, "sample@test/broken_rule"); + assert_eq!(requirement_source, RequirementSource::Unknown); + assert!(reason.contains("invalid regex `[`"), "{reason}"); + Ok(()) + } + #[test] fn deserialize_exec_policy_requirements() -> Result<()> { let toml_str = r#" diff --git a/codex-rs/config/src/constraint.rs b/codex-rs/config/src/constraint.rs index 64b604cbc2..8628f3909c 100644 --- a/codex-rs/config/src/constraint.rs +++ b/codex-rs/config/src/constraint.rs @@ -24,6 +24,15 @@ pub enum ConstraintError { requirement_source: RequirementSource, reason: String, }, + + #[error( + "invalid requirement for MCP server `{server_name}` (set by {requirement_source}): {reason}" + )] + McpServerRequirementParse { + server_name: String, + requirement_source: RequirementSource, + reason: String, + }, } impl ConstraintError { diff --git a/codex-rs/config/src/hook_config.rs b/codex-rs/config/src/hook_config.rs index a7ee820e60..8e18cb935a 100644 --- a/codex-rs/config/src/hook_config.rs +++ b/codex-rs/config/src/hook_config.rs @@ -10,6 +10,8 @@ use serde::Serialize; #[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(deny_unknown_fields)] pub struct HooksFile { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub description: Option, #[serde(default)] pub hooks: HookEventsToml, } diff --git a/codex-rs/config/src/hooks_tests.rs b/codex-rs/config/src/hooks_tests.rs index a2eb2d2322..3d51840fc2 100644 --- a/codex-rs/config/src/hooks_tests.rs +++ b/codex-rs/config/src/hooks_tests.rs @@ -13,6 +13,7 @@ use super::MatcherGroup; fn hooks_file_deserializes_existing_json_shape() { let parsed: HooksFile = serde_json::from_str( r#"{ + "description": "Optional stop-time review gate for Codex Companion.", "hooks": { "PreToolUse": [ { @@ -35,6 +36,7 @@ fn hooks_file_deserializes_existing_json_shape() { assert_eq!( parsed, HooksFile { + description: Some("Optional stop-time review gate for Codex Companion.".to_string()), hooks: HookEventsToml { pre_tool_use: vec![MatcherGroup { matcher: Some("^Bash$".to_string()), diff --git a/codex-rs/config/src/lib.rs b/codex-rs/config/src/lib.rs index afda2cf9f3..df3e45d76d 100644 --- a/codex-rs/config/src/lib.rs +++ b/codex-rs/config/src/lib.rs @@ -12,6 +12,7 @@ mod key_aliases; pub mod loader; mod marketplace_edit; mod mcp_edit; +mod mcp_requirements; mod mcp_types; mod merge; mod overrides; @@ -66,14 +67,14 @@ pub use config_requirements::FilesystemDenyReadPattern; pub use config_requirements::MarketplaceAllowedSourceKind; pub use config_requirements::MarketplaceAllowedSourceToml; pub use config_requirements::MarketplaceRequirementsToml; -pub use config_requirements::McpServerIdentity; -pub use config_requirements::McpServerRequirement; +pub use config_requirements::ModelsRequirementsToml; pub use config_requirements::NetworkConstraints; pub use config_requirements::NetworkDomainPermissionToml; pub use config_requirements::NetworkDomainPermissionsToml; pub use config_requirements::NetworkRequirementsToml; pub use config_requirements::NetworkUnixSocketPermissionToml; pub use config_requirements::NetworkUnixSocketPermissionsToml; +pub use config_requirements::NewThreadModelDefaultsToml; pub use config_requirements::PluginRequirementsToml; pub use config_requirements::RemoteSandboxConfigToml; pub use config_requirements::RequirementSource; @@ -113,6 +114,10 @@ pub use marketplace_edit::remove_user_marketplace; pub use marketplace_edit::remove_user_marketplace_config; pub use mcp_edit::ConfigEditsBuilder; pub use mcp_edit::load_global_mcp_servers; +pub use mcp_requirements::McpServerCommandMatcher; +pub use mcp_requirements::McpServerIdentity; +pub use mcp_requirements::McpServerRequirement; +pub use mcp_requirements::McpServerValueMatcher; pub use mcp_types::AppToolApproval; pub use mcp_types::DEFAULT_MCP_SERVER_ENVIRONMENT_ID; pub use mcp_types::McpServerAuth; diff --git a/codex-rs/config/src/mcp_requirements.rs b/codex-rs/config/src/mcp_requirements.rs new file mode 100644 index 0000000000..b8f9a12d69 --- /dev/null +++ b/codex-rs/config/src/mcp_requirements.rs @@ -0,0 +1,163 @@ +use crate::mcp_types::McpServerConfig; +use crate::mcp_types::McpServerTransportConfig; +use regex_lite::Regex; +use serde::Deserialize; + +#[derive(Deserialize, Debug, Clone, PartialEq, Eq)] +#[serde(untagged)] +pub enum McpServerIdentity { + Command { command: String }, + Url { url: String }, +} + +/// String matching operations available to managed MCP server matchers. +#[derive(Deserialize, Debug, Clone, PartialEq, Eq)] +#[serde(tag = "match", rename_all = "snake_case", deny_unknown_fields)] +pub enum McpServerValueMatcher { + Exact { value: String }, + Prefix { value: String }, + Regex { expression: String }, +} + +impl McpServerValueMatcher { + fn compile_full_regex(expression: &str) -> Result { + Regex::new(&format!(r"\A(?:{expression})\z")).map_err(|err| { + format!("regex `{expression}` cannot be used for full-value matching: {err}") + }) + } + + fn validate(&self) -> Result<(), String> { + let Self::Regex { expression } = self else { + return Ok(()); + }; + + Regex::new(expression).map_err(|err| format!("invalid regex `{expression}`: {err}"))?; + Self::compile_full_regex(expression).map(|_| ()) + } + + fn matches(&self, candidate: &str) -> bool { + match self { + Self::Exact { value } => candidate == value, + Self::Prefix { value } => candidate.starts_with(value), + Self::Regex { expression } => Self::compile_full_regex(expression) + .ok() + .is_some_and(|regex| regex.is_match(candidate)), + } + } +} + +#[derive(Deserialize, Debug, Clone, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct McpServerCommandMatcher { + pub executable: String, + pub args: Vec, +} + +#[derive(Deserialize, Debug, Clone, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +struct RawMcpServerCommandIdentity { + command: McpServerCommandMatcher, +} + +#[derive(Deserialize, Debug, Clone, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +struct RawMcpServerUrlIdentity { + url: McpServerValueMatcher, +} + +/// A requirement for one named MCP server. +/// +/// The `Identity` variant preserves the released exact-match contract. The +/// command and URL variants are the normalized matcher-based forms accepted +/// under the `identity` key. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum McpServerRequirement { + Identity { identity: McpServerIdentity }, + Command(McpServerCommandMatcher), + Url(McpServerValueMatcher), +} + +#[derive(Deserialize)] +struct RawMcpServerRequirement { + identity: RawMcpServerIdentity, +} + +#[derive(Deserialize)] +#[serde(untagged)] +enum RawMcpServerIdentity { + Exact(McpServerIdentity), + Command(RawMcpServerCommandIdentity), + Url(RawMcpServerUrlIdentity), +} + +impl<'de> Deserialize<'de> for McpServerRequirement { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let RawMcpServerRequirement { identity } = + RawMcpServerRequirement::deserialize(deserializer)?; + match identity { + RawMcpServerIdentity::Exact(identity) => Ok(Self::Identity { identity }), + RawMcpServerIdentity::Command(matcher) => Ok(Self::Command(matcher.command)), + RawMcpServerIdentity::Url(matcher) => Ok(Self::Url(matcher.url)), + } + } +} + +impl McpServerRequirement { + pub(crate) fn validate(&self) -> Result<(), String> { + match self { + Self::Identity { .. } => Ok(()), + Self::Command(matcher) => { + for (index, arg) in matcher.args.iter().enumerate() { + arg.validate().map_err(|err| { + format!("invalid argument matcher at index {index}: {err}") + })?; + } + Ok(()) + } + Self::Url(matcher) => matcher.validate(), + } + } + + pub fn matches(&self, server: &McpServerConfig) -> bool { + match (self, &server.transport) { + ( + Self::Identity { + identity: + McpServerIdentity::Command { + command: want_command, + }, + }, + McpServerTransportConfig::Stdio { + command: got_command, + .. + }, + ) => got_command == want_command, + ( + Self::Identity { + identity: McpServerIdentity::Url { url: want_url }, + }, + McpServerTransportConfig::StreamableHttp { url: got_url, .. }, + ) => got_url == want_url, + (Self::Command(matcher), McpServerTransportConfig::Stdio { command, args, .. }) => { + matcher.executable == *command + && matcher.args.len() == args.len() + && matcher + .args + .iter() + .zip(args) + .all(|(matcher, arg)| matcher.matches(arg)) + } + (Self::Url(matcher), McpServerTransportConfig::StreamableHttp { url, .. }) => { + matcher.matches(url) + } + _ => false, + } + } +} + +#[cfg(test)] +#[path = "mcp_requirements_tests.rs"] +mod tests; diff --git a/codex-rs/config/src/mcp_requirements_tests.rs b/codex-rs/config/src/mcp_requirements_tests.rs new file mode 100644 index 0000000000..68b02316c4 --- /dev/null +++ b/codex-rs/config/src/mcp_requirements_tests.rs @@ -0,0 +1,218 @@ +use super::*; +use crate::mcp_types::McpServerConfig; +use pretty_assertions::assert_eq; +use std::collections::HashMap; + +fn stdio_server(command: &str, args: &[&str]) -> McpServerConfig { + McpServerConfig { + auth: Default::default(), + transport: McpServerTransportConfig::Stdio { + command: command.to_string(), + args: args.iter().map(ToString::to_string).collect(), + env: None, + env_vars: Vec::new(), + cwd: None, + }, + environment_id: crate::DEFAULT_MCP_SERVER_ENVIRONMENT_ID.to_string(), + enabled: true, + required: false, + supports_parallel_tool_calls: false, + disabled_reason: None, + startup_timeout_sec: None, + tool_timeout_sec: None, + default_tools_approval_mode: None, + enabled_tools: None, + disabled_tools: None, + scopes: None, + oauth: None, + oauth_resource: None, + tools: HashMap::new(), + } +} + +#[test] +fn command_matcher_matches_exact_positional_arguments() { + let requirement = McpServerRequirement::Command(McpServerCommandMatcher { + executable: "company-cli".to_string(), + args: vec![ + McpServerValueMatcher::Exact { + value: "mcp".to_string(), + }, + McpServerValueMatcher::Regex { + expression: r"https://[a-z]+\.example\.com".to_string(), + }, + ], + }); + + assert!(requirement.matches(&stdio_server( + "company-cli", + &["mcp", "https://pricing.example.com"] + ))); + assert!(!requirement.matches(&stdio_server( + "company-cli", + &["https://pricing.example.com", "mcp"] + ))); + assert!(!requirement.matches(&stdio_server( + "company-cli", + &["mcp", "https://pricing.example.com", "--verbose"] + ))); + assert!(!requirement.matches(&stdio_server( + "/usr/local/bin/company-cli", + &["mcp", "https://pricing.example.com"] + ))); +} + +#[test] +fn regex_matcher_requires_a_full_value_match() { + let matcher = McpServerValueMatcher::Regex { + expression: "mcp".to_string(), + }; + + assert!(matcher.matches("mcp")); + assert!(!matcher.matches("mcp-proxy")); + assert!(!matcher.matches("prefix-mcp")); +} + +#[test] +fn regex_matcher_allows_a_later_alternative_to_match_the_full_value() { + let matcher = McpServerValueMatcher::Regex { + expression: r"https://api\.example\.com|https://api\.example\.com/mcp".to_string(), + }; + + assert!(matcher.matches("https://api.example.com/mcp")); +} + +#[test] +fn regex_matcher_validation_rejects_expression_that_cannot_be_wrapped() { + let matcher = McpServerValueMatcher::Regex { + expression: "(?x)mcp # trailing comment".to_string(), + }; + + let err = matcher + .validate() + .expect_err("expression should not be valid for full-value matching"); + assert!( + err.contains("cannot be used for full-value matching"), + "{err}" + ); +} + +#[test] +fn legacy_command_identity_keeps_ignoring_arguments() { + let requirement: McpServerRequirement = toml::from_str( + r#" +[identity] +command = "company-cli" +"#, + ) + .expect("legacy command identity"); + + assert!(requirement.matches(&stdio_server( + "company-cli", + &["any", "arguments", "remain", "allowed"] + ))); + assert!(!requirement.matches(&stdio_server("different-cli", &[]))); +} + +#[test] +fn requirement_deserializes_command_and_url_matcher_shapes() { + let command: McpServerRequirement = toml::from_str( + r#" +[identity] +command = { executable = "company-cli", args = [ + { match = "exact", value = "mcp" }, + { match = "regex", expression = '^https://[a-z]+\.example\.com$' }, +] } +"#, + ) + .expect("command matcher"); + let url: McpServerRequirement = toml::from_str( + r#" +[identity] +url = { match = "prefix", value = "https://mcp.example.com/" } +"#, + ) + .expect("URL matcher"); + + assert_eq!( + command, + McpServerRequirement::Command(McpServerCommandMatcher { + executable: "company-cli".to_string(), + args: vec![ + McpServerValueMatcher::Exact { + value: "mcp".to_string(), + }, + McpServerValueMatcher::Regex { + expression: r"^https://[a-z]+\.example\.com$".to_string(), + }, + ], + }) + ); + assert_eq!( + url, + McpServerRequirement::Url(McpServerValueMatcher::Prefix { + value: "https://mcp.example.com/".to_string(), + }) + ); +} + +#[test] +fn requirement_rejects_matchers_outside_identity() { + for contents in [ + r#" +command = "company-cli" +"#, + r#" +command = { executable = "company-cli", args = [] } +"#, + r#" +url = { match = "prefix", value = "https://mcp.example.com/" } +"#, + ] { + let err = toml::from_str::(contents) + .expect_err("MCP server requirements should use the identity key"); + assert!( + err.to_string().contains("missing field `identity`"), + "{err}" + ); + } +} + +#[test] +fn matcher_identity_rejects_unknown_fields() { + for contents in [ + r#" +[identity] +unknown = "value" +command = { executable = "company-cli", args = [] } +"#, + r#" +[identity] +command = { executable = "company-cli", args = [], unknown = "value" } +"#, + ] { + toml::from_str::(contents) + .expect_err("matcher identities should reject unknown fields"); + } +} + +#[test] +fn identity_requirement_keeps_ignoring_unrelated_sibling_fields() { + let requirement: McpServerRequirement = toml::from_str( + r#" +unrelated = "ignored" +[identity] +command = "company-cli" +"#, + ) + .expect("legacy identity with unrelated sibling field"); + + assert_eq!( + requirement, + McpServerRequirement::Identity { + identity: McpServerIdentity::Command { + command: "company-cli".to_string(), + }, + } + ); +} diff --git a/codex-rs/config/src/requirements_layers/stack.rs b/codex-rs/config/src/requirements_layers/stack.rs index 3be1d11b5c..9a42650bf3 100644 --- a/codex-rs/config/src/requirements_layers/stack.rs +++ b/codex-rs/config/src/requirements_layers/stack.rs @@ -228,6 +228,7 @@ fn populate_merged_regular_fields_with_sources( enforce_residency, network, permissions, + models, guardian_policy_config, } = requirements; @@ -256,6 +257,7 @@ fn populate_merged_regular_fields_with_sources( set_sourced!(enforce_residency, &["enforce_residency"]); set_sourced!(network, &["experimental_network"]); set_sourced!(permissions, &["permissions"]); + set_sourced!(models, &["models"]); if let Some(guardian_policy_config) = guardian_policy_config.filter(|value| !value.trim().is_empty()) diff --git a/codex-rs/config/src/requirements_layers/stack_tests.rs b/codex-rs/config/src/requirements_layers/stack_tests.rs index a775f68a83..92e6bd8b46 100644 --- a/codex-rs/config/src/requirements_layers/stack_tests.rs +++ b/codex-rs/config/src/requirements_layers/stack_tests.rs @@ -109,6 +109,46 @@ allow_remote_control = false ); } +#[test] +fn new_thread_model_defaults_use_toml_priority() { + let composed = compose(vec![ + layer( + "req_low", + "Low", + r#" +[models.new_thread] +model = "low-priority-model" +model_reasoning_effort = "low" +service_tier = "flex" +"#, + ), + layer( + "req_high", + "High", + r#" +[models.new_thread] +model = "high-priority-model" +model_reasoning_effort = "high" +service_tier = "fast" +"#, + ), + ]) + .expect("compose requirements") + .expect("requirements present"); + + assert_eq!( + composed, + expected_requirements( + r#" +[models.new_thread] +model = "high-priority-model" +model_reasoning_effort = "high" +service_tier = "fast" +"# + ) + ); +} + #[test] fn composition_strategy_applies_to_non_cloud_layers() { let mdm_source = RequirementSource::MdmManagedPreferences { diff --git a/codex-rs/core-plugins/src/discoverable_tests.rs b/codex-rs/core-plugins/src/discoverable_tests.rs index 6e411cdda3..48482ed430 100644 --- a/codex-rs/core-plugins/src/discoverable_tests.rs +++ b/codex-rs/core-plugins/src/discoverable_tests.rs @@ -37,6 +37,13 @@ use wiremock::matchers::query_param; #[tokio::test] async fn returns_fallback_plugins_when_remote_disabled_for_codex_auth() { let codex_home = tempdir().expect("tempdir should succeed"); + write_file( + &codex_home.path().join(CONFIG_TOML_FILE), + r#"[features] +plugins = true +remote_plugin = false +"#, + ); let curated_root = curated_plugins_repo_path(codex_home.path()); write_openai_curated_marketplace(&curated_root, &["sample", "slack", "openai-developers"]); @@ -69,8 +76,7 @@ async fn returns_api_curated_fallback_plugins_for_direct_provider_auth() { let curated_root = curated_plugins_repo_path(codex_home.path()); write_openai_api_curated_marketplace(&curated_root, &["sample", "slack", "openai-developers"]); - let mut plugins = load_plugins_config(codex_home.path(), codex_home.path()).await; - plugins.remote_plugin_enabled = true; + let plugins = load_plugins_config(codex_home.path(), codex_home.path()).await; let plugins_manager = PluginsManager::new(codex_home.path().to_path_buf()); plugins_manager.set_auth_mode(Some(AuthMode::ApiKey)); let auth = CodexAuth::from_api_key("test-api-key"); @@ -153,7 +159,6 @@ async fn omits_openai_curated_but_keeps_configured_marketplaces_for_remote_codex &format!( r#"[features] plugins = true -remote_plugin = true [marketplaces.{bundled_marketplace_name}] source_type = "git" @@ -188,8 +193,7 @@ async fn includes_openai_curated_when_remote_enabled_without_auth() { let curated_root = curated_plugins_repo_path(codex_home.path()); write_openai_curated_marketplace(&curated_root, &["slack"]); - let mut plugins = load_plugins_config(codex_home.path(), codex_home.path()).await; - plugins.remote_plugin_enabled = true; + let plugins = load_plugins_config(codex_home.path(), codex_home.path()).await; let plugins_manager = PluginsManager::new(codex_home.path().to_path_buf()); let discoverable_plugins = list_discoverable_plugins( &plugins_manager, @@ -788,7 +792,6 @@ async fn expands_cached_remote_plugins_by_loaded_apps() { &codex_home.path().join(CONFIG_TOML_FILE), r#"[features] plugins = true -remote_plugin = true "#, ); diff --git a/codex-rs/core-plugins/src/installed_marketplaces.rs b/codex-rs/core-plugins/src/installed_marketplaces.rs index 0377fac9ed..b5bf3b7699 100644 --- a/codex-rs/core-plugins/src/installed_marketplaces.rs +++ b/codex-rs/core-plugins/src/installed_marketplaces.rs @@ -7,6 +7,7 @@ use codex_utils_absolute_path::AbsolutePathBuf; use tracing::warn; use crate::marketplace::find_marketplace_manifest_path; +use crate::marketplace_policy::project_effective_user_config; pub const INSTALLED_MARKETPLACES_DIR: &str = ".tmp/marketplaces"; @@ -18,7 +19,7 @@ pub fn installed_marketplace_roots_from_layer_stack( config_layer_stack: &ConfigLayerStack, codex_home: &Path, ) -> Vec { - let Some(user_config) = config_layer_stack.effective_user_config() else { + let Some(user_config) = project_effective_user_config(config_layer_stack, codex_home) else { return Vec::new(); }; let Some(marketplaces_value) = user_config.get("marketplaces") else { diff --git a/codex-rs/core-plugins/src/lib.rs b/codex-rs/core-plugins/src/lib.rs index 7ee5ddd117..df30305c38 100644 --- a/codex-rs/core-plugins/src/lib.rs +++ b/codex-rs/core-plugins/src/lib.rs @@ -9,6 +9,7 @@ pub mod marketplace_add; mod marketplace_policy; pub mod marketplace_remove; pub mod marketplace_upgrade; +mod npm_source; mod plugin_bundle_archive; mod provider; pub mod remote; @@ -54,6 +55,7 @@ pub use manager::PluginUninstallError; pub use manager::PluginsConfigInput; pub use manager::PluginsManager; pub use manager::RecommendedPluginCandidatesInput; +pub use marketplace_policy::allowed_configured_marketplace_names; pub use marketplace_upgrade::ConfiguredMarketplaceUpgradeError as PluginMarketplaceUpgradeError; pub use marketplace_upgrade::ConfiguredMarketplaceUpgradeOutcome as PluginMarketplaceUpgradeOutcome; pub use provider::ExecutorPluginProvider; diff --git a/codex-rs/core-plugins/src/loader.rs b/codex-rs/core-plugins/src/loader.rs index 49f4580e39..2b5c4782e8 100644 --- a/codex-rs/core-plugins/src/loader.rs +++ b/codex-rs/core-plugins/src/loader.rs @@ -8,8 +8,10 @@ use crate::manifest::PluginManifestPaths; use crate::manifest::load_plugin_manifest; use crate::marketplace::MarketplacePluginSource; use crate::marketplace::find_marketplace_plugin; -use crate::marketplace::list_marketplaces; +use crate::marketplace::list_marketplaces_with_home; use crate::marketplace::load_marketplace; +use crate::marketplace_policy::configured_plugins_from_stack; +use crate::npm_source::materialize_npm_plugin_source; use crate::remote::REMOTE_GLOBAL_MARKETPLACE_NAME; use crate::remote::RemoteInstalledPlugin; use crate::store::PluginStore; @@ -83,6 +85,18 @@ enum NonCuratedCacheRefreshMode { ForceReinstall, } +#[derive(Debug)] +pub(crate) struct NonCuratedCacheRefreshOutcome { + pub(crate) cache_refreshed: bool, + pub(crate) errors: Vec, +} + +#[derive(Debug)] +pub(crate) struct NonCuratedCacheRefreshError { + pub(crate) marketplace_name: String, + pub(crate) message: String, +} + pub(crate) fn log_plugin_load_errors(plugins: &[LoadedPlugin]) { for plugin in plugins.iter().filter(|plugin| plugin.error.is_some()) { if let Some(error) = plugin.error.as_deref() { @@ -128,7 +142,7 @@ async fn load_plugins_from_layer_stack_with_scope( scope: PluginLoadScope<'_>, ) -> Vec> { let configured_plugins = merge_configured_plugins_with_remote_installed( - configured_plugins_from_stack(config_layer_stack), + configured_plugins_from_stack(config_layer_stack, store.codex_home().as_path()), extra_plugins, store, remote_global_catalog_active, @@ -409,24 +423,54 @@ pub fn curated_plugin_cache_version(plugin_version: &str) -> String { } } -pub fn refresh_non_curated_plugin_cache( +#[cfg(test)] +pub(crate) fn refresh_non_curated_plugin_cache( codex_home: &Path, additional_roots: &[AbsolutePathBuf], + configured_plugin_keys: &[String], ) -> Result { + collapse_non_curated_cache_refresh(refresh_non_curated_plugin_cache_detailed( + codex_home, + additional_roots, + configured_plugin_keys, + )) +} + +pub(crate) fn refresh_non_curated_plugin_cache_detailed( + codex_home: &Path, + additional_roots: &[AbsolutePathBuf], + configured_plugin_keys: &[String], +) -> Result { refresh_non_curated_plugin_cache_with_mode( codex_home, additional_roots, + configured_plugin_keys, NonCuratedCacheRefreshMode::IfVersionChanged, ) } -pub fn refresh_non_curated_plugin_cache_force_reinstall( +#[cfg(test)] +pub(crate) fn refresh_non_curated_plugin_cache_force_reinstall( codex_home: &Path, additional_roots: &[AbsolutePathBuf], + configured_plugin_keys: &[String], ) -> Result { + collapse_non_curated_cache_refresh(refresh_non_curated_plugin_cache_force_reinstall_detailed( + codex_home, + additional_roots, + configured_plugin_keys, + )) +} + +pub(crate) fn refresh_non_curated_plugin_cache_force_reinstall_detailed( + codex_home: &Path, + additional_roots: &[AbsolutePathBuf], + configured_plugin_keys: &[String], +) -> Result { refresh_non_curated_plugin_cache_with_mode( codex_home, additional_roots, + configured_plugin_keys, NonCuratedCacheRefreshMode::ForceReinstall, ) } @@ -434,16 +478,32 @@ pub fn refresh_non_curated_plugin_cache_force_reinstall( fn refresh_non_curated_plugin_cache_with_mode( codex_home: &Path, additional_roots: &[AbsolutePathBuf], + configured_plugin_keys: &[String], mode: NonCuratedCacheRefreshMode, -) -> Result { - let configured_non_curated_plugin_ids = - non_curated_plugin_ids_from_config_keys(configured_plugins_from_codex_home( - codex_home, - "failed to read user config while refreshing non-curated plugin cache", - "failed to parse user config while refreshing non-curated plugin cache", - )); +) -> Result { + let mut configured_non_curated_plugin_ids = configured_plugin_keys + .iter() + .filter_map(|plugin_key| match PluginId::parse(plugin_key) { + Ok(plugin_id) if !is_openai_curated_marketplace_name(&plugin_id.marketplace_name) => { + Some(plugin_id) + } + Ok(_) => None, + Err(err) => { + warn!( + plugin_key, + error = %err, + "ignoring invalid plugin key during non-curated cache refresh setup" + ); + None + } + }) + .collect::>(); + configured_non_curated_plugin_ids.sort_unstable_by_key(PluginId::as_key); if configured_non_curated_plugin_ids.is_empty() { - return Ok(false); + return Ok(NonCuratedCacheRefreshOutcome { + cache_refreshed: false, + errors: Vec::new(), + }); } let configured_non_curated_plugin_keys = configured_non_curated_plugin_ids .iter() @@ -451,7 +511,7 @@ fn refresh_non_curated_plugin_cache_with_mode( .collect::>(); let store = PluginStore::try_new(codex_home.to_path_buf()).map_err(|err| err.to_string())?; - let marketplace_outcome = list_marketplaces(additional_roots) + let marketplace_outcome = list_marketplaces_with_home(additional_roots, /*home_dir*/ None) .map_err(|err| format!("failed to discover marketplaces for cache refresh: {err}"))?; let mut plugin_sources = HashMap::)>::new(); @@ -461,14 +521,18 @@ fn refresh_non_curated_plugin_cache_with_mode( } for plugin in marketplace.plugins { - let plugin_id = - PluginId::new(plugin.name.clone(), marketplace.name.clone()).map_err(|err| { - match err { - PluginIdError::Invalid(message) => { - format!("failed to prepare non-curated plugin cache refresh: {message}") - } - } - })?; + let plugin_id = match PluginId::new(plugin.name.clone(), marketplace.name.clone()) { + Ok(plugin_id) => plugin_id, + Err(PluginIdError::Invalid(message)) => { + warn!( + plugin = plugin.name, + marketplace = marketplace.name, + error = %message, + "ignoring invalid plugin entry during cache refresh" + ); + continue; + } + }; let plugin_key = plugin_id.as_key(); if !configured_non_curated_plugin_keys.contains(&plugin_key) { continue; @@ -503,6 +567,7 @@ fn refresh_non_curated_plugin_cache_with_mode( } let mut cache_refreshed = false; + let mut refresh_errors = Vec::new(); for plugin_id in configured_non_curated_plugin_ids { let plugin_key = plugin_id.as_key(); let Some((source, manifest_fallback_contents)) = plugin_sources.get(&plugin_key).cloned() @@ -514,49 +579,70 @@ fn refresh_non_curated_plugin_cache_with_mode( ); continue; }; - let materialized = - materialize_marketplace_plugin_source(codex_home, &source).map_err(|err| { - format!("failed to materialize plugin source for {plugin_key}: {err}") - })?; - let source_path = materialized.path.clone(); - let plugin_version = match manifest_fallback_contents.as_deref() { - Some(manifest_contents) => plugin_version_for_source_with_fallback_manifest( - source_path.as_path(), - manifest_contents, - ), - None => plugin_version_for_source(source_path.as_path()), - } - .map_err(|err| format!("failed to read plugin version for {plugin_key}: {err}"))?; + let refresh_result = (|| -> Result { + let materialized = + materialize_marketplace_plugin_source(codex_home, &source).map_err(|err| { + format!("failed to materialize plugin source for {plugin_key}: {err}") + })?; + let source_path = materialized.path; + let plugin_version = match manifest_fallback_contents.as_deref() { + Some(manifest_contents) => plugin_version_for_source_with_fallback_manifest( + source_path.as_path(), + manifest_contents, + ), + None => plugin_version_for_source(source_path.as_path()), + } + .map_err(|err| format!("failed to read plugin version for {plugin_key}: {err}"))?; - if mode == NonCuratedCacheRefreshMode::IfVersionChanged - && store.active_plugin_version(&plugin_id).as_deref() == Some(plugin_version.as_str()) - { - continue; - } + if mode == NonCuratedCacheRefreshMode::IfVersionChanged + && store.active_plugin_version(&plugin_id).as_deref() + == Some(plugin_version.as_str()) + { + return Ok(false); + } - match manifest_fallback_contents.as_deref() { - Some(manifest_contents) => store.install_with_version_and_fallback_manifest( - source_path, - plugin_id.clone(), - plugin_version, - manifest_contents, - ), - None => store.install_with_version(source_path, plugin_id.clone(), plugin_version), + match manifest_fallback_contents.as_deref() { + Some(manifest_contents) => store.install_with_version_and_fallback_manifest( + source_path, + plugin_id.clone(), + plugin_version, + manifest_contents, + ), + None => store.install_with_version(source_path, plugin_id.clone(), plugin_version), + } + .map_err(|err| format!("failed to refresh plugin cache for {plugin_key}: {err}"))?; + Ok(true) + })(); + match refresh_result { + Ok(refreshed) => cache_refreshed |= refreshed, + Err(message) => refresh_errors.push(NonCuratedCacheRefreshError { + marketplace_name: plugin_id.marketplace_name, + message, + }), } - .map_err(|err| format!("failed to refresh plugin cache for {plugin_key}: {err}"))?; - cache_refreshed = true; } - Ok(cache_refreshed) + Ok(NonCuratedCacheRefreshOutcome { + cache_refreshed, + errors: refresh_errors, + }) } -fn configured_plugins_from_stack( - config_layer_stack: &ConfigLayerStack, -) -> HashMap { - let Some(user_config) = config_layer_stack.effective_user_config() else { - return HashMap::new(); - }; - configured_plugins_from_user_config_value(&user_config) +#[cfg(test)] +fn collapse_non_curated_cache_refresh( + outcome: Result, +) -> Result { + let outcome = outcome?; + if outcome.errors.is_empty() { + Ok(outcome.cache_refreshed) + } else { + Err(outcome + .errors + .into_iter() + .map(|error| error.message) + .collect::>() + .join("; ")) + } } fn is_full_git_sha(value: &str) -> bool { @@ -646,20 +732,6 @@ fn curated_plugin_ids_from_config_keys( configured_curated_plugin_ids } -fn non_curated_plugin_ids_from_config_keys( - configured_plugins: HashMap, -) -> Vec { - let mut configured_non_curated_plugin_ids = configured_plugin_ids( - configured_plugins, - "ignoring invalid plugin key during non-curated cache refresh setup", - ) - .into_iter() - .filter(|plugin_id| !is_openai_curated_marketplace_name(&plugin_id.marketplace_name)) - .collect::>(); - configured_non_curated_plugin_ids.sort_unstable_by_key(PluginId::as_key); - configured_non_curated_plugin_ids -} - pub fn configured_curated_plugin_ids_from_codex_home(codex_home: &Path) -> Vec { curated_plugin_ids_from_config_keys(configured_plugins_from_codex_home( codex_home, @@ -1358,6 +1430,22 @@ pub fn materialize_marketplace_plugin_source( _tempdir: Some(tempdir), }) } + MarketplacePluginSource::Npm { + package, + version, + registry, + } => { + let (path, tempdir) = materialize_npm_plugin_source( + codex_home, + package, + version.as_deref(), + registry.as_deref(), + )?; + Ok(MaterializedMarketplacePluginSource { + path, + _tempdir: Some(tempdir), + }) + } } } diff --git a/codex-rs/core-plugins/src/loader_tests.rs b/codex-rs/core-plugins/src/loader_tests.rs index d4faaa8e7a..8e5a76b4bf 100644 --- a/codex-rs/core-plugins/src/loader_tests.rs +++ b/codex-rs/core-plugins/src/loader_tests.rs @@ -43,7 +43,7 @@ fn configured_plugins_from_stack_merges_user_layers() { ) .expect("valid config layer stack"); - let plugins = configured_plugins_from_stack(&stack); + let plugins = configured_plugins_from_stack(&stack, temp_dir.path()); assert_eq!( plugins, diff --git a/codex-rs/core-plugins/src/manager.rs b/codex-rs/core-plugins/src/manager.rs index b357c5b536..4d215b61d4 100644 --- a/codex-rs/core-plugins/src/manager.rs +++ b/codex-rs/core-plugins/src/manager.rs @@ -16,8 +16,8 @@ use crate::loader::log_plugin_load_errors; use crate::loader::materialize_marketplace_plugin_source; use crate::loader::plugin_capability_summary_from_root; use crate::loader::refresh_curated_plugin_cache; -use crate::loader::refresh_non_curated_plugin_cache; -use crate::loader::refresh_non_curated_plugin_cache_force_reinstall; +use crate::loader::refresh_non_curated_plugin_cache_detailed; +use crate::loader::refresh_non_curated_plugin_cache_force_reinstall_detailed; use crate::loader::remote_installed_plugins_to_config; use crate::manifest::PluginManifestInterface; use crate::manifest::load_plugin_manifest; @@ -32,9 +32,12 @@ use crate::marketplace::MarketplacePluginSource; use crate::marketplace::ResolvedMarketplacePlugin; use crate::marketplace::find_installable_marketplace_plugin; use crate::marketplace::find_marketplace_plugin; -use crate::marketplace::list_marketplaces; +use crate::marketplace::home_dir; +use crate::marketplace::list_marketplaces_with_home; use crate::marketplace::plugin_interface_with_marketplace_category; use crate::marketplace_policy::MarketplacePolicy; +use crate::marketplace_policy::allowed_configured_marketplace_names; +use crate::marketplace_policy::configured_plugins_from_stack; use crate::marketplace_upgrade::ConfiguredMarketplaceUpgradeError; use crate::marketplace_upgrade::ConfiguredMarketplaceUpgradeOutcome; use crate::marketplace_upgrade::upgrade_configured_git_marketplaces; @@ -84,6 +87,7 @@ use codex_utils_absolute_path::AbsolutePathBuf; use codex_utils_plugins::PluginSkillRoot; use std::collections::HashMap; use std::collections::HashSet; +use std::path::Path; use std::path::PathBuf; use std::sync::Arc; use std::sync::RwLock; @@ -195,6 +199,7 @@ pub struct PluginListBackgroundTaskOptions { #[derive(Clone, PartialEq, Eq)] struct NonCuratedCacheRefreshRequest { roots: Vec, + configured_plugin_keys: Vec, mode: NonCuratedCacheRefreshMode, } @@ -388,9 +393,16 @@ struct PluginLoadCacheKey { } impl PluginLoadCacheKey { - fn from_config(config: &PluginsConfigInput, remote_global_catalog_active: bool) -> Self { + fn from_config( + config: &PluginsConfigInput, + codex_home: &Path, + remote_global_catalog_active: bool, + ) -> Self { Self { - configured_plugins: configured_plugins_from_stack(&config.config_layer_stack), + configured_plugins: configured_plugins_from_stack( + &config.config_layer_stack, + codex_home, + ), skill_config_rules: skill_config_rules_from_stack(&config.config_layer_stack), remote_global_catalog_active, } @@ -494,8 +506,11 @@ impl PluginsManager { if !config.plugins_enabled { return None; } - let key = - PluginLoadCacheKey::from_config(config, self.remote_global_catalog_active(config)); + let key = PluginLoadCacheKey::from_config( + config, + self.codex_home.as_path(), + self.remote_global_catalog_active(config), + ); self.loaded_plugins_cache .read() .unwrap_or_else(std::sync::PoisonError::into_inner) @@ -525,7 +540,11 @@ impl PluginsManager { } let remote_global_catalog_active = self.remote_global_catalog_active(config); - let cache_key = PluginLoadCacheKey::from_config(config, remote_global_catalog_active); + let cache_key = PluginLoadCacheKey::from_config( + config, + self.codex_home.as_path(), + remote_global_catalog_active, + ); if !force_reload && let Some(plugins) = self.cached_loaded_plugins(&cache_key) { return self.resolve_loaded_plugins_for_auth(plugins); } @@ -1007,7 +1026,7 @@ impl PluginsManager { options: PluginListBackgroundTaskOptions, on_effective_plugins_changed: Option>, ) { - self.maybe_start_non_curated_plugin_cache_refresh(roots); + self.maybe_start_non_curated_plugin_cache_refresh(config, roots); if options.refresh_global_remote_catalog_cache { self.maybe_start_global_remote_catalog_cache_refresh(config, auth.clone()); } @@ -1547,7 +1566,7 @@ impl PluginsManager { let (installed_plugins, enabled_plugins) = self.configured_plugin_states(config); let marketplace_roots = self.marketplace_roots(config, additional_roots, include_openai_curated); - let marketplace_outcome = list_marketplaces(&marketplace_roots)?; + let marketplace_outcome = self.list_marketplaces_with_policy(config, &marketplace_roots)?; let mut seen_plugin_keys = HashSet::new(); let marketplaces = marketplace_outcome .marketplaces @@ -1578,7 +1597,7 @@ impl PluginsManager { let mut local_version = plugin.local_version; let manifest_fallback = plugin.manifest_fallback.clone(); if installed - && matches!(&plugin.source, MarketplacePluginSource::Git { .. }) + && plugin.source.is_install_materialized() && let Some(plugin_id) = plugin_id.as_ref() && let Some(plugin_root) = self.store.active_plugin_root(plugin_id) && let Some(manifest) = load_plugin_manifest(plugin_root.as_path()) @@ -1636,11 +1655,12 @@ impl PluginsManager { return Ok(MarketplaceListOutcome::default()); } - list_marketplaces(&self.marketplace_roots( + let marketplace_roots = self.marketplace_roots( config, additional_roots, /*include_openai_curated*/ true, - )) + ); + self.list_marketplaces_with_policy(config, &marketplace_roots) } pub(crate) async fn tool_suggest_metadata_for_marketplace_plugin( @@ -1666,6 +1686,17 @@ impl PluginsManager { } let plugin = find_marketplace_plugin(&request.marketplace_path, &request.plugin_name)?; + MarketplacePolicy::from_requirements(config.config_layer_stack.requirements()) + .validate_install( + &config.config_layer_stack, + self.codex_home.as_path(), + &request.marketplace_path, + &plugin.plugin_id.marketplace_name, + ) + .map_err(|message| MarketplaceError::InvalidMarketplaceFile { + path: request.marketplace_path.to_path_buf(), + message, + })?; if !self.restriction_product_matches(plugin.policy.products.as_deref()) { return Err(MarketplaceError::PluginNotFound { plugin_name: plugin.plugin_id.plugin_name, @@ -1741,7 +1772,7 @@ impl PluginsManager { } })?; let plugin_key = plugin_id.as_key(); - if matches!(plugin.source, MarketplacePluginSource::Git { .. }) && !plugin.installed { + if plugin.source.is_install_materialized() && !plugin.installed { let description = remote_plugin_install_required_description(&plugin.source); return Ok(PluginDetail { id: plugin_key, @@ -1766,28 +1797,27 @@ impl PluginsManager { }); } - let source_path = - if matches!(plugin.source, MarketplacePluginSource::Git { .. }) && plugin.installed { - self.store.active_plugin_root(&plugin_id).ok_or_else(|| { - MarketplaceError::InvalidPlugin(format!( - "installed plugin cache entry is missing for {plugin_key}" - )) - })? - } else { - let codex_home = self.codex_home.clone(); - let source = plugin.source.clone(); - let materialized = tokio::task::spawn_blocking(move || { - materialize_marketplace_plugin_source(codex_home.as_path(), &source) - }) - .await - .map_err(|err| { - MarketplaceError::InvalidPlugin(format!( - "failed to materialize plugin source: {err}" - )) - })? - .map_err(MarketplaceError::InvalidPlugin)?; - materialized.path.clone() - }; + let source_path = if plugin.source.is_install_materialized() && plugin.installed { + self.store.active_plugin_root(&plugin_id).ok_or_else(|| { + MarketplaceError::InvalidPlugin(format!( + "installed plugin cache entry is missing for {plugin_key}" + )) + })? + } else { + let codex_home = self.codex_home.clone(); + let source = plugin.source.clone(); + let materialized = tokio::task::spawn_blocking(move || { + materialize_marketplace_plugin_source(codex_home.as_path(), &source) + }) + .await + .map_err(|err| { + MarketplaceError::InvalidPlugin(format!( + "failed to materialize plugin source: {err}" + )) + })? + .map_err(MarketplaceError::InvalidPlugin)?; + materialized.path.clone() + }; if !source_path.as_path().is_dir() { return Err(MarketplaceError::InvalidPlugin( "path does not exist or is not a directory".to_string(), @@ -2024,12 +2054,30 @@ impl PluginsManager { )); } if !outcome.upgraded_roots.is_empty() { - match refresh_non_curated_plugin_cache_force_reinstall( + let mut configured_plugin_keys = configured_plugins_from_stack( + &config.config_layer_stack, + self.codex_home.as_path(), + ) + .into_keys() + .collect::>(); + configured_plugin_keys.sort_unstable(); + match refresh_non_curated_plugin_cache_force_reinstall_detailed( self.codex_home.as_path(), &outcome.upgraded_roots, + &configured_plugin_keys, ) { - Ok(cache_refreshed) => { - self.clear_caches_after_marketplace_source_refresh(cache_refreshed); + Ok(refresh_outcome) => { + self.clear_caches_after_marketplace_source_refresh( + refresh_outcome.cache_refreshed, + ); + outcome + .errors + .extend(refresh_outcome.errors.into_iter().map(|error| { + ConfiguredMarketplaceUpgradeError { + marketplace_name: error.marketplace_name, + message: error.message, + } + })); } Err(err) => { self.clear_cache(); @@ -2049,9 +2097,11 @@ impl PluginsManager { pub fn maybe_start_non_curated_plugin_cache_refresh( self: &Arc, + config: &PluginsConfigInput, roots: &[AbsolutePathBuf], ) { self.schedule_non_curated_plugin_cache_refresh( + config, roots, NonCuratedCacheRefreshMode::IfVersionChanged, ); @@ -2128,38 +2178,81 @@ impl PluginsManager { fn schedule_non_curated_plugin_cache_refresh( self: &Arc, + config: &PluginsConfigInput, roots: &[AbsolutePathBuf], mode: NonCuratedCacheRefreshMode, ) { - let mut roots = roots.to_vec(); + let marketplace_roots = + self.marketplace_roots(config, roots, /*include_openai_curated*/ false); + let outcome = match self.list_marketplaces_with_policy(config, &marketplace_roots) { + Ok(outcome) => outcome, + Err(err) => { + warn!("failed to prepare non-curated plugin cache refresh: {err}"); + return; + } + }; + let policy = MarketplacePolicy::from_requirements(config.config_layer_stack.requirements()); + let mut roots = outcome + .marketplaces + .into_iter() + .filter(|marketplace| !is_openai_curated_marketplace_name(&marketplace.name)) + .filter_map(|marketplace| { + match policy.validate_install( + &config.config_layer_stack, + self.codex_home.as_path(), + &marketplace.path, + &marketplace.name, + ) { + Ok(()) => Some(marketplace.path), + Err(err) => { + warn!( + marketplace = marketplace.name, + path = %marketplace.path.display(), + error = %err, + "skipping marketplace source during plugin cache refresh" + ); + None + } + } + }) + .collect::>(); roots.sort_unstable(); roots.dedup(); - if roots.is_empty() { + let mut configured_plugin_keys = + configured_plugins_from_stack(&config.config_layer_stack, self.codex_home.as_path()) + .into_keys() + .collect::>(); + configured_plugin_keys.sort_unstable(); + if roots.is_empty() || configured_plugin_keys.is_empty() { return; } - let request = NonCuratedCacheRefreshRequest { roots, mode }; + let mut request = NonCuratedCacheRefreshRequest { + roots, + configured_plugin_keys, + mode, + }; let should_spawn = { let mut state = match self.non_curated_cache_refresh_state.write() { Ok(state) => state, Err(err) => err.into_inner(), }; - // Collapse repeated plugin/list requests onto one worker and only queue another pass - // when the requested roots set actually changes. Forced reinstall requests are not - // deduped against the last completed pass because the same marketplace root path can - // point at newly activated files after an auto-upgrade. - if state.requested.as_ref() == Some(&request) - || (mode == NonCuratedCacheRefreshMode::IfVersionChanged - && !state.in_flight - && state.last_refreshed.as_ref() == Some(&request)) - { - return; - } - if mode == NonCuratedCacheRefreshMode::IfVersionChanged + if request.mode == NonCuratedCacheRefreshMode::IfVersionChanged && state.requested.as_ref().is_some_and(|requested| { requested.mode == NonCuratedCacheRefreshMode::ForceReinstall && requested.roots == request.roots }) + { + request.mode = NonCuratedCacheRefreshMode::ForceReinstall; + } + // Collapse repeated plugin/list requests onto one worker and only queue another pass + // when the roots or configured plugin set changes. Forced reinstall requests are not + // deduped against the last completed pass because the same marketplace root path can + // point at newly activated files after an auto-upgrade. + if state.requested.as_ref() == Some(&request) + || (request.mode == NonCuratedCacheRefreshMode::IfVersionChanged + && !state.in_flight + && state.last_refreshed.as_ref() == Some(&request)) { return; } @@ -2350,21 +2443,33 @@ impl PluginsManager { let refresh_result = match request.mode { NonCuratedCacheRefreshMode::IfVersionChanged => { - refresh_non_curated_plugin_cache(self.codex_home.as_path(), &request.roots) - } - NonCuratedCacheRefreshMode::ForceReinstall => { - refresh_non_curated_plugin_cache_force_reinstall( + refresh_non_curated_plugin_cache_detailed( self.codex_home.as_path(), &request.roots, + &request.configured_plugin_keys, + ) + } + NonCuratedCacheRefreshMode::ForceReinstall => { + refresh_non_curated_plugin_cache_force_reinstall_detailed( + self.codex_home.as_path(), + &request.roots, + &request.configured_plugin_keys, ) } }; let refreshed = match refresh_result { - Ok(cache_refreshed) => { - if cache_refreshed { + Ok(refresh_outcome) => { + if refresh_outcome.cache_refreshed { self.clear_cache(); } - true + for error in &refresh_outcome.errors { + warn!( + marketplace = error.marketplace_name, + error = %error.message, + "failed to refresh configured plugin cache" + ); + } + refresh_outcome.errors.is_empty() } Err(err) => { self.clear_cache(); @@ -2392,7 +2497,8 @@ impl PluginsManager { &self, config: &PluginsConfigInput, ) -> (HashSet, HashSet) { - let configured_plugins = configured_plugins_from_stack(&config.config_layer_stack); + let configured_plugins = + configured_plugins_from_stack(&config.config_layer_stack, self.codex_home.as_path()); let installed_plugins = configured_plugins .keys() .filter(|plugin_key| { @@ -2449,6 +2555,27 @@ impl PluginsManager { roots.dedup(); roots } + + fn list_marketplaces_with_policy( + &self, + config: &PluginsConfigInput, + roots: &[AbsolutePathBuf], + ) -> Result { + let mut outcome = list_marketplaces_with_home(roots, home_dir().as_deref())?; + let policy = MarketplacePolicy::from_requirements(config.config_layer_stack.requirements()); + if !policy.is_restricted() { + return Ok(outcome); + } + let allowed_marketplace_names = allowed_configured_marketplace_names( + &config.config_layer_stack, + self.codex_home.as_path(), + ); + outcome.marketplaces.retain(|marketplace| { + is_openai_curated_marketplace_name(&marketplace.name) + || allowed_marketplace_names.contains(&marketplace.name) + }); + Ok(outcome) + } } pub(crate) fn remote_plugin_install_required_description( @@ -2474,10 +2601,29 @@ pub(crate) fn remote_plugin_install_required_description( parts.join(", ") } MarketplacePluginSource::Local { path } => path.as_path().display().to_string(), + MarketplacePluginSource::Npm { + package, + version, + registry, + } => { + let mut parts = vec![package.clone()]; + if let Some(version) = version { + parts.push(format!("version `{version}`")); + } + if let Some(registry) = registry { + parts.push(format!("registry `{registry}`")); + } + parts.join(", ") + } }; + let source_kind = if matches!(source, MarketplacePluginSource::Npm { .. }) { + "an npm plugin" + } else { + "a cross-repo plugin" + }; format!( - "This is a cross-repo plugin. Install it to view more detailed information. The source of the plugin is {source_description}." + "This is {source_kind}. Install it to view more detailed information. The source of the plugin is {source_description}." ) } @@ -2594,31 +2740,6 @@ impl PluginUninstallError { } } -pub(crate) fn configured_plugins_from_stack( - config_layer_stack: &ConfigLayerStack, -) -> HashMap { - // Plugin entries remain persisted user config only. - let Some(user_config) = config_layer_stack.effective_user_config() else { - return HashMap::new(); - }; - configured_plugins_from_user_config_value(&user_config) -} - -fn configured_plugins_from_user_config_value( - user_config: &toml::Value, -) -> HashMap { - let Some(plugins_value) = user_config.get("plugins") else { - return HashMap::new(); - }; - match plugins_value.clone().try_into() { - Ok(plugins) => plugins, - Err(err) => { - warn!("invalid plugins config: {err}"); - HashMap::new() - } - } -} - #[cfg(test)] #[path = "manager_tests.rs"] mod tests; diff --git a/codex-rs/core-plugins/src/manager_tests.rs b/codex-rs/core-plugins/src/manager_tests.rs index cefca287ad..469a9a7fc5 100644 --- a/codex-rs/core-plugins/src/manager_tests.rs +++ b/codex-rs/core-plugins/src/manager_tests.rs @@ -100,6 +100,19 @@ fn config_layer_stack_with_requirements( .expect("build config layer stack") } +fn plugins_config_input_with_requirements( + codex_home: &Path, + user_config: &str, + requirements: &str, +) -> PluginsConfigInput { + PluginsConfigInput::new( + config_layer_stack_with_requirements(codex_home, user_config, requirements), + /*plugins_enabled*/ true, + /*remote_plugin_enabled*/ false, + String::new(), + ) +} + #[test] fn plugins_manager_tracks_auth_mode() { let tmp = TempDir::new().unwrap(); @@ -122,6 +135,184 @@ fn plugins_manager_tracks_auth_mode() { assert_eq!(manager_with_auth.auth_mode(), Some(AuthMode::Chatgpt)); } +#[tokio::test] +async fn marketplace_policy_projection_disables_installed_plugin_and_invalidates_cache() { + let codex_home = TempDir::new().expect("create Codex home"); + write_plugin( + &codex_home.path().join("plugins/cache/company"), + "sample/local", + "sample", + ); + let user_config = r#" +[marketplaces.company] +source_type = "git" +source = "https://github.com/example/company.git" + +[plugins."sample@company"] +enabled = true +"#; + let allowed = plugins_config_input_with_requirements( + codex_home.path(), + user_config, + r#" +[marketplaces] +restrict_to_allowed_sources = true + +[marketplaces.allowed_sources.company] +source = "git" +url = "https://github.com/example/company.git" +"#, + ); + let blocked = plugins_config_input_with_requirements( + codex_home.path(), + user_config, + r#" +[marketplaces] +restrict_to_allowed_sources = true + +[marketplaces.allowed_sources.other] +source = "git" +url = "https://github.com/example/other.git" +"#, + ); + let manager = PluginsManager::new(codex_home.path().to_path_buf()); + + let allowed_outcome = manager.plugins_for_config(&allowed).await; + assert_eq!(allowed_outcome.plugins().len(), 1); + assert_eq!(allowed_outcome.plugins()[0].config_name, "sample@company"); + + let blocked_outcome = manager.plugins_for_config(&blocked).await; + assert_eq!(blocked_outcome, PluginLoadOutcome::default()); +} + +#[tokio::test] +async fn plugin_read_rejects_marketplace_blocked_by_requirements() { + let codex_home = TempDir::new().expect("create Codex home"); + let marketplace_root = codex_home.path().join("marketplace"); + write_plugin(&marketplace_root, "sample", "sample"); + write_file( + &marketplace_root.join(".agents/plugins/marketplace.json"), + r#"{ + "name": "company", + "plugins": [ + { + "name": "sample", + "source": {"source": "local", "path": "./sample"} + } + ] +}"#, + ); + let config = plugins_config_input_with_requirements( + codex_home.path(), + "", + r#" +[marketplaces] +restrict_to_allowed_sources = true +"#, + ); + let marketplace_path = + AbsolutePathBuf::try_from(marketplace_root.join(".agents/plugins/marketplace.json")) + .expect("absolute marketplace path"); + + let err = PluginsManager::new(codex_home.path().to_path_buf()) + .read_plugin_for_config( + &config, + &PluginReadRequest { + plugin_name: "sample".to_string(), + marketplace_path, + }, + ) + .await + .expect_err("blocked marketplace should not be readable"); + assert!(matches!( + err, + MarketplaceError::InvalidMarketplaceFile { .. } + )); +} + +#[test] +fn marketplace_policy_filters_discovered_marketplaces_by_configured_name() { + let codex_home = TempDir::new().expect("create Codex home"); + let repo_root = codex_home.path().join("repo"); + let subdirectory = repo_root.join("worktree/subdirectory"); + fs::create_dir_all(&subdirectory).expect("create input subdirectory"); + write_plugin(&repo_root, "sample", "sample"); + write_file( + &repo_root.join(".agents/plugins/marketplace.json"), + r#"{ + "name": "company", + "plugins": [ + { + "name": "sample", + "source": {"source": "local", "path": "./sample"} + } + ] +}"#, + ); + init_git_repo(&repo_root); + let repo_root = AbsolutePathBuf::try_from(repo_root).expect("absolute repository root"); + let subdirectory = + AbsolutePathBuf::try_from(subdirectory).expect("absolute input subdirectory"); + let manager = PluginsManager::new(codex_home.path().to_path_buf()); + let user_config = format!( + r#" +[marketplaces.company] +source_type = "local" +source = {:?} +"#, + repo_root.as_path() + ); + let allowed = plugins_config_input_with_requirements( + codex_home.path(), + &user_config, + &format!( + r#" +[marketplaces] +restrict_to_allowed_sources = true + +[marketplaces.allowed_sources.company] +source = "local" +path = {:?} +"#, + repo_root.as_path() + ), + ); + let blocked = plugins_config_input_with_requirements( + codex_home.path(), + &user_config, + &format!( + r#" +[marketplaces] +restrict_to_allowed_sources = true + +[marketplaces.allowed_sources.subdirectory] +source = "local" +path = {:?} +"#, + subdirectory.as_path() + ), + ); + + let allowed_outcome = manager + .list_marketplaces_for_config( + &allowed, + std::slice::from_ref(&subdirectory), + /*include_openai_curated*/ false, + ) + .expect("list allowed marketplace"); + assert_eq!(allowed_outcome.marketplaces.len(), 1); + assert_eq!(allowed_outcome.marketplaces[0].name, "company"); + + let blocked_outcome = manager + .list_marketplaces_for_config( + &blocked, + std::slice::from_ref(&subdirectory), + /*include_openai_curated*/ false, + ) + .expect("list blocked marketplace"); + assert_eq!(blocked_outcome.marketplaces, Vec::new()); +} + fn write_auth_projection_plugin(codex_home: &Path, name: &str, include_app: bool) { let plugin_root = codex_home .join("plugins/cache") @@ -586,6 +777,7 @@ fn remote_installed_plugin_in_marketplace( RemoteInstalledPlugin { marketplace_name: marketplace_name.to_string(), id: format!("plugins~Plugin_{name}"), + version: None, name: name.to_string(), enabled: true, install_policy: codex_app_server_protocol::PluginInstallPolicy::Available, @@ -871,7 +1063,6 @@ async fn remote_installed_cache_ignores_plugins_missing_local_cache() { &codex_home.path().join(CONFIG_TOML_FILE), r#"[features] plugins = true -remote_plugin = true "#, ); @@ -1105,7 +1296,6 @@ async fn remote_global_catalog_ignores_local_curated_plugins() { &codex_home.path().join(CONFIG_TOML_FILE), r#"[features] plugins = true -remote_plugin = true [plugins."linear@openai-curated"] enabled = true @@ -1156,7 +1346,6 @@ async fn remote_plugin_feature_keeps_local_curated_without_codex_backend() { &codex_home.path().join(CONFIG_TOML_FILE), r#"[features] plugins = true -remote_plugin = true [plugins."linear@openai-curated"] enabled = true @@ -3348,7 +3537,11 @@ plugins = true .unwrap() .marketplaces .into_iter() - .find(|marketplace| marketplace.name == "debug") + .find(|marketplace| { + marketplace.path + == AbsolutePathBuf::try_from(repo_root.join(".agents/plugins/marketplace.json")) + .unwrap() + }) .unwrap() .plugins .into_iter() @@ -3785,7 +3978,7 @@ enabled = true let marketplaces = PluginsManager::new(tmp.path().to_path_buf()) .list_marketplaces_for_config( &config, - &[AbsolutePathBuf::try_from(repo_root).unwrap()], + &[AbsolutePathBuf::try_from(repo_root.clone()).unwrap()], /*include_openai_curated*/ true, ) .unwrap() @@ -3793,7 +3986,11 @@ enabled = true let marketplace = marketplaces .into_iter() - .find(|marketplace| marketplace.name == "debug") + .find(|marketplace| { + marketplace.path + == AbsolutePathBuf::try_from(repo_root.join(".agents/plugins/marketplace.json")) + .unwrap() + }) .expect("debug marketplace should be listed"); let mut plugins = marketplace.plugins; @@ -4679,7 +4876,6 @@ async fn remote_plugin_caches_refresh_warms_recommended_plugins_cache() { &tmp.path().join(CONFIG_TOML_FILE), r#"[features] plugins = true -remote_plugin = true "#, ); @@ -4740,7 +4936,6 @@ async fn recommended_plugins_mode_deduplicates_concurrent_cache_misses() { &tmp.path().join(CONFIG_TOML_FILE), r#"[features] plugins = true -remote_plugin = true "#, ); @@ -4818,7 +5013,6 @@ async fn recommended_plugin_candidates_filter_installed_and_disabled_plugins() { &tmp.path().join(CONFIG_TOML_FILE), r#"[features] plugins = true -remote_plugin = true "#, ); let server = MockServer::start().await; @@ -4891,7 +5085,6 @@ async fn recommended_plugins_mode_caches_explicit_false() { &tmp.path().join(CONFIG_TOML_FILE), r#"[features] plugins = true -remote_plugin = true "#, ); @@ -4931,7 +5124,6 @@ async fn recommended_plugins_mode_retries_after_fetch_failure() { &tmp.path().join(CONFIG_TOML_FILE), r#"[features] plugins = true -remote_plugin = true "#, ); @@ -5256,6 +5448,7 @@ enabled = true refresh_non_curated_plugin_cache( tmp.path(), &[AbsolutePathBuf::try_from(repo_root).unwrap()], + &["sample-plugin@debug".to_string()], ) .expect("cache refresh should succeed") ); @@ -5308,6 +5501,7 @@ enabled = true refresh_non_curated_plugin_cache( tmp.path(), &[AbsolutePathBuf::try_from(repo_root).unwrap()], + &["sample-plugin@debug".to_string()], ) .expect("cache refresh should reinstall missing configured plugin") ); @@ -5367,6 +5561,7 @@ enabled = true refresh_non_curated_plugin_cache( tmp.path(), &[AbsolutePathBuf::try_from(repo_root).unwrap()], + &["sample-plugin@debug".to_string()], ) .expect("cache refresh should materialize configured Git plugin") ); @@ -5420,6 +5615,7 @@ enabled = true !refresh_non_curated_plugin_cache( tmp.path(), &[AbsolutePathBuf::try_from(repo_root).unwrap()], + &["sample-plugin@debug".to_string()], ) .expect("cache refresh should be a no-op when configured plugins are current") ); @@ -5473,6 +5669,7 @@ enabled = true refresh_non_curated_plugin_cache_force_reinstall( tmp.path(), &[AbsolutePathBuf::try_from(repo_root).unwrap()], + &["sample-plugin@debug".to_string()], ) .expect("cache refresh should reinstall unchanged local version") ); @@ -5531,6 +5728,7 @@ enabled = true refresh_non_curated_plugin_cache( tmp.path(), &[AbsolutePathBuf::try_from(repo_root).unwrap()], + &["sample-plugin@debug".to_string()], ) .expect("cache refresh should ignore unrelated invalid plugin manifests") ); @@ -5542,6 +5740,47 @@ enabled = true ); } +#[test] +fn refresh_non_curated_plugin_cache_continues_after_plugin_error() { + let tmp = tempfile::tempdir().unwrap(); + let repo_root = tmp.path().join("repo"); + fs::create_dir_all(repo_root.join(".git")).unwrap(); + fs::create_dir_all(repo_root.join(".agents/plugins")).unwrap(); + write_plugin_with_version(&repo_root, "z-good", "z-good", Some("1.2.3")); + write_file( + &repo_root.join(".agents/plugins/marketplace.json"), + r#"{ + "name": "debug", + "plugins": [ + { + "name": "a-broken", + "source": { + "source": "local", + "path": "./missing" + } + }, + { + "name": "z-good", + "source": { + "source": "local", + "path": "./z-good" + } + } + ] +}"#, + ); + + let err = refresh_non_curated_plugin_cache( + tmp.path(), + &[AbsolutePathBuf::try_from(repo_root).unwrap()], + &["a-broken@debug".to_string(), "z-good@debug".to_string()], + ) + .expect_err("broken plugin should be reported after refreshing the remaining plugins"); + + assert!(err.contains("a-broken@debug")); + assert!(tmp.path().join("plugins/cache/debug/z-good/1.2.3").is_dir()); +} + #[tokio::test] async fn load_plugins_ignores_project_config_files() { let codex_home = TempDir::new().unwrap(); diff --git a/codex-rs/core-plugins/src/marketplace.rs b/codex-rs/core-plugins/src/marketplace.rs index 3190e0449a..6475d37876 100644 --- a/codex-rs/core-plugins/src/marketplace.rs +++ b/codex-rs/core-plugins/src/marketplace.rs @@ -86,8 +86,8 @@ impl MarketplacePluginManifestFallback { } pub(crate) fn parse_for_listing(&self) -> Option { - // Git sources have no plugin root before install. Parse against a host-native synthetic - // absolute root, then discard path-bearing fields so listings expose metadata only. + // Materialized sources have no plugin root before install. Parse against a host-native + // synthetic absolute root, then discard path-bearing fields so listings expose metadata only. let plugin_root = Path::new(if cfg!(windows) { r"C:\" } else { "/" }); let mut manifest = crate::manifest::parse_plugin_manifest( plugin_root, @@ -133,6 +133,23 @@ pub enum MarketplacePluginSource { ref_name: Option, sha: Option, }, + Npm { + package: String, + version: Option, + registry: Option, + }, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum NpmPackageScope { + Scoped, + Unscoped, +} + +impl MarketplacePluginSource { + pub(crate) fn is_install_materialized(&self) -> bool { + matches!(self, Self::Git { .. } | Self::Npm { .. }) + } } #[derive(Debug, Clone, PartialEq, Eq)] @@ -513,10 +530,12 @@ fn resolve_marketplace_plugin_entry( None } } - MarketplacePluginSource::Git { .. } if manifest_fallback.has_metadata => { + MarketplacePluginSource::Git { .. } | MarketplacePluginSource::Npm { .. } + if manifest_fallback.has_metadata => + { manifest_fallback.parse_for_listing() } - MarketplacePluginSource::Git { .. } => None, + MarketplacePluginSource::Git { .. } | MarketplacePluginSource::Npm { .. } => None, }; let interface = plugin_interface_with_marketplace_category( manifest @@ -610,6 +629,17 @@ fn resolve_plugin_source( ref_name: normalize_optional_git_selector(&ref_name), sha: normalize_optional_git_selector(&sha), }), + RawMarketplaceManifestPluginSource::Object( + RawMarketplaceManifestPluginSourceObject::Npm { + package, + version, + registry, + }, + ) => Ok(MarketplacePluginSource::Npm { + package: normalize_npm_package(marketplace_path, &package)?, + version: normalize_optional_npm_version(marketplace_path, version)?, + registry: normalize_optional_npm_registry(marketplace_path, registry)?, + }), RawMarketplaceManifestPluginSource::Unsupported(_) => { unreachable!("unsupported plugin sources should be filtered before resolution") } @@ -748,6 +778,118 @@ fn normalize_optional_git_selector(value: &Option) -> Option { .map(str::to_string) } +fn normalize_npm_package( + marketplace_path: &AbsolutePathBuf, + package: &str, +) -> Result { + let package = package.trim(); + let package_scope = if package.starts_with('@') { + NpmPackageScope::Scoped + } else { + NpmPackageScope::Unscoped + }; + let segments = if let Some(scoped_package) = package.strip_prefix('@') { + scoped_package.split('/').collect::>() + } else { + package.split('/').collect::>() + }; + let expected_segments = match package_scope { + NpmPackageScope::Scoped => 2, + NpmPackageScope::Unscoped => 1, + }; + if package.is_empty() + || segments.len() != expected_segments + || segments + .iter() + .any(|segment| !is_valid_npm_package_segment(segment, package_scope)) + { + return Err(MarketplaceError::InvalidMarketplaceFile { + path: marketplace_path.to_path_buf(), + message: format!("invalid npm plugin source package: {package}"), + }); + } + Ok(package.to_string()) +} + +fn is_valid_npm_package_segment(segment: &str, package_scope: NpmPackageScope) -> bool { + !segment.is_empty() + && segment != "." + && segment != ".." + && (package_scope == NpmPackageScope::Scoped + || !matches!(segment.chars().next(), Some('.' | '_'))) + && segment + .chars() + .all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '-' | '_' | '.')) +} + +fn normalize_optional_npm_version( + marketplace_path: &AbsolutePathBuf, + version: Option, +) -> Result, MarketplaceError> { + let Some(version) = normalize_optional_npm_source_field(marketplace_path, version, "version")? + else { + return Ok(None); + }; + if !is_registry_npm_version_selector(&version) { + return Err(MarketplaceError::InvalidMarketplaceFile { + path: marketplace_path.to_path_buf(), + message: format!("npm plugin source version must use the registry: {version}"), + }); + } + Ok(Some(version)) +} + +fn is_registry_npm_version_selector(version: &str) -> bool { + version != "." && version != ".." && !version.chars().any(|ch| matches!(ch, '/' | '\\' | ':')) +} + +fn normalize_optional_npm_registry( + marketplace_path: &AbsolutePathBuf, + registry: Option, +) -> Result, MarketplaceError> { + let Some(registry) = + normalize_optional_npm_source_field(marketplace_path, registry, "registry")? + else { + return Ok(None); + }; + let parsed = + url::Url::parse(®istry).map_err(|_| MarketplaceError::InvalidMarketplaceFile { + path: marketplace_path.to_path_buf(), + message: format!("invalid npm plugin source registry: {registry}"), + })?; + if parsed.scheme() != "https" + || parsed.host_str().is_none() + || !parsed.username().is_empty() + || parsed.password().is_some() + || parsed.query().is_some() + || parsed.fragment().is_some() + { + return Err(MarketplaceError::InvalidMarketplaceFile { + path: marketplace_path.to_path_buf(), + message: format!("invalid npm plugin source registry: {registry}"), + }); + } + Ok(Some(registry)) +} + +fn normalize_optional_npm_source_field( + marketplace_path: &AbsolutePathBuf, + value: Option, + field: &str, +) -> Result, MarketplaceError> { + let Some(value) = value else { + return Ok(None); + }; + let value = value.trim(); + if value.is_empty() { + return Err(MarketplaceError::InvalidMarketplaceFile { + path: marketplace_path.to_path_buf(), + message: format!("npm plugin source {field} must not be empty"), + }); + } + Ok(Some(value.to_string())) +} + fn normalize_github_git_url(url: &str) -> String { if url.starts_with("https://github.com/") && !url.ends_with(".git") { format!("{url}.git") @@ -886,6 +1028,11 @@ enum RawMarketplaceManifestPluginSourceObject { ref_name: Option, sha: Option, }, + Npm { + package: String, + version: Option, + registry: Option, + }, } fn resolve_marketplace_interface( diff --git a/codex-rs/core-plugins/src/marketplace_policy.rs b/codex-rs/core-plugins/src/marketplace_policy.rs index 8e41fb893e..a4c99a81cf 100644 --- a/codex-rs/core-plugins/src/marketplace_policy.rs +++ b/codex-rs/core-plugins/src/marketplace_policy.rs @@ -18,9 +18,13 @@ use codex_config::MarketplaceAllowedSourceToml; use codex_config::RequirementSource; use codex_config::types::MarketplaceConfig; use codex_config::types::MarketplaceSourceType; +use codex_config::types::PluginConfig; +use codex_plugin::PluginId; use codex_utils_absolute_path::AbsolutePathBuf; use codex_utils_path::paths_match_after_normalization; use regex::Regex; +use std::collections::HashMap; +use std::collections::HashSet; use std::path::Path; use std::path::PathBuf; use url::Url; @@ -70,7 +74,7 @@ impl MarketplacePolicy { } } - fn is_restricted(&self) -> bool { + pub(crate) fn is_restricted(&self) -> bool { self.restricted.is_some() } @@ -198,6 +202,104 @@ impl AllowedMarketplaceSource { } } +pub(crate) fn project_effective_user_config( + config_layer_stack: &ConfigLayerStack, + codex_home: &Path, +) -> Option { + let mut user_config = config_layer_stack.effective_user_config()?; + let policy = MarketplacePolicy::from_requirements(config_layer_stack.requirements()); + if !policy.is_restricted() { + return Some(user_config); + } + let allowed_marketplace_names = + allowed_configured_marketplace_names_with_policy(&user_config, &policy, codex_home); + let configured_marketplace_names = user_config + .get("marketplaces") + .and_then(toml::Value::as_table) + .map(|marketplaces| marketplaces.keys().cloned().collect::>()) + .unwrap_or_default(); + + if let Some(marketplaces) = user_config + .get_mut("marketplaces") + .and_then(toml::Value::as_table_mut) + { + marketplaces + .retain(|marketplace_name, _| allowed_marketplace_names.contains(marketplace_name)); + } + if let Some(plugins) = user_config + .get_mut("plugins") + .and_then(toml::Value::as_table_mut) + { + plugins.retain(|plugin_key, _| { + let Ok(plugin_id) = PluginId::parse(plugin_key) else { + return false; + }; + (is_openai_curated_marketplace_name(&plugin_id.marketplace_name) + && !configured_marketplace_names.contains(&plugin_id.marketplace_name)) + || allowed_marketplace_names.contains(&plugin_id.marketplace_name) + }); + } + Some(user_config) +} + +pub fn allowed_configured_marketplace_names( + config_layer_stack: &ConfigLayerStack, + codex_home: &Path, +) -> HashSet { + let Some(user_config) = config_layer_stack.effective_user_config() else { + return HashSet::new(); + }; + let policy = MarketplacePolicy::from_requirements(config_layer_stack.requirements()); + allowed_configured_marketplace_names_with_policy(&user_config, &policy, codex_home) +} + +fn allowed_configured_marketplace_names_with_policy( + user_config: &toml::Value, + policy: &MarketplacePolicy, + codex_home: &Path, +) -> HashSet { + let Some(marketplaces) = user_config + .get("marketplaces") + .and_then(toml::Value::as_table) + else { + return HashSet::new(); + }; + if !policy.is_restricted() { + return marketplaces.keys().cloned().collect(); + } + marketplaces + .iter() + .filter_map(|(marketplace_name, marketplace)| { + let allowed = match managed_marketplace_config_name(codex_home, marketplace) { + Some(expected_name) => expected_name == marketplace_name, + None => policy + .validate_configured_marketplace(marketplace_name, marketplace) + .is_ok(), + }; + allowed.then(|| marketplace_name.clone()) + }) + .collect() +} + +pub(crate) fn configured_plugins_from_stack( + config_layer_stack: &ConfigLayerStack, + codex_home: &Path, +) -> HashMap { + let Some(user_config) = project_effective_user_config(config_layer_stack, codex_home) else { + return HashMap::new(); + }; + let Some(plugins_value) = user_config.get("plugins") else { + return HashMap::new(); + }; + match plugins_value.clone().try_into() { + Ok(plugins) => plugins, + Err(err) => { + tracing::warn!("invalid plugins config: {err}"); + HashMap::new() + } + } +} + pub(crate) fn validate_marketplace_source_for_add( codex_home: &Path, requirements: &ConfigRequirements, @@ -358,6 +460,21 @@ fn managed_marketplace_name( managed_local_marketplace_name(codex_home, root.as_path()) } +fn managed_marketplace_config_name( + codex_home: &Path, + marketplace: &toml::Value, +) -> Option<&'static str> { + if marketplace.get("source_type").and_then(toml::Value::as_str) != Some("local") { + return None; + } + let path = marketplace + .get("source") + .and_then(toml::Value::as_str) + .map(Path::new) + .filter(|path| path.is_absolute())?; + managed_local_marketplace_name(codex_home, path) +} + fn managed_local_marketplace_name(codex_home: &Path, root: &Path) -> Option<&'static str> { for marketplace_name in [ OPENAI_BUNDLED_MARKETPLACE_NAME, diff --git a/codex-rs/core-plugins/src/marketplace_policy_tests.rs b/codex-rs/core-plugins/src/marketplace_policy_tests.rs index 63518dfb3d..75c0a2f17c 100644 --- a/codex-rs/core-plugins/src/marketplace_policy_tests.rs +++ b/codex-rs/core-plugins/src/marketplace_policy_tests.rs @@ -383,6 +383,197 @@ restrict_to_allowed_sources = true assert!(validate_marketplace_name_for_add(expected_name, "other").is_err()); } +#[test] +fn projected_user_config_removes_blocked_marketplaces_and_plugins() { + let codex_home = TempDir::new().expect("create Codex home"); + let config_file = AbsolutePathBuf::try_from(codex_home.path().join("config.toml")) + .expect("absolute config path"); + let stack = config_layer_stack_with_user_config( + r#" +[marketplaces] +restrict_to_allowed_sources = true + +[marketplaces.allowed_sources.company] +source = "git" +url = "https://github.com/example/allowed.git" +"#, + Some(( + r#" +[marketplaces.allowed] +source_type = "git" +source = "https://github.com/example/allowed.git" + +[marketplaces.blocked] +source_type = "git" +source = "https://github.com/example/blocked.git" + +[plugins."sample@allowed"] +enabled = true + +[plugins."sample@blocked"] +enabled = true +"#, + config_file, + )), + ); + + let projected = + project_effective_user_config(&stack, codex_home.path()).expect("project user config"); + assert_eq!( + projected["marketplaces"] + .as_table() + .expect("projected marketplaces") + .keys() + .cloned() + .collect::>(), + vec!["allowed".to_string()] + ); + assert_eq!( + configured_plugins_from_stack(&stack, codex_home.path()) + .into_keys() + .collect::>(), + vec!["sample@allowed".to_string()] + ); + + let raw = stack.effective_user_config().expect("raw user config"); + assert!(raw["marketplaces"]["blocked"].is_table()); + assert!(raw["plugins"]["sample@blocked"].is_table()); +} + +#[test] +fn managed_bundled_config_is_retained_only_at_its_owned_path() { + let codex_home = TempDir::new().expect("create Codex home"); + let bundled_root = codex_home + .path() + .join(".tmp/bundled-marketplaces") + .join(crate::OPENAI_BUNDLED_MARKETPLACE_NAME); + let config_file = AbsolutePathBuf::try_from(codex_home.path().join("config.toml")) + .expect("absolute config path"); + let stack = config_layer_stack_with_user_config( + r#" +[marketplaces] +restrict_to_allowed_sources = true +"#, + Some(( + &format!( + r#" +[marketplaces.openai-bundled] +source_type = "local" +source = {bundled_root:?} + +[marketplaces.openai-bundled-alpha] +source_type = "local" +source = "/tmp/not-managed" + +[marketplaces.evil] +source_type = "local" +source = {bundled_root:?} + +[plugins."sample@openai-bundled"] +enabled = true + +[plugins."sample@openai-bundled-alpha"] +enabled = true + +[plugins."sample@evil"] +enabled = true +"# + ), + config_file, + )), + ); + + let projected = + project_effective_user_config(&stack, codex_home.path()).expect("project user config"); + + assert_eq!( + projected["marketplaces"] + .as_table() + .expect("projected marketplaces") + .keys() + .cloned() + .collect::>(), + vec![crate::OPENAI_BUNDLED_MARKETPLACE_NAME.to_string()] + ); + assert_eq!( + projected["plugins"] + .as_table() + .expect("projected plugins") + .keys() + .cloned() + .collect::>(), + vec![format!("sample@{}", crate::OPENAI_BUNDLED_MARKETPLACE_NAME)] + ); +} + +#[test] +fn allowlisted_config_names_are_not_globally_reserved() { + let codex_home = TempDir::new().expect("create Codex home"); + let source_root = TempDir::new().expect("create marketplace root"); + let source_root = source_root + .path() + .canonicalize() + .expect("canonical marketplace root"); + let config_file = AbsolutePathBuf::try_from(codex_home.path().join("config.toml")) + .expect("absolute config path"); + let stack = config_layer_stack_with_user_config( + &format!( + r#" +[marketplaces] +restrict_to_allowed_sources = true + +[marketplaces.allowed_sources.local] +source = "local" +path = {source_root:?} +"# + ), + Some(( + &format!( + r#" +[marketplaces.openai-bundled] +source_type = "local" +source = {source_root:?} + +[marketplaces.openai-curated] +source_type = "local" +source = {source_root:?} + +[plugins."sample@openai-bundled"] +enabled = true + +[plugins."sample@openai-curated"] +enabled = true +"# + ), + config_file, + )), + ); + + let projected = + project_effective_user_config(&stack, codex_home.path()).expect("project user config"); + assert_eq!( + projected["marketplaces"] + .as_table() + .expect("projected marketplaces") + .keys() + .cloned() + .collect::>(), + vec!["openai-bundled".to_string(), "openai-curated".to_string()] + ); + assert_eq!( + projected["plugins"] + .as_table() + .expect("projected plugins") + .keys() + .cloned() + .collect::>(), + vec![ + "sample@openai-bundled".to_string(), + "sample@openai-curated".to_string() + ] + ); +} + #[test] fn blocked_upgrade_is_rejected_before_marketplace_installation() { let codex_home = TempDir::new().expect("create Codex home"); diff --git a/codex-rs/core-plugins/src/marketplace_tests.rs b/codex-rs/core-plugins/src/marketplace_tests.rs index 3ea2c9d606..d30e9ad1da 100644 --- a/codex-rs/core-plugins/src/marketplace_tests.rs +++ b/codex-rs/core-plugins/src/marketplace_tests.rs @@ -6,7 +6,6 @@ use tempfile::tempdir; const ALTERNATE_MARKETPLACE_RELATIVE_PATH: &str = ".claude-plugin/marketplace.json"; const ALTERNATE_PLUGIN_MANIFEST_RELATIVE_PATH: &str = ".claude-plugin/plugin.json"; - fn write_alternate_marketplace(repo_root: &Path, contents: &str) -> AbsolutePathBuf { let marketplace_path = repo_root.join(ALTERNATE_MARKETPLACE_RELATIVE_PATH); fs::create_dir_all(marketplace_path.parent().unwrap()).unwrap(); @@ -225,6 +224,248 @@ fn find_marketplace_plugin_omits_interface_asset_paths_for_git_sources() { assert!(interface.screenshots.is_empty()); } +#[test] +fn find_marketplace_plugin_supports_npm_sources() { + let tmp = tempdir().unwrap(); + let repo_root = tmp.path().join("repo"); + fs::create_dir_all(repo_root.join(".git")).unwrap(); + fs::create_dir_all(repo_root.join(".agents/plugins")).unwrap(); + fs::write( + repo_root.join(".agents/plugins/marketplace.json"), + r#"{ + "name": "codex-curated", + "plugins": [ + { + "name": "npm-plugin", + "source": { + "source": "npm", + "package": "@acme/codex-plugin", + "version": "^1.2.0", + "registry": "https://npm.example.com" + } + } + ] +}"#, + ) + .unwrap(); + + let resolved = find_marketplace_plugin( + &AbsolutePathBuf::try_from(repo_root.join(".agents/plugins/marketplace.json")).unwrap(), + "npm-plugin", + ) + .unwrap(); + + assert_eq!( + resolved, + ResolvedMarketplacePlugin { + plugin_id: PluginId::new("npm-plugin".to_string(), "codex-curated".to_string()) + .unwrap(), + source: MarketplacePluginSource::Npm { + package: "@acme/codex-plugin".to_string(), + version: Some("^1.2.0".to_string()), + registry: Some("https://npm.example.com".to_string()), + }, + policy: MarketplacePluginPolicy { + installation: MarketplacePluginInstallPolicy::Available, + authentication: MarketplacePluginAuthPolicy::OnInstall, + products: None, + }, + interface: None, + manifest: None, + manifest_fallback: minimal_manifest_fallback("npm-plugin"), + } + ); +} + +#[test] +fn find_marketplace_plugin_skips_unsafe_npm_sources() { + let tmp = tempdir().unwrap(); + let repo_root = tmp.path().join("repo"); + fs::create_dir_all(repo_root.join(".git")).unwrap(); + let marketplace_path = write_alternate_marketplace( + &repo_root, + r#"{ + "name": "codex-curated", + "plugins": [ + { + "name": "remote-version", + "source": { + "source": "npm", + "package": "@acme/codex-plugin", + "version": "https://attacker.example/plugin.tgz", + "registry": "https://npm.example.com" + } + }, + { + "name": "local-version", + "source": { + "source": "npm", + "package": "@acme/codex-plugin", + "version": ".", + "registry": "https://npm.example.com" + } + }, + { + "name": "plaintext-registry", + "source": { + "source": "npm", + "package": "@acme/codex-plugin", + "version": "1.2.0", + "registry": "http://npm.example.com" + } + }, + { + "name": "credential-registry", + "source": { + "source": "npm", + "package": "@acme/codex-plugin", + "version": "1.2.0", + "registry": "https://user:password@npm.example.com" + } + }, + { + "name": "dot-package", + "source": { + "source": "npm", + "package": ".codex-plugin", + "registry": "https://npm.example.com" + } + }, + { + "name": "underscore-package", + "source": { + "source": "npm", + "package": "_codex-plugin", + "registry": "https://npm.example.com" + } + } + ] +}"#, + ); + + assert_eq!( + load_marketplace(&marketplace_path).unwrap().plugins, + Vec::new() + ); +} + +#[test] +fn find_marketplace_plugin_supports_npm_registry_version_selectors() { + let tmp = tempdir().unwrap(); + let repo_root = tmp.path().join("repo"); + fs::create_dir_all(repo_root.join(".git")).unwrap(); + let marketplace_path = write_alternate_marketplace( + &repo_root, + r#"{ + "name": "codex-curated", + "plugins": [ + { + "name": "dist-tag", + "source": { + "source": "npm", + "package": "@acme/codex-plugin", + "version": "latest" + } + }, + { + "name": "comparator-range", + "source": { + "source": "npm", + "package": "@acme/codex-plugin", + "version": ">=1.2.7 <1.3.0" + } + }, + { + "name": "x-range", + "source": { + "source": "npm", + "package": "@acme/codex-plugin", + "version": "1.2.x" + } + }, + { + "name": "or-range", + "source": { + "source": "npm", + "package": "@acme/codex-plugin", + "version": "1.2.7 || >=1.2.9 <2.0.0" + } + } + ] +}"#, + ); + + assert_eq!( + load_marketplace(&marketplace_path) + .unwrap() + .plugins + .into_iter() + .map(|plugin| plugin.source) + .collect::>(), + vec![ + MarketplacePluginSource::Npm { + package: "@acme/codex-plugin".to_string(), + version: Some("latest".to_string()), + registry: None, + }, + MarketplacePluginSource::Npm { + package: "@acme/codex-plugin".to_string(), + version: Some(">=1.2.7 <1.3.0".to_string()), + registry: None, + }, + MarketplacePluginSource::Npm { + package: "@acme/codex-plugin".to_string(), + version: Some("1.2.x".to_string()), + registry: None, + }, + MarketplacePluginSource::Npm { + package: "@acme/codex-plugin".to_string(), + version: Some("1.2.7 || >=1.2.9 <2.0.0".to_string()), + registry: None, + }, + ] + ); +} + +#[test] +fn find_marketplace_plugin_supports_npm_sources_without_optional_fields() { + let tmp = tempdir().unwrap(); + let repo_root = tmp.path().join("repo"); + fs::create_dir_all(repo_root.join(".git")).unwrap(); + fs::create_dir_all(repo_root.join(".agents/plugins")).unwrap(); + fs::write( + repo_root.join(".agents/plugins/marketplace.json"), + r#"{ + "name": "codex-curated", + "plugins": [ + { + "name": "npm-plugin", + "source": { + "source": "npm", + "package": "@acme/codex-plugin" + } + } + ] +}"#, + ) + .unwrap(); + + let resolved = find_marketplace_plugin( + &AbsolutePathBuf::try_from(repo_root.join(".agents/plugins/marketplace.json")).unwrap(), + "npm-plugin", + ) + .unwrap(); + + assert_eq!( + resolved.source, + MarketplacePluginSource::Npm { + package: "@acme/codex-plugin".to_string(), + version: None, + registry: None, + } + ); +} + #[test] fn find_marketplace_plugin_builds_manifest_fallback_from_entry() { let tmp = tempdir().unwrap(); diff --git a/codex-rs/core-plugins/src/npm_source.rs b/codex-rs/core-plugins/src/npm_source.rs new file mode 100644 index 0000000000..3d728bd0f7 --- /dev/null +++ b/codex-rs/core-plugins/src/npm_source.rs @@ -0,0 +1,188 @@ +use crate::plugin_bundle_archive::unpack_plugin_bundle_tar_gz; +use codex_utils_absolute_path::AbsolutePathBuf; +use serde::Deserialize; +use std::ffi::OsStr; +use std::fs; +use std::path::Path; +use std::path::PathBuf; +use std::process::Command; +use tempfile::TempDir; + +const NPM_PLUGIN_SOURCE_STAGING_DIR: &str = "plugins/.marketplace-plugin-source-staging"; +const NPM_PLUGIN_SOURCE_MAX_ARCHIVE_BYTES: u64 = 50 * 1024 * 1024; +const NPM_PLUGIN_SOURCE_MAX_EXTRACTED_BYTES: u64 = 250 * 1024 * 1024; +const NPM_PACKAGE_ARCHIVE_ROOT: &str = "package"; + +pub(crate) fn materialize_npm_plugin_source( + codex_home: &Path, + package: &str, + version: Option<&str>, + registry: Option<&str>, +) -> Result<(AbsolutePathBuf, TempDir), String> { + materialize_npm_plugin_source_with_command( + codex_home, + package, + version, + registry, + OsStr::new(npm_command()), + ) +} + +fn materialize_npm_plugin_source_with_command( + codex_home: &Path, + package: &str, + version: Option<&str>, + registry: Option<&str>, + npm_command: &OsStr, +) -> Result<(AbsolutePathBuf, TempDir), String> { + let staging_root = codex_home.join(NPM_PLUGIN_SOURCE_STAGING_DIR); + fs::create_dir_all(&staging_root).map_err(|err| { + format!( + "failed to create marketplace plugin source staging directory {}: {err}", + staging_root.display() + ) + })?; + let tempdir = tempfile::Builder::new() + .prefix("marketplace-plugin-source-") + .tempdir_in(&staging_root) + .map_err(|err| { + format!( + "failed to create marketplace plugin source staging directory in {}: {err}", + staging_root.display() + ) + })?; + + pack_npm_package(tempdir.path(), package, version, registry, npm_command)?; + let archive_path = find_npm_package_archive(tempdir.path())?; + let archive_bytes = read_npm_package_archive(&archive_path)?; + + let extraction_root = tempdir.path().join("extracted"); + unpack_plugin_bundle_tar_gz( + &archive_bytes, + &extraction_root, + NPM_PLUGIN_SOURCE_MAX_EXTRACTED_BYTES, + ) + .map_err(|err| format!("failed to extract npm plugin package: {err}"))?; + let plugin_root = extraction_root.join(NPM_PACKAGE_ARCHIVE_ROOT); + if !plugin_root.is_dir() { + return Err(format!( + "npm pack completed without creating plugin package directory {}", + plugin_root.display() + )); + } + validate_npm_package_metadata(&plugin_root, package)?; + let plugin_root = AbsolutePathBuf::try_from(plugin_root) + .map_err(|err| format!("failed to resolve materialized plugin source path: {err}"))?; + Ok((plugin_root, tempdir)) +} + +fn pack_npm_package( + destination: &Path, + package: &str, + version: Option<&str>, + registry: Option<&str>, + npm_command: &OsStr, +) -> Result<(), String> { + let package_spec = version.map_or_else( + || package.to_string(), + |version| format!("{package}@{version}"), + ); + let mut command = Command::new(npm_command); + command + .current_dir(destination) + .arg("pack") + .arg("--ignore-scripts") + .arg("--pack-destination") + .arg(destination); + if let Some(registry) = registry { + command.arg("--registry").arg(registry); + } + command.arg("--").arg(package_spec); + + let output = command + .output() + .map_err(|err| format!("failed to run npm pack: {err}"))?; + if output.status.success() { + return Ok(()); + } + + Err(format!( + "npm pack failed with status {}\nstdout:\n{}\nstderr:\n{}", + output.status, + String::from_utf8_lossy(&output.stdout).trim(), + String::from_utf8_lossy(&output.stderr).trim() + )) +} + +fn find_npm_package_archive(destination: &Path) -> Result { + let mut archives = fs::read_dir(destination) + .map_err(|err| format!("failed to read npm pack destination: {err}"))? + .filter_map(std::result::Result::ok) + .filter_map(|entry| { + let path = entry.path(); + let is_file = entry.file_type().is_ok_and(|file_type| file_type.is_file()); + (is_file && path.extension() == Some(OsStr::new("tgz"))).then_some(path) + }) + .collect::>(); + if archives.len() != 1 { + return Err(format!( + "npm pack completed with {} package archives; expected exactly one", + archives.len() + )); + } + Ok(archives.remove(0)) +} + +fn read_npm_package_archive(archive_path: &Path) -> Result, String> { + let archive_size = fs::metadata(archive_path) + .map_err(|err| format!("failed to inspect npm package archive: {err}"))? + .len(); + if archive_size > NPM_PLUGIN_SOURCE_MAX_ARCHIVE_BYTES { + return Err(format!( + "npm package archive is {archive_size} bytes, exceeding maximum size of {NPM_PLUGIN_SOURCE_MAX_ARCHIVE_BYTES} bytes" + )); + } + fs::read(archive_path).map_err(|err| format!("failed to read npm package archive: {err}")) +} + +fn validate_npm_package_metadata(plugin_root: &Path, package: &str) -> Result<(), String> { + #[derive(Deserialize)] + struct NpmPackageMetadata { + name: String, + } + + let package_json_path = plugin_root.join("package.json"); + let package_json = fs::read_to_string(&package_json_path).map_err(|err| { + format!( + "failed to read npm plugin package metadata {}: {err}", + package_json_path.display() + ) + })?; + let metadata: NpmPackageMetadata = serde_json::from_str(&package_json).map_err(|err| { + format!( + "failed to parse npm plugin package metadata {}: {err}", + package_json_path.display() + ) + })?; + if metadata.name != package { + return Err(format!( + "npm plugin package name '{}' does not match requested package '{package}'", + metadata.name + )); + } + Ok(()) +} + +#[cfg(windows)] +fn npm_command() -> &'static str { + "npm.cmd" +} + +#[cfg(not(windows))] +fn npm_command() -> &'static str { + "npm" +} + +#[cfg(all(test, unix))] +#[path = "npm_source_tests.rs"] +mod tests; diff --git a/codex-rs/core-plugins/src/npm_source_tests.rs b/codex-rs/core-plugins/src/npm_source_tests.rs new file mode 100644 index 0000000000..3c2dcd94ac --- /dev/null +++ b/codex-rs/core-plugins/src/npm_source_tests.rs @@ -0,0 +1,111 @@ +use super::*; +use flate2::Compression; +use flate2::write::GzEncoder; +use pretty_assertions::assert_eq; +use std::io::Cursor; +use std::io::Write; + +#[cfg(unix)] +#[test] +fn materialize_npm_plugin_source_uses_packed_package_root() { + use std::os::unix::fs::PermissionsExt; + + let codex_home = tempfile::tempdir().expect("create codex home"); + let fake_npm_dir = tempfile::tempdir().expect("create fake npm directory"); + let archive_bytes = + npm_package_archive_bytes("@acme/plugin", "1.2.0").expect("build fixture archive"); + let archive_path = fake_npm_dir.path().join("fixture.tgz"); + fs::write(&archive_path, &archive_bytes).expect("write fixture archive"); + let fake_npm = fake_npm_dir.path().join("npm"); + fs::write( + &fake_npm, + format!( + r#"#!/bin/sh +destination="" +previous="" +for argument in "$@"; do + if [ "$previous" = "--pack-destination" ]; then + destination="$argument" + fi + previous="$argument" +done +cp "{}" "$destination/acme-plugin-1.2.0.tgz" +printf '%s\n' "$@" > "$destination/args.txt" +pwd > "$destination/pwd.txt" +"#, + archive_path.display() + ), + ) + .expect("write fake npm"); + let mut permissions = fs::metadata(&fake_npm) + .expect("read fake npm metadata") + .permissions(); + permissions.set_mode(0o755); + fs::set_permissions(&fake_npm, permissions).expect("make fake npm executable"); + + let (plugin_root, tempdir) = materialize_npm_plugin_source_with_command( + codex_home.path(), + "@acme/plugin", + Some("^1.2.0"), + Some("https://npm.example.com"), + fake_npm.as_os_str(), + ) + .expect("materialize npm source"); + + assert_eq!( + plugin_root.as_path(), + tempdir.path().join("extracted/package") + ); + assert!( + plugin_root + .as_path() + .join(".codex-plugin/plugin.json") + .is_file() + ); + let args = fs::read_to_string(tempdir.path().join("args.txt")).expect("read npm arguments"); + assert!(args.contains("pack")); + assert!(args.contains("--ignore-scripts")); + assert!(args.contains("--registry")); + assert!(args.contains("https://npm.example.com")); + assert!(args.contains("@acme/plugin@^1.2.0")); + assert!(!args.contains("install")); + let npm_working_directory = fs::canonicalize( + fs::read_to_string(tempdir.path().join("pwd.txt")) + .expect("read npm working directory") + .trim(), + ) + .expect("canonicalize npm working directory"); + assert_eq!( + npm_working_directory, + fs::canonicalize(tempdir.path()).expect("canonicalize tempdir") + ); +} + +fn npm_package_archive_bytes(package: &str, version: &str) -> std::io::Result> { + let encoder = GzEncoder::new(Vec::new(), Compression::default()); + let mut archive = tar::Builder::new(encoder); + append_archive_file( + &mut archive, + "package/package.json", + format!(r#"{{"name":"{package}","version":"{version}"}}"#).as_bytes(), + )?; + append_archive_file( + &mut archive, + "package/.codex-plugin/plugin.json", + br#"{"name":"plugin"}"#, + )?; + let encoder = archive.into_inner()?; + encoder.finish() +} + +fn append_archive_file( + archive: &mut tar::Builder, + path: &str, + contents: &[u8], +) -> std::io::Result<()> { + let mut header = tar::Header::new_gnu(); + header.set_size(contents.len() as u64); + header.set_mode(0o644); + header.set_cksum(); + archive.append_data(&mut header, path, Cursor::new(contents)) +} diff --git a/codex-rs/core-plugins/src/remote.rs b/codex-rs/core-plugins/src/remote.rs index 515bbb71c0..4c9bd2e672 100644 --- a/codex-rs/core-plugins/src/remote.rs +++ b/codex-rs/core-plugins/src/remote.rs @@ -148,6 +148,7 @@ pub enum RemoteMarketplaceSource { pub struct RemoteInstalledPlugin { pub marketplace_name: String, pub id: String, + pub version: Option, pub name: String, pub enabled: bool, pub install_policy: PluginInstallPolicy, @@ -161,6 +162,7 @@ pub struct RemoteInstalledPlugin { pub struct RemotePluginSummary { pub id: String, pub remote_plugin_id: String, + pub version: Option, pub local_version: Option, pub name: String, pub share_context: Option, @@ -1041,6 +1043,7 @@ pub fn group_remote_installed_plugins_by_marketplaces( let plugin_summary = RemotePluginSummary { id: plugin_id.as_key(), remote_plugin_id: plugin.id.clone(), + version: plugin.version.clone(), local_version: None, name: plugin.name.clone(), share_context: None, @@ -1472,6 +1475,7 @@ fn build_remote_plugin_summary( Ok(RemotePluginSummary { id: plugin_id.as_key(), remote_plugin_id: plugin.id.clone(), + version: plugin.release.version.clone(), local_version: installed_plugin .and_then(|installed| installed.plugin.release.version.clone()), name: plugin.name.clone(), @@ -1554,6 +1558,7 @@ fn remote_installed_plugin_to_cache_entry( Ok(RemoteInstalledPlugin { marketplace_name: remote_plugin_canonical_marketplace_name(plugin)?.to_string(), id: plugin.id.clone(), + version: plugin.release.version.clone(), name: plugin.name.clone(), enabled: installed_plugin.enabled, install_policy: plugin.installation_policy, diff --git a/codex-rs/core-plugins/src/remote/share/tests.rs b/codex-rs/core-plugins/src/remote/share/tests.rs index c8d06b234d..d7744681b9 100644 --- a/codex-rs/core-plugins/src/remote/share/tests.rs +++ b/codex-rs/core-plugins/src/remote/share/tests.rs @@ -618,6 +618,7 @@ async fn list_remote_plugin_shares_fetches_created_workspace_plugins() { summary: RemotePluginSummary { id: "demo-plugin@workspace-shared-with-me".to_string(), remote_plugin_id: "plugins_123".to_string(), + version: Some("0.1.0".to_string()), local_version: None, name: "demo-plugin".to_string(), share_context: Some(RemotePluginShareContext { @@ -658,6 +659,7 @@ async fn list_remote_plugin_shares_fetches_created_workspace_plugins() { summary: RemotePluginSummary { id: "demo-plugin@workspace-shared-with-me".to_string(), remote_plugin_id: "plugins_456".to_string(), + version: Some("0.1.0".to_string()), local_version: Some("0.1.0".to_string()), name: "demo-plugin".to_string(), share_context: Some(RemotePluginShareContext { diff --git a/codex-rs/core-plugins/src/store.rs b/codex-rs/core-plugins/src/store.rs index 5d78fefa90..bb4929dedb 100644 --- a/codex-rs/core-plugins/src/store.rs +++ b/codex-rs/core-plugins/src/store.rs @@ -37,6 +37,7 @@ pub struct PluginInstallResult { #[derive(Debug, Clone)] pub struct PluginStore { + codex_home: AbsolutePathBuf, root: AbsolutePathBuf, data_root: AbsolutePathBuf, } @@ -59,14 +60,24 @@ impl PluginStore { let data_root = AbsolutePathBuf::from_absolute_path_checked(codex_home.join(PLUGINS_DATA_DIR)) .map_err(|err| PluginStoreError::io("failed to resolve plugin data root", err))?; + let codex_home = AbsolutePathBuf::from_absolute_path_checked(codex_home) + .map_err(|err| PluginStoreError::io("failed to resolve Codex home", err))?; - Ok(Self { root, data_root }) + Ok(Self { + codex_home, + root, + data_root, + }) } pub fn root(&self) -> &AbsolutePathBuf { &self.root } + pub(crate) fn codex_home(&self) -> &AbsolutePathBuf { + &self.codex_home + } + pub fn plugin_base_root(&self, plugin_id: &PluginId) -> AbsolutePathBuf { self.root .join(&plugin_id.marketplace_name) diff --git a/codex-rs/core-plugins/src/test_support.rs b/codex-rs/core-plugins/src/test_support.rs index 19148df323..9e06de9d16 100644 --- a/codex-rs/core-plugins/src/test_support.rs +++ b/codex-rs/core-plugins/src/test_support.rs @@ -149,7 +149,7 @@ pub(crate) async fn load_plugins_config(codex_home: &Path, cwd: &Path) -> Plugin feature_enabled( &effective_config, "remote_plugin", - /*default_enabled*/ false, + /*default_enabled*/ true, ), "https://chatgpt.com/backend-api/".to_string(), ) diff --git a/codex-rs/core-skills/src/lib.rs b/codex-rs/core-skills/src/lib.rs index 53ffcddd07..dcfc6568a5 100644 --- a/codex-rs/core-skills/src/lib.rs +++ b/codex-rs/core-skills/src/lib.rs @@ -24,7 +24,6 @@ pub use render::AvailableSkills; pub use render::SKILLS_HOW_TO_USE_WITH_ABSOLUTE_PATHS; pub use render::SKILLS_HOW_TO_USE_WITH_ALIASES; pub use render::SKILLS_INTRO_WITH_ABSOLUTE_PATHS; -pub use render::SKILLS_INTRO_WITH_ALIASES; pub use render::SkillMetadataBudget; pub use render::SkillRenderReport; pub use render::build_available_skills; diff --git a/codex-rs/core-skills/src/loader/environment.rs b/codex-rs/core-skills/src/loader/environment.rs index 5dc29ebfb2..3d50bafe1f 100644 --- a/codex-rs/core-skills/src/loader/environment.rs +++ b/codex-rs/core-skills/src/loader/environment.rs @@ -10,6 +10,7 @@ use codex_utils_path_uri::PathUri; use codex_utils_plugins::DISCOVERABLE_PLUGIN_MANIFEST_PATHS; use codex_utils_plugins::plugin_namespace_for_root_uri; use codex_utils_plugins::plugin_namespace_for_skill_uri; +use futures::StreamExt; use futures::future::join_all; use crate::model::SkillDependencies; @@ -22,7 +23,6 @@ use super::ParsedSkillFrontmatter; use super::SKILLS_FILENAME; use super::SKILLS_METADATA_DIR; use super::SKILLS_METADATA_FILENAME; -use super::SkillFileDiscovery; use super::SkillMetadataFile; use super::parse_skill_frontmatter_metadata_inner; use super::resolve_dependencies; @@ -31,6 +31,34 @@ use super::sanitize_single_line; use super::validate_len; const MAX_SKILLS_ENTRIES_PER_ROOT: usize = 20_000; +const MAX_CONCURRENT_SKILL_LOADS: usize = 64; + +struct EnvironmentSkillDiscovery { + skills: Vec, + plugin_roots: HashSet, + namespace_roots: HashSet, + warnings: Vec, +} + +struct DiscoveredEnvironmentSkill { + path: PathUri, + metadata: SkillMetadataDiscovery, +} + +struct ParsedEnvironmentSkill { + path_to_skills_md: PathUri, + base_name: String, + description: String, + short_description: Option, + dependencies: Option, + policy: Option, +} + +enum SkillMetadataDiscovery { + Present(PathUri), + Absent, + Probe(PathUri), +} /// URI-native metadata for one skill owned by an execution environment. #[derive(Clone, Debug, PartialEq, Eq)] @@ -62,32 +90,44 @@ impl EnvironmentSkillMetadata { None => true, } } +} - async fn parse( +impl ParsedEnvironmentSkill { + async fn load( file_system: &dyn ExecutorFileSystem, - path: &PathUri, - plugin_namespace: Option<&str>, + skill: &DiscoveredEnvironmentSkill, ) -> Result { - let contents = file_system - .read_file_text(path, /*sandbox*/ None) - .await - .map_err(|err| format!("failed to read file: {err}"))?; + let (contents, discovered_metadata) = match &skill.metadata { + SkillMetadataDiscovery::Present(metadata_path) => { + let (contents, metadata) = tokio::join!( + read_skill_contents(file_system, &skill.path), + read_skill_metadata(file_system, metadata_path), + ); + (contents?, metadata) + } + SkillMetadataDiscovery::Absent | SkillMetadataDiscovery::Probe(_) => ( + read_skill_contents(file_system, &skill.path).await?, + (None, None), + ), + }; let ParsedSkillFrontmatter { name: base_name, description, short_description, - } = parse_skill_frontmatter_metadata_inner(&contents, || default_skill_name(path)) + } = parse_skill_frontmatter_metadata_inner(&contents, || default_skill_name(&skill.path)) .map_err(|err| err.to_string())?; - let name = plugin_namespace - .map(|namespace| format!("{namespace}:{base_name}")) - .unwrap_or(base_name); - validate_len(&name, MAX_QUALIFIED_NAME_LEN, "qualified name") - .map_err(|err| err.to_string())?; - let (dependencies, policy) = load_skill_metadata(file_system, path).await; + let (dependencies, policy) = match &skill.metadata { + SkillMetadataDiscovery::Present(_) | SkillMetadataDiscovery::Absent => { + discovered_metadata + } + SkillMetadataDiscovery::Probe(metadata_path) => { + probe_skill_metadata(file_system, metadata_path).await + } + }; Ok(Self { - path_to_skills_md: path.clone(), - name, + path_to_skills_md: skill.path.clone(), + base_name, description, short_description, dependencies, @@ -103,6 +143,12 @@ pub struct EnvironmentSkillLoadOutcome { } /// Discovers skills without converting environment-owned paths to host paths. +#[tracing::instrument( + name = "skills.environment.load", + level = "info", + skip_all, + fields(skill_count = tracing::field::Empty) +)] pub async fn load_environment_skills_from_root( file_system: &dyn ExecutorFileSystem, root: &PathUri, @@ -123,6 +169,7 @@ pub async fn load_environment_skills_from_root( .await { Ok(walk) => { + let inventory_complete = !walk.truncated && walk.errors.is_empty(); let mut warnings = walk .errors .into_iter() @@ -139,10 +186,13 @@ pub async fn load_environment_skills_from_root( )); } let mut skill_files = Vec::new(); + let mut file_paths = HashSet::new(); + let mut directory_paths = HashSet::new(); let mut plugin_roots = HashSet::new(); for entry in walk.entries { match entry.kind { WalkEntryKind::Directory => { + directory_paths.insert(entry.path.clone()); if DISCOVERABLE_PLUGIN_MANIFEST_PATHS .iter() .any(|path| path.split('/').next() == entry.path.basename().as_deref()) @@ -152,40 +202,54 @@ pub async fn load_environment_skills_from_root( } } WalkEntryKind::File => { + file_paths.insert(entry.path.clone()); if entry.path.basename().as_deref() == Some(SKILLS_FILENAME) { skill_files.push(entry.path); } } } } - SkillFileDiscovery { - skill_files, + let skills = skill_files + .into_iter() + .map(|path| DiscoveredEnvironmentSkill { + metadata: discover_skill_metadata( + &path, + &file_paths, + &directory_paths, + inventory_complete, + ), + path, + }) + .collect(); + EnvironmentSkillDiscovery { + skills, plugin_roots, namespace_roots: HashSet::from([root.clone()]), warnings, } } - Err(error) if error.kind() == io::ErrorKind::NotFound => SkillFileDiscovery { - skill_files: Vec::new(), + Err(error) if error.kind() == io::ErrorKind::NotFound => EnvironmentSkillDiscovery { + skills: Vec::new(), plugin_roots: HashSet::new(), namespace_roots: HashSet::new(), warnings: Vec::new(), }, - Err(error) => SkillFileDiscovery { - skill_files: Vec::new(), + Err(error) => EnvironmentSkillDiscovery { + skills: Vec::new(), plugin_roots: HashSet::new(), namespace_roots: HashSet::new(), warnings: vec![format!("failed to walk skills root {root}: {error:#}")], }, }; + tracing::Span::current().record("skill_count", discovery.skills.len()); outcome.warnings.extend(discovery.warnings); - if discovery.skill_files.is_empty() { + if discovery.skills.is_empty() { return outcome; } let mut skill_ancestors = HashSet::new(); - for skill_path in &discovery.skill_files { - let mut ancestor = skill_path.parent(); + for skill in &discovery.skills { + let mut ancestor = skill.path.parent(); while let Some(path) = ancestor { skill_ancestors.insert(path.clone()); ancestor = path.parent(); @@ -193,51 +257,80 @@ pub async fn load_environment_skills_from_root( } let namespace_roots = discovery.namespace_roots; - let namespace_lookups = join_all(namespace_roots.iter().map(|namespace_root| async { - ( - namespace_root.clone(), - plugin_namespace_for_skill_uri(file_system, namespace_root).await, - ) - })) - .await; - let plugin_lookups = join_all( - discovery - .plugin_roots - .iter() - .filter(|plugin_root| skill_ancestors.contains(*plugin_root)) - .filter(|plugin_root| !namespace_roots.contains(*plugin_root)) - .map(|plugin_root| async { - ( - plugin_root.clone(), - plugin_namespace_for_root_uri(file_system, plugin_root).await, - ) - }), - ) - .await; - let plugin_namespaces = namespace_lookups - .into_iter() - .chain(plugin_lookups) - .filter_map(|(plugin_root, namespace)| namespace.map(|namespace| (plugin_root, namespace))) - .collect::>(); + let plugin_namespaces = async { + let namespace_lookups = join_all(namespace_roots.iter().map(|namespace_root| async { + ( + namespace_root.clone(), + plugin_namespace_for_skill_uri(file_system, namespace_root).await, + ) + })); + let plugin_lookups = join_all( + discovery + .plugin_roots + .iter() + .filter(|plugin_root| skill_ancestors.contains(*plugin_root)) + .filter(|plugin_root| !namespace_roots.contains(*plugin_root)) + .map(|plugin_root| async { + ( + plugin_root.clone(), + plugin_namespace_for_root_uri(file_system, plugin_root).await, + ) + }), + ); + let (namespace_lookups, plugin_lookups) = tokio::join!(namespace_lookups, plugin_lookups); + namespace_lookups + .into_iter() + .chain(plugin_lookups) + .filter_map(|(plugin_root, namespace)| { + namespace.map(|namespace| (plugin_root, namespace)) + }) + .collect::>() + }; - for path in discovery.skill_files { - let mut ancestor = path.parent(); - let plugin_namespace = loop { - let Some(current) = ancestor else { - break None; - }; - if let Some(namespace) = plugin_namespaces.get(¤t) { - break Some(namespace.as_str()); + // Remote executors can multiplex these independent per-skill reads, so polling a bounded + // number together allows the I/O for each skill and its metadata to happen concurrently. + let skill_results = futures::stream::iter(discovery.skills) + .map(|skill| { + let path = skill.path.clone(); + async move { + ( + path, + ParsedEnvironmentSkill::load(file_system, &skill).await, + ) } - ancestor = current.parent(); - }; - match EnvironmentSkillMetadata::parse( - file_system, - &path, - /*plugin_namespace*/ plugin_namespace, - ) - .await - { + }) + .buffered(MAX_CONCURRENT_SKILL_LOADS) + .collect::>(); + let (plugin_namespaces, skill_results) = tokio::join!(plugin_namespaces, skill_results); + + for (path, result) in skill_results { + let result = result.and_then(|skill| { + let mut ancestor = skill.path_to_skills_md.parent(); + let plugin_namespace = loop { + let Some(current) = ancestor else { + break None; + }; + if let Some(namespace) = plugin_namespaces.get(¤t) { + break Some(namespace.as_str()); + } + ancestor = current.parent(); + }; + let name = plugin_namespace + .map(|namespace| format!("{namespace}:{}", skill.base_name)) + .unwrap_or(skill.base_name); + validate_len(&name, MAX_QUALIFIED_NAME_LEN, "qualified name") + .map_err(|err| err.to_string())?; + + Ok(EnvironmentSkillMetadata { + path_to_skills_md: skill.path_to_skills_md, + name, + description: skill.description, + short_description: skill.short_description, + dependencies: skill.dependencies, + policy: skill.policy, + }) + }); + match result { Ok(skill) if skill.matches_product_restriction(restriction_product) => { outcome.skills.push(skill); } @@ -257,20 +350,48 @@ pub async fn load_environment_skills_from_root( outcome } -async fn load_skill_metadata( +fn discover_skill_metadata( + skill_path: &PathUri, + file_paths: &HashSet, + directory_paths: &HashSet, + inventory_complete: bool, +) -> SkillMetadataDiscovery { + let Some(skill_dir) = skill_path.parent() else { + return SkillMetadataDiscovery::Absent; + }; + let Ok(metadata_dir) = skill_dir.join(SKILLS_METADATA_DIR) else { + return SkillMetadataDiscovery::Absent; + }; + let Ok(metadata_path) = metadata_dir.join(SKILLS_METADATA_FILENAME) else { + return SkillMetadataDiscovery::Absent; + }; + if file_paths.contains(&metadata_path) { + SkillMetadataDiscovery::Present(metadata_path) + } else if inventory_complete && !directory_paths.contains(&metadata_dir) { + SkillMetadataDiscovery::Absent + } else { + // The walk can omit entries after an error or traversal limit. It also omits file + // symlinks, so keep the existing probe when the metadata directory itself was observed. + SkillMetadataDiscovery::Probe(metadata_path) + } +} + +async fn read_skill_contents( file_system: &dyn ExecutorFileSystem, skill_path: &PathUri, +) -> Result { + file_system + .read_file_text(skill_path, /*sandbox*/ None) + .await + .map_err(|err| format!("failed to read file: {err}")) +} + +async fn probe_skill_metadata( + file_system: &dyn ExecutorFileSystem, + metadata_path: &PathUri, ) -> (Option, Option) { - let Some(skill_dir) = skill_path.parent() else { - return (None, None); - }; - let Ok(metadata_path) = - skill_dir.join(&format!("{SKILLS_METADATA_DIR}/{SKILLS_METADATA_FILENAME}")) - else { - return (None, None); - }; match file_system - .get_metadata(&metadata_path, /*sandbox*/ None) + .get_metadata(metadata_path, /*sandbox*/ None) .await { Ok(metadata) if metadata.is_file => {} @@ -281,8 +402,15 @@ async fn load_skill_metadata( return (None, None); } } + read_skill_metadata(file_system, metadata_path).await +} + +async fn read_skill_metadata( + file_system: &dyn ExecutorFileSystem, + metadata_path: &PathUri, +) -> (Option, Option) { let contents = match file_system - .read_file_text(&metadata_path, /*sandbox*/ None) + .read_file_text(metadata_path, /*sandbox*/ None) .await { Ok(contents) => contents, diff --git a/codex-rs/core-skills/src/render.rs b/codex-rs/core-skills/src/render.rs index 89ea1675d0..3f26e8caf1 100644 --- a/codex-rs/core-skills/src/render.rs +++ b/codex-rs/core-skills/src/render.rs @@ -26,7 +26,7 @@ pub const SKILL_DESCRIPTION_TRUNCATED_WARNING_WITH_PERCENT: &str = "Skill descri pub const SKILL_DESCRIPTIONS_REMOVED_WARNING_PREFIX: &str = "Exceeded skills context budget. All skill descriptions were removed and"; pub const SKILLS_INTRO_WITH_ABSOLUTE_PATHS: &str = "A skill is a set of instructions provided through a `SKILL.md` source. Below is the list of skills that can be used. Each entry includes a name, description, and source locator. `file` locators are on the host filesystem, `environment resource` locators are owned by an execution environment, `orchestrator resource` locators are opaque non-filesystem resources, and `custom resource` locators use their provider's access mechanism."; -pub const SKILLS_INTRO_WITH_ALIASES: &str = "A skill is a set of local instructions to follow that is stored in a `SKILL.md` file. Below is the list of skills that can be used. Each entry includes a name, description, and a short path that can be expanded into an absolute path using the skill roots table."; +const SKILLS_INTRO_WITH_ALIASES: &str = "A skill is a set of local instructions to follow that is stored in a `SKILL.md` file. Below is the list of skills that can be used. Each entry includes a name, description, and a short path that can be expanded into an absolute path using the skill roots table."; pub const SKILLS_HOW_TO_USE_WITH_ABSOLUTE_PATHS: &str = r###"- Discovery: The list above is the skills available in this session (name + description + source locator). `file` entries live on the host filesystem, `environment resource` entries are owned by their execution environment, `orchestrator resource` entries must be accessed through `skills.list` and `skills.read`, and `custom resource` entries use their provider's access mechanism. - Trigger rules: If the user names a skill (with `$SkillName` or plain text) OR the task clearly matches a skill's description shown above, you must use that skill for that turn. Multiple mentions mean use them all. Do not carry skills across turns unless re-mentioned. - Missing/blocked: If a named skill isn't in the list or its source can't be read, say so briefly and continue with the best fallback. @@ -75,14 +75,6 @@ pub fn render_available_skills_body(skill_root_lines: &[String], skill_lines: &[ lines.push("### Available skills".to_string()); lines.extend(skill_lines.iter().cloned()); - lines.push("### How to use skills".to_string()); - let how_to_use = if skill_root_lines.is_empty() { - SKILLS_HOW_TO_USE_WITH_ABSOLUTE_PATHS - } else { - SKILLS_HOW_TO_USE_WITH_ALIASES - }; - lines.push(how_to_use.to_string()); - format!("\n{}\n", lines.join("\n")) } diff --git a/codex-rs/core-skills/tests/environment_loader.rs b/codex-rs/core-skills/tests/environment_loader.rs index 42d2446083..74efc6e1a3 100644 --- a/codex-rs/core-skills/tests/environment_loader.rs +++ b/codex-rs/core-skills/tests/environment_loader.rs @@ -1,5 +1,9 @@ use std::fs; use std::sync::Mutex; +use std::sync::atomic::AtomicBool; +use std::sync::atomic::AtomicUsize; +use std::sync::atomic::Ordering; +use std::time::Duration; use codex_core_skills::loader::EnvironmentSkillMetadata; use codex_core_skills::loader::load_environment_skills_from_root; @@ -13,28 +17,70 @@ use codex_exec_server::FileSystemSandboxContext; use codex_exec_server::LOCAL_FS; use codex_exec_server::ReadDirectoryEntry; use codex_exec_server::RemoveOptions; +use codex_exec_server::WalkOptions; +use codex_exec_server::WalkOutcome; use codex_utils_path_uri::PathUri; use pretty_assertions::assert_eq; use tempfile::tempdir; +use tokio::sync::Notify; + +#[derive(Clone, Copy)] +enum ManifestMetadataBehavior { + Immediate, + WaitForSkillRead, +} struct RecordingFileSystem<'a> { inner: &'a dyn ExecutorFileSystem, read_files: Mutex>, + metadata_files: Mutex>, + walks: AtomicUsize, + manifest_metadata_behavior: ManifestMetadataBehavior, + skill_read_started: AtomicBool, + skill_read_started_notify: Notify, +} + +#[derive(Debug, PartialEq, Eq)] +struct FileSystemCalls { + walks: usize, + read_files: Vec, + metadata_files: Vec, } impl<'a> RecordingFileSystem<'a> { - fn new(inner: &'a dyn ExecutorFileSystem) -> Self { + fn new( + inner: &'a dyn ExecutorFileSystem, + manifest_metadata_behavior: ManifestMetadataBehavior, + ) -> Self { Self { inner, read_files: Mutex::new(Vec::new()), + metadata_files: Mutex::new(Vec::new()), + walks: AtomicUsize::new(0), + manifest_metadata_behavior, + skill_read_started: AtomicBool::new(false), + skill_read_started_notify: Notify::new(), } } - fn read_files(&self) -> Vec { - self.read_files + fn calls(&self) -> FileSystemCalls { + let mut read_files = self + .read_files .lock() .unwrap_or_else(std::sync::PoisonError::into_inner) - .clone() + .clone(); + read_files.sort_by_key(ToString::to_string); + let mut metadata_files = self + .metadata_files + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clone(); + metadata_files.sort_by_key(ToString::to_string); + FileSystemCalls { + walks: self.walks.load(Ordering::Relaxed), + read_files, + metadata_files, + } } } @@ -56,6 +102,10 @@ impl ExecutorFileSystem for RecordingFileSystem<'_> { .lock() .unwrap_or_else(std::sync::PoisonError::into_inner) .push(path.clone()); + if path.basename().as_deref() == Some("SKILL.md") { + self.skill_read_started.store(true, Ordering::Release); + self.skill_read_started_notify.notify_waiters(); + } self.inner.read_file(path, sandbox) } @@ -90,6 +140,26 @@ impl ExecutorFileSystem for RecordingFileSystem<'_> { path: &'a PathUri, sandbox: Option<&'a FileSystemSandboxContext>, ) -> ExecutorFileSystemFuture<'a, FileMetadata> { + self.metadata_files + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .push(path.clone()); + if matches!( + self.manifest_metadata_behavior, + ManifestMetadataBehavior::WaitForSkillRead + ) && path.basename().as_deref() == Some("plugin.json") + { + return Box::pin(async move { + loop { + let notified = self.skill_read_started_notify.notified(); + if self.skill_read_started.load(Ordering::Acquire) { + break; + } + notified.await; + } + self.inner.get_metadata(path, sandbox).await + }); + } self.inner.get_metadata(path, sandbox) } @@ -101,6 +171,16 @@ impl ExecutorFileSystem for RecordingFileSystem<'_> { self.inner.read_directory(path, sandbox) } + fn walk<'a>( + &'a self, + path: &'a PathUri, + options: WalkOptions, + sandbox: Option<&'a FileSystemSandboxContext>, + ) -> ExecutorFileSystemFuture<'a, WalkOutcome> { + self.walks.fetch_add(1, Ordering::Relaxed); + self.inner.walk(path, options, sandbox) + } + fn remove<'a>( &'a self, path: &'a PathUri, @@ -159,7 +239,8 @@ async fn loads_nearest_plugin_namespaces_without_reading_unused_sibling_manifest .expect("skill"); } - let file_system = RecordingFileSystem::new(LOCAL_FS.as_ref()); + let file_system = + RecordingFileSystem::new(LOCAL_FS.as_ref(), ManifestMetadataBehavior::Immediate); let root_uri = PathUri::from_host_native_path(root.path()).expect("root URI"); let outcome = load_environment_skills_from_root( &file_system, @@ -200,7 +281,8 @@ async fn loads_nearest_plugin_namespaces_without_reading_unused_sibling_manifest ); let mut manifest_reads = file_system - .read_files() + .calls() + .read_files .into_iter() .filter(|path| path.basename().as_deref() == Some("plugin.json")) .collect::>(); @@ -214,3 +296,118 @@ async fn loads_nearest_plugin_namespaces_without_reading_unused_sibling_manifest expected_manifest_reads.sort_by_key(ToString::to_string); assert_eq!(manifest_reads, expected_manifest_reads); } + +#[tokio::test] +async fn reuses_walk_inventory_for_missing_skill_metadata() { + const SKILL_COUNT: usize = 66; + + let root = tempdir().expect("tempdir"); + let manifest_path = root.path().join(".codex-plugin/plugin.json"); + fs::create_dir_all(manifest_path.parent().expect("manifest parent")).expect("manifest dir"); + fs::write(&manifest_path, r#"{"name":"inventory"}"#).expect("manifest"); + + let mut skill_paths = Vec::new(); + for index in 0..SKILL_COUNT { + let name = format!("skill-{index}"); + let skill_path = root.path().join(&name).join("SKILL.md"); + fs::create_dir_all(skill_path.parent().expect("skill parent")).expect("skill dir"); + fs::write( + &skill_path, + format!("---\nname: {name}\ndescription: {name} skill.\n---\n"), + ) + .expect("skill"); + skill_paths.push(skill_path); + } + + let file_system = + RecordingFileSystem::new(LOCAL_FS.as_ref(), ManifestMetadataBehavior::Immediate); + let root_uri = PathUri::from_host_native_path(root.path()).expect("root URI"); + let outcome = load_environment_skills_from_root( + &file_system, + &root_uri, + /*restriction_product*/ None, + ) + .await; + + let mut expected_skills = skill_paths + .iter() + .enumerate() + .map(|(index, skill_path)| EnvironmentSkillMetadata { + path_to_skills_md: PathUri::from_host_native_path(skill_path).unwrap(), + name: format!("inventory:skill-{index}"), + description: format!("skill-{index} skill."), + short_description: None, + dependencies: None, + policy: None, + }) + .collect::>(); + expected_skills.sort_by(|left, right| { + left.name.cmp(&right.name).then_with(|| { + left.path_to_skills_md + .to_string() + .cmp(&right.path_to_skills_md.to_string()) + }) + }); + assert_eq!(outcome.skills, expected_skills); + assert_eq!(outcome.warnings, Vec::::new()); + + let mut expected_read_files = skill_paths + .iter() + .map(|path| PathUri::from_host_native_path(path).unwrap()) + .collect::>(); + let manifest_uri = PathUri::from_host_native_path(manifest_path).unwrap(); + expected_read_files.push(manifest_uri.clone()); + expected_read_files.sort_by_key(ToString::to_string); + assert_eq!( + file_system.calls(), + FileSystemCalls { + walks: 1, + read_files: expected_read_files, + metadata_files: vec![manifest_uri], + } + ); +} + +#[tokio::test] +async fn reads_skill_files_while_resolving_plugin_namespaces() { + let root = tempdir().expect("tempdir"); + let manifest_path = root.path().join(".codex-plugin/plugin.json"); + fs::create_dir_all(manifest_path.parent().expect("manifest parent")).expect("manifest dir"); + fs::write(&manifest_path, r#"{"name":"parallel"}"#).expect("manifest"); + let skill_path = root.path().join("demo/SKILL.md"); + fs::create_dir_all(skill_path.parent().expect("skill parent")).expect("skill dir"); + fs::write( + &skill_path, + "---\nname: demo\ndescription: demo skill.\n---\n", + ) + .expect("skill"); + + let file_system = RecordingFileSystem::new( + LOCAL_FS.as_ref(), + ManifestMetadataBehavior::WaitForSkillRead, + ); + let root_uri = PathUri::from_host_native_path(root.path()).expect("root URI"); + let outcome = tokio::time::timeout( + Duration::from_secs(5), + load_environment_skills_from_root( + &file_system, + &root_uri, + /*restriction_product*/ None, + ), + ) + .await + .expect("skill reads should start before namespace resolution finishes"); + + assert_eq!(outcome.warnings, Vec::::new()); + assert_eq!( + outcome.skills, + vec![EnvironmentSkillMetadata { + path_to_skills_md: PathUri::from_host_native_path(skill_path).unwrap(), + name: "parallel:demo".to_string(), + description: "demo skill.".to_string(), + short_description: None, + dependencies: None, + policy: None, + }] + ); +} diff --git a/codex-rs/core/config.schema.json b/codex-rs/core/config.schema.json index 92a45d1978..876180dd7e 100644 --- a/codex-rs/core/config.schema.json +++ b/codex-rs/core/config.schema.json @@ -448,6 +448,9 @@ "code_mode": { "$ref": "#/definitions/FeatureToml_for_CodeModeConfigToml" }, + "code_mode_host": { + "type": "boolean" + }, "code_mode_only": { "type": "boolean" }, @@ -810,12 +813,15 @@ "clock_source": { "$ref": "#/definitions/CurrentTimeSource" }, + "delivery_mode": { + "$ref": "#/definitions/CurrentTimeReminderDeliveryMode" + }, "enabled": { "type": "boolean" }, "reminder_interval_seconds": { "format": "uint64", - "minimum": 1.0, + "minimum": 0.0, "type": "integer" }, "sleep_tool": { @@ -825,6 +831,25 @@ }, "type": "object" }, + "CurrentTimeReminderDeliveryMode": { + "description": "Which inference boundaries may receive current-time reminders.", + "oneOf": [ + { + "description": "Allow a reminder before any inference request once the interval is due.", + "enum": [ + "any_inference" + ], + "type": "string" + }, + { + "description": "Allow reminders after user input or tool output; new context windows still force one.", + "enum": [ + "after_user_or_tool_output" + ], + "type": "string" + } + ] + }, "CurrentTimeSource": { "enum": [ "system", @@ -1725,6 +1750,9 @@ "minimum": 0.0, "type": "integer" }, + "multi_agent_mode_hint_text": { + "type": "string" + }, "non_code_mode_only": { "type": "boolean" }, @@ -4789,6 +4817,9 @@ "code_mode": { "$ref": "#/definitions/FeatureToml_for_CodeModeConfigToml" }, + "code_mode_host": { + "type": "boolean" + }, "code_mode_only": { "type": "boolean" }, diff --git a/codex-rs/core/src/agent/control.rs b/codex-rs/core/src/agent/control.rs index 49bbab0f87..85e54e2fb9 100644 --- a/codex-rs/core/src/agent/control.rs +++ b/codex-rs/core/src/agent/control.rs @@ -4,6 +4,8 @@ use crate::agent::registry::AgentRegistry; use crate::agent::role::DEFAULT_ROLE_NAME; use crate::agent::role::resolve_role_config; use crate::agent::status::is_final; +use crate::agent_communication::AgentCommunicationContext; +use crate::agent_communication::AgentCommunicationKind; use crate::codex_thread::ThreadConfigSnapshot; use crate::config::Config; use crate::config::RolloutBudgetConfig; @@ -139,12 +141,12 @@ impl AgentControl { pub(crate) async fn send_input( &self, agent_id: ThreadId, - initial_operation: Op, + input: Vec, ) -> CodexResult { let state = self.upgrade()?; - self.ensure_execution_capacity_for_op(agent_id, &initial_operation) + self.ensure_execution_capacity_for_turn_start(agent_id, /*starts_turn*/ true) .await?; - self.send_input_after_capacity_check(agent_id, &state, initial_operation) + self.send_input_after_capacity_check(agent_id, &state, input) .await } @@ -152,19 +154,14 @@ impl AgentControl { &self, agent_id: ThreadId, state: &Arc, - initial_operation: Op, + input: Vec, ) -> CodexResult { - let last_task_message = match &initial_operation { - Op::InterAgentCommunication { communication } => { - last_task_message_from_communication(communication) - } - _ => non_empty_task_message(render_input_preview(&initial_operation)), - }; + let last_task_message = non_empty_task_message(render_input_preview(&input)); let result = self .handle_thread_request_result( agent_id, state, - state.send_op(agent_id, initial_operation).await, + state.send_op(agent_id, input.into()).await, ) .await; if result.is_ok() { @@ -182,14 +179,60 @@ impl AgentControl { &self, agent_id: ThreadId, communication: InterAgentCommunication, + agent_communication_context: AgentCommunicationContext, + ) -> CodexResult { + let state = self.upgrade()?; + self.ensure_execution_capacity_for_turn_start(agent_id, communication.trigger_turn) + .await?; + self.send_inter_agent_communication_after_capacity_check( + agent_id, + &state, + communication, + agent_communication_context, + ) + .await + } + + async fn send_inter_agent_communication_after_capacity_check( + &self, + agent_id: ThreadId, + state: &Arc, + communication: InterAgentCommunication, + context: AgentCommunicationContext, + ) -> CodexResult { + self.submit_inter_agent_communication(agent_id, state, communication, context) + .await + } + + async fn submit_inter_agent_communication( + &self, + agent_id: ThreadId, + state: &Arc, + communication: InterAgentCommunication, + context: AgentCommunicationContext, ) -> CodexResult { let last_task_message = last_task_message_from_communication(&communication); - let state = self.upgrade()?; - let op = Op::InterAgentCommunication { communication }; - self.ensure_execution_capacity_for_op(agent_id, &op).await?; + let communication_for_log = + crate::agent_communication::logging_enabled().then(|| communication.clone()); let result = self - .handle_thread_request_result(agent_id, &state, state.send_op(agent_id, op).await) + .handle_thread_request_result( + agent_id, + state, + state + .send_op(agent_id, Op::InterAgentCommunication { communication }) + .await, + ) .await; + if let (Some(communication), Ok(communication_id)) = + (communication_for_log, result.as_ref()) + { + crate::agent_communication::emit_agent_communication_send( + communication_id, + &context, + &communication, + agent_id, + ); + } if result.is_ok() { match last_task_message { Some(last_task_message) => self @@ -480,8 +523,10 @@ impl AgentControl { message, /*trigger_turn*/ false, ); + let context = + AgentCommunicationContext::new(AgentCommunicationKind::Result, child_thread_id); let _ = control - .send_inter_agent_communication(parent_thread_id, communication) + .send_inter_agent_communication(parent_thread_id, communication, context) .await; return; } @@ -706,27 +751,23 @@ fn agent_matches_prefix(agent_path: Option<&AgentPath>, prefix: &AgentPath) -> b }) } -pub(crate) fn render_input_preview(initial_operation: &Op) -> String { - match initial_operation { - Op::UserInput { items, .. } => items - .iter() - .map(|item| match item { - UserInput::Text { text, .. } => text.clone(), - UserInput::Image { .. } => "[image]".to_string(), - UserInput::LocalImage { path, .. } => { - format!("[local_image:{}]", path.display()) - } - UserInput::Skill { name, path, .. } => { - format!("[skill:${name}]({})", path.display()) - } - UserInput::Mention { name, path, .. } => format!("[mention:${name}]({path})"), - _ => "[input]".to_string(), - }) - .collect::>() - .join("\n"), - Op::InterAgentCommunication { communication } => communication.content.clone(), - _ => String::new(), - } +pub(crate) fn render_input_preview(input: &[UserInput]) -> String { + input + .iter() + .map(|item| match item { + UserInput::Text { text, .. } => text.clone(), + UserInput::Image { .. } => "[image]".to_string(), + UserInput::LocalImage { path, .. } => { + format!("[local_image:{}]", path.display()) + } + UserInput::Skill { name, path, .. } => { + format!("[skill:${name}]({})", path.display()) + } + UserInput::Mention { name, path, .. } => format!("[mention:${name}]({path})"), + _ => "[input]".to_string(), + }) + .collect::>() + .join("\n") } fn last_task_message_from_communication(communication: &InterAgentCommunication) -> Option { diff --git a/codex-rs/core/src/agent/control/execution.rs b/codex-rs/core/src/agent/control/execution.rs index 42aaa50e02..d8559c1499 100644 --- a/codex-rs/core/src/agent/control/execution.rs +++ b/codex-rs/core/src/agent/control/execution.rs @@ -32,7 +32,16 @@ impl AgentControl { thread_id: ThreadId, op: &Op, ) -> CodexResult<()> { - if !op_starts_turn(op) { + self.ensure_execution_capacity_for_turn_start(thread_id, op_starts_turn(op)) + .await + } + + pub(super) async fn ensure_execution_capacity_for_turn_start( + &self, + thread_id: ThreadId, + starts_turn: bool, + ) -> CodexResult<()> { + if !starts_turn { return Ok(()); } let state = self.upgrade()?; diff --git a/codex-rs/core/src/agent/control/spawn.rs b/codex-rs/core/src/agent/control/spawn.rs index 6919d8c627..9078434f1f 100644 --- a/codex-rs/core/src/agent/control/spawn.rs +++ b/codex-rs/core/src/agent/control/spawn.rs @@ -1,5 +1,6 @@ use super::residency::is_v2_resident_session_source; use super::*; +use codex_extension_api::ExtensionDataInit; const AGENT_NAMES: &str = include_str!("../agent_names.txt"); @@ -8,6 +9,17 @@ struct SpawnAgentThreadInheritance { exec_policy: Option>, } +/// Initial input delivered after a spawned agent acquires execution capacity. +/// +/// V2 communication spawns keep the communication and its context paired so centralized +/// submission and lifecycle logging cannot receive one without the other. Other spawn sources +/// provide user input directly, making an uncontextualized inter-agent communication +/// unrepresentable. +enum SpawnInitialInput { + UserInput(Vec), + InterAgentCommunication(InterAgentCommunication, AgentCommunicationContext), +} + fn default_agent_nickname_list() -> Vec<&'static str> { AGENT_NAMES .lines() @@ -88,12 +100,12 @@ impl AgentControl { pub(crate) async fn spawn_agent( &self, config: Config, - initial_operation: Op, + initial_input: Vec, session_source: Option, ) -> CodexResult { let spawned_agent = Box::pin(self.spawn_agent_internal( config, - initial_operation, + SpawnInitialInput::UserInput(initial_input), session_source, SpawnAgentOptions::default(), )) @@ -105,12 +117,34 @@ impl AgentControl { pub(crate) async fn spawn_agent_with_metadata( &self, config: Config, - initial_operation: Op, + initial_input: Vec, session_source: Option, options: SpawnAgentOptions, // TODO(jif) drop with new fork. ) -> CodexResult { - Box::pin(self.spawn_agent_internal(config, initial_operation, session_source, options)) - .await + Box::pin(self.spawn_agent_internal( + config, + SpawnInitialInput::UserInput(initial_input), + session_source, + options, + )) + .await + } + + pub(crate) async fn spawn_agent_with_communication( + &self, + config: Config, + communication: InterAgentCommunication, + context: AgentCommunicationContext, + session_source: Option, + options: SpawnAgentOptions, + ) -> CodexResult { + Box::pin(self.spawn_agent_internal( + config, + SpawnInitialInput::InterAgentCommunication(communication, context), + session_source, + options, + )) + .await } pub(crate) async fn ensure_v2_agent_loaded( @@ -196,7 +230,7 @@ impl AgentControl { async fn spawn_agent_internal( &self, config: Config, - initial_operation: Op, + initial_input: SpawnInitialInput, session_source: Option, options: SpawnAgentOptions, ) -> CodexResult { @@ -355,8 +389,21 @@ impl AgentControl { ) .await; - self.send_input_after_capacity_check(new_thread.thread_id, &state, initial_operation) - .await?; + match initial_input { + SpawnInitialInput::UserInput(input) => { + self.send_input_after_capacity_check(new_thread.thread_id, &state, input) + .await?; + } + SpawnInitialInput::InterAgentCommunication(communication, context) => { + self.send_inter_agent_communication_after_capacity_check( + new_thread.thread_id, + &state, + communication, + context, + ) + .await?; + } + } if multi_agent_version != MultiAgentVersion::V2 { let child_reference = agent_metadata .agent_path @@ -433,6 +480,16 @@ impl AgentControl { )) })?; + let selected_capability_roots = parent_history + .items + .iter() + .find_map(|item| { + let RolloutItem::SessionMeta(meta_line) = item else { + return None; + }; + Some(meta_line.meta.selected_capability_roots.clone()) + }) + .unwrap_or_default(); let mut forked_rollout_items = parent_history.items; if let SpawnAgentForkMode::LastNTurns(last_n_turns) = fork_mode { forked_rollout_items = @@ -504,6 +561,8 @@ impl AgentControl { { forked_rollout_items.push(RolloutItem::ResponseItem(subagent_usage_hint_message)); } + let mut thread_extension_init = ExtensionDataInit::new(); + thread_extension_init.insert(selected_capability_roots); state .fork_thread_with_source( @@ -517,6 +576,7 @@ impl AgentControl { inherited_environments, inherited_exec_policy, options.environments.clone(), + thread_extension_init, ) .await } diff --git a/codex-rs/core/src/agent/control_tests.rs b/codex-rs/core/src/agent/control_tests.rs index e00a8624f4..4741c3c101 100644 --- a/codex-rs/core/src/agent/control_tests.rs +++ b/codex-rs/core/src/agent/control_tests.rs @@ -3,6 +3,8 @@ use crate::CodexThread; use crate::StateDbHandle; use crate::ThreadManager; use crate::agent::agent_status_from_event; +use crate::agent_communication::AgentCommunicationContext; +use crate::agent_communication::AgentCommunicationKind; use crate::config::AgentRoleConfig; use crate::config::Config; use crate::config::ConfigBuilder; @@ -17,6 +19,8 @@ use codex_features::Feature; use codex_login::AuthManager; use codex_login::CodexAuth; use codex_protocol::AgentPath; +use codex_protocol::capabilities::CapabilityRootLocation; +use codex_protocol::capabilities::SelectedCapabilityRoot; use codex_protocol::config_types::ModeKind; use codex_protocol::models::ContentItem; use codex_protocol::models::MessagePhase; @@ -38,6 +42,7 @@ use codex_thread_store::InMemoryThreadStore; use codex_thread_store::LocalThreadStore; use codex_thread_store::LocalThreadStoreConfig; use codex_thread_store::ThreadStore; +use codex_utils_path_uri::PathUri; use pretty_assertions::assert_eq; use tempfile::TempDir; use tokio::time::Duration; @@ -62,12 +67,11 @@ async fn test_config() -> (TempDir, Config) { test_config_with_cli_overrides(Vec::new()).await } -fn text_input(text: &str) -> Op { +fn text_input(text: &str) -> Vec { vec![UserInput::Text { text: text.to_string(), text_elements: Vec::new(), }] - .into() } fn assistant_message(text: &str, phase: Option) -> ResponseItem { @@ -308,8 +312,7 @@ async fn send_input_errors_when_manager_dropped() { vec![UserInput::Text { text: "hello".to_string(), text_elements: Vec::new(), - }] - .into(), + }], ) .await .expect_err("send_input should fail without a manager"); @@ -420,8 +423,7 @@ async fn send_input_errors_when_thread_missing() { vec![UserInput::Text { text: "hello".to_string(), text_elements: Vec::new(), - }] - .into(), + }], ) .await .expect_err("send_input should fail for missing thread"); @@ -487,8 +489,7 @@ async fn send_input_submits_user_message() { vec![UserInput::Text { text: "hello from tests".to_string(), text_elements: Vec::new(), - }] - .into(), + }], ) .await .expect("send_input should succeed"); @@ -528,7 +529,11 @@ async fn send_inter_agent_communication_without_turn_queues_message_without_trig let submission_id = harness .control - .send_inter_agent_communication(thread_id, communication.clone()) + .send_inter_agent_communication( + thread_id, + communication.clone(), + AgentCommunicationContext::new(AgentCommunicationKind::Message, ThreadId::new()), + ) .await .expect("send_inter_agent_communication should succeed"); assert!(!submission_id.is_empty()); @@ -653,7 +658,11 @@ async fn ensure_v2_agent_loaded_reloads_registered_unloaded_agent() { ); harness .control - .send_inter_agent_communication(spawned_agent.thread_id, communication.clone()) + .send_inter_agent_communication( + spawned_agent.thread_id, + communication.clone(), + AgentCommunicationContext::new(AgentCommunicationKind::Message, ThreadId::new()), + ) .await .expect("send_inter_agent_communication should succeed after reload"); let expected = ( @@ -800,7 +809,11 @@ async fn encrypted_inter_agent_communication_clears_existing_last_task_message() ); harness .control - .send_inter_agent_communication(spawned_agent.thread_id, communication) + .send_inter_agent_communication( + spawned_agent.thread_id, + communication, + AgentCommunicationContext::new(AgentCommunicationKind::Followup, ThreadId::new()), + ) .await .expect("send_inter_agent_communication should succeed"); @@ -1446,7 +1459,35 @@ async fn spawn_agent_fork_last_n_turns_keeps_only_recent_turns() { #[tokio::test] async fn spawn_agent_fork_last_n_turns_drops_parent_startup_prefix_when_under_limit() { let harness = AgentControlHarness::new().await; - let (parent_thread_id, parent_thread) = harness.start_thread().await; + let selected_capability_roots = vec![SelectedCapabilityRoot { + id: "demo@1".to_string(), + location: CapabilityRootLocation::Environment { + environment_id: "build".to_string(), + path: PathUri::parse("file:///plugins/demo").expect("plugin root URI"), + }, + }]; + let mut thread_extension_init = ExtensionDataInit::new(); + thread_extension_init.insert(selected_capability_roots.clone()); + let parent = harness + .manager + .start_thread_with_options(StartThreadOptions { + config: harness.config.clone(), + allow_provider_model_fallback: false, + initial_history: InitialHistory::New, + history_mode: None, + session_source: None, + thread_source: None, + dynamic_tools: Vec::new(), + metrics_service_name: None, + parent_trace: None, + environments: Vec::new(), + thread_extension_init, + supports_openai_form_elicitation: false, + }) + .await + .expect("start parent thread"); + let parent_thread_id = parent.thread_id; + let parent_thread = parent.thread; let startup_turn_context = parent_thread.codex.session.new_default_turn().await; parent_thread .codex @@ -1525,6 +1566,14 @@ async fn spawn_agent_fork_last_n_turns_drops_parent_startup_prefix_when_under_li !history_contains_text(history.raw_items(), "parent startup developer context"), "bounded fork should drop parent startup context even when fewer turns exist than requested" ); + assert_eq!( + &child_thread + .codex + .session + .services + .selected_capability_roots, + &selected_capability_roots + ); assert!( child_thread .codex @@ -2213,7 +2262,9 @@ async fn spawn_thread_subagents_persist_parent_originator_across_new_and_truncat .manager .start_thread_with_options(StartThreadOptions { config: harness.config.clone(), + allow_provider_model_fallback: false, initial_history: InitialHistory::New, + history_mode: None, session_source: None, thread_source: None, dynamic_tools: Vec::new(), diff --git a/codex-rs/core/src/agent_communication.rs b/codex-rs/core/src/agent_communication.rs new file mode 100644 index 0000000000..e71f50ad96 --- /dev/null +++ b/codex-rs/core/src/agent_communication.rs @@ -0,0 +1,79 @@ +use codex_protocol::ThreadId; +use codex_protocol::protocol::InterAgentCommunication; + +const AGENT_COMMUNICATION_TARGET: &str = "codex_otel.agent_communication"; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum AgentCommunicationKind { + Spawn, + Message, + Followup, + Result, +} + +impl AgentCommunicationKind { + fn as_str(self) -> &'static str { + match self { + Self::Spawn => "spawn", + Self::Message => "message", + Self::Followup => "followup", + Self::Result => "result", + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct AgentCommunicationContext { + kind: AgentCommunicationKind, + sender_thread_id: ThreadId, +} + +impl AgentCommunicationContext { + pub(crate) fn new(kind: AgentCommunicationKind, sender_thread_id: ThreadId) -> Self { + Self { + kind, + sender_thread_id, + } + } +} + +pub(crate) fn logging_enabled() -> bool { + tracing::enabled!(target: AGENT_COMMUNICATION_TARGET, tracing::Level::INFO) +} + +pub(crate) fn emit_agent_communication_send( + communication_id: &str, + context: &AgentCommunicationContext, + communication: &InterAgentCommunication, + receiver_thread_id: ThreadId, +) { + tracing::info!( + target: AGENT_COMMUNICATION_TARGET, + { + event.name = "codex.agent_communication", + communication_id, + kind = context.kind.as_str(), + state = "send", + sender_thread_id = %context.sender_thread_id, + receiver_thread_id = %receiver_thread_id, + content = if communication.content.is_empty() { + communication.encrypted_content.as_deref().unwrap_or_default() + } else { + communication.content.as_str() + }, + }, + "agent communication" + ); +} + +pub(crate) fn emit_agent_communication_receive(communication_id: &str) { + tracing::info!( + target: AGENT_COMMUNICATION_TARGET, + { + event.name = "codex.agent_communication", + communication_id, + state = "receive", + }, + "agent communication" + ); +} diff --git a/codex-rs/core/src/apps/render.rs b/codex-rs/core/src/apps/render.rs index 2331e13f77..b3d47913f6 100644 --- a/codex-rs/core/src/apps/render.rs +++ b/codex-rs/core/src/apps/render.rs @@ -5,7 +5,10 @@ use codex_protocol::protocol::APPS_INSTRUCTIONS_CLOSE_TAG; use codex_protocol::protocol::APPS_INSTRUCTIONS_OPEN_TAG; pub(crate) fn render_apps_section(connectors: &[AppInfo]) -> Option { - AppsInstructions::from_connectors(connectors).map(|instructions| instructions.render()) + connectors + .iter() + .any(|connector| connector.is_accessible && connector.is_enabled) + .then(|| AppsInstructions.render()) } #[cfg(test)] diff --git a/codex-rs/core/src/client.rs b/codex-rs/core/src/client.rs index ffa8ae8e20..43a758b0e8 100644 --- a/codex-rs/core/src/client.rs +++ b/codex-rs/core/src/client.rs @@ -169,7 +169,7 @@ pub(crate) struct CompactConversationRequestSettings { fn reasoning_effort_for_request(effort: ReasoningEffortConfig) -> ReasoningEffortConfig { match effort { - ReasoningEffortConfig::Ultra => ReasoningEffortConfig::Custom("max".to_string()), + ReasoningEffortConfig::Ultra => ReasoningEffortConfig::Max, effort => effort, } } @@ -1123,42 +1123,48 @@ impl ModelClientSession { } } + /// Checks whether the current request is an incremental extension of the previous request. + /// We only reuse an incremental input delta when non-input request fields are unchanged and + /// `input` is a strict extension of the previous known input. Server-returned output items + /// are treated as part of the baseline so we do not resend them. fn get_incremental_items( &self, request: &ResponsesApiRequest, last_response: Option<&LastResponse>, allow_empty_delta: bool, ) -> Option> { - // Checks whether the current request is an incremental extension of the previous request. - // We only reuse an incremental input delta when non-input request fields are unchanged and - // `input` is a strict - // extension of the previous known input. Server-returned output items are treated as part - // of the baseline so we do not resend them. let previous_request = self.websocket_session.last_request.as_ref()?; if !responses_request_properties_match(previous_request, request) { trace!("incremental request failed, websocket reuse properties didn't match"); return None; } - let Some(after_previous_input) = request - .input - .strip_prefix(previous_request.input.as_slice()) - else { - trace!("incremental request failed, items didn't match"); - return None; - }; - let mut response_items = - last_response.map_or_else(Vec::new, |response| response.items_added.clone()); - if !self.client.state.provider.info().is_openai() { - response_items - .iter_mut() - .for_each(ResponseItem::clear_internal_chat_message_metadata_passthrough); + // To compare the inputs, we concatenate the previous request items with the response items, + // then compare that against the equivalent slice of request items, ignoring metadata. If + // they match, we can consider the remaining items the incremental request. + let mut previous_items = previous_request.input.clone(); + if let Some(response) = last_response { + previous_items.extend_from_slice(&response.items_added); } - let Some(incremental_items) = after_previous_input.strip_prefix(response_items.as_slice()) + previous_items + .iter_mut() + .for_each(ResponseItem::clear_internal_chat_message_metadata_passthrough); + + let Some((request_items_to_compare, incremental_items)) = + request.input.split_at_checked(previous_items.len()) else { - trace!("incremental request failed, items didn't match"); + trace!("incremental request failed, incompatible request length"); return None; }; + let mut request_prefix = request_items_to_compare.to_vec(); + request_prefix + .iter_mut() + .for_each(ResponseItem::clear_internal_chat_message_metadata_passthrough); + + if previous_items != request_prefix { + trace!("incremental request failed, items didn't match"); + return None; + } if !allow_empty_delta && incremental_items.is_empty() { return None; } @@ -1912,6 +1918,7 @@ where let mut logged_error = false; let mut tx_last_response = Some(tx_last_response); let mut items_added: Vec = Vec::new(); + let (request_start, mut ttft_ms) = (Instant::now(), None); let mut api_stream = api_stream; let upstream_request_id = upstream_request_id.as_deref(); if let Some(upstream_request_id) = upstream_request_id { @@ -1961,6 +1968,7 @@ where Some(usage.cached_input_tokens), Some(usage.reasoning_output_tokens), usage.total_tokens, + ttft_ms, ); } inference_trace_attempt.record_completed( @@ -1988,6 +1996,11 @@ where } } Ok(event) => { + if matches!(&event, ResponseEvent::OutputItemAdded(_)) && ttft_ms.is_none() { + ttft_ms = Some( + i64::try_from(request_start.elapsed().as_millis()).unwrap_or(i64::MAX), + ); + } if tx_event.send(Ok(event)).await.is_err() { inference_trace_attempt.record_cancelled( STREAM_DROPPED_REASON, diff --git a/codex-rs/core/src/client_tests.rs b/codex-rs/core/src/client_tests.rs index dd2b1079f7..82c464625d 100644 --- a/codex-rs/core/src/client_tests.rs +++ b/codex-rs/core/src/client_tests.rs @@ -292,10 +292,7 @@ fn ultra_reasoning_uses_max_for_requests() { super::reasoning_effort_for_request(ReasoningEffort::Ultra), super::reasoning_effort_for_request(ReasoningEffort::High), ), - ( - ReasoningEffort::Custom("max".to_string()), - ReasoningEffort::High, - ) + (ReasoningEffort::Max, ReasoningEffort::High,) ); } diff --git a/codex-rs/core/src/codex_delegate.rs b/codex-rs/core/src/codex_delegate.rs index 6fbd195fce..4cba4bddc5 100644 --- a/codex-rs/core/src/codex_delegate.rs +++ b/codex-rs/core/src/codex_delegate.rs @@ -40,6 +40,7 @@ use crate::guardian::spawn_approval_request_review; use crate::mcp_tool_call::MCP_TOOL_APPROVAL_ACCEPT; use crate::mcp_tool_call::MCP_TOOL_APPROVAL_ACCEPT_FOR_SESSION; use crate::mcp_tool_call::MCP_TOOL_APPROVAL_DECLINE_SYNTHETIC; +use crate::mcp_tool_call::McpToolApprovalMetadata; use crate::mcp_tool_call::build_guardian_mcp_tool_review_request; use crate::mcp_tool_call::is_mcp_tool_approval_question_id; use crate::mcp_tool_call::lookup_mcp_tool_metadata; @@ -60,6 +61,12 @@ use codex_protocol::protocol::MultiAgentVersion; #[cfg(test)] use crate::session::completed_session_loop_termination; +#[derive(Clone)] +struct PendingMcpInvocation { + invocation: McpInvocation, + metadata: Option, +} + /// Start an interactive sub-Codex thread and return IO channels. /// /// The returned `events_rx` yields non-approval events emitted by the sub-agent. @@ -86,6 +93,7 @@ pub(crate) async fn run_codex_thread_interactive( }; let CodexSpawnOk { codex, .. } = Box::pin(Codex::spawn(CodexSpawnArgs { config, + allow_provider_model_fallback: false, user_instructions, installation_id: parent_session.installation_id.clone(), auth_manager, @@ -97,8 +105,10 @@ pub(crate) async fn run_codex_thread_interactive( skills_service: Arc::clone(&parent_session.services.skills_service), plugins_manager: Arc::clone(&parent_session.services.plugins_manager), mcp_manager: Arc::clone(&parent_session.services.mcp_manager), + code_mode_session_provider: parent_session.services.code_mode_service.session_provider(), extensions: Arc::clone(&parent_session.services.extensions), conversation_history, + requested_history_mode: None, session_source: SessionSource::SubAgent(subagent_source.clone()), forked_from_thread_id, parent_thread_id: Some(parent_session.thread_id), @@ -148,10 +158,10 @@ pub(crate) async fn run_codex_thread_interactive( let parent_session_clone = Arc::clone(&parent_session); let parent_ctx_clone = Arc::clone(&parent_ctx); let codex_for_events = Arc::clone(&codex); - // Cache delegated MCP invocations so guardian can recover the full tool call - // context when the later legacy RequestUserInput approval event only carries - // a call_id plus approval question metadata. - let pending_mcp_invocations = Arc::new(Mutex::new(HashMap::::new())); + // Cache the child call's MCP metadata at begin time. The later legacy + // RequestUserInput approval event only carries a call_id and question metadata. + let pending_mcp_invocations = + Arc::new(Mutex::new(HashMap::::new())); tokio::spawn(async move { forward_events( codex_for_events, @@ -269,7 +279,7 @@ async fn forward_events( tx_sub: Sender, parent_session: Arc, parent_ctx: Arc, - pending_mcp_invocations: Arc>>, + pending_mcp_invocations: Arc>>, cancel_token: CancellationToken, ) { let cancelled = cancel_token.cancelled(); @@ -289,11 +299,11 @@ async fn forward_events( match event { Event { id: _, - msg: EventMsg::TokenCount(_), - } => {} - Event { - id: _, - msg: EventMsg::SessionConfigured(_), + msg: + EventMsg::TokenCount(_) + | EventMsg::SessionConfigured(_) + | EventMsg::McpStartupUpdate(_) + | EventMsg::McpStartupComplete(_), } => {} Event { id, @@ -356,10 +366,34 @@ async fn forward_events( id, msg: EventMsg::McpToolCallBegin(event), } => { + // Runtime refreshes are published before a request step is captured, so + // the child runtime at call begin is the one executing this invocation. + // Cache its metadata now; the later approval event has only a call ID. + let metadata = if let Some(turn_context) = + codex.session.turn_context_for_sub_id(&id).await + { + let mcp = codex.session.services.latest_mcp_runtime(); + lookup_mcp_tool_metadata( + codex.session.as_ref(), + turn_context.as_ref(), + mcp.manager(), + &event.invocation.server, + &event.invocation.tool, + ) + .await + } else { + None + }; pending_mcp_invocations .lock() .await - .insert(event.call_id.clone(), event.invocation.clone()); + .insert( + event.call_id.clone(), + PendingMcpInvocation { + invocation: event.invocation.clone(), + metadata, + }, + ); if !forward_event_or_shutdown( &codex, &tx_sub, @@ -647,7 +681,7 @@ async fn handle_request_user_input( id: String, parent_session: &Arc, parent_ctx: &Arc, - pending_mcp_invocations: &Arc>>, + pending_mcp_invocations: &Arc>>, event: RequestUserInputEvent, cancel_token: &CancellationToken, ) { @@ -685,12 +719,12 @@ async fn handle_request_user_input( /// programmatically after running the guardian review. /// /// The RequestUserInput event only carries `call_id` plus approval question -/// metadata, so this helper joins it back to the cached `McpToolCallBegin` -/// invocation in order to rebuild the full guardian review request. +/// metadata, so this helper joins it back to the child runtime metadata cached at +/// `McpToolCallBegin` in order to rebuild the full guardian review request. async fn maybe_auto_review_mcp_request_user_input( parent_session: &Arc, parent_ctx: &Arc, - pending_mcp_invocations: &Arc>>, + pending_mcp_invocations: &Arc>>, event: &RequestUserInputEvent, cancel_token: &CancellationToken, ) -> Option { @@ -701,18 +735,13 @@ async fn maybe_auto_review_mcp_request_user_input( .questions .iter() .find(|question| is_mcp_tool_approval_question_id(&question.id))?; - let invocation = pending_mcp_invocations + let pending = pending_mcp_invocations .lock() .await .get(&event.call_id) .cloned()?; - let metadata = lookup_mcp_tool_metadata( - parent_session.as_ref(), - parent_ctx.as_ref(), - &invocation.server, - &invocation.tool, - ) - .await; + let invocation = pending.invocation; + let metadata = pending.metadata; let approvals_reviewer = mcp_approvals_reviewer(parent_ctx, &invocation.server, metadata.as_ref()); if !routes_approval_to_guardian_with_reviewer(parent_ctx, approvals_reviewer) { diff --git a/codex-rs/core/src/codex_delegate_tests.rs b/codex-rs/core/src/codex_delegate_tests.rs index 6d51a0032b..5cb2251a4a 100644 --- a/codex-rs/core/src/codex_delegate_tests.rs +++ b/codex-rs/core/src/codex_delegate_tests.rs @@ -14,6 +14,9 @@ use codex_protocol::protocol::GuardianAssessmentAction; use codex_protocol::protocol::GuardianAssessmentStatus; use codex_protocol::protocol::GuardianCommandSource; use codex_protocol::protocol::McpInvocation; +use codex_protocol::protocol::McpStartupCompleteEvent; +use codex_protocol::protocol::McpStartupStatus; +use codex_protocol::protocol::McpStartupUpdateEvent; use codex_protocol::protocol::RawResponseItemEvent; use codex_protocol::protocol::ReviewDecision; use codex_protocol::protocol::TurnAbortReason; @@ -34,8 +37,8 @@ use tokio::sync::watch; use tokio::time::timeout; #[tokio::test] -async fn forward_events_cancelled_while_send_blocked_shuts_down_delegate() { - let (tx_events, rx_events) = bounded(1); +async fn forward_events_filters_private_events_before_blocked_send_is_cancelled() { + let (tx_events, rx_events) = bounded(SUBMISSION_CHANNEL_CAPACITY); let (tx_sub, rx_sub) = bounded(SUBMISSION_CHANNEL_CAPACITY); let (_agent_status_tx, agent_status) = watch::channel(AgentStatus::PendingInit); let (session, ctx, _rx_evt) = crate::session::tests::make_session_and_context_with_rx().await; @@ -71,32 +74,53 @@ async fn forward_events_cancelled_while_send_blocked_shuts_down_delegate() { cancel.clone(), )); - tx_events - .send(Event { - id: "evt".to_string(), - msg: EventMsg::RawResponseItem(RawResponseItemEvent { - item: ResponseItem::CustomToolCall { - id: None, - status: None, - call_id: "call-1".to_string(), - name: "tool".to_string(), - input: "{}".to_string(), - internal_chat_message_metadata_passthrough: None, - }, - }), - }) - .await - .unwrap(); + for msg in [ + EventMsg::McpStartupUpdate(McpStartupUpdateEvent { + server: "pending".to_string(), + status: McpStartupStatus::Starting, + }), + EventMsg::McpStartupComplete(McpStartupCompleteEvent::default()), + ] { + tx_events + .send(Event { + id: "delegate-startup".to_string(), + msg, + }) + .await + .unwrap(); + } + let visible_msg = EventMsg::RawResponseItem(RawResponseItemEvent { + item: ResponseItem::CustomToolCall { + id: None, + status: None, + call_id: "call-1".to_string(), + name: "tool".to_string(), + namespace: None, + input: "{}".to_string(), + internal_chat_message_metadata_passthrough: None, + }, + }); + for id in ["visible-1", "visible-2", "blocked"] { + tx_events + .send(Event { + id: id.to_string(), + msg: visible_msg.clone(), + }) + .await + .unwrap(); + } drop(tx_events); + let received = rx_out.recv().await.expect("prefilled event missing"); + assert_eq!(received.id, "full"); + let received = rx_out.recv().await.expect("visible event missing"); + assert_eq!(received.id, "visible-1"); cancel.cancel(); timeout(std::time::Duration::from_millis(1000), forward) .await .expect("forward_events hung") .expect("forward_events join error"); - let received = rx_out.recv().await.expect("prefilled event missing"); - assert_eq!("full", received.id); let mut ops = Vec::new(); while let Ok(sub) = rx_sub.try_recv() { ops.push(sub.op); @@ -411,10 +435,13 @@ async fn delegated_mcp_guardian_abort_returns_synthetic_decline_answer() { let pending_mcp_invocations = Arc::new(Mutex::new(HashMap::from([( "call-1".to_string(), - McpInvocation { - server: "custom_server".to_string(), - tool: "dangerous_tool".to_string(), - arguments: None, + PendingMcpInvocation { + invocation: McpInvocation { + server: "custom_server".to_string(), + tool: "dangerous_tool".to_string(), + arguments: None, + }, + metadata: None, }, )]))); let cancel_token = CancellationToken::new(); @@ -460,10 +487,13 @@ async fn delegated_mcp_user_reviewer_returns_none_without_metadata() { crate::session::tests::make_session_and_context_with_rx().await; let pending_mcp_invocations = Arc::new(Mutex::new(HashMap::from([( "call-1".to_string(), - McpInvocation { - server: CODEX_APPS_MCP_SERVER_NAME.to_string(), - tool: "dangerous_tool".to_string(), - arguments: None, + PendingMcpInvocation { + invocation: McpInvocation { + server: CODEX_APPS_MCP_SERVER_NAME.to_string(), + tool: "dangerous_tool".to_string(), + arguments: None, + }, + metadata: None, }, )]))); let cancel_token = CancellationToken::new(); diff --git a/codex-rs/core/src/codex_thread.rs b/codex-rs/core/src/codex_thread.rs index 25f47ce45d..c80268e2f9 100644 --- a/codex-rs/core/src/codex_thread.rs +++ b/codex-rs/core/src/codex_thread.rs @@ -1,5 +1,6 @@ use crate::agent::AgentStatus; use crate::config::ConstraintResult; +use crate::elicitation::ElicitationRegistration; use crate::session::Codex; use crate::session::SessionSettingsUpdate; use crate::session::SteerInputError; @@ -29,6 +30,7 @@ use codex_protocol::protocol::SandboxPolicy; use codex_protocol::protocol::SessionConfiguredEvent; use codex_protocol::protocol::SessionSource; use codex_protocol::protocol::Submission; +use codex_protocol::protocol::ThreadHistoryMode; use codex_protocol::protocol::ThreadMemoryMode; use codex_protocol::protocol::ThreadSource; use codex_protocol::protocol::TokenUsageInfo; @@ -72,6 +74,7 @@ pub struct ThreadConfigSnapshot { pub personality: Option, pub collaboration_mode: CollaborationMode, pub session_source: SessionSource, + pub history_mode: ThreadHistoryMode, pub forked_from_thread_id: Option, pub parent_thread_id: Option, pub thread_source: Option, @@ -160,7 +163,13 @@ pub struct CodexThread { pub(crate) session_source: SessionSource, session_configured: SessionConfiguredEvent, rollout_path: Option, - out_of_band_elicitation_count: Mutex, + out_of_band_elicitations: Mutex, +} + +#[derive(Default)] +struct OutOfBandElicitations { + count: i64, + registration: Option, } #[derive(Debug, Eq, PartialEq)] @@ -185,7 +194,7 @@ impl CodexThread { session_source, session_configured, rollout_path, - out_of_band_elicitation_count: Mutex::new(0), + out_of_band_elicitations: Mutex::new(OutOfBandElicitations::default()), } } @@ -594,6 +603,17 @@ impl CodexThread { self.codex.session.runtime_mcp_config(config).await } + /// Returns the exact MCP config, environment bindings, and manager most recently published. + pub async fn current_mcp_runtime(&self) -> Arc { + let turn_context = self.codex.session.new_default_turn().await; + self.codex + .session + .capture_step_context(turn_context) + .await + .mcp + .clone() + } + pub fn multi_agent_version(&self) -> Option { self.codex.session.multi_agent_version() } @@ -615,8 +635,9 @@ impl CodexThread { uri: &str, ) -> anyhow::Result { let result = self - .codex - .session + .current_mcp_runtime() + .await + .manager_arc() .read_resource(server, ReadResourceRequestParams::new(uri)) .await?; @@ -630,8 +651,9 @@ impl CodexThread { arguments: Option, meta: Option, ) -> anyhow::Result { - self.codex - .session + self.current_mcp_runtime() + .await + .manager_arc() .call_tool(server, tool, arguments, meta) .await } @@ -640,38 +662,30 @@ impl CodexThread { self.codex.enabled(feature) } - pub async fn increment_out_of_band_elicitation_count(&self) -> CodexResult { - let mut guard = self.out_of_band_elicitation_count.lock().await; - let was_zero = *guard == 0; - *guard = guard.checked_add(1).ok_or_else(|| { + pub async fn increment_out_of_band_elicitation_count(&self) -> CodexResult { + let mut elicitations = self.out_of_band_elicitations.lock().await; + let incremented = elicitations.count.checked_add(1).ok_or_else(|| { CodexErr::Fatal("out-of-band elicitation count overflowed".to_string()) })?; - - if was_zero { - self.codex - .session - .set_out_of_band_elicitation_pause_state(/*paused*/ true); + if elicitations.count == 0 { + elicitations.registration = Some(self.codex.session.services.elicitations.register()); } - - Ok(*guard) + elicitations.count = incremented; + Ok(incremented) } - pub async fn decrement_out_of_band_elicitation_count(&self) -> CodexResult { - let mut guard = self.out_of_band_elicitation_count.lock().await; - if *guard == 0 { + pub async fn decrement_out_of_band_elicitation_count(&self) -> CodexResult { + let mut elicitations = self.out_of_band_elicitations.lock().await; + if elicitations.count == 0 { return Err(CodexErr::InvalidRequest( "out-of-band elicitation count is already zero".to_string(), )); } - *guard -= 1; - let now_zero = *guard == 0; - if now_zero { - self.codex - .session - .set_out_of_band_elicitation_pause_state(/*paused*/ false); + elicitations.count -= 1; + if elicitations.count == 0 { + elicitations.registration = None; } - - Ok(*guard) + Ok(elicitations.count) } } diff --git a/codex-rs/core/src/config/config_tests.rs b/codex-rs/core/src/config/config_tests.rs index a178dfae04..6156b9b9ea 100644 --- a/codex-rs/core/src/config/config_tests.rs +++ b/codex-rs/core/src/config/config_tests.rs @@ -6,6 +6,10 @@ use codex_config::CONFIG_TOML_FILE; use codex_config::ConfigLayerEntry; use codex_config::ConfigLayerSource; use codex_config::ConfigLayerStack; +use codex_config::McpServerCommandMatcher; +use codex_config::McpServerIdentity; +use codex_config::McpServerRequirement; +use codex_config::McpServerValueMatcher; use codex_config::ProfileV2Name; use codex_config::RequirementSource; use codex_config::Sourced; @@ -113,11 +117,15 @@ use std::time::Duration; use tempfile::TempDir; fn stdio_mcp(command: &str) -> McpServerConfig { + stdio_mcp_with_args(command, &[]) +} + +fn stdio_mcp_with_args(command: &str, args: &[&str]) -> McpServerConfig { McpServerConfig { auth: Default::default(), transport: McpServerTransportConfig::Stdio { command: command.to_string(), - args: Vec::new(), + args: args.iter().map(ToString::to_string).collect(), env: None, env_vars: Vec::new(), cwd: None, @@ -625,13 +633,15 @@ current_time_reminder = true r#" [features.current_time_reminder] enabled = true -reminder_interval_seconds = 4 +reminder_interval_seconds = 0 clock_source = "external" +delivery_mode = "after_user_or_tool_output" sleep_tool = true "#, CurrentTimeReminderConfig { - reminder_interval_seconds: 4, + reminder_interval_seconds: 0, clock_source: CurrentTimeSource::External, + delivery_mode: CurrentTimeReminderDeliveryMode::AfterUserOrToolOutput, sleep_tool: true, }, ), @@ -643,26 +653,6 @@ sleep_tool = true Ok(()) } -#[tokio::test] -async fn load_config_rejects_zero_current_time_reminder_interval() -> std::io::Result<()> { - let error = load_current_time_reminder_config( - r#" -[features.current_time_reminder] -enabled = true -reminder_interval_seconds = 0 -"#, - ) - .await - .expect_err("zero reminder interval should be rejected"); - - assert_eq!(error.kind(), std::io::ErrorKind::InvalidInput); - assert_eq!( - error.to_string(), - "features.current_time_reminder.reminder_interval_seconds must be positive" - ); - Ok(()) -} - async fn load_current_time_reminder_config(config_toml: &str) -> std::io::Result { let codex_home = tempdir()?; let config_toml = toml::from_str(config_toml).expect("TOML should deserialize"); @@ -4146,7 +4136,7 @@ fn filter_mcp_servers_by_allowlist_enforces_identity_rules() { BTreeMap::from([ ( MISMATCHED_URL_SERVER.to_string(), - McpServerRequirement { + McpServerRequirement::Identity { identity: McpServerIdentity::Url { url: "https://example.com/other".to_string(), }, @@ -4154,7 +4144,7 @@ fn filter_mcp_servers_by_allowlist_enforces_identity_rules() { ), ( MISMATCHED_COMMAND_SERVER.to_string(), - McpServerRequirement { + McpServerRequirement::Identity { identity: McpServerIdentity::Command { command: "other-cmd".to_string(), }, @@ -4162,7 +4152,7 @@ fn filter_mcp_servers_by_allowlist_enforces_identity_rules() { ), ( MATCHED_URL_SERVER.to_string(), - McpServerRequirement { + McpServerRequirement::Identity { identity: McpServerIdentity::Url { url: GOOD_URL.to_string(), }, @@ -4170,7 +4160,7 @@ fn filter_mcp_servers_by_allowlist_enforces_identity_rules() { ), ( MATCHED_COMMAND_SERVER.to_string(), - McpServerRequirement { + McpServerRequirement::Identity { identity: McpServerIdentity::Command { command: GOOD_CMD.to_string(), }, @@ -4227,6 +4217,117 @@ fn filter_mcp_servers_by_allowlist_allows_all_when_unset() { ); } +#[test] +fn filter_mcp_servers_by_matchers_enforces_command_and_positional_args() { + let mut servers = HashMap::from([ + ( + "internal_mcp_proxy".to_string(), + stdio_mcp_with_args( + "company-cli", + &[ + "mcp", + "proxy", + "--server", + "https://pricing.mcp.internal.example.com", + ], + ), + ), + ( + "unlisted".to_string(), + stdio_mcp_with_args( + "company-cli", + &[ + "mcp", + "proxy", + "--server", + "https://pricing.mcp.internal.example.com", + ], + ), + ), + ( + "wrong-order".to_string(), + stdio_mcp_with_args( + "company-cli", + &[ + "proxy", + "mcp", + "--server", + "https://pricing.mcp.internal.example.com", + ], + ), + ), + ( + "trailing-arg".to_string(), + stdio_mcp_with_args( + "company-cli", + &[ + "mcp", + "proxy", + "--server", + "https://pricing.mcp.internal.example.com", + "--verbose", + ], + ), + ), + ( + "wrong-host".to_string(), + stdio_mcp_with_args( + "company-cli", + &["mcp", "proxy", "--server", "https://mcp.example.com"], + ), + ), + ]); + let source = RequirementSource::LegacyManagedConfigTomlFromMdm; + let requirement = McpServerRequirement::Command(McpServerCommandMatcher { + executable: "company-cli".to_string(), + args: vec![ + McpServerValueMatcher::Exact { + value: "mcp".to_string(), + }, + McpServerValueMatcher::Exact { + value: "proxy".to_string(), + }, + McpServerValueMatcher::Exact { + value: "--server".to_string(), + }, + McpServerValueMatcher::Regex { + expression: + r"^https://[A-Za-z0-9-]+\.mcp\.internal\.example\.com(?::443)?(?:/.*)?$" + .to_string(), + }, + ], + }); + let requirements = Sourced::new( + BTreeMap::from([ + ("internal_mcp_proxy".to_string(), requirement.clone()), + ("wrong-order".to_string(), requirement.clone()), + ("trailing-arg".to_string(), requirement.clone()), + ("wrong-host".to_string(), requirement), + ]), + source.clone(), + ); + + filter_mcp_servers_by_requirements(&mut servers, Some(&requirements)); + + let reason = Some(McpServerDisabledReason::Requirements { source }); + assert_eq!( + servers + .iter() + .map(|(name, server)| ( + name.clone(), + (server.enabled, server.disabled_reason.clone()) + )) + .collect::)>>(), + HashMap::from([ + ("internal_mcp_proxy".to_string(), (true, None)), + ("unlisted".to_string(), (false, reason.clone())), + ("wrong-order".to_string(), (false, reason.clone())), + ("trailing-arg".to_string(), (false, reason.clone())), + ("wrong-host".to_string(), (false, reason)), + ]) + ); +} + #[test] fn filter_mcp_servers_by_allowlist_blocks_all_when_empty() { let mut servers = HashMap::from([ @@ -4277,7 +4378,7 @@ fn filter_plugin_mcp_servers_by_allowlist_enforces_plugin_and_identity_rules() { mcp_servers: Some(BTreeMap::from([ ( MATCHED_SERVER.to_string(), - McpServerRequirement { + McpServerRequirement::Identity { identity: McpServerIdentity::Command { command: GOOD_CMD.to_string(), }, @@ -4285,7 +4386,7 @@ fn filter_plugin_mcp_servers_by_allowlist_enforces_plugin_and_identity_rules() { ), ( MISMATCHED_SERVER.to_string(), - McpServerRequirement { + McpServerRequirement::Identity { identity: McpServerIdentity::Command { command: GOOD_CMD.to_string(), }, @@ -4326,7 +4427,7 @@ fn filter_plugin_mcp_servers_by_allowlist_blocks_unlisted_plugin() { codex_config::PluginRequirementsToml { mcp_servers: Some(BTreeMap::from([( "server-a".to_string(), - McpServerRequirement { + McpServerRequirement::Identity { identity: McpServerIdentity::Command { command: "cmd-a".to_string(), }, @@ -4357,6 +4458,65 @@ fn filter_plugin_mcp_servers_by_allowlist_blocks_unlisted_plugin() { ); } +#[test] +fn filter_plugin_mcp_servers_by_matchers_enforces_name_and_invocation() { + const MATCHED_SERVER: &str = "matched"; + const MISMATCHED_SERVER: &str = "mismatched"; + const UNLISTED_SERVER: &str = "unlisted"; + + let mut servers = HashMap::from([ + ( + MATCHED_SERVER.to_string(), + stdio_mcp_with_args("company-cli", &["approved"]), + ), + ( + MISMATCHED_SERVER.to_string(), + stdio_mcp_with_args("company-cli", &["rejected"]), + ), + ( + UNLISTED_SERVER.to_string(), + stdio_mcp_with_args("company-cli", &["approved"]), + ), + ]); + let source = RequirementSource::LegacyManagedConfigTomlFromMdm; + let requirement = McpServerRequirement::Command(McpServerCommandMatcher { + executable: "company-cli".to_string(), + args: vec![McpServerValueMatcher::Exact { + value: "approved".to_string(), + }], + }); + let requirements = Sourced::new( + BTreeMap::from([( + "sample@test".to_string(), + codex_config::PluginRequirementsToml { + mcp_servers: Some(BTreeMap::from([ + (MATCHED_SERVER.to_string(), requirement.clone()), + (MISMATCHED_SERVER.to_string(), requirement), + ])), + }, + )]), + source.clone(), + ); + + filter_plugin_mcp_servers_by_requirements("sample@test", &mut servers, Some(&requirements)); + + let reason = Some(McpServerDisabledReason::Requirements { source }); + assert_eq!( + servers + .iter() + .map(|(name, server)| ( + name.clone(), + (server.enabled, server.disabled_reason.clone()) + )) + .collect::)>>(), + HashMap::from([ + (MATCHED_SERVER.to_string(), (true, None)), + (MISMATCHED_SERVER.to_string(), (false, reason.clone())), + (UNLISTED_SERVER.to_string(), (false, reason)), + ]) + ); +} + #[tokio::test] async fn rebuild_preserving_session_layers_refreshes_requirements() -> std::io::Result<()> { let codex_home = TempDir::new()?; @@ -4366,7 +4526,7 @@ async fn rebuild_preserving_session_layers_refreshes_requirements() -> std::io:: let mcp_requirements = BTreeMap::from([ ( "session_overrides_user".to_string(), - McpServerRequirement { + McpServerRequirement::Identity { identity: McpServerIdentity::Command { command: "session-command".to_string(), }, @@ -4374,7 +4534,7 @@ async fn rebuild_preserving_session_layers_refreshes_requirements() -> std::io:: ), ( "managed_overrides_session".to_string(), - McpServerRequirement { + McpServerRequirement::Identity { identity: McpServerIdentity::Command { command: "managed-command".to_string(), }, @@ -4382,7 +4542,7 @@ async fn rebuild_preserving_session_layers_refreshes_requirements() -> std::io:: ), ( "fresh_global".to_string(), - McpServerRequirement { + McpServerRequirement::Identity { identity: McpServerIdentity::Command { command: "fresh-global-command".to_string(), }, @@ -4390,7 +4550,7 @@ async fn rebuild_preserving_session_layers_refreshes_requirements() -> std::io:: ), ( "fresh_project".to_string(), - McpServerRequirement { + McpServerRequirement::Identity { identity: McpServerIdentity::Command { command: "fresh-project-command".to_string(), }, @@ -8707,6 +8867,7 @@ async fn test_requirements_web_search_mode_allowlist_does_not_warn_when_unset() enforce_residency: None, network: None, permissions: None, + models: None, guardian_policy_config: None, }; let requirement_source = codex_config::RequirementSource::Unknown; @@ -10146,6 +10307,7 @@ default_wait_timeout_ms = 30000 usage_hint_text = "Custom delegation guidance." root_agent_usage_hint_text = "Root guidance." subagent_usage_hint_text = "Subagent guidance." +multi_agent_mode_hint_text = "Custom mode guidance." tool_namespace = "agents" hide_spawn_agent_metadata = true non_code_mode_only = true @@ -10182,6 +10344,10 @@ non_code_mode_only = true config.multi_agent_v2.subagent_usage_hint_text.as_deref(), Some("Subagent guidance.") ); + assert_eq!( + config.multi_agent_v2.multi_agent_mode_hint_text.as_deref(), + Some("Custom mode guidance.") + ); assert_eq!( config.multi_agent_v2.tool_namespace.as_deref(), Some("agents") @@ -10244,6 +10410,22 @@ max_concurrent_threads_per_session = 17 ); } +#[test] +fn multi_agent_v2_preserves_empty_mode_hint_override() { + let config_toml = toml::from_str( + r#"[features.multi_agent_v2] +multi_agent_mode_hint_text = "" +"#, + ) + .expect("multi-agent v2 config should parse"); + + let expected = MultiAgentV2Config { + multi_agent_mode_hint_text: Some(String::new()), + ..Default::default() + }; + assert_eq!(resolve_multi_agent_v2_config(&config_toml), expected); +} + #[tokio::test] async fn multi_agent_v2_empty_usage_hint_overrides_clear_default_hints() -> std::io::Result<()> { let codex_home = TempDir::new()?; diff --git a/codex-rs/core/src/config/mod.rs b/codex-rs/core/src/config/mod.rs index 78188d92e2..54fbb97f69 100644 --- a/codex-rs/core/src/config/mod.rs +++ b/codex-rs/core/src/config/mod.rs @@ -14,7 +14,6 @@ use codex_config::ConfigRequirements; use codex_config::ConfigRequirementsToml; use codex_config::ConstrainedWithSource; use codex_config::FeatureRequirementsToml; -use codex_config::McpServerIdentity; use codex_config::McpServerRequirement; use codex_config::PluginRequirementsToml; use codex_config::ProfileV2Name; @@ -40,7 +39,6 @@ use codex_config::types::AuthKeyringBackendKind; use codex_config::types::History; use codex_config::types::McpServerConfig; use codex_config::types::McpServerDisabledReason; -use codex_config::types::McpServerTransportConfig; use codex_config::types::MemoriesConfig; use codex_config::types::ModelAvailabilityNuxConfig; use codex_config::types::Notice; @@ -54,11 +52,13 @@ use codex_config::types::TuiNotificationSettings; use codex_config::types::TuiPetAnchor; use codex_config::types::UriBasedFileOpener; use codex_config::types::WindowsSandboxModeToml; +use codex_core_plugins::PluginLoadOutcome; use codex_core_plugins::PluginsConfigInput; use codex_exec_server::ExecutorFileSystem; use codex_exec_server::LOCAL_FS; use codex_features::CodeModeConfigToml; use codex_features::CurrentTimeReminderConfigToml; +use codex_features::CurrentTimeReminderDeliveryMode; use codex_features::CurrentTimeSource; use codex_features::Feature; use codex_features::FeatureConfigSource; @@ -1120,6 +1120,7 @@ pub struct RolloutBudgetConfig { pub struct CurrentTimeReminderConfig { pub reminder_interval_seconds: u64, pub clock_source: CurrentTimeSource, + pub delivery_mode: CurrentTimeReminderDeliveryMode, /// Whether to expose the input-interruptible `clock.sleep` tool. pub sleep_tool: bool, } @@ -1129,6 +1130,7 @@ impl Default for CurrentTimeReminderConfig { Self { reminder_interval_seconds: 1, clock_source: CurrentTimeSource::System, + delivery_mode: CurrentTimeReminderDeliveryMode::AnyInference, sleep_tool: false, } } @@ -1143,6 +1145,7 @@ pub struct MultiAgentV2Config { pub usage_hint_text: Option, pub root_agent_usage_hint_text: Option, pub subagent_usage_hint_text: Option, + pub multi_agent_mode_hint_text: Option, pub tool_namespace: Option, pub hide_spawn_agent_metadata: bool, pub non_code_mode_only: bool, @@ -1164,6 +1167,7 @@ impl MultiAgentV2Config { DEFAULT_MULTI_AGENT_V2_SUBAGENT_USAGE_HINT_TEXT, max_concurrent_threads_per_session, )), + multi_agent_mode_hint_text: None, tool_namespace: Some(DEFAULT_MULTI_AGENT_V2_TOOL_NAMESPACE.to_string()), hide_spawn_agent_metadata: true, non_code_mode_only: true, @@ -1525,6 +1529,14 @@ impl Config { ) -> McpConfig { let plugins_input = self.plugins_config_input(); let loaded_plugins = plugins_manager.plugins_for_config(&plugins_input).await; + self.to_mcp_config_with_loaded_plugins(&loaded_plugins, additional_plugin_registrations) + } + + pub(crate) fn to_mcp_config_with_loaded_plugins( + &self, + loaded_plugins: &PluginLoadOutcome, + additional_plugin_registrations: impl IntoIterator, + ) -> McpConfig { let mut catalog = ResolvedMcpCatalog::builder(); for (plugin_order, plugin) in loaded_plugins .plugins() @@ -1887,7 +1899,7 @@ fn filter_mcp_servers_by_requirements( let allowed = allowlist .value .get(name) - .is_some_and(|requirement| mcp_server_matches_requirement(requirement, server)); + .is_some_and(|requirement| requirement.matches(server)); if allowed { server.disabled_reason = None; } else { @@ -1916,7 +1928,7 @@ fn filter_plugin_mcp_servers_by_requirements( for (name, server) in mcp_servers.iter_mut() { let allowed = plugin_mcp_requirements .and_then(|mcp_requirements| mcp_requirements.get(name)) - .is_some_and(|requirement| mcp_server_matches_requirement(requirement, server)); + .is_some_and(|requirement| requirement.matches(server)); if allowed { server.disabled_reason = None; } else { @@ -1979,26 +1991,6 @@ where Ok(false) } -fn mcp_server_matches_requirement( - requirement: &McpServerRequirement, - server: &McpServerConfig, -) -> bool { - match &requirement.identity { - McpServerIdentity::Command { - command: want_command, - } => matches!( - &server.transport, - McpServerTransportConfig::Stdio { command: got_command, .. } - if got_command == want_command - ), - McpServerIdentity::Url { url: want_url } => matches!( - &server.transport, - McpServerTransportConfig::StreamableHttp { url: got_url, .. } - if got_url == want_url - ), - } -} - pub async fn load_global_mcp_servers( codex_home: &Path, ) -> std::io::Result> { @@ -2520,6 +2512,10 @@ fn resolve_multi_agent_v2_config(config_toml: &ConfigToml) -> MultiAgentV2Config base.map(|config| &config.subagent_usage_hint_text), default.subagent_usage_hint_text, ); + let multi_agent_mode_hint_text = base + .and_then(|config| config.multi_agent_mode_hint_text.as_ref()) + .cloned() + .or(default.multi_agent_mode_hint_text); let tool_namespace = base .and_then(|config| config.tool_namespace.as_ref()) .cloned() @@ -2539,6 +2535,7 @@ fn resolve_multi_agent_v2_config(config_toml: &ConfigToml) -> MultiAgentV2Config usage_hint_text, root_agent_usage_hint_text, subagent_usage_hint_text, + multi_agent_mode_hint_text, tool_namespace, hide_spawn_agent_metadata, non_code_mode_only, @@ -2685,18 +2682,15 @@ fn resolve_current_time_reminder_config( let reminder_interval_seconds = base .and_then(|config| config.reminder_interval_seconds) .unwrap_or(default.reminder_interval_seconds); - if reminder_interval_seconds == 0 { - return Err(std::io::Error::new( - std::io::ErrorKind::InvalidInput, - "features.current_time_reminder.reminder_interval_seconds must be positive", - )); - } Ok(Some(CurrentTimeReminderConfig { reminder_interval_seconds, clock_source: base .and_then(|config| config.clock_source) .unwrap_or(default.clock_source), + delivery_mode: base + .and_then(|config| config.delivery_mode) + .unwrap_or(default.delivery_mode), sleep_tool: base .and_then(|config| config.sleep_tool) .unwrap_or(default.sleep_tool), diff --git a/codex-rs/core/src/connectors.rs b/codex-rs/core/src/connectors.rs index 79978da998..6e643f34d5 100644 --- a/codex-rs/core/src/connectors.rs +++ b/codex-rs/core/src/connectors.rs @@ -80,19 +80,6 @@ pub async fn list_accessible_connectors_from_mcp_tools( ) } -pub(crate) async fn list_accessible_and_enabled_connectors_from_manager( - mcp_connection_manager: &McpConnectionManager, - config: &Config, -) -> Vec { - with_app_enabled_state( - accessible_connectors_from_mcp_tools(&mcp_connection_manager.list_all_tools().await), - config, - ) - .into_iter() - .filter(|connector| connector.is_accessible && connector.is_enabled) - .collect() -} - #[instrument(level = "trace", skip_all)] pub(crate) async fn list_tool_suggest_discoverable_tools_with_auth( config: &Config, @@ -246,11 +233,14 @@ pub async fn list_accessible_connectors_from_mcp_tools_with_mcp_manager( }); } + let runtime_context = + McpRuntimeContext::new(Arc::clone(&environment_manager), config.cwd.to_path_buf()); let auth_status_entries = compute_auth_statuses( mcp_servers.iter(), config.mcp_oauth_credentials_store_mode, config.auth_keyring_backend_kind(), auth.as_ref(), + &runtime_context, ) .await; @@ -270,8 +260,9 @@ pub async fn list_accessible_connectors_from_mcp_tools_with_mcp_manager( PermissionProfile::default(), // Connector discovery is threadless. Use an actually configured env if // one exists, but do not reintroduce the old hidden-local fallback. - McpRuntimeContext::new(environment_manager, config.cwd.to_path_buf()), + runtime_context, config.codex_home.to_path_buf(), + mcp_manager.codex_apps_tools_cache(), codex_apps_tools_cache_key(auth.as_ref()), mcp_config.prefix_mcp_tool_names, mcp_config.client_elicitation_capability, @@ -279,6 +270,8 @@ pub async fn list_accessible_connectors_from_mcp_tools_with_mcp_manager( ToolPluginProvenance::default(), auth.as_ref(), /*elicitation_reviewer*/ None, + /*elicitation_lifecycle*/ None, + codex_mcp::ElicitationRequestRouter::default(), ) .await; diff --git a/codex-rs/core/src/context/apps_instructions.rs b/codex-rs/core/src/context/apps_instructions.rs index 761187c794..71d836d4ea 100644 --- a/codex-rs/core/src/context/apps_instructions.rs +++ b/codex-rs/core/src/context/apps_instructions.rs @@ -1,4 +1,3 @@ -use crate::connectors::AppInfo; use codex_mcp::CODEX_APPS_MCP_SERVER_NAME; use codex_protocol::protocol::APPS_INSTRUCTIONS_CLOSE_TAG; use codex_protocol::protocol::APPS_INSTRUCTIONS_OPEN_TAG; @@ -8,15 +7,6 @@ use super::ContextualUserFragment; #[derive(Debug, Clone, PartialEq)] pub(crate) struct AppsInstructions; -impl AppsInstructions { - pub(crate) fn from_connectors(connectors: &[AppInfo]) -> Option { - connectors - .iter() - .any(|connector| connector.is_accessible && connector.is_enabled) - .then_some(Self) - } -} - impl ContextualUserFragment for AppsInstructions { fn role(&self) -> &'static str { "developer" diff --git a/codex-rs/core/src/context/available_plugins_instructions.rs b/codex-rs/core/src/context/available_plugins_instructions.rs index 683d939aa0..a31d61d7fd 100644 --- a/codex-rs/core/src/context/available_plugins_instructions.rs +++ b/codex-rs/core/src/context/available_plugins_instructions.rs @@ -1,25 +1,10 @@ -use codex_plugin::PluginCapabilitySummary; use codex_protocol::protocol::PLUGINS_INSTRUCTIONS_CLOSE_TAG; use codex_protocol::protocol::PLUGINS_INSTRUCTIONS_OPEN_TAG; use super::ContextualUserFragment; #[derive(Debug, Clone, PartialEq)] -pub(crate) struct AvailablePluginsInstructions { - plugins: Vec, -} - -impl AvailablePluginsInstructions { - pub(crate) fn from_plugins(plugins: &[PluginCapabilitySummary]) -> Option { - if plugins.is_empty() { - return None; - } - - Some(Self { - plugins: plugins.to_vec(), - }) - } -} +pub(crate) struct AvailablePluginsInstructions; impl ContextualUserFragment for AvailablePluginsInstructions { fn role(&self) -> &'static str { diff --git a/codex-rs/core/src/context/available_skills_instructions.rs b/codex-rs/core/src/context/available_skills_instructions.rs index 4e166d436c..a96f2195cc 100644 --- a/codex-rs/core/src/context/available_skills_instructions.rs +++ b/codex-rs/core/src/context/available_skills_instructions.rs @@ -1,4 +1,6 @@ use codex_core_skills::AvailableSkills; +use codex_core_skills::SKILLS_HOW_TO_USE_WITH_ABSOLUTE_PATHS; +use codex_core_skills::SKILLS_HOW_TO_USE_WITH_ALIASES; use codex_core_skills::render_available_skills_body; use codex_protocol::protocol::SKILLS_INSTRUCTIONS_CLOSE_TAG; use codex_protocol::protocol::SKILLS_INSTRUCTIONS_OPEN_TAG; @@ -20,13 +22,24 @@ impl AvailableSkillsInstructions { skill_lines, } } -} -impl From for AvailableSkillsInstructions { - fn from(available_skills: AvailableSkills) -> Self { + pub fn from_available_skills( + available_skills: &AvailableSkills, + include_skills_usage_instructions: bool, + ) -> Self { + let mut skill_lines = available_skills.skill_lines.clone(); + if include_skills_usage_instructions { + skill_lines.push("### How to use skills".to_string()); + let instructions = if available_skills.skill_root_lines.is_empty() { + SKILLS_HOW_TO_USE_WITH_ABSOLUTE_PATHS + } else { + SKILLS_HOW_TO_USE_WITH_ALIASES + }; + skill_lines.push(instructions.to_string()); + } Self { - skill_root_lines: available_skills.skill_root_lines, - skill_lines: available_skills.skill_lines, + skill_root_lines: available_skills.skill_root_lines.clone(), + skill_lines, } } } diff --git a/codex-rs/core/src/context/multi_agent_mode_instructions.rs b/codex-rs/core/src/context/multi_agent_mode_instructions.rs index fc4debe5a2..9c8f5ad5ed 100644 --- a/codex-rs/core/src/context/multi_agent_mode_instructions.rs +++ b/codex-rs/core/src/context/multi_agent_mode_instructions.rs @@ -3,8 +3,7 @@ use codex_protocol::config_types::MultiAgentMode; use codex_protocol::protocol::MULTI_AGENT_MODE_CLOSE_TAG; use codex_protocol::protocol::MULTI_AGENT_MODE_OPEN_TAG; -const EXPLICIT_REQUEST_ONLY_MULTI_AGENT_MODE_TEXT: &str = "Do not spawn sub-agents unless the user explicitly asks for sub-agents, delegation, or parallel agent work."; -const NO_MULTI_AGENT_MODE_TEXT: &str = "Multi-agent delegation mode instructions are inactive. Any earlier multi-agent mode developer message no longer applies."; +const EXPLICIT_REQUEST_ONLY_MULTI_AGENT_MODE_TEXT: &str = "Do not spawn sub-agents unless the user or applicable AGENTS.md/skill instructions explicitly ask for sub-agents, delegation, or parallel agent work."; const PROACTIVE_MULTI_AGENT_MODE_TEXT: &str = "Proactive multi-agent delegation is active. Any earlier instruction requiring an explicit user request before spawning sub-agents no longer applies. Use sub-agents when parallel work would materially improve speed or quality. This mode remains active until a later multi-agent mode developer message changes it."; #[derive(Debug, Clone, PartialEq, Eq)] @@ -32,8 +31,8 @@ impl ContextualUserFragment for MultiAgentModeInstructions { } fn body(&self) -> String { - match self.multi_agent_mode { - MultiAgentMode::None => NO_MULTI_AGENT_MODE_TEXT.to_string(), + match &self.multi_agent_mode { + MultiAgentMode::Custom(hint_text) => hint_text.clone(), MultiAgentMode::ExplicitRequestOnly => { EXPLICIT_REQUEST_ONLY_MULTI_AGENT_MODE_TEXT.to_string() } diff --git a/codex-rs/core/src/context/world_state/agents_md.rs b/codex-rs/core/src/context/world_state/agents_md.rs index f713b2aff8..1b9a168010 100644 --- a/codex-rs/core/src/context/world_state/agents_md.rs +++ b/codex-rs/core/src/context/world_state/agents_md.rs @@ -1,3 +1,4 @@ +use super::PreviousSectionState; use super::WorldStateSection; use crate::agents_md::LoadedAgentsMd; use crate::context::ContextualUserFragment; @@ -44,27 +45,35 @@ impl WorldStateSection for AgentsMdState { } } + fn matches_legacy_fragment(role: &str, text: &str) -> bool { + role == "user" && UserInstructions::matches_text(text) + } + fn render_diff( &self, - previous: Option<&Self::Snapshot>, + previous: PreviousSectionState<'_, Self::Snapshot>, ) -> Option> { let current = self.snapshot(); - if previous == Some(¤t) { + if matches!(previous, PreviousSectionState::Known(previous) if previous == ¤t) { return None; } - let previous_instructions = previous.and_then(|state| state.text.as_ref()); - let instructions = match (&self.instructions, previous_instructions) { - (Some(instructions), Some(_)) => UserInstructions { + let previous_may_contain_instructions = match previous { + PreviousSectionState::Known(previous) => previous.text.is_some(), + PreviousSectionState::Unknown => true, + PreviousSectionState::Absent => false, + }; + let instructions = match (&self.instructions, previous_may_contain_instructions) { + (Some(instructions), true) => UserInstructions { directory: instructions.directory.clone(), text: format!("{REPLACEMENT_NOTICE}\n\n{}", instructions.text), }, - (Some(instructions), None) => instructions.clone(), - (None, Some(_)) => UserInstructions { + (Some(instructions), false) => instructions.clone(), + (None, true) => UserInstructions { directory: None, text: REMOVAL_NOTICE.to_string(), }, - (None, None) => return None, + (None, false) => return None, }; Some(Box::new(instructions)) } diff --git a/codex-rs/core/src/context/world_state/agents_md_tests.rs b/codex-rs/core/src/context/world_state/agents_md_tests.rs index f4c171b74a..d39fe6c727 100644 --- a/codex-rs/core/src/context/world_state/agents_md_tests.rs +++ b/codex-rs/core/src/context/world_state/agents_md_tests.rs @@ -53,6 +53,34 @@ fn changed_and_removed_state_supersedes_previous_instructions() { ); } +#[test] +fn unknown_previous_state_is_explicitly_superseded() { + let loaded = LoadedAgentsMd::from_text_for_testing("current instructions"); + let current = AgentsMdState::new(Some(&loaded)); + assert_eq!( + vec![user_message( + "# AGENTS.md instructions\n\n\nThese AGENTS.md instructions replace all previously provided AGENTS.md instructions.\n\ncurrent instructions\n", + )], + render_fragments(vec![ + WorldStateSection::render_diff(¤t, PreviousSectionState::Unknown) + .expect("unknown state should be replaced"), + ]), + ); + + assert_eq!( + vec![user_message( + "# AGENTS.md instructions\n\n\nThe previously provided AGENTS.md instructions no longer apply.\n", + )], + render_fragments(vec![ + WorldStateSection::render_diff( + &AgentsMdState::default(), + PreviousSectionState::Unknown, + ) + .expect("unknown state should be removed"), + ]), + ); +} + fn render_fragments(fragments: Vec>) -> Vec { fragments .into_iter() diff --git a/codex-rs/core/src/context/world_state/apps_instructions.rs b/codex-rs/core/src/context/world_state/apps_instructions.rs new file mode 100644 index 0000000000..767e1c584e --- /dev/null +++ b/codex-rs/core/src/context/world_state/apps_instructions.rs @@ -0,0 +1,55 @@ +use super::PreviousSectionState; +use super::WorldStateSection; +use crate::context::AppsInstructions; +use crate::context::ContextualUserFragment; + +/// Whether generic Apps usage guidance should be visible to the model. +#[derive(Clone, Copy, Debug, Default)] +pub(crate) struct AppsInstructionsState { + available: bool, +} + +impl AppsInstructionsState { + pub(crate) fn new(available: bool) -> Self { + Self { available } + } +} + +impl WorldStateSection for AppsInstructionsState { + const ID: &'static str = "apps_instructions"; + type Snapshot = bool; + + fn snapshot(&self) -> Self::Snapshot { + self.available + } + + fn matches_legacy_fragment(role: &str, text: &str) -> bool { + role == "developer" && AppsInstructions::matches_text(text) + } + + fn has_retained_fragment_matcher() -> bool { + true + } + + fn matches_retained_fragment(role: &str, text: &str) -> bool { + Self::matches_legacy_fragment(role, text) + } + + fn render_diff( + &self, + previous: PreviousSectionState<'_, Self::Snapshot>, + ) -> Option> { + if !self.available + || matches!(previous, PreviousSectionState::Known(previous) if *previous) + || matches!(previous, PreviousSectionState::Unknown) + { + return None; + } + + Some(Box::new(AppsInstructions)) + } +} + +#[cfg(test)] +#[path = "apps_instructions_tests.rs"] +mod tests; diff --git a/codex-rs/core/src/context/world_state/apps_instructions_tests.rs b/codex-rs/core/src/context/world_state/apps_instructions_tests.rs new file mode 100644 index 0000000000..92f6613594 --- /dev/null +++ b/codex-rs/core/src/context/world_state/apps_instructions_tests.rs @@ -0,0 +1,69 @@ +use super::*; +use crate::context::ContextualUserFragment; +use codex_protocol::models::ResponseItem; +use pretty_assertions::assert_eq; + +fn render(state: AppsInstructionsState, previous: PreviousSectionState<'_, bool>) -> Vec { + state + .render_diff(previous) + .into_iter() + .map(|fragment| fragment.render()) + .collect() +} + +#[test] +fn renders_only_when_apps_become_available() { + let unavailable = AppsInstructionsState::new(/*available*/ false); + let available = AppsInstructionsState::new(/*available*/ true); + let false_snapshot = false; + let true_snapshot = true; + + assert_eq!( + render(unavailable, PreviousSectionState::Absent), + Vec::::new() + ); + assert_eq!(render(available, PreviousSectionState::Absent).len(), 1); + assert_eq!( + render(available, PreviousSectionState::Known(&false_snapshot)).len(), + 1 + ); + assert_eq!( + render(available, PreviousSectionState::Known(&true_snapshot)), + Vec::::new() + ); + assert_eq!( + render(unavailable, PreviousSectionState::Known(&true_snapshot)), + Vec::::new() + ); +} + +#[test] +fn legacy_guidance_is_not_injected_again() { + let mut world_state = super::super::WorldState::default(); + world_state.add_section(AppsInstructionsState::new(/*available*/ true)); + let legacy: ResponseItem = ContextualUserFragment::into(AppsInstructions); + + assert!( + world_state + .render_history_diff(/*previous*/ None, &[legacy]) + .is_empty() + ); +} + +#[test] +fn persisted_guidance_is_restored_only_when_missing_from_history() { + let mut world_state = super::super::WorldState::default(); + world_state.add_section(AppsInstructionsState::new(/*available*/ true)); + let snapshot = world_state.snapshot(); + let retained: ResponseItem = ContextualUserFragment::into(AppsInstructions); + + assert_eq!( + world_state.render_history_diff(Some(&snapshot), &[]).len(), + 1 + ); + assert!( + world_state + .render_history_diff(Some(&snapshot), &[retained]) + .is_empty() + ); +} diff --git a/codex-rs/core/src/context/world_state/environment.rs b/codex-rs/core/src/context/world_state/environment.rs index 97ce78f1fd..d34124ecf5 100644 --- a/codex-rs/core/src/context/world_state/environment.rs +++ b/codex-rs/core/src/context/world_state/environment.rs @@ -1,3 +1,4 @@ +use super::PreviousSectionState; use super::WorldStateSection; use crate::context::ContextualUserFragment; use crate::context::environment_context::FileSystemContext; @@ -95,11 +96,14 @@ impl WorldStateSection for EnvironmentsState { fn render_diff( &self, - previous: Option<&Self::Snapshot>, + previous: PreviousSectionState<'_, Self::Snapshot>, ) -> Option> { let current = self.snapshot(); let empty = EnvironmentsSnapshot::default(); - let previous = previous.unwrap_or(&empty); + let previous = match previous { + PreviousSectionState::Known(previous) => previous, + PreviousSectionState::Absent | PreviousSectionState::Unknown => &empty, + }; let turn_context_values_changed = current.current_date != previous.current_date || current.timezone != previous.timezone || current.network != previous.network diff --git a/codex-rs/core/src/context/world_state/environment_tests.rs b/codex-rs/core/src/context/world_state/environment_tests.rs index e96a6c3091..70a1561219 100644 --- a/codex-rs/core/src/context/world_state/environment_tests.rs +++ b/codex-rs/core/src/context/world_state/environment_tests.rs @@ -220,7 +220,10 @@ fn single_environment_diff_ignores_unknown_shell() -> Result<()> { assert_eq!( None, - render_fragment(WorldStateSection::render_diff(¤t, Some(&previous))) + render_fragment(WorldStateSection::render_diff( + ¤t, + PreviousSectionState::Known(&previous), + )) ); Ok(()) } @@ -248,7 +251,7 @@ fn removed_legacy_environment_renders_unavailable() -> Result<()> { )), render_fragment(WorldStateSection::render_diff( &EnvironmentsState::default(), - Some(&previous), + PreviousSectionState::Known(&previous), )), ); Ok(()) diff --git a/codex-rs/core/src/context/world_state/mod.rs b/codex-rs/core/src/context/world_state/mod.rs index 541efbe74c..ccdc976c4c 100644 --- a/codex-rs/core/src/context/world_state/mod.rs +++ b/codex-rs/core/src/context/world_state/mod.rs @@ -1,7 +1,14 @@ mod agents_md; +mod apps_instructions; mod environment; +mod plugins_instructions; use crate::context::ContextualUserFragment; +use codex_extension_api::PreviousWorldStateSection; +use codex_extension_api::RenderedWorldStateFragment; +use codex_extension_api::WorldStateSectionContribution; +use codex_protocol::models::ContentItem; +use codex_protocol::models::ResponseItem; use indexmap::IndexMap; use serde::Serialize; use serde::de::DeserializeOwned; @@ -11,12 +18,23 @@ use std::collections::BTreeMap; use std::fmt; pub(crate) use agents_md::AgentsMdState; +pub(crate) use apps_instructions::AppsInstructionsState; pub(crate) use environment::EnvironmentsState; +pub(crate) use plugins_instructions::PluginsInstructionsState; trait ErasedWorldStateSection: Send + Sync { fn snapshot(&self) -> Option; - fn render_diff(&self, previous: Option<&Value>) -> Option>; + fn matches_legacy_fragment(&self, role: &str, text: &str) -> bool; + + fn has_retained_fragment_matcher(&self) -> bool; + + fn matches_retained_fragment(&self, role: &str, text: &str) -> bool; + + fn render_diff( + &self, + previous: PreviousSectionState<'_, Value>, + ) -> Option>; } impl ErasedWorldStateSection for S { @@ -43,20 +61,111 @@ impl ErasedWorldStateSection for S { Some(snapshot) } - fn render_diff(&self, previous: Option<&Value>) -> Option> { - let previous = previous.and_then(|previous| { - serde_json::from_value::(previous.clone()) - .inspect_err(|err| { - tracing::warn!( - section_id = S::ID, - %err, - "failed to restore world-state section snapshot" - ); - }) - .ok() - }); - WorldStateSection::render_diff(self, previous.as_ref()) + fn matches_legacy_fragment(&self, role: &str, text: &str) -> bool { + S::matches_legacy_fragment(role, text) } + + fn has_retained_fragment_matcher(&self) -> bool { + S::has_retained_fragment_matcher() + } + + fn matches_retained_fragment(&self, role: &str, text: &str) -> bool { + S::matches_retained_fragment(role, text) + } + + fn render_diff( + &self, + previous: PreviousSectionState<'_, Value>, + ) -> Option> { + let typed_snapshot; + let previous = match previous { + PreviousSectionState::Known(previous) => { + match serde_json::from_value::(previous.clone()) { + Ok(previous) => { + typed_snapshot = previous; + PreviousSectionState::Known(&typed_snapshot) + } + Err(err) => { + tracing::warn!( + section_id = S::ID, + %err, + "failed to restore world-state section snapshot" + ); + PreviousSectionState::Unknown + } + } + } + PreviousSectionState::Absent => PreviousSectionState::Absent, + PreviousSectionState::Unknown => PreviousSectionState::Unknown, + }; + WorldStateSection::render_diff(self, previous) + } +} + +struct ExtensionWorldStateSection(WorldStateSectionContribution); + +impl ErasedWorldStateSection for ExtensionWorldStateSection { + fn snapshot(&self) -> Option { + let mut snapshot = self.0.snapshot().clone(); + remove_null_object_fields(&mut snapshot); + (!snapshot.is_null()).then_some(snapshot) + } + + fn matches_legacy_fragment(&self, role: &str, text: &str) -> bool { + self.0.matches_legacy_fragment(role, text) + } + + fn has_retained_fragment_matcher(&self) -> bool { + self.0.has_retained_fragment_matcher() + } + + fn matches_retained_fragment(&self, role: &str, text: &str) -> bool { + self.0.matches_retained_fragment(role, text) + } + + fn render_diff( + &self, + previous: PreviousSectionState<'_, Value>, + ) -> Option> { + let previous = match previous { + PreviousSectionState::Absent => PreviousWorldStateSection::Absent, + PreviousSectionState::Unknown => PreviousWorldStateSection::Unknown, + PreviousSectionState::Known(previous) => PreviousWorldStateSection::Known(previous), + }; + self.0 + .render_diff(previous) + .map(|fragment| Box::new(WorldStateContextFragment(fragment)) as _) + } +} + +struct WorldStateContextFragment(RenderedWorldStateFragment); + +impl ContextualUserFragment for WorldStateContextFragment { + fn role(&self) -> &'static str { + self.0.role() + } + + fn markers(&self) -> (&'static str, &'static str) { + self.0.markers() + } + + fn body(&self) -> String { + self.0.body().to_string() + } + + fn type_markers() -> (&'static str, &'static str) { + ("", "") + } +} + +/// What is known about a section's previously model-visible state. +pub(crate) enum PreviousSectionState<'a, T> { + /// No persisted snapshot or matching fragment exists in retained history. + Absent, + /// Retained history contains the section, but its typed snapshot is unavailable. + Unknown, + /// The exact persisted snapshot is available. + Known(&'a T), } /// A typed portion of the state visible to the model. @@ -65,16 +174,32 @@ impl ErasedWorldStateSection for S { /// earlier snapshot of the same section. `ID` is persisted in rollouts and /// must remain stable. `Snapshot` should contain only the comparison data /// needed to decide what the model must be told next, and must not serialize -/// to null because merge-patch nulls represent deletion. +/// to null because merge-patch nulls represent deletion. Sections migrated +/// from older context can recognize their previous fragments through +/// `matches_legacy_fragment`. pub(crate) trait WorldStateSection: Send + Sync + 'static { const ID: &'static str; type Snapshot: DeserializeOwned + Serialize; fn snapshot(&self) -> Self::Snapshot; + fn matches_legacy_fragment(_role: &str, _text: &str) -> bool { + false + } + + /// Whether retained history must still contain this section's rendered fragment. + fn has_retained_fragment_matcher() -> bool { + false + } + + /// Recognizes this section's rendered fragment in retained model history. + fn matches_retained_fragment(_role: &str, _text: &str) -> bool { + false + } + fn render_diff( &self, - previous: Option<&Self::Snapshot>, + previous: PreviousSectionState<'_, Self::Snapshot>, ) -> Option>; } @@ -129,6 +254,16 @@ impl WorldState { self.sections.insert(id, Box::new(section)); } + pub(crate) fn add_extension_section(&mut self, section: WorldStateSectionContribution) { + let id = section.id(); + assert!( + !self.sections.contains_key(id), + "duplicate world-state section ID: {id}" + ); + self.sections + .insert(id, Box::new(ExtensionWorldStateSection(section))); + } + pub(crate) fn snapshot(&self) -> WorldStateSnapshot { WorldStateSnapshot { sections: self @@ -143,21 +278,87 @@ impl WorldState { } } + /// Renders every section as new, without any known previous state. pub(crate) fn render_full(&self) -> Vec> { - self.render_diff(&WorldStateSnapshot::default()) + self.render_with(|_, _| PreviousSectionState::Absent) } + /// Renders each section against the exact persisted snapshot when available. pub(crate) fn render_diff( &self, previous: &WorldStateSnapshot, + ) -> Vec> { + self.render_with(|id, _| match previous.sections.get(id) { + Some(previous) => PreviousSectionState::Known(previous), + None => PreviousSectionState::Absent, + }) + } + + /// Falls back to retained model history when no exact persisted snapshot is available. + pub(crate) fn render_history_diff( + &self, + previous: Option<&WorldStateSnapshot>, + items: &[ResponseItem], + ) -> Vec> { + self.render_with(|id, section| { + if let Some(previous) = previous.and_then(|previous| previous.sections.get(id)) { + if section.has_retained_fragment_matcher() && !has_retained_fragment(items, section) + { + PreviousSectionState::Absent + } else { + PreviousSectionState::Known(previous) + } + } else if has_legacy_fragment(items, section) { + PreviousSectionState::Unknown + } else { + PreviousSectionState::Absent + } + }) + } + + fn render_with<'a>( + &self, + mut previous: impl FnMut(&str, &dyn ErasedWorldStateSection) -> PreviousSectionState<'a, Value>, ) -> Vec> { self.sections .iter() - .filter_map(|(id, section)| section.render_diff(previous.sections.get(*id))) + .filter_map(|(id, section)| section.render_diff(previous(id, section.as_ref()))) .collect() } } +fn has_retained_fragment(items: &[ResponseItem], section: &dyn ErasedWorldStateSection) -> bool { + items.iter().any(|item| { + matches!( + item, + ResponseItem::Message { role, content, .. } + if content.iter().any(|content| { + matches!( + content, + ContentItem::InputText { text } + if section.matches_retained_fragment(role, text) + ) + }) + ) + }) +} + +fn has_legacy_fragment(items: &[ResponseItem], section: &dyn ErasedWorldStateSection) -> bool { + items.iter().any(|item| { + matches!( + item, + ResponseItem::Message { role, content, .. } + if content.iter().any(|content| { + matches!( + content, + ContentItem::InputText { text } + if section.matches_legacy_fragment(role, text) + ) + }) + ) + }) +} + fn remove_null_object_fields(value: &mut Value) { // RFC 7386 reserves object-valued nulls for deletion, but arrays are replaced whole. match value { diff --git a/codex-rs/core/src/context/world_state/plugins_instructions.rs b/codex-rs/core/src/context/world_state/plugins_instructions.rs new file mode 100644 index 0000000000..303307317d --- /dev/null +++ b/codex-rs/core/src/context/world_state/plugins_instructions.rs @@ -0,0 +1,55 @@ +use super::PreviousSectionState; +use super::WorldStateSection; +use crate::context::AvailablePluginsInstructions; +use crate::context::ContextualUserFragment; + +/// Whether generic plugin usage guidance should be visible to the model. +#[derive(Clone, Copy, Debug, Default)] +pub(crate) struct PluginsInstructionsState { + available: bool, +} + +impl PluginsInstructionsState { + pub(crate) fn new(available: bool) -> Self { + Self { available } + } +} + +impl WorldStateSection for PluginsInstructionsState { + const ID: &'static str = "plugins_instructions"; + type Snapshot = bool; + + fn snapshot(&self) -> Self::Snapshot { + self.available + } + + fn matches_legacy_fragment(role: &str, text: &str) -> bool { + role == "developer" && AvailablePluginsInstructions::matches_text(text) + } + + fn has_retained_fragment_matcher() -> bool { + true + } + + fn matches_retained_fragment(role: &str, text: &str) -> bool { + Self::matches_legacy_fragment(role, text) + } + + fn render_diff( + &self, + previous: PreviousSectionState<'_, Self::Snapshot>, + ) -> Option> { + if !self.available + || matches!(previous, PreviousSectionState::Known(previous) if *previous) + || matches!(previous, PreviousSectionState::Unknown) + { + return None; + } + + Some(Box::new(AvailablePluginsInstructions)) + } +} + +#[cfg(test)] +#[path = "plugins_instructions_tests.rs"] +mod tests; diff --git a/codex-rs/core/src/context/world_state/plugins_instructions_tests.rs b/codex-rs/core/src/context/world_state/plugins_instructions_tests.rs new file mode 100644 index 0000000000..a688ff93ca --- /dev/null +++ b/codex-rs/core/src/context/world_state/plugins_instructions_tests.rs @@ -0,0 +1,72 @@ +use super::*; +use crate::context::ContextualUserFragment; +use codex_protocol::models::ResponseItem; +use pretty_assertions::assert_eq; + +fn render( + state: PluginsInstructionsState, + previous: PreviousSectionState<'_, bool>, +) -> Vec { + state + .render_diff(previous) + .into_iter() + .map(|fragment| fragment.render()) + .collect() +} + +#[test] +fn renders_only_when_plugins_become_available() { + let unavailable = PluginsInstructionsState::new(/*available*/ false); + let available = PluginsInstructionsState::new(/*available*/ true); + let false_snapshot = false; + let true_snapshot = true; + + assert_eq!( + render(unavailable, PreviousSectionState::Absent), + Vec::::new() + ); + assert_eq!(render(available, PreviousSectionState::Absent).len(), 1); + assert_eq!( + render(available, PreviousSectionState::Known(&false_snapshot)).len(), + 1 + ); + assert_eq!( + render(available, PreviousSectionState::Known(&true_snapshot)), + Vec::::new() + ); + assert_eq!( + render(unavailable, PreviousSectionState::Known(&true_snapshot)), + Vec::::new() + ); +} + +#[test] +fn legacy_guidance_is_not_injected_again() { + let mut world_state = super::super::WorldState::default(); + world_state.add_section(PluginsInstructionsState::new(/*available*/ true)); + let legacy: ResponseItem = ContextualUserFragment::into(AvailablePluginsInstructions); + + assert!( + world_state + .render_history_diff(/*previous*/ None, &[legacy]) + .is_empty() + ); +} + +#[test] +fn persisted_guidance_is_restored_only_when_missing_from_history() { + let mut world_state = super::super::WorldState::default(); + world_state.add_section(PluginsInstructionsState::new(/*available*/ true)); + let snapshot = world_state.snapshot(); + let retained: ResponseItem = ContextualUserFragment::into(AvailablePluginsInstructions); + + assert_eq!( + world_state.render_history_diff(Some(&snapshot), &[]).len(), + 1 + ); + assert!( + world_state + .render_history_diff(Some(&snapshot), &[retained]) + .is_empty() + ); +} diff --git a/codex-rs/core/src/context/world_state/world_state_tests.rs b/codex-rs/core/src/context/world_state/world_state_tests.rs index a3aec922c6..9432672423 100644 --- a/codex-rs/core/src/context/world_state/world_state_tests.rs +++ b/codex-rs/core/src/context/world_state/world_state_tests.rs @@ -21,11 +21,15 @@ impl WorldStateSection for TestSection { fn render_diff( &self, - previous: Option<&Self::Snapshot>, + previous: PreviousSectionState<'_, Self::Snapshot>, ) -> Option> { - let previous = previous?; - (self.value != previous.value) - .then(|| Box::new(TestFragment(self.value.clone())) as Box) + match previous { + PreviousSectionState::Known(previous) if self.value != previous.value => { + Some(Box::new(TestFragment(self.value.clone()))) + } + PreviousSectionState::Unknown => Some(Box::new(TestFragment("unknown".to_string()))), + PreviousSectionState::Absent | PreviousSectionState::Known(_) => None, + } } } @@ -59,7 +63,7 @@ impl WorldStateSection for DuplicateTestSection { fn render_diff( &self, - _previous: Option<&Self::Snapshot>, + _previous: PreviousSectionState<'_, Self::Snapshot>, ) -> Option> { None } @@ -106,6 +110,114 @@ fn render_diff_restores_the_typed_section_snapshot() { ); } +#[test] +fn extension_owned_section_uses_its_snapshot_and_renderer() { + let mut world_state = WorldState::default(); + world_state.add_extension_section(WorldStateSectionContribution::new( + "extension_test", + json!({"value": "after", "optional": null}), + |previous| match previous { + PreviousWorldStateSection::Known(previous) + if previous == &json!({"value": "before"}) => + { + Some(RenderedWorldStateFragment::new( + "developer", + ("", ""), + "after", + )) + } + PreviousWorldStateSection::Absent + | PreviousWorldStateSection::Unknown + | PreviousWorldStateSection::Known(_) => None, + }, + )); + let previous = WorldStateSnapshot { + sections: BTreeMap::from([("extension_test".to_string(), json!({"value": "before"}))]), + }; + + let rendered = world_state.render_diff(&previous); + + assert_eq!( + serde_json::to_value(world_state.snapshot()).expect("serialize world-state snapshot"), + json!({"extension_test": {"value": "after"}}) + ); + assert_eq!(rendered.len(), 1); + assert_eq!(rendered[0].role(), "developer"); + assert_eq!( + rendered[0].render(), + "after" + ); +} + +#[test] +fn missing_retained_fragment_is_rendered_again() { + let mut world_state = WorldState::default(); + world_state.add_extension_section( + WorldStateSectionContribution::new( + "extension_test", + json!({"body": "current catalog"}), + |previous| match previous { + PreviousWorldStateSection::Absent => Some(RenderedWorldStateFragment::new( + "developer", + ("", ""), + "current catalog", + )), + PreviousWorldStateSection::Unknown | PreviousWorldStateSection::Known(_) => None, + }, + ) + .with_retained_fragment_matcher(|role, text| { + role == "developer" && text.contains("current catalog") + }), + ); + let previous = world_state.snapshot(); + let retained = ResponseItem::Message { + id: None, + role: "developer".to_string(), + content: vec![ContentItem::InputText { + text: "current catalog".to_string(), + }], + phase: None, + internal_chat_message_metadata_passthrough: None, + }; + + assert_eq!( + world_state + .render_history_diff(Some(&previous), &[]) + .into_iter() + .map(|fragment| fragment.body()) + .collect::>(), + vec!["current catalog"] + ); + assert!( + world_state + .render_history_diff(Some(&previous), &[retained]) + .is_empty() + ); +} + +#[test] +fn unreadable_section_snapshot_is_treated_as_unknown() { + let mut current = WorldState::default(); + current.add_section(TestSection { + value: "current".to_string(), + optional: None, + array: Vec::new(), + }); + let previous = WorldStateSnapshot { + sections: BTreeMap::from([("test".to_string(), json!({"invalid": true}))]), + }; + + let rendered = current.render_diff(&previous); + + assert_eq!( + vec!["unknown"], + rendered + .into_iter() + .map(|fragment| fragment.body()) + .collect::>() + ); +} + #[test] #[should_panic(expected = "duplicate world-state section ID: test")] fn duplicate_section_ids_are_rejected() { diff --git a/codex-rs/core/src/context_manager/history.rs b/codex-rs/core/src/context_manager/history.rs index defdbeb389..318c5d4d2f 100644 --- a/codex-rs/core/src/context_manager/history.rs +++ b/codex-rs/core/src/context_manager/history.rs @@ -90,20 +90,14 @@ impl ContextManager { world_state: &WorldState, ) -> (Vec>, Option) { let snapshot = world_state.snapshot(); - let (fragments, rollout_item) = self.world_state_baseline.as_ref().map_or_else( - || { - ( - world_state.render_full(), - Some(WorldStateItem::full(snapshot.clone().into_value())), - ) - }, + let fragments = + world_state.render_history_diff(self.world_state_baseline.as_ref(), &self.items); + let rollout_item = self.world_state_baseline.as_ref().map_or_else( + || Some(WorldStateItem::full(snapshot.clone().into_value())), |previous| { - ( - world_state.render_diff(previous), - snapshot - .merge_patch_from(previous) - .map(WorldStateItem::patch), - ) + snapshot + .merge_patch_from(previous) + .map(WorldStateItem::patch) }, ); self.world_state_baseline = Some(snapshot); diff --git a/codex-rs/core/src/context_manager/history_tests.rs b/codex-rs/core/src/context_manager/history_tests.rs index 396b495d7a..679e1a69f7 100644 --- a/codex-rs/core/src/context_manager/history_tests.rs +++ b/codex-rs/core/src/context_manager/history_tests.rs @@ -20,8 +20,10 @@ use codex_protocol::models::ReasoningItemContent; use codex_protocol::models::ReasoningItemReasoningSummary; use codex_protocol::openai_models::InputModality; use codex_protocol::openai_models::default_input_modalities; +use codex_protocol::protocol::APPS_INSTRUCTIONS_OPEN_TAG; use codex_protocol::protocol::AskForApproval; use codex_protocol::protocol::InterAgentCommunication; +use codex_protocol::protocol::PLUGINS_INSTRUCTIONS_OPEN_TAG; use codex_protocol::protocol::SandboxPolicy; use codex_protocol::protocol::TurnContextItem; use codex_utils_absolute_path::AbsolutePathBuf; @@ -86,16 +88,25 @@ impl WorldStateSection for TestWorldStateSection { true } + fn matches_legacy_fragment(role: &str, text: &str) -> bool { + role == "user" && UserInstructions::matches_text(text) + } + fn render_diff( &self, - previous: Option<&Self::Snapshot>, + previous: crate::context::world_state::PreviousSectionState<'_, Self::Snapshot>, ) -> Option> { - (previous != Some(&true)).then(|| { - Box::new(UserInstructions { - directory: None, - text: "test".to_string(), - }) as Box + let text = match previous { + crate::context::world_state::PreviousSectionState::Known(true) => return None, + crate::context::world_state::PreviousSectionState::Unknown => "unknown", + crate::context::world_state::PreviousSectionState::Absent + | crate::context::world_state::PreviousSectionState::Known(false) => "test", + }; + Some(Box::new(UserInstructions { + directory: None, + text: text.to_string(), }) + as Box) } } @@ -123,6 +134,31 @@ fn world_state_baseline_deduplicates_until_history_is_replaced() { assert!(replacement_item.is_some_and(|item| item.full)); } +#[test] +fn world_state_reconciles_matching_legacy_history_once() { + let item = crate::context::ContextualUserFragment::into(UserInstructions { + directory: None, + text: "legacy".to_string(), + }); + let mut history = create_history_with_items(vec![item]); + let mut world_state = WorldState::default(); + world_state.add_section(TestWorldStateSection); + + let (fragments, rollout_item) = history.update_world_state(&world_state); + assert_eq!( + vec!["\n\n\nunknown\n"], + fragments + .into_iter() + .map(|fragment| fragment.body()) + .collect::>() + ); + assert!(rollout_item.is_some_and(|item| item.full)); + + let (fragments, rollout_item) = history.update_world_state(&world_state); + assert!(fragments.is_empty()); + assert_eq!(rollout_item, None); +} + fn user_msg(text: &str) -> ResponseItem { ResponseItem::Message { id: None, @@ -479,6 +515,7 @@ fn for_prompt_strips_images_when_model_does_not_support_images() { status: None, call_id: "tool-1".to_string(), name: "js_repl".to_string(), + namespace: None, input: "view_image".to_string(), internal_chat_message_metadata_passthrough: None, }, @@ -548,6 +585,7 @@ fn for_prompt_strips_images_when_model_does_not_support_images() { status: None, call_id: "tool-1".to_string(), name: "js_repl".to_string(), + namespace: None, input: "view_image".to_string(), internal_chat_message_metadata_passthrough: None, }, @@ -985,6 +1023,12 @@ fn drop_last_n_user_turns_trims_context_updates_above_rolled_back_turn() { user_input_text_msg("turn 1 user"), assistant_msg("turn 1 assistant"), developer_msg("Generated images are saved to /tmp as /tmp/image-1.png by default."), + developer_msg(&format!( + "{APPS_INSTRUCTIONS_OPEN_TAG}\nROLLED_BACK_APPS_INSTRUCTIONS" + )), + developer_msg(&format!( + "{PLUGINS_INSTRUCTIONS_OPEN_TAG}\nROLLED_BACK_PLUGIN_INSTRUCTIONS" + )), developer_msg("ROLLED_BACK_DEV_INSTRUCTIONS"), developer_msg("ROLLED_BACK_MULTI_AGENT_MODE"), user_input_text_msg( @@ -1056,6 +1100,7 @@ fn remove_first_item_handles_custom_tool_pair() { status: None, call_id: "tool-1".to_string(), name: "my_tool".to_string(), + namespace: None, input: "{}".to_string(), internal_chat_message_metadata_passthrough: None, }, @@ -1344,6 +1389,7 @@ fn normalize_adds_missing_output_for_custom_tool_call() { status: None, call_id: "tool-x".to_string(), name: "custom".to_string(), + namespace: None, input: "{}".to_string(), internal_chat_message_metadata_passthrough: None, }]; @@ -1359,6 +1405,7 @@ fn normalize_adds_missing_output_for_custom_tool_call() { status: None, call_id: "tool-x".to_string(), name: "custom".to_string(), + namespace: None, input: "{}".to_string(), internal_chat_message_metadata_passthrough: None, }, @@ -1478,6 +1525,7 @@ fn normalize_mixed_inserts_and_removals() { status: None, call_id: "t1".to_string(), name: "tool".to_string(), + namespace: None, input: "{}".to_string(), internal_chat_message_metadata_passthrough: None, }, @@ -1522,6 +1570,7 @@ fn normalize_mixed_inserts_and_removals() { status: None, call_id: "t1".to_string(), name: "tool".to_string(), + namespace: None, input: "{}".to_string(), internal_chat_message_metadata_passthrough: None, }, @@ -1588,6 +1637,49 @@ fn normalize_adds_missing_output_for_function_call_inserts_output() { ); } +#[test] +fn for_prompt_assigns_stable_id_to_synthetic_output_without_reordering_history() { + let items = vec![ + ResponseItem::FunctionCall { + id: Some("fc_existing".to_string()), + name: "do_it".to_string(), + namespace: None, + arguments: "{}".to_string(), + call_id: "call-x".to_string(), + internal_chat_message_metadata_passthrough: None, + }, + ResponseItem::Message { + id: Some("msg_later".to_string()), + role: "user".to_string(), + content: vec![ContentItem::InputText { + text: "later turn".to_string(), + }], + phase: None, + internal_chat_message_metadata_passthrough: None, + }, + ]; + + let first = create_history_with_items(items.clone()).for_prompt(&default_input_modalities()); + let second = create_history_with_items(items).for_prompt(&default_input_modalities()); + + assert_eq!( + first, second, + "repeated prompt projections should assign the same ID to the synthetic output" + ); + let [ + ResponseItem::FunctionCall { .. }, + ResponseItem::FunctionCallOutput { id: Some(id), .. }, + ResponseItem::Message { .. }, + ] = first.as_slice() + else { + panic!("expected the synthetic output between its call and the later message"); + }; + assert!( + id.starts_with("fco_"), + "the synthetic function call output should use the Responses API output ID prefix" + ); +} + #[test] fn normalize_adds_missing_output_for_tool_search_call() { let items = vec![ResponseItem::ToolSearchCall { @@ -1634,6 +1726,7 @@ fn normalize_adds_missing_output_for_custom_tool_call_panics_in_debug() { status: None, call_id: "tool-x".to_string(), name: "custom".to_string(), + namespace: None, input: "{}".to_string(), internal_chat_message_metadata_passthrough: None, }]; @@ -1776,6 +1869,7 @@ fn normalize_mixed_inserts_and_removals_panics_in_debug() { status: None, call_id: "t1".to_string(), name: "tool".to_string(), + namespace: None, input: "{}".to_string(), internal_chat_message_metadata_passthrough: None, }, diff --git a/codex-rs/core/src/context_manager/normalize.rs b/codex-rs/core/src/context_manager/normalize.rs index d11cffe4e9..3fd1a59a8f 100644 --- a/codex-rs/core/src/context_manager/normalize.rs +++ b/codex-rs/core/src/context_manager/normalize.rs @@ -4,12 +4,15 @@ use codex_protocol::models::FunctionCallOutputPayload; use codex_protocol::models::ResponseItem; use codex_protocol::openai_models::InputModality; use std::collections::HashSet; +use uuid::Uuid; use crate::util::error_or_panic; use tracing::info; const IMAGE_CONTENT_OMITTED_PLACEHOLDER: &str = "image content omitted because you do not support image input"; +// Changing this value would change model-visible IDs and invalidate prompt caches. +const SYNTHETIC_OUTPUT_ID_NAMESPACE: Uuid = Uuid::from_u128(0x90d38d3e_6a5b_4d52_bfe2_2f1e634bfac4); pub(crate) fn ensure_call_outputs_present(items: &mut Vec) { let mut function_output_ids = HashSet::new(); @@ -40,14 +43,14 @@ pub(crate) fn ensure_call_outputs_present(items: &mut Vec) { for (idx, item) in items.iter().enumerate() { match item { - ResponseItem::FunctionCall { call_id, .. } + ResponseItem::FunctionCall { id, call_id, .. } if !function_output_ids.contains(call_id.as_str()) => { info!("Function call output is missing for call id: {call_id}"); missing_outputs_to_insert.push(( idx, ResponseItem::FunctionCallOutput { - id: None, + id: synthetic_output_id("fco", id.as_deref()), call_id: call_id.clone(), output: FunctionCallOutputPayload::from_text("aborted".to_string()), internal_chat_message_metadata_passthrough: None, @@ -55,6 +58,7 @@ pub(crate) fn ensure_call_outputs_present(items: &mut Vec) { )); } ResponseItem::ToolSearchCall { + id, call_id: Some(call_id), .. } if !tool_search_output_ids.contains(call_id.as_str()) => { @@ -62,7 +66,7 @@ pub(crate) fn ensure_call_outputs_present(items: &mut Vec) { missing_outputs_to_insert.push(( idx, ResponseItem::ToolSearchOutput { - id: None, + id: synthetic_output_id("tso", id.as_deref()), call_id: Some(call_id.clone()), status: "completed".to_string(), execution: "client".to_string(), @@ -71,7 +75,7 @@ pub(crate) fn ensure_call_outputs_present(items: &mut Vec) { }, )); } - ResponseItem::CustomToolCall { call_id, .. } + ResponseItem::CustomToolCall { id, call_id, .. } if !custom_tool_output_ids.contains(call_id.as_str()) => { error_or_panic(format!( @@ -80,7 +84,7 @@ pub(crate) fn ensure_call_outputs_present(items: &mut Vec) { missing_outputs_to_insert.push(( idx, ResponseItem::CustomToolCallOutput { - id: None, + id: synthetic_output_id("ctco", id.as_deref()), call_id: call_id.clone(), name: None, output: FunctionCallOutputPayload::from_text("aborted".to_string()), @@ -90,6 +94,7 @@ pub(crate) fn ensure_call_outputs_present(items: &mut Vec) { } // LocalShellCall is represented in upstream streams by a FunctionCallOutput ResponseItem::LocalShellCall { + id, call_id: Some(call_id), .. } if !function_output_ids.contains(call_id.as_str()) => { @@ -99,7 +104,7 @@ pub(crate) fn ensure_call_outputs_present(items: &mut Vec) { missing_outputs_to_insert.push(( idx, ResponseItem::FunctionCallOutput { - id: None, + id: synthetic_output_id("fco", id.as_deref()), call_id: call_id.clone(), output: FunctionCallOutputPayload::from_text("aborted".to_string()), internal_chat_message_metadata_passthrough: None, @@ -121,6 +126,21 @@ pub(crate) fn ensure_call_outputs_present(items: &mut Vec) { } } +/// Derives a stable ID for a prompt-only output from its source call's item ID. +/// +/// Prompt normalization can run repeatedly without persisting its synthetic +/// outputs, so the namespace and name format must remain stable across retries +/// and resumes to preserve prompt-cache reuse. Returning `None` when the source +/// call has no ID preserves the legacy behavior for older history items. +fn synthetic_output_id(prefix: &str, item_id: Option<&str>) -> Option { + let source_id = item_id.filter(|id| !id.is_empty())?; + let name = format!("{prefix}:{source_id}"); + Some(format!( + "{prefix}_{}", + Uuid::new_v5(&SYNTHETIC_OUTPUT_ID_NAMESPACE, name.as_bytes()) + )) +} + pub(crate) fn remove_orphan_outputs(items: &mut Vec) { let function_call_ids: HashSet = items .iter() diff --git a/codex-rs/core/src/context_manager/updates.rs b/codex-rs/core/src/context_manager/updates.rs index 5e81bc2a7d..a520e6f595 100644 --- a/codex-rs/core/src/context_manager/updates.rs +++ b/codex-rs/core/src/context_manager/updates.rs @@ -85,9 +85,6 @@ fn build_multi_agent_mode_update_item( } match effective_multi_agent_mode { - Some(MultiAgentMode::None) => { - Some(MultiAgentModeInstructions::new(MultiAgentMode::None).render()) - } Some(multi_agent_mode) => Some(MultiAgentModeInstructions::new(multi_agent_mode).render()), None if previous.multi_agent_mode == Some(MultiAgentMode::Proactive) => { Some(MultiAgentModeInstructions::new(MultiAgentMode::ExplicitRequestOnly).render()) diff --git a/codex-rs/core/src/elicitation.rs b/codex-rs/core/src/elicitation.rs new file mode 100644 index 0000000000..b29207af46 --- /dev/null +++ b/codex-rs/core/src/elicitation.rs @@ -0,0 +1,100 @@ +use std::sync::Arc; +use std::sync::Mutex; + +use tokio::sync::watch; + +/// Coordinates user elicitations that pause tool-result delivery for a session. +/// +/// Registrations are counted so concurrent elicitations keep the session paused until all of them +/// finish. Consumers can subscribe to pause timeout progress or wait before returning an already +/// captured result. +#[derive(Clone)] +pub(crate) struct ElicitationService { + inner: Arc, +} + +struct Inner { + state: Mutex, + paused: watch::Sender, +} + +#[derive(Default)] +struct State { + outstanding: i64, +} + +pub(crate) struct ElicitationRegistration { + service: ElicitationService, +} + +impl ElicitationService { + pub(crate) fn new() -> Self { + let (paused, _paused_rx) = watch::channel(false); + Self { + inner: Arc::new(Inner { + state: Mutex::new(State::default()), + paused, + }), + } + } + + pub(crate) fn register(&self) -> ElicitationRegistration { + self.increment(); + ElicitationRegistration { + service: self.clone(), + } + } + + fn increment(&self) { + let mut state = self + .inner + .state + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let was_clear = state.outstanding == 0; + assert_ne!( + state.outstanding, + i64::MAX, + "outstanding elicitation count overflowed" + ); + state.outstanding += 1; + if was_clear { + self.inner.paused.send_replace(true); + } + } + + pub(crate) fn subscribe(&self) -> watch::Receiver { + self.inner.paused.subscribe() + } + + pub(crate) async fn wait_until_clear(&self) { + let mut paused = self.subscribe(); + let _ = paused.wait_for(|paused| !*paused).await; + } + + fn decrement(&self) { + let mut state = self + .inner + .state + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + assert!( + state.outstanding > 0, + "elicitation registration count underflowed" + ); + state.outstanding -= 1; + if state.outstanding == 0 { + self.inner.paused.send_replace(false); + } + } +} + +impl Drop for ElicitationRegistration { + fn drop(&mut self) { + self.service.decrement(); + } +} + +#[cfg(test)] +#[path = "elicitation_tests.rs"] +mod tests; diff --git a/codex-rs/core/src/elicitation_tests.rs b/codex-rs/core/src/elicitation_tests.rs new file mode 100644 index 0000000000..eefe1f5394 --- /dev/null +++ b/codex-rs/core/src/elicitation_tests.rs @@ -0,0 +1,19 @@ +use super::*; + +#[tokio::test] +async fn wait_until_clear_waits_for_every_registration() { + let service = ElicitationService::new(); + let first = service.register(); + let second = service.register(); + let waiting = tokio::spawn({ + let service = service.clone(); + async move { service.wait_until_clear().await } + }); + + drop(first); + tokio::task::yield_now().await; + assert!(!waiting.is_finished()); + + drop(second); + waiting.await.expect("elicitation waiter should complete"); +} diff --git a/codex-rs/core/src/environment_selection.rs b/codex-rs/core/src/environment_selection.rs index fa86dded55..ef6a3ec5ea 100644 --- a/codex-rs/core/src/environment_selection.rs +++ b/codex-rs/core/src/environment_selection.rs @@ -1,3 +1,4 @@ +use std::collections::HashMap; use std::collections::HashSet; use std::fmt; use std::sync::Arc; @@ -227,6 +228,24 @@ pub(crate) struct TurnEnvironmentSnapshot { } impl TurnEnvironmentSnapshot { + /// Maps each captured environment to its exact ready handle, or `None` when it was starting. + pub(crate) fn captured_environments(&self) -> HashMap>> { + self.turn_environments + .iter() + .map(|environment| { + ( + environment.environment_id.clone(), + Some(Arc::clone(&environment.environment)), + ) + }) + .chain( + self.starting + .iter() + .map(|environment| (environment.selection.environment_id.clone(), None)), + ) + .collect() + } + pub(crate) fn primary(&self) -> Option<&TurnEnvironment> { self.turn_environments.first() } diff --git a/codex-rs/core/src/event_mapping.rs b/codex-rs/core/src/event_mapping.rs index 392203c079..ef8f0fad5b 100644 --- a/codex-rs/core/src/event_mapping.rs +++ b/codex-rs/core/src/event_mapping.rs @@ -14,10 +14,12 @@ use codex_protocol::models::is_image_close_tag_text; use codex_protocol::models::is_image_open_tag_text; use codex_protocol::models::is_local_image_close_tag_text; use codex_protocol::models::is_local_image_open_tag_text; +use codex_protocol::protocol::APPS_INSTRUCTIONS_OPEN_TAG; use codex_protocol::protocol::COLLABORATION_MODE_OPEN_TAG; use codex_protocol::protocol::CONTEXT_WINDOW_GUIDANCE_OPEN_TAG; use codex_protocol::protocol::CONTEXT_WINDOW_OPEN_TAG; use codex_protocol::protocol::MULTI_AGENT_MODE_OPEN_TAG; +use codex_protocol::protocol::PLUGINS_INSTRUCTIONS_OPEN_TAG; use codex_protocol::protocol::REALTIME_CONVERSATION_OPEN_TAG; use codex_protocol::protocol::SKILLS_INSTRUCTIONS_OPEN_TAG; use codex_protocol::user_input::UserInput; @@ -31,8 +33,10 @@ use crate::web_search::web_search_action_detail; const CONTEXTUAL_DEVELOPER_PREFIXES: &[&str] = &[ "", "", + APPS_INSTRUCTIONS_OPEN_TAG, COLLABORATION_MODE_OPEN_TAG, MULTI_AGENT_MODE_OPEN_TAG, + PLUGINS_INSTRUCTIONS_OPEN_TAG, REALTIME_CONVERSATION_OPEN_TAG, SKILLS_INSTRUCTIONS_OPEN_TAG, "", diff --git a/codex-rs/core/src/exec_env.rs b/codex-rs/core/src/exec_env.rs index 938667b12e..f33061698b 100644 --- a/codex-rs/core/src/exec_env.rs +++ b/codex-rs/core/src/exec_env.rs @@ -2,11 +2,16 @@ use codex_protocol::ThreadId; #[cfg(test)] use codex_protocol::config_types::EnvironmentVariablePattern; use codex_protocol::config_types::ShellEnvironmentPolicy; +use codex_protocol::models::ActivePermissionProfile; use codex_protocol::shell_environment; use std::collections::HashMap; pub use codex_protocol::shell_environment::CODEX_THREAD_ID_ENV_VAR; +/// Informational name of the active permission profile. Child processes can +/// overwrite this value, so it must not be treated as proof of enforcement. +pub const CODEX_PERMISSION_PROFILE_ENV_VAR: &str = "CODEX_PERMISSION_PROFILE"; + /// Construct an environment map based on the rules in the specified policy. The /// resulting map can be passed directly to `Command::envs()` after calling /// `env_clear()` to ensure no unintended variables are leaked to the spawned @@ -25,6 +30,27 @@ pub fn create_env( shell_environment::create_env(policy, thread_id.as_deref()) } +/// Injects the selected named permission profile into a shell tool's environment. +/// +/// This is applied after the shell environment policy so the runtime-selected +/// profile wins over inherited or configured values. +pub(crate) fn inject_permission_profile_env( + env: &mut HashMap, + active_permission_profile: Option<&ActivePermissionProfile>, +) { + if cfg!(windows) { + env.retain(|key, _| !key.eq_ignore_ascii_case(CODEX_PERMISSION_PROFILE_ENV_VAR)); + } else { + env.remove(CODEX_PERMISSION_PROFILE_ENV_VAR); + } + if let Some(active_permission_profile) = active_permission_profile { + env.insert( + CODEX_PERMISSION_PROFILE_ENV_VAR.to_string(), + active_permission_profile.id.clone(), + ); + } +} + #[cfg(all(test, target_os = "windows"))] fn create_env_from_vars( vars: I, diff --git a/codex-rs/core/src/exec_env_tests.rs b/codex-rs/core/src/exec_env_tests.rs index 725edd8cc5..73c0944c14 100644 --- a/codex-rs/core/src/exec_env_tests.rs +++ b/codex-rs/core/src/exec_env_tests.rs @@ -10,6 +10,59 @@ fn make_vars(pairs: &[(&str, &str)]) -> Vec<(String, String)> { .collect() } +#[test] +fn inject_permission_profile_env_overrides_policy_value() { + let mut env = HashMap::from([( + CODEX_PERMISSION_PROFILE_ENV_VAR.to_string(), + "stale-profile".to_string(), + )]); + + inject_permission_profile_env( + &mut env, + Some(&ActivePermissionProfile::new("current-profile")), + ); + + assert_eq!( + env.get(CODEX_PERMISSION_PROFILE_ENV_VAR) + .map(String::as_str), + Some("current-profile") + ); +} + +#[test] +fn inject_permission_profile_env_removes_stale_value_without_active_profile() { + let mut env = HashMap::from([( + CODEX_PERMISSION_PROFILE_ENV_VAR.to_string(), + "stale-profile".to_string(), + )]); + + inject_permission_profile_env(&mut env, /*active_permission_profile*/ None); + + assert_eq!(env.get(CODEX_PERMISSION_PROFILE_ENV_VAR), None); +} + +#[cfg(target_os = "windows")] +#[test] +fn inject_permission_profile_env_replaces_differently_cased_windows_key() { + let mut env = HashMap::from([( + "codex_permission_profile".to_string(), + "stale-profile".to_string(), + )]); + + inject_permission_profile_env( + &mut env, + Some(&ActivePermissionProfile::new("current-profile")), + ); + + assert_eq!( + env, + HashMap::from([( + CODEX_PERMISSION_PROFILE_ENV_VAR.to_string(), + "current-profile".to_string(), + )]) + ); +} + #[test] fn test_core_inherit_defaults_keep_sensitive_vars() { let vars = make_vars(&[ diff --git a/codex-rs/core/src/lib.rs b/codex-rs/core/src/lib.rs index ad5f54123f..91ccbd6e63 100644 --- a/codex-rs/core/src/lib.rs +++ b/codex-rs/core/src/lib.rs @@ -31,6 +31,7 @@ pub use codex_thread::TryStartTurnIfIdleError; pub use codex_thread::TryStartTurnIfIdleRejectionReason; pub use session::turn_context::TurnContext; mod agent; +mod agent_communication; mod attestation; mod codex_delegate; mod command_canonicalization; @@ -39,6 +40,7 @@ pub mod connectors; pub mod context; mod context_manager; mod current_time; +mod elicitation; mod environment_selection; pub mod exec; pub mod exec_env; @@ -145,6 +147,7 @@ pub(crate) mod state_db_bridge; pub use state_db_bridge::StateDbHandle; pub use state_db_bridge::init_state_db; mod thread_rollout_truncation; +pub use thread_rollout_truncation::truncate_rollout_after_turn_id; mod tools; pub(crate) mod turn_diff_tracker; mod turn_metadata; diff --git a/codex-rs/core/src/mcp.rs b/codex-rs/core/src/mcp.rs index 90e6b01e75..835c47ff51 100644 --- a/codex-rs/core/src/mcp.rs +++ b/codex-rs/core/src/mcp.rs @@ -3,13 +3,17 @@ use std::sync::Arc; use crate::config::Config; use codex_config::McpServerConfig; +use codex_connectors::ConnectorSnapshot; +use codex_connectors::PluginConnectorSource; use codex_core_plugins::PluginsManager; +use codex_extension_api::ExtensionData; use codex_extension_api::ExtensionDataInit; use codex_extension_api::ExtensionRegistry; use codex_extension_api::McpServerContribution; use codex_extension_api::McpServerContributionContext; use codex_login::CodexAuth; use codex_mcp::CODEX_APPS_MCP_SERVER_NAME; +use codex_mcp::CodexAppsToolsCache; use codex_mcp::EffectiveMcpServer; use codex_mcp::McpConfig; use codex_mcp::McpPluginAttribution; @@ -17,9 +21,16 @@ use codex_mcp::McpServerRegistration; use codex_mcp::codex_apps_mcp_server_config; use codex_mcp::configured_mcp_servers; use codex_mcp::effective_mcp_servers; +use codex_plugin::AppConnectorId; const LEGACY_CODEX_APPS_REGISTRATION_ID: &str = "legacy_codex_apps"; +/// MCP configuration and capability availability derived from the same inputs. +pub(crate) struct McpRuntimeProjection { + pub(crate) config: McpConfig, + pub(crate) plugins_available: bool, +} + enum OrderedMcpOverlay { Set { contributor_id: &'static str, @@ -38,14 +49,15 @@ enum OrderedMcpOverlay { pub struct McpManager { plugins_manager: Arc, extensions: Arc>, + codex_apps_tools_cache: CodexAppsToolsCache, } impl McpManager { pub fn new(plugins_manager: Arc) -> Self { - Self { + Self::new_with_extensions( plugins_manager, - extensions: codex_extension_api::empty_extension_registry(), - } + codex_extension_api::empty_extension_registry(), + ) } /// Creates a manager that resolves host-installed MCP contributions. @@ -56,34 +68,45 @@ impl McpManager { Self { plugins_manager, extensions, + codex_apps_tools_cache: CodexAppsToolsCache::default(), } } + pub fn codex_apps_tools_cache(&self) -> CodexAppsToolsCache { + self.codex_apps_tools_cache.clone() + } + /// Returns the MCP config after applying compatibility built-ins and /// runtime-only extension overlays. pub async fn runtime_config(&self, config: &Config) -> McpConfig { - self.runtime_config_with_context(config, /*thread_init*/ None) + self.runtime_config_with_context(McpServerContributionContext::global(config)) .await + .config } - pub(crate) async fn runtime_config_for_thread( + pub(crate) async fn runtime_config_for_step( &self, config: &Config, thread_init: &ExtensionDataInit, - ) -> McpConfig { - self.runtime_config_with_context(config, Some(thread_init)) - .await + thread_store: &ExtensionData, + available_environment_ids: &[String], + ) -> McpRuntimeProjection { + self.runtime_config_with_context(McpServerContributionContext::for_step( + config, + thread_init, + thread_store, + available_environment_ids, + )) + .await } async fn runtime_config_with_context( &self, - config: &Config, - thread_init: Option<&ExtensionDataInit>, - ) -> McpConfig { - let context = match thread_init { - Some(thread_init) => McpServerContributionContext::for_thread(config, thread_init), - None => McpServerContributionContext::global(config), - }; + context: McpServerContributionContext<'_, Config>, + ) -> McpRuntimeProjection { + let config = context.config(); + let mut selected_plugin_available = false; + let mut selected_plugin_connector_sources = Vec::new(); let mut selected_plugin_registrations = Vec::new(); let mut overlays = Vec::new(); // A contributor can emit multiple ordered actions, so order each action globally rather @@ -114,6 +137,22 @@ impl McpManager { *config, ), ), + McpServerContribution::SelectedPluginPackage { + plugin_id, + plugin_display_name, + connector_ids, + } => { + selected_plugin_available = true; + if !connector_ids.is_empty() { + selected_plugin_connector_sources.push( + PluginConnectorSource::from_connector_ids( + plugin_id, + plugin_display_name, + connector_ids.into_iter().map(AppConnectorId), + ), + ); + } + } McpServerContribution::Remove { name } => { overlays.push(OrderedMcpOverlay::Remove { contributor_id: contributor.id(), @@ -126,12 +165,14 @@ impl McpManager { } } - let mut mcp_config = config - .to_mcp_config_with_plugin_registrations( - self.plugins_manager.as_ref(), - selected_plugin_registrations, - ) + let loaded_plugins = self + .plugins_manager + .plugins_for_config(&config.plugins_config_input()) .await; + let plugins_available = + selected_plugin_available || !loaded_plugins.capability_summaries().is_empty(); + let mut mcp_config = config + .to_mcp_config_with_loaded_plugins(&loaded_plugins, selected_plugin_registrations); let mut catalog = mcp_config.mcp_server_catalog.to_builder(); if mcp_config.apps_enabled { catalog.register(McpServerRegistration::from_compatibility( @@ -179,7 +220,16 @@ impl McpManager { ); } mcp_config.mcp_server_catalog = catalog; - mcp_config + mcp_config.connector_snapshot = + mcp_config + .connector_snapshot + .merged_with(&ConnectorSnapshot::from_plugin_sources( + selected_plugin_connector_sources, + )); + McpRuntimeProjection { + config: mcp_config, + plugins_available, + } } /// Returns config- and plugin-backed servers without runtime contributions. diff --git a/codex-rs/core/src/mcp_skill_dependencies.rs b/codex-rs/core/src/mcp_skill_dependencies.rs index 9b2b85aa2d..61d5c54a59 100644 --- a/codex-rs/core/src/mcp_skill_dependencies.rs +++ b/codex-rs/core/src/mcp_skill_dependencies.rs @@ -199,15 +199,8 @@ pub(crate) async fn maybe_install_mcp_dependencies( warn!("failed to refresh MCP dependencies for mentioned skills: {err}"); return; } - let refresh_servers = sess.runtime_mcp_servers(&refresh_config).await; - sess.refresh_mcp_servers_now( - turn_context, - refresh_servers, - config.mcp_oauth_credentials_store_mode, - config.auth_keyring_backend_kind(), - elicitation_reviewer, - ) - .await; + sess.refresh_mcp_servers_now(turn_context, &refresh_config, elicitation_reviewer) + .await; } async fn should_install_mcp_dependencies( diff --git a/codex-rs/core/src/mcp_tool_call.rs b/codex-rs/core/src/mcp_tool_call.rs index df2a375e35..d03f232d12 100644 --- a/codex-rs/core/src/mcp_tool_call.rs +++ b/codex-rs/core/src/mcp_tool_call.rs @@ -18,6 +18,7 @@ use crate::mcp_openai_file::rewrite_mcp_tool_arguments_for_openai_files; use crate::mcp_tool_approval_templates::RenderedMcpToolApprovalParam; use crate::mcp_tool_approval_templates::render_mcp_tool_approval_template; use crate::session::session::Session; +use crate::session::step_context::StepContext; use crate::session::turn_context::TurnContext; use crate::tools::hook_names::HookToolName; use crate::tools::sandboxing::PermissionRequestPayload; @@ -35,6 +36,7 @@ use codex_features::Feature; use codex_hooks::PermissionRequestDecision; use codex_mcp::CODEX_APPS_MCP_SERVER_NAME; use codex_mcp::MCP_TOOL_CODEX_APPS_META_KEY; +use codex_mcp::McpConnectionManager; use codex_mcp::McpPermissionPromptAutoApproveContext; use codex_mcp::SandboxState; use codex_mcp::auth_elicitation_completed_result; @@ -111,13 +113,15 @@ const MCP_TOOL_CALL_EVENT_RESULT_MAX_BYTES: usize = DEFAULT_OUTPUT_BYTES_CAP; /// item lifecycle events to the `Session`. pub(crate) async fn handle_mcp_tool_call( sess: Arc, - turn_context: &Arc, + step_context: &Arc, call_id: String, server: String, tool_name: String, hook_tool_name: HookToolName, arguments: String, ) -> HandledMcpToolCall { + let turn_context = &step_context.turn; + let manager = step_context.mcp.manager(); // Parse the `arguments` as JSON. An empty string is OK, but invalid JSON // is not. let arguments_value = if arguments.trim().is_empty() { @@ -141,8 +145,14 @@ pub(crate) async fn handle_mcp_tool_call( arguments: arguments_value.clone(), }; - let metadata = - lookup_mcp_tool_metadata(sess.as_ref(), turn_context.as_ref(), &server, &tool_name).await; + let metadata = lookup_mcp_tool_metadata( + sess.as_ref(), + turn_context.as_ref(), + manager, + &server, + &tool_name, + ) + .await; let item_metadata = McpToolCallItemMetadata::from_tool_metadata(&server, metadata.as_ref()); let app_tool_policy = if server == CODEX_APPS_MCP_SERVER_NAME { let annotations = metadata @@ -169,7 +179,6 @@ pub(crate) async fn handle_mcp_tool_call( } else if let Some(approval_mode) = { // Selected-plugin registrations are absent from config.toml and the legacy plugin manager, // so their resolved catalog entry is the authoritative source for tool approval policy. - let manager = sess.services.mcp_connection_manager.load(); manager .is_selected_plugin_mcp_server(&server) .then(|| manager.tool_approval_mode(&server, &tool_name)) @@ -226,7 +235,7 @@ pub(crate) async fn handle_mcp_tool_call( if let Some(decision) = maybe_request_mcp_tool_approval( &sess, - turn_context, + step_context, &call_id, &invocation, &hook_tool_name, @@ -241,7 +250,7 @@ pub(crate) async fn handle_mcp_tool_call( | McpToolApprovalDecision::AcceptAndRemember => { return handle_approved_mcp_tool_call( sess.as_ref(), - turn_context.as_ref(), + step_context.as_ref(), &call_id, invocation, metadata.as_ref(), @@ -298,7 +307,7 @@ pub(crate) async fn handle_mcp_tool_call( handle_approved_mcp_tool_call( sess.as_ref(), - turn_context.as_ref(), + step_context.as_ref(), &call_id, invocation, metadata.as_ref(), @@ -317,6 +326,9 @@ struct McpToolCallItemMetadata { connector_id: Option, link_id: Option, mcp_app_resource_uri: Option, + app_name: Option, + template_id: Option, + action_name: Option, plugin_id: Option, } @@ -333,6 +345,15 @@ impl McpToolCallItemMetadata { link_id: trusted_mcp_app_metadata.and_then(|metadata| metadata.link_id.clone()), mcp_app_resource_uri: metadata .and_then(|metadata| metadata.mcp_app_resource_uri.clone()), + app_name: trusted_mcp_app_metadata.and_then(|metadata| metadata.connector_name.clone()), + template_id: trusted_mcp_app_metadata.and_then(|metadata| metadata.template_id.clone()), + action_name: trusted_mcp_app_metadata + .and_then(|metadata| metadata.codex_apps_meta.as_ref()) + .and_then(|meta| meta.get(MCP_TOOL_RESOURCE_URI_META_KEY)) + .and_then(serde_json::Value::as_str) + .and_then(|resource_uri| resource_uri.trim_matches('/').rsplit('/').next()) + .filter(|action_name| !action_name.is_empty()) + .map(str::to_string), plugin_id: metadata.and_then(|metadata| metadata.plugin_id.clone()), } } @@ -340,24 +361,21 @@ impl McpToolCallItemMetadata { async fn handle_approved_mcp_tool_call( sess: &Session, - turn_context: &TurnContext, + step_context: &StepContext, call_id: &str, invocation: McpInvocation, metadata: Option<&McpToolApprovalMetadata>, item_metadata: McpToolCallItemMetadata, ) -> HandledMcpToolCall { + let turn_context = step_context.turn.as_ref(); + let manager = step_context.mcp.manager(); let server = invocation.server.clone(); - maybe_mark_thread_memory_mode_polluted(sess, turn_context, &server).await; + maybe_mark_thread_memory_mode_polluted(sess, turn_context, manager, &server).await; let tool_name = invocation.tool.clone(); let arguments_value = invocation.arguments.clone(); let connector_id = metadata.and_then(|metadata| metadata.connector_id.as_deref()); let connector_name = metadata.and_then(|metadata| metadata.connector_name.as_deref()); - let server_origin = { - let mcp_connection_manager = sess.services.mcp_connection_manager.load_full(); - mcp_connection_manager - .server_origin(&server) - .map(str::to_string) - }; + let server_origin = manager.server_origin(&server).map(str::to_string); let start = Instant::now(); let rewrite = rewrite_mcp_tool_arguments_for_openai_files( @@ -380,7 +398,7 @@ async fn handle_approved_mcp_tool_call( build_mcp_tool_call_request_meta(turn_context, &server, call_id, metadata); execute_mcp_tool_call( sess, - turn_context, + step_context, call_id, &invocation, rewritten_arguments, @@ -420,7 +438,7 @@ async fn handle_approved_mcp_tool_call( truncate_mcp_tool_result_for_event(&result), ) .await; - maybe_track_codex_app_used(sess, turn_context, &server, &tool_name).await; + maybe_track_codex_app_used(sess, turn_context, manager, &server, &tool_name).await; let outcome = mcp_call_metric_outcome(&result); emit_mcp_call_metrics( @@ -547,17 +565,19 @@ fn truncate_str_to_char_boundary(value: &str, max_chars: usize) -> &str { async fn execute_mcp_tool_call( sess: &Session, - turn_context: &TurnContext, + step_context: &StepContext, call_id: &str, invocation: &McpInvocation, rewritten_arguments: Option, metadata: Option<&McpToolApprovalMetadata>, request_meta: Option, ) -> Result { + let turn_context = step_context.turn.as_ref(); + let manager = step_context.mcp.manager(); let request_meta = with_mcp_tool_call_thread_id_meta(request_meta, &sess.thread_id.to_string()); let request_meta = augment_mcp_tool_request_meta_with_sandbox_state( - sess, - turn_context, + step_context, + manager, &invocation.server, request_meta, ) @@ -568,7 +588,7 @@ async fn execute_mcp_tool_call( .rollout_thread_trace .start_mcp_call_trace(call_id); let request_meta = mcp_call_trace.add_request_meta(request_meta); - let result = sess + let result = manager .call_tool( &invocation.server, &invocation.tool, @@ -587,6 +607,7 @@ async fn execute_mcp_tool_call( Ok(maybe_request_codex_apps_auth_elicitation( sess, turn_context, + manager, call_id, &invocation.server, metadata, @@ -598,17 +619,13 @@ async fn execute_mcp_tool_call( async fn maybe_request_codex_apps_auth_elicitation( sess: &Session, turn_context: &TurnContext, + manager: &McpConnectionManager, call_id: &str, server: &str, metadata: Option<&McpToolApprovalMetadata>, result: CallToolResult, ) -> CallToolResult { - if !sess - .services - .mcp_connection_manager - .load_full() - .is_host_owned_codex_apps_server(server) - { + if !manager.is_host_owned_codex_apps_server(server) { return result; } @@ -666,15 +683,16 @@ async fn maybe_request_codex_apps_auth_elicitation( return result; } - refresh_codex_apps_after_connector_auth(sess, turn_context).await; + refresh_codex_apps_after_connector_auth(sess, turn_context, manager).await; auth_elicitation_completed_result(&plan.auth_failure, result.meta) } -async fn refresh_codex_apps_after_connector_auth(sess: &Session, turn_context: &TurnContext) { - let mcp_tools_result = { - let manager = sess.services.mcp_connection_manager.load_full(); - manager.hard_refresh_codex_apps_tools_cache().await - }; +async fn refresh_codex_apps_after_connector_auth( + sess: &Session, + turn_context: &TurnContext, + manager: &McpConnectionManager, +) { + let mcp_tools_result = manager.hard_refresh_codex_apps_tools_cache().await; match mcp_tools_result { Ok(mcp_tools) => { @@ -692,13 +710,13 @@ async fn refresh_codex_apps_after_connector_auth(sess: &Session, turn_context: & } async fn augment_mcp_tool_request_meta_with_sandbox_state( - sess: &Session, - turn_context: &TurnContext, + step_context: &StepContext, + manager: &McpConnectionManager, server: &str, mut meta: Option, ) -> anyhow::Result> { - let mcp_connection_manager = sess.services.mcp_connection_manager.load_full(); - let supports_sandbox_state_meta = mcp_connection_manager + let turn_context = step_context.turn.as_ref(); + let supports_sandbox_state_meta = manager .server_supports_sandbox_state_meta_capability(server) .await .unwrap_or(false); @@ -706,18 +724,18 @@ async fn augment_mcp_tool_request_meta_with_sandbox_state( return Ok(meta); } - let server_environment_id = mcp_connection_manager + let server_environment_id = manager .server_environment_id(server) .unwrap_or(codex_config::DEFAULT_MCP_SERVER_ENVIRONMENT_ID); - let Some(sandbox_cwd) = sandbox_cwd_for_mcp_server(turn_context, server_environment_id) else { + let Some(sandbox_cwd) = sandbox_cwd_for_mcp_server(step_context, server_environment_id) else { return Ok(meta); }; let permission_profile = turn_context.permission_profile(); let sandbox_state = serde_json::to_value(SandboxState { permission_profile, - codex_linux_sandbox_exe: turn_context.config.codex_linux_sandbox_exe.clone(), + codex_linux_sandbox_exe: step_context.mcp.config().codex_linux_sandbox_exe.clone(), sandbox_cwd, - use_legacy_landlock: turn_context.config.features.use_legacy_landlock(), + use_legacy_landlock: step_context.mcp.config().use_legacy_landlock, })?; match meta.as_mut() { @@ -741,8 +759,8 @@ async fn augment_mcp_tool_request_meta_with_sandbox_state( Ok(meta) } -fn sandbox_cwd_for_mcp_server(turn_context: &TurnContext, environment_id: &str) -> Option { - if let Some(environment) = turn_context +fn sandbox_cwd_for_mcp_server(step_context: &StepContext, environment_id: &str) -> Option { + if let Some(environment) = step_context .environments .turn_environments .iter() @@ -753,7 +771,7 @@ fn sandbox_cwd_for_mcp_server(turn_context: &TurnContext, environment_id: &str) if environment_id == codex_config::DEFAULT_MCP_SERVER_ENVIRONMENT_ID { #[allow(deprecated)] - return Some(PathUri::from_abs_path(&turn_context.cwd)); + return Some(PathUri::from_abs_path(&step_context.turn.cwd)); } None @@ -762,16 +780,13 @@ fn sandbox_cwd_for_mcp_server(turn_context: &TurnContext, environment_id: &str) async fn maybe_mark_thread_memory_mode_polluted( sess: &Session, turn_context: &TurnContext, + manager: &McpConnectionManager, server: &str, ) { if !turn_context.config.memories.disable_on_external_context { return; } - let pollutes_memory = sess - .services - .mcp_connection_manager - .load_full() - .server_pollutes_memory(server); + let pollutes_memory = manager.server_pollutes_memory(server); if !pollutes_memory { return; } @@ -878,6 +893,9 @@ async fn notify_mcp_tool_call_started( connector_id: item_metadata.connector_id, mcp_app_resource_uri: item_metadata.mcp_app_resource_uri, link_id: item_metadata.link_id, + app_name: item_metadata.app_name, + template_id: item_metadata.template_id, + action_name: item_metadata.action_name, plugin_id: item_metadata.plugin_id, status: McpToolCallStatus::InProgress, result: None, @@ -920,6 +938,9 @@ async fn notify_mcp_tool_call_completed( connector_id: item_metadata.connector_id, mcp_app_resource_uri: item_metadata.mcp_app_resource_uri, link_id: item_metadata.link_id, + app_name: item_metadata.app_name, + template_id: item_metadata.template_id, + action_name: item_metadata.action_name, plugin_id: item_metadata.plugin_id, status, result, @@ -937,13 +958,14 @@ struct McpAppUsageMetadata { async fn maybe_track_codex_app_used( sess: &Session, turn_context: &TurnContext, + manager: &McpConnectionManager, server: &str, tool_name: &str, ) { if server != CODEX_APPS_MCP_SERVER_NAME { return; } - let metadata = lookup_mcp_app_usage_metadata(sess, server, tool_name).await; + let metadata = lookup_mcp_app_usage_metadata(manager, server, tool_name).await; let (connector_id, app_name) = metadata .map(|metadata| (metadata.connector_id, metadata.app_name)) .unwrap_or((None, None)); @@ -983,6 +1005,7 @@ enum McpToolApprovalDecision { Cancel, } +#[derive(Clone)] pub(crate) struct McpToolApprovalMetadata { annotations: Option, connector_id: Option, @@ -994,6 +1017,7 @@ pub(crate) struct McpToolApprovalMetadata { tool_title: Option, tool_description: Option, mcp_app_resource_uri: Option, + template_id: Option, codex_apps_meta: Option>, openai_file_input_params: Option>, } @@ -1004,6 +1028,8 @@ const MCP_TOOL_LINK_ID_META_KEY: &str = "link_id"; const MCP_TOOL_PLUGIN_ID_META_KEY: &str = "plugin_id"; const MCP_TOOL_THREAD_ID_META_KEY: &str = "threadId"; const MCP_TOOL_CONNECTED_ACCOUNT_EMAIL_META_KEY: &str = "connected_account_email"; +const MCP_TOOL_TEMPLATE_ID_META_KEY: &str = "template_id"; +const MCP_TOOL_RESOURCE_URI_META_KEY: &str = "resource_uri"; async fn custom_mcp_tool_approval_mode( sess: &Session, @@ -1176,13 +1202,15 @@ fn mcp_tool_approval_prompt_options( async fn maybe_request_mcp_tool_approval( sess: &Arc, - turn_context: &Arc, + step_context: &Arc, call_id: &str, invocation: &McpInvocation, hook_tool_name: &HookToolName, metadata: Option<&McpToolApprovalMetadata>, approval_mode: AppToolApproval, ) -> Option { + let turn_context = &step_context.turn; + let manager = step_context.mcp.manager(); let approvals_reviewer = mcp_approvals_reviewer(turn_context, &invocation.server, metadata); if mcp_permission_prompt_is_auto_approved( turn_context.approval_policy.value(), @@ -1201,12 +1229,7 @@ async fn maybe_request_mcp_tool_approval( } let session_approval_key = session_mcp_tool_approval_key(invocation, metadata, approval_mode); - let persistent_approval_key = if sess - .services - .mcp_connection_manager - .load() - .is_selected_plugin_mcp_server(&invocation.server) - { + let persistent_approval_key = if manager.is_selected_plugin_mcp_server(&invocation.server) { None } else { persistent_mcp_tool_approval_key(invocation, metadata, approval_mode) @@ -1452,10 +1475,10 @@ async fn mcp_tool_approval_decision_from_guardian( pub(crate) async fn lookup_mcp_tool_metadata( sess: &Session, turn_context: &TurnContext, + manager: &McpConnectionManager, server: &str, tool_name: &str, ) -> Option { - let manager = sess.services.mcp_connection_manager.load_full(); let plugin_id = manager .plugin_id_for_mcp_server_name(server) .map(str::to_string); @@ -1471,9 +1494,15 @@ pub(crate) async fn lookup_mcp_tool_metadata( { Some(connectors) => Some(connectors), None => { - connectors::list_accessible_connectors_from_mcp_tools(turn_context.config.as_ref()) - .await - .ok() + connectors::list_accessible_connectors_from_mcp_tools_with_mcp_manager( + turn_context.config.as_ref(), + /*force_refetch*/ false, + sess.services.turn_environments.environment_manager(), + Arc::clone(&sess.services.mcp_manager), + ) + .await + .ok() + .map(|status| status.connectors) } }; connectors.and_then(|connectors| { @@ -1523,6 +1552,11 @@ pub(crate) async fn lookup_mcp_tool_metadata( tool_title: tool_info.tool.title, tool_description: tool_info.tool.description.map(std::borrow::Cow::into_owned), mcp_app_resource_uri: get_mcp_app_resource_uri(tool_info.tool.meta.as_deref()), + template_id: codex_apps_meta + .as_ref() + .and_then(|meta| meta.get(MCP_TOOL_TEMPLATE_ID_META_KEY)) + .and_then(serde_json::Value::as_str) + .map(str::to_string), codex_apps_meta, // Disallow custom MCPs from uploading files via fileParams. openai_file_input_params: openai_file_input_params_for_server( @@ -1562,16 +1596,11 @@ fn get_mcp_app_resource_uri( } async fn lookup_mcp_app_usage_metadata( - sess: &Session, + manager: &McpConnectionManager, server: &str, tool_name: &str, ) -> Option { - let tools = sess - .services - .mcp_connection_manager - .load_full() - .list_all_tools() - .await; + let tools = manager.list_all_tools().await; tools.into_iter().find_map(|tool_info| { if tool_info.server_name == server && tool_info.tool.name == tool_name { diff --git a/codex-rs/core/src/mcp_tool_call_tests.rs b/codex-rs/core/src/mcp_tool_call_tests.rs index 387987c9f6..f68d32ff4e 100644 --- a/codex-rs/core/src/mcp_tool_call_tests.rs +++ b/codex-rs/core/src/mcp_tool_call_tests.rs @@ -1,6 +1,7 @@ use super::*; use crate::config::ConfigBuilder; use crate::config::ManagedFeatures; +use crate::session::step_context::StepContext; use crate::session::tests::make_session_and_context; use crate::session::tests::make_session_and_context_with_rx; use crate::session::turn_context::TurnEnvironment; @@ -86,6 +87,7 @@ fn approval_metadata( tool_title: tool_title.map(str::to_string), tool_description: tool_description.map(str::to_string), mcp_app_resource_uri: None, + template_id: None, codex_apps_meta: None, openai_file_input_params: None, } @@ -157,10 +159,12 @@ async fn execute_mcp_tool_call_records_replayable_correlation() -> anyhow::Resul }) }); assert!(dispatch_trace.is_enabled()); + let turn_context = Arc::new(turn_context); + let step_context = StepContext::for_test(Arc::clone(&turn_context)); let result = execute_mcp_tool_call( &session, - &turn_context, + step_context.as_ref(), "mcp-call", &McpInvocation { server: "docs".to_string(), @@ -1147,7 +1151,8 @@ async fn mcp_sandbox_cwd_uses_matching_server_environment_uri() -> anyhow::Resul /*shell*/ None, )); - let sandbox_cwd = sandbox_cwd_for_mcp_server(&turn_context, "remote"); + let step_context = StepContext::for_test(Arc::new(turn_context)); + let sandbox_cwd = sandbox_cwd_for_mcp_server(&step_context, "remote"); assert_eq!(sandbox_cwd, Some(secondary_cwd)); Ok(()) @@ -1157,7 +1162,8 @@ async fn mcp_sandbox_cwd_uses_matching_server_environment_uri() -> anyhow::Resul async fn mcp_sandbox_cwd_is_none_for_unselected_server_environment() -> anyhow::Result<()> { let (_, turn_context) = make_session_and_context().await; - let sandbox_cwd = sandbox_cwd_for_mcp_server(&turn_context, "remote"); + let step_context = StepContext::for_test(Arc::new(turn_context)); + let sandbox_cwd = sandbox_cwd_for_mcp_server(&step_context, "remote"); assert_eq!(sandbox_cwd, None); Ok(()) @@ -1190,12 +1196,22 @@ async fn plugin_mcp_tool_call_request_meta_includes_plugin_id() { fn mcp_tool_call_item_metadata_only_trusts_codex_apps_identity() { let mut metadata = approval_metadata( Some("asdk_app_0123456789abcdef0123456789abcdef"), - /*connector_name*/ None, + Some("Calendar"), /*connector_description*/ None, - /*tool_title*/ None, + Some("Create a calendar event"), /*tool_description*/ None, ); metadata.link_id = Some("link_fedcba9876543210fedcba9876543210".to_string()); + metadata.template_id = Some("calendar_template".to_string()); + metadata.codex_apps_meta = Some( + serde_json::json!({ + "resource_uri": "/asdk_app_0123456789abcdef0123456789abcdef/link_fedcba9876543210fedcba9876543210/create_event", + "template_id": "calendar_template", + }) + .as_object() + .cloned() + .expect("_codex_apps metadata should be an object"), + ); assert_eq!( McpToolCallItemMetadata::from_tool_metadata(CODEX_APPS_MCP_SERVER_NAME, Some(&metadata),), @@ -1203,6 +1219,9 @@ fn mcp_tool_call_item_metadata_only_trusts_codex_apps_identity() { connector_id: Some("asdk_app_0123456789abcdef0123456789abcdef".to_string()), link_id: Some("link_fedcba9876543210fedcba9876543210".to_string()), mcp_app_resource_uri: None, + app_name: Some("Calendar".to_string()), + template_id: Some("calendar_template".to_string()), + action_name: Some("create_event".to_string()), plugin_id: None, } ); @@ -1212,6 +1231,9 @@ fn mcp_tool_call_item_metadata_only_trusts_codex_apps_identity() { connector_id: None, link_id: None, mcp_app_resource_uri: None, + app_name: None, + template_id: None, + action_name: None, plugin_id: None, } ); @@ -1234,6 +1256,9 @@ async fn mcp_tool_call_item_includes_app_identity() { connector_id: Some("asdk_app_0123456789abcdef0123456789abcdef".to_string()), link_id: Some("link_fedcba9876543210fedcba9876543210".to_string()), mcp_app_resource_uri: None, + app_name: Some("Calendar".to_string()), + template_id: Some("calendar_template".to_string()), + action_name: Some("create_event".to_string()), plugin_id: Some("sample@test".to_string()), }, ) @@ -1259,6 +1284,8 @@ async fn mcp_tool_call_item_includes_app_identity() { Some("link_fedcba9876543210fedcba9876543210") ); assert_eq!(item.plugin_id.as_deref(), Some("sample@test")); + assert_eq!(item.app_name.as_deref(), Some("Calendar")); + assert_eq!(item.action_name.as_deref(), Some("create_event")); } #[tokio::test] @@ -1279,6 +1306,7 @@ async fn codex_apps_tool_call_request_meta_includes_turn_metadata_and_codex_apps tool_title: Some("Create Event".to_string()), tool_description: Some("Create a calendar event.".to_string()), mcp_app_resource_uri: None, + template_id: None, codex_apps_meta: Some( serde_json::json!({ "resource_uri": "connector://calendar/tools/calendar_create_event", @@ -1370,7 +1398,10 @@ fn codex_apps_auth_failure_metadata() -> McpToolApprovalMetadata { ) } -async fn install_host_owned_codex_apps_manager(session: &Session, turn_context: &TurnContext) { +async fn host_owned_codex_apps_manager( + session: &Session, + turn_context: &TurnContext, +) -> Arc { let auth = session.services.auth_manager.auth().await; let startup_cancellation_token = CancellationToken::new(); startup_cancellation_token.cancel(); @@ -1400,6 +1431,7 @@ async fn install_host_owned_codex_apps_manager(session: &Session, turn_context: }, ), turn_context.config.codex_home.to_path_buf(), + session.services.mcp_manager.codex_apps_tools_cache(), codex_mcp::codex_apps_tools_cache_key(auth.as_ref()), turn_context.config.prefix_mcp_tool_names(), rmcp::model::ElicitationCapability::default(), @@ -1407,24 +1439,24 @@ async fn install_host_owned_codex_apps_manager(session: &Session, turn_context: codex_mcp::ToolPluginProvenance::default(), auth.as_ref(), /*elicitation_reviewer*/ None, + /*elicitation_lifecycle*/ None, + codex_mcp::ElicitationRequestRouter::default(), ) .await; - session - .services - .mcp_connection_manager - .store(Arc::new(manager)); + Arc::new(manager) } #[tokio::test] async fn codex_apps_auth_elicitation_feature_disabled_returns_original_result() { let (session, turn_context, rx_event) = make_session_and_context_with_rx().await; - install_host_owned_codex_apps_manager(&session, &turn_context).await; + let manager = host_owned_codex_apps_manager(&session, &turn_context).await; let result = codex_apps_auth_failure_result(); let metadata = codex_apps_auth_failure_metadata(); let returned = maybe_request_codex_apps_auth_elicitation( &session, &turn_context, + manager.as_ref(), "call_123", CODEX_APPS_MCP_SERVER_NAME, Some(&metadata), @@ -1445,10 +1477,12 @@ async fn codex_apps_auth_elicitation_non_host_owned_server_returns_original_resu Arc::make_mut(&mut turn_context.config).features = ManagedFeatures::from(features); let result = codex_apps_auth_failure_result(); let metadata = codex_apps_auth_failure_metadata(); + let manager = session.services.latest_mcp_runtime().manager_arc(); let returned = maybe_request_codex_apps_auth_elicitation( &session, turn_context, + manager.as_ref(), "call_123", CODEX_APPS_MCP_SERVER_NAME, Some(&metadata), @@ -1463,7 +1497,7 @@ async fn codex_apps_auth_elicitation_non_host_owned_server_returns_original_resu #[tokio::test] async fn codex_apps_auth_elicitation_disallowed_by_policy_returns_original_result() { let (session, mut turn_context, rx_event) = make_session_and_context_with_rx().await; - install_host_owned_codex_apps_manager(&session, &turn_context).await; + let manager = host_owned_codex_apps_manager(&session, &turn_context).await; let mut features = Features::with_defaults(); features.enable(Feature::AuthElicitation); let turn_context = Arc::get_mut(&mut turn_context).expect("single turn context ref"); @@ -1478,6 +1512,7 @@ async fn codex_apps_auth_elicitation_disallowed_by_policy_returns_original_resul let returned = maybe_request_codex_apps_auth_elicitation( &session, turn_context, + manager.as_ref(), "call_123", CODEX_APPS_MCP_SERVER_NAME, Some(&metadata), @@ -1492,7 +1527,7 @@ async fn codex_apps_auth_elicitation_disallowed_by_policy_returns_original_resul #[tokio::test] async fn codex_apps_auth_elicitation_granular_mcp_disabled_returns_original_result() { let (session, mut turn_context, rx_event) = make_session_and_context_with_rx().await; - install_host_owned_codex_apps_manager(&session, &turn_context).await; + let manager = host_owned_codex_apps_manager(&session, &turn_context).await; let mut features = Features::with_defaults(); features.enable(Feature::AuthElicitation); let turn_context = Arc::get_mut(&mut turn_context).expect("single turn context ref"); @@ -1513,6 +1548,7 @@ async fn codex_apps_auth_elicitation_granular_mcp_disabled_returns_original_resu let returned = maybe_request_codex_apps_auth_elicitation( &session, turn_context, + manager.as_ref(), "call_123", CODEX_APPS_MCP_SERVER_NAME, Some(&metadata), @@ -1527,7 +1563,7 @@ async fn codex_apps_auth_elicitation_granular_mcp_disabled_returns_original_resu #[tokio::test] async fn codex_apps_auth_elicitation_feature_enabled_requests_elicitation() { let (session, mut turn_context, rx_event) = make_session_and_context_with_rx().await; - install_host_owned_codex_apps_manager(&session, &turn_context).await; + let manager = host_owned_codex_apps_manager(&session, &turn_context).await; *session.active_turn.lock().await = Some(ActiveTurn::default()); let mut features = Features::with_defaults(); features.enable(Feature::AuthElicitation); @@ -1541,10 +1577,12 @@ async fn codex_apps_auth_elicitation_feature_enabled_requests_elicitation() { let request_task = tokio::spawn({ let session = Arc::clone(&session); let turn_context = Arc::clone(&turn_context); + let manager = Arc::clone(&manager); async move { maybe_request_codex_apps_auth_elicitation( &session, &turn_context, + manager.as_ref(), "call_123", CODEX_APPS_MCP_SERVER_NAME, Some(&metadata), @@ -1757,6 +1795,7 @@ fn guardian_mcp_review_request_includes_annotations_when_present() { tool_title: None, tool_description: None, mcp_app_resource_uri: None, + template_id: None, codex_apps_meta: None, openai_file_input_params: None, }; @@ -2459,13 +2498,14 @@ async fn approve_mode_skips_when_annotations_do_not_require_approval() { tool_title: Some("Read Only Tool".to_string()), tool_description: None, mcp_app_resource_uri: None, + template_id: None, codex_apps_meta: None, openai_file_input_params: None, }; let decision = maybe_request_mcp_tool_approval( &session, - &turn_context, + &StepContext::for_test(Arc::clone(&turn_context)), "call-1", &invocation, &HookToolName::new("mcp__test__tool"), @@ -2535,13 +2575,14 @@ async fn guardian_mode_skips_auto_when_annotations_do_not_require_approval() { tool_title: Some("Read Only Tool".to_string()), tool_description: None, mcp_app_resource_uri: None, + template_id: None, codex_apps_meta: None, openai_file_input_params: None, }; let decision = maybe_request_mcp_tool_approval( &session, - &turn_context, + &StepContext::for_test(Arc::clone(&turn_context)), "call-guardian", &invocation, &HookToolName::new("mcp__test__tool"), @@ -2594,13 +2635,14 @@ async fn permission_request_hook_allows_mcp_tool_call() { tool_title: Some("Create entities".to_string()), tool_description: None, mcp_app_resource_uri: None, + template_id: None, codex_apps_meta: None, openai_file_input_params: None, }; let decision = maybe_request_mcp_tool_approval( &session, - &turn_context, + &StepContext::for_test(Arc::clone(&turn_context)), "call-mcp-hook", &invocation, &HookToolName::new("mcp__memory__create_entities"), @@ -2662,7 +2704,7 @@ async fn permission_request_hook_uses_hook_tool_name_without_metadata() { let decision = maybe_request_mcp_tool_approval( &session, - &turn_context, + &StepContext::for_test(Arc::clone(&turn_context)), "call-mcp-hook-no-metadata", &invocation, &HookToolName::new("mcp__memory__create_entities"), @@ -2732,6 +2774,7 @@ async fn permission_request_hook_runs_after_remembered_mcp_approval() { tool_title: Some("Create entities".to_string()), tool_description: None, mcp_app_resource_uri: None, + template_id: None, codex_apps_meta: None, openai_file_input_params: None, }; @@ -2744,7 +2787,7 @@ async fn permission_request_hook_runs_after_remembered_mcp_approval() { let turn_context = Arc::new(turn_context); let decision = maybe_request_mcp_tool_approval( &session, - &turn_context, + &StepContext::for_test(Arc::clone(&turn_context)), "call-mcp-remembered", &invocation, &HookToolName::new("mcp__memory__create_entities"), @@ -2821,13 +2864,14 @@ async fn guardian_mode_mcp_denial_returns_rationale_message() { tool_title: Some("Dangerous Tool".to_string()), tool_description: Some("Reads calendar data.".to_string()), mcp_app_resource_uri: None, + template_id: None, codex_apps_meta: None, openai_file_input_params: None, }; let decision = maybe_request_mcp_tool_approval( &session, - &turn_context, + &StepContext::for_test(Arc::clone(&turn_context)), "call-guardian-deny", &invocation, &HookToolName::new("mcp__test__tool"), @@ -2877,6 +2921,7 @@ async fn prompt_mode_waits_for_approval_when_annotations_do_not_require_approval tool_title: Some("Read Only Tool".to_string()), tool_description: None, mcp_app_resource_uri: None, + template_id: None, codex_apps_meta: None, openai_file_input_params: None, }; @@ -2887,7 +2932,7 @@ async fn prompt_mode_waits_for_approval_when_annotations_do_not_require_approval tokio::spawn(async move { maybe_request_mcp_tool_approval( &session, - &turn_context, + &StepContext::for_test(Arc::clone(&turn_context)), "call-prompt", &invocation, &HookToolName::new("mcp__test__tool"), @@ -2934,6 +2979,7 @@ async fn full_access_mode_skips_mcp_tool_approval_for_all_approval_modes() { tool_title: Some("Dangerous Tool".to_string()), tool_description: Some("Performs a risky action.".to_string()), mcp_app_resource_uri: None, + template_id: None, codex_apps_meta: None, openai_file_input_params: None, }; @@ -2945,7 +2991,7 @@ async fn full_access_mode_skips_mcp_tool_approval_for_all_approval_modes() { ] { let decision = maybe_request_mcp_tool_approval( &session, - &turn_context, + &StepContext::for_test(Arc::clone(&turn_context)), "call-2", &invocation, &HookToolName::new("mcp__test__tool"), @@ -2989,6 +3035,7 @@ async fn approve_mode_skips_guardian_in_every_permission_mode() { tool_title: Some("Dangerous Tool".to_string()), tool_description: Some("Performs a risky action.".to_string()), mcp_app_resource_uri: None, + template_id: None, codex_apps_meta: None, openai_file_input_params: None, }; @@ -3034,7 +3081,7 @@ async fn approve_mode_skips_guardian_in_every_permission_mode() { let turn_context = Arc::new(turn_context); let decision = maybe_request_mcp_tool_approval( &session, - &turn_context, + &StepContext::for_test(Arc::clone(&turn_context)), "call-3", &invocation, &HookToolName::new("mcp__test__tool"), diff --git a/codex-rs/core/src/network_policy_decision_tests.rs b/codex-rs/core/src/network_policy_decision_tests.rs index 0640041847..8888bbe9c2 100644 --- a/codex-rs/core/src/network_policy_decision_tests.rs +++ b/codex-rs/core/src/network_policy_decision_tests.rs @@ -161,6 +161,7 @@ fn denied_network_policy_message_requires_deny_decision() { method: Some("GET".to_string()), mode: None, protocol: "http".to_string(), + execution_id: None, decision: Some("ask".to_string()), source: Some("decider".to_string()), port: Some(80), @@ -178,6 +179,7 @@ fn denied_network_policy_message_for_denylist_block_is_explicit() { method: Some("GET".to_string()), mode: None, protocol: "http".to_string(), + execution_id: None, decision: Some("deny".to_string()), source: Some("baseline_policy".to_string()), port: Some(80), diff --git a/codex-rs/core/src/personality_migration_tests.rs b/codex-rs/core/src/personality_migration_tests.rs index dd0da327c4..80f5b3c020 100644 --- a/codex-rs/core/src/personality_migration_tests.rs +++ b/codex-rs/core/src/personality_migration_tests.rs @@ -59,7 +59,9 @@ async fn write_rollout_with_user_event(dir: &Path, thread_id: ThreadId) -> io::R model_provider: None, base_instructions: None, dynamic_tools: None, + selected_capability_roots: Vec::new(), memory_mode: None, + history_mode: Default::default(), multi_agent_version: None, context_window: None, }, diff --git a/codex-rs/core/src/plugins/discoverable_tests.rs b/codex-rs/core/src/plugins/discoverable_tests.rs index 2d92e70964..53d010d190 100644 --- a/codex-rs/core/src/plugins/discoverable_tests.rs +++ b/codex-rs/core/src/plugins/discoverable_tests.rs @@ -69,7 +69,6 @@ async fn list_tool_suggest_discoverable_plugins_includes_cached_remote_global_pl &codex_home.path().join(crate::config::CONFIG_TOML_FILE), r#"[features] plugins = true -remote_plugin = true "#, ); @@ -306,7 +305,6 @@ remote_plugin = true &codex_home.path().join(crate::config::CONFIG_TOML_FILE), r#"[features] plugins = true -remote_plugin = true [tool_suggest] disabled_tools = [ diff --git a/codex-rs/core/src/plugins/render.rs b/codex-rs/core/src/plugins/render.rs index fc197dbbea..4e7575bd62 100644 --- a/codex-rs/core/src/plugins/render.rs +++ b/codex-rs/core/src/plugins/render.rs @@ -6,7 +6,7 @@ use crate::plugins::PluginCapabilitySummary; #[cfg(test)] pub(crate) fn render_plugins_section(plugins: &[PluginCapabilitySummary]) -> Option { - AvailablePluginsInstructions::from_plugins(plugins).map(|instructions| instructions.render()) + (!plugins.is_empty()).then(|| AvailablePluginsInstructions.render()) } pub(crate) fn render_explicit_plugin_instructions( diff --git a/codex-rs/core/src/prompt_debug.rs b/codex-rs/core/src/prompt_debug.rs index a866116a24..cac9bb41ae 100644 --- a/codex-rs/core/src/prompt_debug.rs +++ b/codex-rs/core/src/prompt_debug.rs @@ -64,7 +64,7 @@ pub async fn build_prompt_input( ); let thread = thread_manager.start_thread(config).await?; - let output = build_prompt_input_from_session(thread.thread.codex.session.as_ref(), input).await; + let output = build_prompt_input_from_session(&thread.thread.codex.session, input).await; let shutdown = thread.thread.shutdown_and_wait().await; let _removed = thread_manager.remove_thread(&thread.thread_id).await; @@ -73,7 +73,7 @@ pub async fn build_prompt_input( } pub(crate) async fn build_prompt_input_from_session( - sess: &Session, + sess: &Arc, input: Vec, ) -> CodexResult> { let turn_context = sess.new_default_turn().await; diff --git a/codex-rs/core/src/realtime_context_tests.rs b/codex-rs/core/src/realtime_context_tests.rs index e74d5b3e06..23f0b0c03a 100644 --- a/codex-rs/core/src/realtime_context_tests.rs +++ b/codex-rs/core/src/realtime_context_tests.rs @@ -55,6 +55,7 @@ fn stored_thread(cwd: &str, title: &str, first_user_message: &str) -> StoredThre cwd: PathBuf::from(cwd), cli_version: "test".to_string(), source: SessionSource::Cli, + history_mode: Default::default(), thread_source: None, agent_nickname: None, agent_role: None, diff --git a/codex-rs/core/src/realtime_conversation.rs b/codex-rs/core/src/realtime_conversation.rs index 0c81133be1..3915c8190d 100644 --- a/codex-rs/core/src/realtime_conversation.rs +++ b/codex-rs/core/src/realtime_conversation.rs @@ -48,6 +48,7 @@ use codex_protocol::protocol::RealtimeConversationSdpEvent; use codex_protocol::protocol::RealtimeConversationStartedEvent; use codex_protocol::protocol::RealtimeHandoffRequested; use codex_protocol::protocol::RealtimeOutputModality; +use codex_protocol::protocol::RealtimeTranscriptEntry; use codex_protocol::protocol::RealtimeVoice; use codex_protocol::protocol::RealtimeVoicesList; use http::HeaderMap; @@ -59,6 +60,7 @@ use std::sync::atomic::AtomicBool; use std::sync::atomic::Ordering; use tokio::sync::Mutex; use tokio::task::JoinHandle; +use tokio_util::sync::CancellationToken; use tracing::debug; use tracing::error; use tracing::info; @@ -80,6 +82,7 @@ const REALTIME_V2_STEER_ACKNOWLEDGEMENT: &str = "This was sent to steer the previous background agent task."; const REALTIME_ACTIVE_RESPONSE_ERROR_PREFIX: &str = "Conversation already has an active response in progress:"; +const REALTIME_SESSION_ENDED_HANDOFF_INSTRUCTION: &str = "The user just ended their realtime session. Here is the remaining handoff/transcript tail. You probably do not have to do anything; acknowledge the handoff unless the transcript itself asks for something."; #[derive(Clone, Copy, Debug, PartialEq, Eq)] enum RealtimeConversationEnd { @@ -89,7 +92,7 @@ enum RealtimeConversationEnd { } enum RealtimeFanoutTaskStop { - Abort, + Await, Detach, } @@ -203,6 +206,9 @@ struct RealtimeInputTask { handoff_state: RealtimeHandoffState, session_kind: RealtimeSessionKind, event_parser: RealtimeEventParser, + flush_transcript_tail_on_session_end: bool, + transcript_tail_tx: Sender, + stop_token: CancellationToken, } struct RealtimeInputChannels { @@ -242,12 +248,14 @@ struct ConversationState { input_task: JoinHandle<()>, fanout_task: Option>, realtime_active: Arc, + stop_token: CancellationToken, } struct RealtimeStart { api_provider: ApiProvider, extra_headers: Option, client_managed_handoffs: bool, + flush_transcript_tail_on_session_end: bool, codex_responses_as_items: bool, codex_response_item_prefix: Option, codex_response_handoff_prefix: Option, @@ -260,6 +268,7 @@ struct RealtimeStart { struct RealtimeStartOutput { realtime_active: Arc, events_rx: Receiver, + transcript_tail_rx: Receiver, sdp: Option, } @@ -294,7 +303,7 @@ impl RealtimeConversationManager { guard.take() }; if let Some(state) = previous_state { - stop_conversation_state(state, RealtimeFanoutTaskStop::Abort).await; + stop_conversation_state(state, RealtimeFanoutTaskStop::Await).await; } self.start_inner(start).await @@ -305,6 +314,7 @@ impl RealtimeConversationManager { api_provider, extra_headers, client_managed_handoffs, + flush_transcript_tail_on_session_end, codex_responses_as_items, codex_response_item_prefix, codex_response_handoff_prefix, @@ -327,8 +337,10 @@ impl RealtimeConversationManager { async_channel::bounded::(HANDOFF_OUT_QUEUE_CAPACITY); let (events_tx, events_rx) = async_channel::bounded::(OUTPUT_EVENTS_QUEUE_CAPACITY); + let (transcript_tail_tx, transcript_tail_rx) = async_channel::bounded::(1); let realtime_active = Arc::new(AtomicBool::new(true)); + let stop_token = CancellationToken::new(); let handoff = RealtimeHandoffState::new( handoff_output_tx, client_managed_handoffs, @@ -364,6 +376,9 @@ impl RealtimeConversationManager { session_kind, event_parser, realtime_active: Arc::clone(&realtime_active), + flush_transcript_tail_on_session_end, + transcript_tail_tx, + stop_token: stop_token.clone(), }); (task, Some(call.sdp)) } else { @@ -385,6 +400,9 @@ impl RealtimeConversationManager { handoff_state: handoff.clone(), session_kind, event_parser, + flush_transcript_tail_on_session_end, + transcript_tail_tx, + stop_token: stop_token.clone(), }); (task, None) }; @@ -398,10 +416,12 @@ impl RealtimeConversationManager { input_task: task, fanout_task: None, realtime_active: Arc::clone(&realtime_active), + stop_token, }); Ok(RealtimeStartOutput { realtime_active, events_rx, + transcript_tail_rx, sdp, }) } @@ -652,7 +672,7 @@ impl RealtimeConversationManager { }; if let Some(state) = state { - stop_conversation_state(state, RealtimeFanoutTaskStop::Abort).await; + stop_conversation_state(state, RealtimeFanoutTaskStop::Await).await; } Ok(()) } @@ -663,13 +683,12 @@ async fn stop_conversation_state( fanout_task_stop: RealtimeFanoutTaskStop, ) { state.realtime_active.store(false, Ordering::Relaxed); - state.input_task.abort(); + state.stop_token.cancel(); let _ = state.input_task.await; if let Some(fanout_task) = state.fanout_task.take() { match fanout_task_stop { - RealtimeFanoutTaskStop::Abort => { - fanout_task.abort(); + RealtimeFanoutTaskStop::Await => { let _ = fanout_task.await; } RealtimeFanoutTaskStop::Detach => {} @@ -716,6 +735,7 @@ struct PreparedRealtimeConversationStart { api_provider: ApiProvider, extra_headers: Option, client_managed_handoffs: bool, + flush_transcript_tail_on_session_end: bool, codex_responses_as_items: bool, codex_response_item_prefix: Option, codex_response_handoff_prefix: Option, @@ -800,6 +820,7 @@ async fn prepare_realtime_start( api_provider, extra_headers, client_managed_handoffs: params.client_managed_handoffs, + flush_transcript_tail_on_session_end: params.flush_transcript_tail_on_session_end, codex_responses_as_items: params.codex_responses_as_items, codex_response_item_prefix: params.codex_response_item_prefix, codex_response_handoff_prefix: params.codex_response_handoff_prefix, @@ -966,6 +987,7 @@ async fn handle_start_inner( api_provider, extra_headers, client_managed_handoffs, + flush_transcript_tail_on_session_end, codex_responses_as_items, codex_response_item_prefix, codex_response_handoff_prefix, @@ -984,6 +1006,7 @@ async fn handle_start_inner( api_provider, extra_headers, client_managed_handoffs, + flush_transcript_tail_on_session_end, codex_responses_as_items, codex_response_item_prefix, codex_response_handoff_prefix, @@ -1008,6 +1031,7 @@ async fn handle_start_inner( let RealtimeStartOutput { realtime_active, events_rx, + transcript_tail_rx, sdp, } = start_output; if let Some(sdp) = sdp { @@ -1027,10 +1051,8 @@ async fn handle_start_inner( msg, }; let mut end = RealtimeConversationEnd::TransportClosed; + // Drain already-parsed events so a queued handoff is routed before the final tail. while let Ok(event) = events_rx.recv().await { - if !fanout_realtime_active.load(Ordering::Relaxed) { - break; - } match &event { RealtimeEvent::AudioOut(_) => {} _ => { @@ -1054,9 +1076,6 @@ async fn handle_start_inner( let sess_for_routed_text = Arc::clone(&sess_clone); sess_for_routed_text.route_realtime_text_input(text).await; } - if !fanout_realtime_active.load(Ordering::Relaxed) { - break; - } sess_clone .send_event_raw(ev(EventMsg::RealtimeConversationRealtime( RealtimeConversationRealtimeEvent { @@ -1065,6 +1084,9 @@ async fn handle_start_inner( ))) .await; } + if let Ok(text) = transcript_tail_rx.recv().await { + sess_clone.route_realtime_text_input(text).await; + } if fanout_realtime_active.swap(false, Ordering::Relaxed) { match end { RealtimeConversationEnd::TransportClosed => { @@ -1103,8 +1125,11 @@ pub(crate) async fn handle_audio( } fn realtime_transcript_delta_from_handoff(handoff: &RealtimeHandoffRequested) -> Option { - let active_transcript = handoff - .active_transcript + realtime_transcript_delta(&handoff.active_transcript) +} + +fn realtime_transcript_delta(active_transcript: &[RealtimeTranscriptEntry]) -> Option { + let active_transcript = active_transcript .iter() .map(|entry| format!("{role}: {text}", role = entry.role, text = entry.text)) .collect::>() @@ -1254,6 +1279,9 @@ struct RealtimeWebrtcSidebandInputTask { session_kind: RealtimeSessionKind, event_parser: RealtimeEventParser, realtime_active: Arc, + flush_transcript_tail_on_session_end: bool, + transcript_tail_tx: Sender, + stop_token: CancellationToken, } fn spawn_webrtc_sideband_input_task(input: RealtimeWebrtcSidebandInputTask) -> JoinHandle<()> { @@ -1268,6 +1296,9 @@ fn spawn_webrtc_sideband_input_task(input: RealtimeWebrtcSidebandInputTask) -> J session_kind, event_parser, realtime_active, + flush_transcript_tail_on_session_end, + transcript_tail_tx, + stop_token, } = input; tokio::spawn(async move { @@ -1275,15 +1306,15 @@ fn spawn_webrtc_sideband_input_task(input: RealtimeWebrtcSidebandInputTask) -> J return; } - let connection = match client - .connect_webrtc_sideband( + let connection = match tokio::select! { + connection = client.connect_webrtc_sideband( session_config, &call_id, sideband_headers, default_headers(), - ) - .await - { + ) => connection, + _ = stop_token.cancelled() => return, + } { Ok(connection) => connection, Err(err) => { if realtime_active.load(Ordering::Relaxed) { @@ -1311,6 +1342,9 @@ fn spawn_webrtc_sideband_input_task(input: RealtimeWebrtcSidebandInputTask) -> J handoff_state, session_kind, event_parser, + flush_transcript_tail_on_session_end, + transcript_tail_tx, + stop_token, }) .await; }) @@ -1327,6 +1361,9 @@ async fn run_realtime_input_task(input: RealtimeInputTask) { handoff_state, session_kind, event_parser, + flush_transcript_tail_on_session_end, + transcript_tail_tx, + stop_token, } = input; let mut output_audio_state: Option = None; @@ -1334,6 +1371,7 @@ async fn run_realtime_input_task(input: RealtimeInputTask) { loop { let result = tokio::select! { + _ = stop_token.cancelled() => break, // Text input that should be sent into realtime. text = text_rx.recv() => { handle_text_input( @@ -1378,6 +1416,18 @@ async fn run_realtime_input_task(input: RealtimeInputTask) { break; } } + + if flush_transcript_tail_on_session_end + && let Some(transcript_delta) = + realtime_transcript_delta(&events.take_transcript_tail().await) + { + let _ = transcript_tail_tx + .send(wrap_realtime_delegation_input( + REALTIME_SESSION_ENDED_HANDOFF_INSTRUCTION, + Some(&transcript_delta), + )) + .await; + } } async fn handle_text_input( diff --git a/codex-rs/core/src/session/config_lock.rs b/codex-rs/core/src/session/config_lock.rs index b1adf94466..606d9d2be0 100644 --- a/codex-rs/core/src/session/config_lock.rs +++ b/codex-rs/core/src/session/config_lock.rs @@ -361,6 +361,7 @@ mod tests { enabled: Some(true), reminder_interval_seconds: Some(1), clock_source: Some(codex_features::CurrentTimeSource::System), + delivery_mode: Some(codex_features::CurrentTimeReminderDeliveryMode::AnyInference), sleep_tool: Some(false), })) ); diff --git a/codex-rs/core/src/session/handlers.rs b/codex-rs/core/src/session/handlers.rs index 401e82977d..0b2d60f911 100644 --- a/codex-rs/core/src/session/handlers.rs +++ b/codex-rs/core/src/session/handlers.rs @@ -288,6 +288,7 @@ pub async fn inter_agent_communication( sess.input_queue .enqueue_mailbox_communication(communication) .await; + crate::agent_communication::emit_agent_communication_receive(&sub_id); if trigger_turn { sess.maybe_start_turn_for_pending_work_with_sub_id(sub_id) .await; @@ -613,8 +614,8 @@ async fn shutdown_session_runtime(sess: &Arc) { if let Some(startup_prewarm) = sess.take_session_startup_prewarm().await { startup_prewarm.abort().await; } - sess.abort_all_tasks(TurnAbortReason::Interrupted).await; let _ = sess.conversation.shutdown().await; + sess.abort_all_tasks(TurnAbortReason::Interrupted).await; sess.services .unified_exec_manager .terminate_all_processes() @@ -623,8 +624,8 @@ async fn shutdown_session_runtime(sess: &Arc) { warn!("failed to shutdown code mode session: {err}"); } sess.services - .mcp_connection_manager - .load_full() + .latest_mcp_runtime() + .manager_arc() .shutdown() .await; sess.guardian_review_session.shutdown().await; @@ -879,6 +880,11 @@ pub(super) async fn submission_loop( if !shutdown_received { shutdown_session_runtime(&sess).await; emit_thread_stop_lifecycle(sess.as_ref()).await; + if let Some(live_thread) = sess.live_thread() + && let Err(err) = live_thread.shutdown().await + { + warn!("failed to shutdown thread persistence after submission channel closed: {err}"); + } } debug!("Agent loop exited"); } diff --git a/codex-rs/core/src/session/mcp.rs b/codex-rs/core/src/session/mcp.rs index 2d251fec3b..1e486cd34f 100644 --- a/codex-rs/core/src/session/mcp.rs +++ b/codex-rs/core/src/session/mcp.rs @@ -1,8 +1,12 @@ use super::*; +use crate::mcp::McpRuntimeProjection; use anyhow::Context; +use codex_exec_server::ResolvedSelectedCapabilityRoot; use codex_mcp::ElicitationReviewRequest; use codex_mcp::ElicitationReviewer; use codex_mcp::ElicitationReviewerHandle; +use codex_mcp::effective_mcp_servers_from_configured; +use codex_protocol::capabilities::CapabilityRootLocation; use codex_protocol::config_types::ApprovalsReviewer; use codex_protocol::mcp_approval_meta::APPROVAL_KIND_KEY as MCP_ELICITATION_APPROVAL_KIND_KEY; use codex_protocol::mcp_approval_meta::APPROVAL_KIND_MCP_TOOL_CALL as MCP_ELICITATION_APPROVAL_KIND_MCP_TOOL_CALL; @@ -76,10 +80,22 @@ impl ElicitationReviewer for GuardianMcpElicitationReviewer { impl Session { pub(crate) async fn runtime_mcp_config(&self, config: &Config) -> McpConfig { + let environments = self.services.turn_environments.snapshot().await; + let selected_capability_roots = self + .resolve_selected_capability_roots_for_step(&environments) + .await; + let available_environment_ids = + Self::available_selected_environment_ids(&selected_capability_roots); self.services .mcp_manager - .runtime_config_for_thread(config, &self.services.mcp_thread_init) + .runtime_config_for_step( + config, + &self.services.mcp_thread_init, + &self.services.thread_extension_data, + &available_environment_ids, + ) .await + .config } pub(crate) async fn runtime_mcp_servers( @@ -89,10 +105,103 @@ impl Session { codex_mcp::configured_mcp_servers(&self.runtime_mcp_config(config).await) } + #[expect( + clippy::await_holding_invalid_type, + reason = "MCP runtime comparison and publication must remain serialized" + )] + pub(crate) async fn mcp_runtime_for_step( + self: &Arc, + turn_context: &TurnContext, + environments: &TurnEnvironmentSnapshot, + selected_capability_roots: &[ResolvedSelectedCapabilityRoot], + ) -> Arc { + let available_environment_ids = + Self::available_selected_environment_ids(selected_capability_roots); + let current = self.services.latest_mcp_runtime(); + if current.available_environment_ids() == available_environment_ids { + return current; + } + + let _guard = self.services.mcp_projection_lock.lock().await; + let current = self.services.latest_mcp_runtime(); + if current.available_environment_ids() == available_environment_ids { + return current; + } + let mcp_projection = self + .services + .mcp_manager + .runtime_config_for_step( + &turn_context.config, + &self.services.mcp_thread_init, + &self.services.thread_extension_data, + &available_environment_ids, + ) + .await; + let mcp_config = &mcp_projection.config; + let changed_environment_is_used_by_mcp = mcp_config + .mcp_server_catalog + .configured_servers() + .values() + .any(|server| { + let was_available = current + .available_environment_ids() + .contains(&server.environment_id); + let is_available = available_environment_ids.contains(&server.environment_id); + server.enabled && was_available != is_available + }); + if !changed_environment_is_used_by_mcp + && current + .config() + .mcp_server_catalog + .has_same_servers(&mcp_config.mcp_server_catalog) + && current.config().connector_snapshot == mcp_config.connector_snapshot + { + // Availability is only an input to the MCP projection. When that input changes but + // the projected servers and connectors do not, advance the input key without + // replacing the live manager and restarting its processes. + let runtime = Arc::new(McpRuntimeSnapshot::new( + Arc::new(current.config().clone()), + mcp_projection.plugins_available, + current.manager_arc(), + current.runtime_context().clone(), + available_environment_ids, + )); + self.services.mcp_runtime.store(Some(Arc::clone(&runtime))); + return runtime; + } + self.refresh_mcp_servers_inner( + turn_context, + mcp_projection, + environments, + &available_environment_ids, + Some(self.mcp_elicitation_reviewer()), + ) + .await + } + + pub(crate) async fn resolve_selected_capability_roots_for_step( + &self, + environments: &TurnEnvironmentSnapshot, + ) -> Vec { + self.services + .turn_environments + .environment_manager() + .resolve_selected_capability_roots( + &self.services.selected_capability_roots, + &environments.captured_environments(), + ) + .await + } + pub(crate) fn mcp_elicitation_reviewer(self: &Arc) -> ElicitationReviewerHandle { Arc::new(GuardianMcpElicitationReviewer::new(self)) } + pub(crate) fn mcp_elicitation_lifecycle(&self) -> codex_mcp::ElicitationLifecycle { + let elicitations = self.services.elicitations.clone(); + codex_mcp::ElicitationLifecycle::new(move || elicitations.register()) + } + #[expect( clippy::await_holding_invalid_type, reason = "active turn checks and turn state updates must remain atomic" @@ -106,8 +215,8 @@ impl Session { ) -> McpServerElicitationOutcome { if self .services - .mcp_connection_manager - .load_full() + .latest_mcp_runtime() + .manager() .elicitations_auto_deny() { return McpServerElicitationOutcome { @@ -120,6 +229,7 @@ impl Session { }; } + let _elicitation = self.services.elicitations.register(); let (tx_response, rx_response) = oneshot::channel(); let prev_entry = { let mut active = self.active_turn.lock().await; @@ -202,88 +312,32 @@ impl Session { } self.services - .mcp_connection_manager - .load_full() + .latest_mcp_runtime() + .manager_arc() .resolve_elicitation(server_name, id, response) .await } - pub async fn list_resources( - &self, - server: &str, - params: Option, - ) -> anyhow::Result { - self.services - .mcp_connection_manager - .load_full() - .list_resources(server, params) - .await - } - - pub async fn list_resource_templates( - &self, - server: &str, - params: Option, - ) -> anyhow::Result { - self.services - .mcp_connection_manager - .load_full() - .list_resource_templates(server, params) - .await - } - - pub async fn read_resource( - &self, - server: &str, - params: ReadResourceRequestParams, - ) -> anyhow::Result { - self.services - .mcp_connection_manager - .load_full() - .read_resource(server, params) - .await - } - - pub async fn call_tool( - &self, - server: &str, - tool: &str, - arguments: Option, - meta: Option, - ) -> anyhow::Result { - self.services - .mcp_connection_manager - .load_full() - .call_tool(server, tool, arguments, meta) - .await - } - async fn refresh_mcp_servers_inner( &self, turn_context: &TurnContext, - mcp_servers: HashMap, - store_mode: OAuthCredentialsStoreMode, - keyring_backend_kind: AuthKeyringBackendKind, + mcp_projection: McpRuntimeProjection, + environments: &TurnEnvironmentSnapshot, + available_environment_ids: &[String], elicitation_reviewer: Option, - ) { + ) -> Arc { let auth = self.services.auth_manager.auth().await; - let config = self.get_config().await; - let mcp_config = self.runtime_mcp_config(config.as_ref()).await; + let McpRuntimeProjection { + config: mcp_config, + plugins_available, + } = mcp_projection; + let mcp_config = Arc::new(mcp_config); let tool_plugin_provenance = codex_mcp::tool_plugin_provenance(&mcp_config); - let mcp_servers = - effective_mcp_servers_from_configured(mcp_servers, &mcp_config, auth.as_ref()); - let auth_statuses = compute_auth_statuses( - mcp_servers.iter(), - store_mode, - keyring_backend_kind, - auth.as_ref(), - ) - .await; + let mcp_servers = effective_mcp_servers(&mcp_config, auth.as_ref()); let environment_manager = self.services.turn_environments.environment_manager(); // TODO(anp): Migrate MCP runtime cwd plumbing to PathUri so foreign environment cwd // values can be used without falling back to the legacy host cwd. - let cwd = turn_context - .environments + let cwd = environments .primary() .and_then(|turn_environment| turn_environment.cwd().to_abs_path().ok()) .map(|cwd| cwd.to_path_buf()) @@ -292,47 +346,64 @@ impl Session { turn_context.cwd.to_path_buf() }); let mcp_runtime_context = McpRuntimeContext::new(environment_manager, cwd); + let auth_statuses = compute_auth_statuses( + mcp_servers.iter(), + mcp_config.mcp_oauth_credentials_store_mode, + mcp_config.auth_keyring_backend_kind, + auth.as_ref(), + &mcp_runtime_context, + ) + .await; let mcp_startup_cancellation_token = { let mut guard = self.services.mcp_startup_cancellation_token.lock().await; - guard.cancel(); + // The previous runtime owns the old token and may still be serving an in-flight step. + // Its manager cancels that token when the last runtime handle is dropped. let cancellation_token = CancellationToken::new(); *guard = cancellation_token.clone(); cancellation_token }; + let current_runtime = self.services.latest_mcp_runtime(); let refreshed_manager = McpConnectionManager::new( &mcp_servers, - store_mode, - keyring_backend_kind, + mcp_config.mcp_oauth_credentials_store_mode, + mcp_config.auth_keyring_backend_kind, auth_statuses, &turn_context.approval_policy, turn_context.sub_id.clone(), self.get_tx_event(), mcp_startup_cancellation_token, turn_context.permission_profile(), - mcp_runtime_context, - config.codex_home.to_path_buf(), + mcp_runtime_context.clone(), + mcp_config.codex_home.clone(), + self.services.mcp_manager.codex_apps_tools_cache(), codex_apps_tools_cache_key(auth.as_ref()), mcp_config.prefix_mcp_tool_names, - mcp_config.client_elicitation_capability, + mcp_config.client_elicitation_capability.clone(), self.services .supports_openai_form_elicitation .load(std::sync::atomic::Ordering::Relaxed), tool_plugin_provenance, auth.as_ref(), elicitation_reviewer, + Some(self.mcp_elicitation_lifecycle()), + current_runtime.manager().elicitation_router(), ) .await; - { - let current_manager = self.services.mcp_connection_manager.load_full(); - refreshed_manager.set_elicitations_auto_deny(current_manager.elicitations_auto_deny()); - } - let superseded_manager = self - .services - .mcp_connection_manager - .swap(Arc::new(refreshed_manager)); - superseded_manager.shutdown().await; + refreshed_manager + .set_elicitations_auto_deny(current_runtime.manager().elicitations_auto_deny()); + self.services.publish_mcp_runtime( + mcp_config, + plugins_available, + mcp_runtime_context, + available_environment_ids.to_vec(), + refreshed_manager, + ) } + #[expect( + clippy::await_holding_invalid_type, + reason = "MCP runtime refresh and publication must remain serialized" + )] pub(crate) async fn refresh_mcp_servers_if_requested( &self, turn_context: &TurnContext, @@ -375,11 +446,45 @@ impl Session { } }; + let mut refresh_config = self.get_config().await.as_ref().clone(); + refresh_config.mcp_oauth_credentials_store_mode = store_mode; + let secret_auth_storage_enabled = match keyring_backend_kind { + AuthKeyringBackendKind::Direct => false, + AuthKeyringBackendKind::Secrets => true, + }; + if let Err(err) = refresh_config + .features + .set_enabled(Feature::SecretAuthStorage, secret_auth_storage_enabled) + { + warn!("failed to apply MCP auth keyring backend refresh config: {err}"); + return; + } + + let _guard = self.services.mcp_projection_lock.lock().await; + let available_environment_ids = self + .services + .latest_mcp_runtime() + .available_environment_ids() + .to_vec(); + let mut mcp_projection = self + .services + .mcp_manager + .runtime_config_for_step( + &refresh_config, + &self.services.mcp_thread_init, + &self.services.thread_extension_data, + &available_environment_ids, + ) + .await; + mcp_projection.config.mcp_server_catalog = mcp_projection + .config + .mcp_server_catalog + .with_materialized_servers(mcp_servers); self.refresh_mcp_servers_inner( turn_context, - mcp_servers, - store_mode, - keyring_backend_kind, + mcp_projection, + &turn_context.environments, + &available_environment_ids, elicitation_reviewer, ) .await; @@ -413,24 +518,46 @@ impl Session { Ok(()) } + #[expect( + clippy::await_holding_invalid_type, + reason = "MCP runtime refresh and publication must remain serialized" + )] pub(crate) async fn refresh_mcp_servers_now( &self, turn_context: &TurnContext, - mcp_servers: HashMap, - store_mode: OAuthCredentialsStoreMode, - keyring_backend_kind: AuthKeyringBackendKind, + refresh_config: &Config, elicitation_reviewer: Option, ) { + let _guard = self.services.mcp_projection_lock.lock().await; + let available_environment_ids = self + .services + .latest_mcp_runtime() + .available_environment_ids() + .to_vec(); + let mcp_projection = self + .services + .mcp_manager + .runtime_config_for_step( + refresh_config, + &self.services.mcp_thread_init, + &self.services.thread_extension_data, + &available_environment_ids, + ) + .await; self.refresh_mcp_servers_inner( turn_context, - mcp_servers, - store_mode, - keyring_backend_kind, + mcp_projection, + &turn_context.environments, + &available_environment_ids, elicitation_reviewer, ) .await; } + #[expect( + clippy::await_holding_invalid_type, + reason = "MCP runtime replacement and publication must remain serialized" + )] pub(crate) async fn refresh_mcp_server_now( &self, update: codex_protocol::protocol::McpServerUpdateConfig, @@ -450,15 +577,17 @@ impl Session { let server = effective_servers.remove(&server_name).ok_or_else(|| { anyhow::anyhow!("MCP server update did not resolve server `{server_name}`") })?; + let _guard = self.services.mcp_projection_lock.lock().await; + let current_runtime = self.services.latest_mcp_runtime(); let mut auth_entries = compute_auth_statuses( std::iter::once((&server_name, &server)), - config.mcp_oauth_credentials_store_mode, - config.auth_keyring_backend_kind(), + current_runtime.config().mcp_oauth_credentials_store_mode, + current_runtime.config().auth_keyring_backend_kind, auth.as_ref(), + current_runtime.runtime_context(), ) .await; - - let current_manager = self.services.mcp_connection_manager.load_full(); + let current_manager = current_runtime.manager_arc(); let replacement_manager = Arc::new( current_manager .prepare_server_replacement( @@ -469,20 +598,32 @@ impl Session { ) .await?, ); - let superseded_manager = self - .services - .mcp_connection_manager - .swap(Arc::clone(&replacement_manager)); + self.services + .publish_replacement_mcp_manager(¤t_runtime, Arc::clone(&replacement_manager)); // Publish first, then transfer cancellation ownership while both manager // snapshots are still held here. Dropping an uncommitted replacement // cancels only the client it prepared. - replacement_manager.take_cancellation_ownership_from(&superseded_manager); - drop(current_manager); + replacement_manager.take_cancellation_ownership_from(¤t_manager); + drop(current_runtime); - tokio::spawn(superseded_manager.shutdown_server_after_readers_release(server_name)); + tokio::spawn(current_manager.shutdown_server_after_readers_release(server_name)); Ok(()) } + fn available_selected_environment_ids( + selected_capability_roots: &[ResolvedSelectedCapabilityRoot], + ) -> Vec { + let mut available = Vec::new(); + for root in selected_capability_roots { + let CapabilityRootLocation::Environment { environment_id, .. } = + &root.selected_root().location; + if !available.contains(environment_id) { + available.push(environment_id.clone()); + } + } + available + } + #[cfg(test)] pub(crate) async fn mcp_startup_cancellation_token(&self) -> CancellationToken { self.services diff --git a/codex-rs/core/src/session/mcp_runtime.rs b/codex-rs/core/src/session/mcp_runtime.rs new file mode 100644 index 0000000000..523cade153 --- /dev/null +++ b/codex-rs/core/src/session/mcp_runtime.rs @@ -0,0 +1,120 @@ +use std::fmt; +use std::sync::Arc; + +use codex_mcp::McpConfig; +use codex_mcp::McpConnectionManager; +use codex_mcp::McpRuntimeContext; + +/// MCP config, plugin availability, exact environment bindings, and manager for one request. +pub struct McpRuntimeSnapshot { + config: Arc, + plugins_available: bool, + manager: Arc, + runtime_context: McpRuntimeContext, + available_environment_ids: Vec, +} + +impl McpRuntimeSnapshot { + pub(crate) fn new( + config: Arc, + plugins_available: bool, + manager: Arc, + runtime_context: McpRuntimeContext, + available_environment_ids: Vec, + ) -> Self { + Self { + config, + plugins_available, + manager, + runtime_context, + available_environment_ids, + } + } + + pub fn config(&self) -> &McpConfig { + self.config.as_ref() + } + + pub(crate) fn plugins_available(&self) -> bool { + self.plugins_available + } + + pub fn manager(&self) -> &McpConnectionManager { + self.manager.as_ref() + } + + pub(crate) fn manager_arc(&self) -> Arc { + Arc::clone(&self.manager) + } + + pub(crate) fn with_manager(&self, manager: Arc) -> Self { + Self { + config: Arc::clone(&self.config), + plugins_available: self.plugins_available, + manager, + runtime_context: self.runtime_context.clone(), + available_environment_ids: self.available_environment_ids.clone(), + } + } + + pub fn runtime_context(&self) -> &McpRuntimeContext { + &self.runtime_context + } + + pub(crate) fn available_environment_ids(&self) -> &[String] { + &self.available_environment_ids + } + + #[cfg(test)] + pub(crate) fn new_uninitialized_for_test(config: &crate::config::Config) -> Arc { + use codex_exec_server::EnvironmentManager; + use codex_features::Feature; + use codex_mcp::ResolvedMcpCatalog; + use rmcp::model::ElicitationCapability; + + let mcp_config = McpConfig { + chatgpt_base_url: config.chatgpt_base_url.clone(), + apps_mcp_product_sku: config.apps_mcp_product_sku.clone(), + codex_home: config.codex_home.to_path_buf(), + mcp_oauth_credentials_store_mode: config.mcp_oauth_credentials_store_mode, + auth_keyring_backend_kind: config.auth_keyring_backend_kind(), + mcp_oauth_callback_port: config.mcp_oauth_callback_port, + mcp_oauth_callback_url: config.mcp_oauth_callback_url.clone(), + skill_mcp_dependency_install_enabled: config + .features + .enabled(Feature::SkillMcpDependencyInstall), + approval_policy: config.permissions.approval_policy.clone(), + codex_linux_sandbox_exe: config.codex_linux_sandbox_exe.clone(), + use_legacy_landlock: config.features.use_legacy_landlock(), + apps_enabled: config.features.enabled(Feature::Apps), + prefix_mcp_tool_names: config.prefix_mcp_tool_names(), + client_elicitation_capability: ElicitationCapability::default(), + mcp_server_catalog: ResolvedMcpCatalog::default(), + connector_snapshot: codex_connectors::ConnectorSnapshot::default(), + }; + let manager = McpConnectionManager::new_uninitialized_with_permission_profile( + &config.permissions.approval_policy, + config.permissions.permission_profile(), + config.prefix_mcp_tool_names(), + ); + let runtime_context = McpRuntimeContext::new( + Arc::new(EnvironmentManager::default_for_tests()), + config.cwd.to_path_buf(), + ); + Arc::new(Self::new( + Arc::new(mcp_config), + /*plugins_available*/ false, + Arc::new(manager), + runtime_context, + Vec::new(), + )) + } +} + +impl fmt::Debug for McpRuntimeSnapshot { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("McpRuntimeSnapshot") + .finish_non_exhaustive() + } +} diff --git a/codex-rs/core/src/session/mod.rs b/codex-rs/core/src/session/mod.rs index be12e61e08..ba8eacb395 100644 --- a/codex-rs/core/src/session/mod.rs +++ b/codex-rs/core/src/session/mod.rs @@ -14,15 +14,14 @@ use crate::agent::AgentControl; use crate::agent::AgentStatus; use crate::agent::agent_status_from_event; use crate::agent::status::is_final; +use crate::agent_communication::AgentCommunicationContext; +use crate::agent_communication::AgentCommunicationKind; use crate::attestation::AttestationProvider; use crate::build_available_skills; use crate::compact; use crate::config::ManagedFeatures; use crate::config::resolve_tool_suggest_config_from_layer_stack; -use crate::connectors; use crate::context::ApprovedCommandPrefixSaved; -use crate::context::AppsInstructions; -use crate::context::AvailablePluginsInstructions; use crate::context::AvailableSkillsInstructions; use crate::context::CollaborationModeInstructions; use crate::context::ContextualUserFragment; @@ -92,7 +91,6 @@ use codex_protocol::approvals::NetworkPolicyRuleAction; use codex_protocol::config_types::ApprovalsReviewer; use codex_protocol::config_types::AutoCompactTokenLimitScope; use codex_protocol::config_types::ModeKind; -use codex_protocol::config_types::MultiAgentMode; use codex_protocol::config_types::SERVICE_TIER_DEFAULT_REQUEST_VALUE; use codex_protocol::config_types::Settings; use codex_protocol::config_types::WebSearchMode; @@ -100,7 +98,6 @@ use codex_protocol::dynamic_tools::DynamicToolResponse; use codex_protocol::dynamic_tools::DynamicToolSpec; use codex_protocol::items::TurnItem; use codex_protocol::items::UserMessageItem; -use codex_protocol::mcp::CallToolResult; use codex_protocol::models::ActivePermissionProfile; use codex_protocol::models::AdditionalPermissionProfile; use codex_protocol::models::BaseInstructions; @@ -123,6 +120,7 @@ use codex_protocol::protocol::ReviewRequest; use codex_protocol::protocol::RolloutItem; use codex_protocol::protocol::SessionSource; use codex_protocol::protocol::SubAgentSource; +use codex_protocol::protocol::ThreadHistoryMode; use codex_protocol::protocol::ThreadSource; use codex_protocol::protocol::TurnAbortReason; use codex_protocol::protocol::TurnContextItem; @@ -158,15 +156,7 @@ use codex_utils_path_uri::PathUri; use futures::future::BoxFuture; use futures::future::Shared; use futures::prelude::*; -use rmcp::model::ElicitationCapability; -use rmcp::model::FormElicitationCapability; -use rmcp::model::ListResourceTemplatesResult; -use rmcp::model::ListResourcesResult; -use rmcp::model::PaginatedRequestParams; -use rmcp::model::ReadResourceRequestParams; -use rmcp::model::ReadResourceResult; use rmcp::model::RequestId; -use rmcp::model::UrlElicitationCapability; use serde_json::Value; use tokio::sync::Mutex; use tokio::sync::RwLock; @@ -214,6 +204,7 @@ mod handlers; mod inject; mod input_queue; mod mcp; +mod mcp_runtime; pub(crate) mod multi_agents; mod review; mod rollout_budget; @@ -235,6 +226,7 @@ use self::handlers::submission_loop; pub(crate) use self::input_queue::InputQueueActivity; pub(crate) use self::input_queue::TurnInput; pub(crate) use self::input_queue::TurnInputQueue; +pub use self::mcp_runtime::McpRuntimeSnapshot; use self::review::spawn_review_thread; use self::session::AppServerClientMetadata; use self::session::Session; @@ -340,7 +332,7 @@ use codex_core_plugins::RecommendedPluginCandidatesInput; use codex_git_utils::get_git_repo_root; use codex_mcp::McpConfig; use codex_mcp::compute_auth_statuses; -use codex_mcp::effective_mcp_servers_from_configured; +use codex_mcp::effective_mcp_servers; use codex_otel::SessionTelemetry; use codex_otel::THREAD_STARTED_METRIC; use codex_otel::TelemetryAuthMode; @@ -414,6 +406,7 @@ pub struct CodexSpawnOk { pub(crate) struct CodexSpawnArgs { pub(crate) config: Config, + pub(crate) allow_provider_model_fallback: bool, pub(crate) user_instructions: LoadedUserInstructions, pub(crate) installation_id: String, pub(crate) auth_manager: Arc, @@ -422,8 +415,10 @@ pub(crate) struct CodexSpawnArgs { pub(crate) skills_service: Arc, pub(crate) plugins_manager: Arc, pub(crate) mcp_manager: Arc, + pub(crate) code_mode_session_provider: Arc, pub(crate) extensions: Arc>, pub(crate) conversation_history: InitialHistory, + pub(crate) requested_history_mode: Option, pub(crate) session_source: SessionSource, pub(crate) forked_from_thread_id: Option, pub(crate) parent_thread_id: Option, @@ -503,6 +498,7 @@ impl Codex { async fn spawn_internal(args: CodexSpawnArgs) -> CodexResult { let CodexSpawnArgs { mut config, + allow_provider_model_fallback, user_instructions, installation_id, auth_manager, @@ -511,8 +507,10 @@ impl Codex { skills_service, plugins_manager, mcp_manager, + code_mode_session_provider, extensions, conversation_history, + requested_history_mode, session_source, forked_from_thread_id, parent_thread_id, @@ -576,8 +574,23 @@ impl Codex { let _ = models_manager.list_models(refresh_strategy).await; } let model = models_manager - .get_default_model(&config.model, refresh_strategy) + .get_default_model( + &config.model, + allow_provider_model_fallback, + refresh_strategy, + ) .await; + if allow_provider_model_fallback + && let Some(requested_model) = config.model.as_ref() + && model != *requested_model + { + info!( + model_provider = %config.model_provider_id, + requested_model, + fallback_model = %model, + "replaced unavailable requested model with provider default" + ); + } // Resolve base instructions for the session. Priority order: // 1. config.base_instructions override @@ -588,6 +601,9 @@ impl Codex { .await; let multi_agent_version = resolve_multi_agent_version(&conversation_history, inherited_multi_agent_version); + let history_mode = conversation_history.get_history_mode( + requested_history_mode.unwrap_or_else(|| thread_store.default_history_mode()), + ); config .validate_multi_agent_v2_config() .map_err(|err| CodexErr::InvalidRequest(err.to_string()))?; @@ -643,6 +659,7 @@ impl Codex { app_server_client_name: None, app_server_client_version: None, session_source, + history_mode, forked_from_thread_id, parent_thread_id, thread_source, @@ -670,6 +687,7 @@ impl Codex { skills_service, plugins_manager, mcp_manager.clone(), + code_mode_session_provider, extensions, thread_extension_init, supports_openai_form_elicitation, @@ -823,8 +841,11 @@ impl Codex { ..Default::default() }) .await?; - let mcp_connection_manager = self.session.services.mcp_connection_manager.load_full(); - mcp_connection_manager.set_elicitations_auto_deny(mcp_elicitations_auto_deny); + self.session + .services + .latest_mcp_runtime() + .manager() + .set_elicitations_auto_deny(mcp_elicitations_auto_deny); Ok(()) } @@ -1118,12 +1139,8 @@ impl Session { state.session_configuration.codex_home().clone() } - pub(crate) fn subscribe_out_of_band_elicitation_pause_state(&self) -> watch::Receiver { - self.out_of_band_elicitation_paused.subscribe() - } - - pub(crate) fn set_out_of_band_elicitation_pause_state(&self, paused: bool) { - self.out_of_band_elicitation_paused.send_replace(paused); + pub(crate) fn subscribe_elicitation_pause_state(&self) -> watch::Receiver { + self.services.elicitations.subscribe() } pub(crate) fn get_tx_event(&self) -> Sender { @@ -1844,10 +1861,12 @@ impl Session { message, /*trigger_turn*/ false, ); + let context = + AgentCommunicationContext::new(AgentCommunicationKind::Result, self.thread_id); if let Err(err) = self .services .agent_control - .send_inter_agent_communication(parent_thread_id, communication) + .send_inter_agent_communication(parent_thread_id, communication, context) .await { debug!("failed to notify parent thread {parent_thread_id}: {err}"); @@ -2758,6 +2777,7 @@ impl Session { let items = items.as_ref(); { let mut state = self.state.lock().await; + state.current_time_reminder.note_recorded_items(items); state.record_items( items.iter(), turn_context.model_info.truncation_policy.into(), @@ -2807,8 +2827,9 @@ impl Session { /// This may refresh filesystem-derived state. Normal turns should call it only from /// `run_turn` and pass the result down; standalone request or history boundaries may capture /// their own step. + #[tracing::instrument(name = "step_context.capture", level = "info", skip_all)] pub(crate) async fn capture_step_context( - &self, + self: &Arc, turn_context: Arc, ) -> Arc { let deferred_executor_enabled = turn_context @@ -2828,9 +2849,21 @@ impl Session { .await; } let loaded_agents_md = self.services.agents_md_manager.get_loaded().await; + let selected_capability_roots = self + .resolve_selected_capability_roots_for_step(&environments) + .await; + let mcp = self + .mcp_runtime_for_step( + turn_context.as_ref(), + &environments, + &selected_capability_roots, + ) + .await; Arc::new(StepContext::new( turn_context, environments, + selected_capability_roots, + mcp, loaded_agents_md, )) } @@ -2850,6 +2883,7 @@ impl Session { let response_item = items[0].clone(); { let mut state = self.state.lock().await; + state.current_time_reminder.note_recorded_items(items); state.record_items( items.iter(), turn_context.model_info.truncation_policy.into(), @@ -3096,6 +3130,17 @@ impl Session { &self, turn_context: &TurnContext, world_state: &WorldState, + ) -> Vec { + let mcp = self.services.latest_mcp_runtime(); + self.build_initial_context_with_world_state_and_mcp(turn_context, world_state, &mcp) + .await + } + + async fn build_initial_context_with_world_state_and_mcp( + &self, + turn_context: &TurnContext, + world_state: &WorldState, + mcp: &McpRuntimeSnapshot, ) -> Vec { let mut developer_sections = Vec::::with_capacity(8); let mut contextual_user_sections = Vec::::with_capacity(2); @@ -3188,20 +3233,6 @@ impl Session { .push(PersonalitySpecInstructions::new(personality_message).render()); } } - if turn_context.config.include_apps_instructions && turn_context.apps_enabled() { - let mcp_connection_manager = self.services.mcp_connection_manager.load_full(); - let accessible_and_enabled_connectors = - connectors::list_accessible_and_enabled_connectors_from_manager( - &mcp_connection_manager, - &turn_context.config, - ) - .await; - if let Some(apps_instructions) = - AppsInstructions::from_connectors(&accessible_and_enabled_connectors) - { - developer_sections.push(apps_instructions.render()); - } - } if turn_context.config.include_skill_instructions { let available_skills = build_available_skills( turn_context.turn_skills.snapshot.outcome(), @@ -3212,7 +3243,10 @@ impl Session { ); if let Some(available_skills) = available_skills { let warning_message = available_skills.warning_message.clone(); - let skills_instructions = AvailableSkillsInstructions::from(available_skills); + let skills_instructions = AvailableSkillsInstructions::from_available_skills( + &available_skills, + turn_context.model_info.include_skills_usage_instructions, + ); if let Some(warning_message) = warning_message { self.send_event_raw(Event { id: String::new(), @@ -3253,11 +3287,6 @@ impl Session { { contextual_user_sections.push(recommended_plugins.render()); } - if let Some(plugin_instructions) = - AvailablePluginsInstructions::from_plugins(loaded_plugins.capability_summaries()) - { - developer_sections.push(plugin_instructions.render()); - } let context_contributors = self.services.extensions.context_contributors().to_vec(); for contributor in &context_contributors { for fragment in contributor @@ -3299,7 +3328,8 @@ impl Session { if turn_context.config.features.enabled(Feature::TokenBudget) && turn_context.model_context_window().is_some() { - let mcp_result = self + let mcp_result = mcp + .manager() .call_tool( "notes", "thread_hint", @@ -3375,16 +3405,10 @@ impl Session { { items.push(usage_hint_message); } - match multi_agents::effective_multi_agent_mode(turn_context) { - Some( - multi_agent_mode - @ (MultiAgentMode::ExplicitRequestOnly | MultiAgentMode::Proactive), - ) => { - items.push(ContextualUserFragment::into( - MultiAgentModeInstructions::new(multi_agent_mode), - )); - } - Some(MultiAgentMode::None) | None => {} + if let Some(multi_agent_mode) = multi_agents::effective_multi_agent_mode(turn_context) { + items.push(ContextualUserFragment::into( + MultiAgentModeInstructions::new(multi_agent_mode), + )); } if let Some(contextual_user_message) = crate::context_manager::updates::build_contextual_user_message(contextual_user_sections) @@ -3514,7 +3538,11 @@ impl Session { // Full initial context resets the baseline; later turns persist only its changes. let (mut context_items, world_state_item) = if should_inject_full_context { let context_items = self - .build_initial_context_with_world_state(turn_context, world_state.as_ref()) + .build_initial_context_with_world_state_and_mcp( + turn_context, + world_state.as_ref(), + step_context.mcp.as_ref(), + ) .await; let snapshot = world_state.snapshot(); self.state diff --git a/codex-rs/core/src/session/multi_agents.rs b/codex-rs/core/src/session/multi_agents.rs index 385f2281ba..c2f139e68b 100644 --- a/codex-rs/core/src/session/multi_agents.rs +++ b/codex-rs/core/src/session/multi_agents.rs @@ -41,9 +41,18 @@ pub(crate) fn effective_multi_agent_mode(turn_context: &TurnContext) -> Option MultiAgentMode::Proactive, - _ => MultiAgentMode::ExplicitRequestOnly, + // A configured hint, including an empty string, defines a custom policy instead of an + // effort-derived built-in policy. + let multi_agent_mode = match &turn_context + .config + .multi_agent_v2 + .multi_agent_mode_hint_text + { + Some(hint_text) => MultiAgentMode::Custom(hint_text.clone()), + None => match turn_context.effective_reasoning_effort() { + Some(ReasoningEffort::Ultra) => MultiAgentMode::Proactive, + _ => MultiAgentMode::ExplicitRequestOnly, + }, }; match &turn_context.session_source { diff --git a/codex-rs/core/src/session/session.rs b/codex-rs/core/src/session/session.rs index 5988834ee5..2261eaa495 100644 --- a/codex-rs/core/src/session/session.rs +++ b/codex-rs/core/src/session/session.rs @@ -10,11 +10,13 @@ use crate::state::ActiveTurn; use codex_extension_api::ExtensionDataInit; use codex_login::auth::AgentIdentityAuthPolicy; use codex_protocol::SessionId; +use codex_protocol::capabilities::SelectedCapabilityRoot; use codex_protocol::config_types::SERVICE_TIER_DEFAULT_REQUEST_VALUE; use codex_protocol::config_types::ServiceTier; use codex_protocol::permissions::FileSystemPath; use codex_protocol::permissions::FileSystemSpecialPath; use codex_protocol::protocol::MultiAgentVersion; +use codex_protocol::protocol::ThreadHistoryMode; use codex_protocol::protocol::ThreadSource; use codex_protocol::protocol::TurnEnvironmentSelections; use std::sync::OnceLock; @@ -28,7 +30,6 @@ pub(crate) struct Session { pub(crate) installation_id: String, pub(super) tx_event: Sender, pub(super) agent_status: watch::Sender, - pub(super) out_of_band_elicitation_paused: watch::Sender, pub(super) state: Mutex, /// Serializes rebuild/apply cycles for the running proxy; each cycle /// rebuilds from the current SessionState while holding this lock. @@ -95,6 +96,8 @@ pub(crate) struct SessionConfiguration { pub(super) app_server_client_version: Option, /// Source of the session (cli, vscode, exec, mcp, ...) pub(super) session_source: SessionSource, + /// Persisted thread history contract selected when this thread was created. + pub(super) history_mode: ThreadHistoryMode, /// Immediate history source copied into this thread, when this thread was forked. pub(super) forked_from_thread_id: Option, /// Immediate control/spawn parent for this thread, when it has one. @@ -191,6 +194,7 @@ impl SessionConfiguration { personality: self.personality, collaboration_mode: self.collaboration_mode.clone(), session_source: self.session_source.clone(), + history_mode: self.history_mode, forked_from_thread_id: self.forked_from_thread_id, parent_thread_id: self.parent_thread_id, thread_source: self.thread_source.clone(), @@ -488,8 +492,9 @@ impl Session { skills_service: Arc, plugins_manager: Arc, mcp_manager: Arc, + code_mode_session_provider: Arc, extensions: Arc>, - thread_extension_init: ExtensionDataInit, + mut thread_extension_init: ExtensionDataInit, supports_openai_form_elicitation: bool, agent_control: AgentControl, environment_manager: Arc, @@ -555,6 +560,17 @@ impl Session { config.current_time_reminder.as_ref(), external_time_provider, )?; + let selected_capability_roots = + match thread_extension_init.get::>() { + Some(roots) => roots.as_ref().clone(), + None => { + let roots = initial_history.get_selected_capability_roots(); + if !roots.is_empty() { + thread_extension_init.insert(roots.clone()); + } + roots + } + }; let mcp_thread_init = thread_extension_init.clone(); let thread_extension_data = codex_extension_api::ExtensionData::new_with_init( thread_id.to_string(), @@ -584,7 +600,9 @@ impl Session { text: session_configuration.base_instructions.clone(), }, dynamic_tools: session_configuration.dynamic_tools.clone(), + selected_capability_roots: selected_capability_roots.clone(), multi_agent_version: initial_multi_agent_version, + history_mode: session_configuration.history_mode, initial_window_id: initial_auto_compact_window_ids .window_id .to_string(), @@ -648,21 +666,44 @@ impl Session { let config_for_mcp = Arc::clone(&config); let mcp_manager_for_mcp = Arc::clone(&mcp_manager); let mcp_thread_init_for_startup = &mcp_thread_init; + let thread_extension_data_for_mcp = &thread_extension_data; + let mcp_runtime_cwd = session_configuration + .environment_selections() + .first() + .and_then(|environment| environment.cwd.to_abs_path().ok()) + .map(|cwd| cwd.to_path_buf()) + .unwrap_or_else(|| session_configuration.cwd().to_path_buf()); + let mcp_runtime_context = + McpRuntimeContext::new(Arc::clone(&environment_manager), mcp_runtime_cwd); + let mcp_runtime_context_for_auth = mcp_runtime_context.clone(); let auth_and_mcp_fut = async move { let auth = auth_manager_clone.auth().await; - let mcp_config = mcp_manager_for_mcp - .runtime_config_for_thread(&config_for_mcp, mcp_thread_init_for_startup) + let mcp_projection = mcp_manager_for_mcp + .runtime_config_for_step( + &config_for_mcp, + mcp_thread_init_for_startup, + thread_extension_data_for_mcp, + /*available_environment_ids*/ &[], + ) .await; - let mcp_servers = codex_mcp::effective_mcp_servers(&mcp_config, auth.as_ref()); - let tool_plugin_provenance = codex_mcp::tool_plugin_provenance(&mcp_config); + let mcp_config = &mcp_projection.config; + let mcp_servers = codex_mcp::effective_mcp_servers(mcp_config, auth.as_ref()); + let tool_plugin_provenance = codex_mcp::tool_plugin_provenance(mcp_config); let auth_statuses = compute_auth_statuses( mcp_servers.iter(), config_for_mcp.mcp_oauth_credentials_store_mode, config_for_mcp.auth_keyring_backend_kind(), auth.as_ref(), + &mcp_runtime_context_for_auth, ) .await; - (auth, mcp_servers, auth_statuses, tool_plugin_provenance) + ( + auth, + mcp_projection, + mcp_servers, + auth_statuses, + tool_plugin_provenance, + ) } .instrument(info_span!( "session_init.auth_mcp", @@ -673,7 +714,7 @@ impl Session { let ( thread_persistence_result, state_db_ctx, - (auth, mcp_servers, auth_statuses, tool_plugin_provenance), + (auth, mcp_projection, mcp_servers, auth_statuses, tool_plugin_provenance), ) = tokio::join!(thread_persistence_fut, state_db_fut, auth_and_mcp_fut); let mut live_thread_init = @@ -1009,10 +1050,13 @@ impl Session { // changing this to use Option or OnceCell, though the current // setup is straightforward enough and performs well. mcp_connection_manager, + mcp_runtime: arc_swap::ArcSwapOption::empty(), + mcp_projection_lock: Mutex::new(()), mcp_startup_cancellation_token: Mutex::new(CancellationToken::new()), unified_exec_manager: UnifiedExecProcessManager::new( config.background_terminal_max_timeout, ), + elicitations: crate::elicitation::ElicitationService::new(), shell_zsh_path: config.zsh_path.clone(), main_execve_wrapper_exe: config.main_execve_wrapper_exe.clone(), analytics_events_client, @@ -1036,6 +1080,7 @@ impl Session { // TODO(jif): extract session to share between sub-agents session_extension_data, thread_extension_data, + selected_capability_roots, mcp_thread_init, supports_openai_form_elicitation: std::sync::atomic::AtomicBool::new( supports_openai_form_elicitation, @@ -1074,19 +1119,17 @@ impl Session { session_configuration.parent_thread_id, ), ), - code_mode_service: crate::tools::code_mode::CodeModeService::new(), + code_mode_service: crate::tools::code_mode::CodeModeService::new(Arc::clone( + &code_mode_session_provider, + )), tool_search_handler_cache: Default::default(), turn_environments: Arc::clone(&turn_environments), }; - let (out_of_band_elicitation_paused, _out_of_band_elicitation_paused_rx) = - watch::channel(false); - let sess = Arc::new(Session { thread_id, installation_id, tx_event: tx_event.clone(), agent_status, - out_of_band_elicitation_paused, state: Mutex::new(state), managed_network_proxy_refresh_lock: Semaphore::new(/*permits*/ 1), features: config.features.clone(), @@ -1140,14 +1183,6 @@ impl Session { sess.send_event_raw(event).await; } - let client_elicitation_capability = if config.features.enabled(Feature::AuthElicitation) { - ElicitationCapability { - form: Some(FormElicitationCapability::default()), - url: Some(UrlElicitationCapability::default()), - } - } else { - ElicitationCapability::default() - }; let mcp_startup_cancellation_token = { let mut cancel_guard = sess.services.mcp_startup_cancellation_token.lock().await; cancel_guard.cancel(); @@ -1155,20 +1190,6 @@ impl Session { *cancel_guard = cancel_token.clone(); cancel_token }; - let mcp_runtime_context = { - let turn_environments = sess.services.turn_environments.snapshot().await; - // TODO(anp): Migrate MCP runtime cwd plumbing to PathUri so foreign environment - // cwd values can be used without falling back to the session host cwd. - let cwd = turn_environments - .primary() - .and_then(|turn_environment| turn_environment.cwd().to_abs_path().ok()) - .map(|cwd| cwd.to_path_buf()) - .unwrap_or_else(|| session_configuration.cwd().to_path_buf()); - McpRuntimeContext::new( - sess.services.turn_environments.environment_manager(), - cwd, - ) - }; let mcp_connection_manager = McpConnectionManager::new( &mcp_servers, config.mcp_oauth_credentials_store_mode, @@ -1179,17 +1200,23 @@ impl Session { tx_event.clone(), mcp_startup_cancellation_token, session_configuration.permission_profile(), - mcp_runtime_context, + mcp_runtime_context.clone(), config.codex_home.to_path_buf(), + sess.services.mcp_manager.codex_apps_tools_cache(), codex_apps_tools_cache_key(auth), config.prefix_mcp_tool_names(), - client_elicitation_capability, + mcp_projection + .config + .client_elicitation_capability + .clone(), sess.services .supports_openai_form_elicitation .load(std::sync::atomic::Ordering::Relaxed), tool_plugin_provenance, auth, Some(sess.mcp_elicitation_reviewer()), + Some(sess.mcp_elicitation_lifecycle()), + codex_mcp::ElicitationRequestRouter::default(), ) .instrument(info_span!( "session_init.mcp_manager_init", @@ -1197,7 +1224,13 @@ impl Session { )) .await; sess.services - .install_mcp_connection_manager(mcp_connection_manager) + .install_mcp_connection_manager( + Arc::new(mcp_projection.config), + mcp_projection.plugins_available, + mcp_runtime_context, + /*available_environment_ids*/ Vec::new(), + mcp_connection_manager, + ) .await?; sess.schedule_startup_prewarm(session_configuration.base_instructions.clone()) .await; diff --git a/codex-rs/core/src/session/step_context.rs b/codex-rs/core/src/session/step_context.rs index bf672db6e7..3210eb0286 100644 --- a/codex-rs/core/src/session/step_context.rs +++ b/codex-rs/core/src/session/step_context.rs @@ -2,13 +2,19 @@ use std::sync::Arc; use crate::agents_md::LoadedAgentsMd; use crate::environment_selection::TurnEnvironmentSnapshot; +use crate::session::McpRuntimeSnapshot; use crate::session::turn_context::TurnContext; +use codex_exec_server::ResolvedSelectedCapabilityRoot; /// Request-scoped state that may change between model sampling requests. #[derive(Debug)] pub(crate) struct StepContext { pub(crate) turn: Arc, pub(crate) environments: TurnEnvironmentSnapshot, + /// Capability roots bound to ready environments in this exact step. + pub(crate) selected_capability_roots: Vec, + /// The exact MCP config and manager used to advertise and execute tools for this step. + pub(crate) mcp: Arc, /// The canonical AGENTS.md value observed with this environment snapshot. pub(crate) loaded_agents_md: Option>, } @@ -17,11 +23,15 @@ impl StepContext { pub(crate) fn new( turn: Arc, environments: TurnEnvironmentSnapshot, + selected_capability_roots: Vec, + mcp: Arc, loaded_agents_md: Option>, ) -> Self { Self { turn, environments, + selected_capability_roots, + mcp, loaded_agents_md, } } diff --git a/codex-rs/core/src/session/tests.rs b/codex-rs/core/src/session/tests.rs index 4553d54a0c..6cc223cd7d 100644 --- a/codex-rs/core/src/session/tests.rs +++ b/codex-rs/core/src/session/tests.rs @@ -18,6 +18,7 @@ use crate::test_support::models_manager_with_provider; use crate::tools::format_exec_output_str; use codex_config::ConfigLayerStack; use codex_config::ConfigLayerStackOrdering; +use codex_config::DEFAULT_MCP_SERVER_ENVIRONMENT_ID; use codex_config::LoaderOverrides; use codex_config::NetworkConstraints; use codex_config::NetworkDomainPermissionToml; @@ -25,6 +26,8 @@ use codex_config::NetworkDomainPermissionsToml; use codex_config::RequirementSource; use codex_config::Sourced; use codex_config::loader::project_trust_key; +use codex_config::types::McpServerConfig; +use codex_config::types::McpServerTransportConfig; use codex_config::types::ToolSuggestDisabledTool; use codex_core_skills::HostSkillsSnapshot; use core_test_support::test_codex::local_selections; @@ -188,8 +191,10 @@ impl StepContext { pub(crate) fn for_test(turn: Arc) -> Arc { let environments = turn.environments.clone(); Arc::new(Self::new( - turn, + Arc::clone(&turn), environments, + Vec::new(), + crate::session::McpRuntimeSnapshot::new_uninitialized_for_test(&turn.config), /*loaded_agents_md*/ None, )) } @@ -378,8 +383,8 @@ async fn request_mcp_server_elicitation_auto_accepts_when_auto_deny_is_enabled() let (session, turn_context, rx) = make_session_and_context_with_rx().await; session .services - .mcp_connection_manager - .load_full() + .latest_mcp_runtime() + .manager() .set_elicitations_auto_deny(/*auto_deny*/ true); let response = session @@ -1034,6 +1039,22 @@ async fn danger_full_access_tool_attempts_do_not_enforce_managed_network() -> an ) -> futures::future::BoxFuture<'a, ReviewDecision> { Box::pin(async { ReviewDecision::Approved }) } + + fn approval_action( + &self, + _req: &(), + ctx: &crate::tools::sandboxing::ApprovalCtx<'_>, + ) -> std::io::Result { + Ok(crate::tools::sandboxing::ApprovalAction::Shell { + id: ctx.call_id.to_string(), + command: Vec::new(), + #[allow(deprecated)] + cwd: ctx.turn.cwd.clone(), + sandbox_permissions: crate::sandboxing::SandboxPermissions::UseDefault, + additional_permissions: None, + justification: None, + }) + } } impl crate::tools::sandboxing::Sandboxable for ProbeToolRuntime { @@ -3681,6 +3702,7 @@ async fn set_rate_limits_retains_previous_credits() { app_server_client_name: None, app_server_client_version: None, session_source: SessionSource::Exec, + history_mode: Default::default(), forked_from_thread_id: None, parent_thread_id: None, thread_source: None, @@ -3787,6 +3809,7 @@ async fn set_rate_limits_updates_plan_type_when_present() { app_server_client_name: None, app_server_client_version: None, session_source: SessionSource::Exec, + history_mode: Default::default(), forked_from_thread_id: None, parent_thread_id: None, thread_source: None, @@ -4043,7 +4066,9 @@ async fn attach_thread_persistence(session: &mut Session) -> PathBuf { originator: "test_originator".to_string(), base_instructions: BaseInstructions::default(), dynamic_tools: Vec::new(), + selected_capability_roots: Vec::new(), multi_agent_version: None, + history_mode: Default::default(), initial_window_id: Uuid::now_v7().to_string(), metadata: ThreadPersistenceMetadata { cwd: Some(config.cwd.to_path_buf()), @@ -4317,6 +4342,7 @@ pub(crate) async fn make_session_configuration_for_tests() -> SessionConfigurati app_server_client_name: None, app_server_client_version: None, session_source: SessionSource::Exec, + history_mode: Default::default(), forked_from_thread_id: None, parent_thread_id: None, thread_source: None, @@ -5187,6 +5213,7 @@ async fn session_new_fails_when_zsh_fork_enabled_without_packaged_zsh() { app_server_client_name: None, app_server_client_version: None, session_source: SessionSource::Exec, + history_mode: Default::default(), forked_from_thread_id: None, parent_thread_id: None, thread_source: None, @@ -5219,6 +5246,7 @@ async fn session_new_fails_when_zsh_fork_enabled_without_packaged_zsh() { skills_service, plugins_manager, mcp_manager, + Arc::new(codex_code_mode::InProcessCodeModeSessionProvider), Arc::new(codex_extension_api::ExtensionRegistryBuilder::new().build()), codex_extension_api::ExtensionDataInit::default(), /*supports_openai_form_elicitation*/ false, @@ -5317,6 +5345,7 @@ pub(crate) async fn make_session_and_context() -> (Session, TurnContext) { app_server_client_name: None, app_server_client_version: None, session_source: SessionSource::Exec, + history_mode: Default::default(), forked_from_thread_id: None, parent_thread_id: None, thread_source: None, @@ -5361,18 +5390,17 @@ pub(crate) async fn make_session_and_context() -> (Session, TurnContext) { /*bundled_skills_enabled*/ true, )); let network_approval = Arc::new(NetworkApprovalService::default()); + let mcp_runtime = + crate::session::McpRuntimeSnapshot::new_uninitialized_for_test(config.as_ref()); let services = SessionServices { - mcp_connection_manager: Arc::new(arc_swap::ArcSwap::from_pointee( - McpConnectionManager::new_uninitialized_with_permission_profile( - &config.permissions.approval_policy, - config.permissions.permission_profile(), - config.prefix_mcp_tool_names(), - ), - )), + mcp_connection_manager: Arc::new(arc_swap::ArcSwap::from(mcp_runtime.manager_arc())), + mcp_runtime: arc_swap::ArcSwapOption::from(Some(mcp_runtime)), + mcp_projection_lock: Mutex::new(()), mcp_startup_cancellation_token: Mutex::new(CancellationToken::new()), unified_exec_manager: UnifiedExecProcessManager::new( config.background_terminal_max_timeout, ), + elicitations: crate::elicitation::ElicitationService::new(), shell_zsh_path: None, main_execve_wrapper_exe: config.main_execve_wrapper_exe.clone(), analytics_events_client: AnalyticsEventsClient::new( @@ -5404,6 +5432,7 @@ pub(crate) async fn make_session_and_context() -> (Session, TurnContext) { agent_control.session_id().to_string(), ), thread_extension_data: codex_extension_api::ExtensionData::new(thread_id.to_string()), + selected_capability_roots: Vec::new(), mcp_thread_init: codex_extension_api::ExtensionDataInit::default(), supports_openai_form_elicitation: std::sync::atomic::AtomicBool::new(false), agent_control, @@ -5433,7 +5462,9 @@ pub(crate) async fn make_session_and_context() -> (Session, TurnContext) { /*item_ids_enabled*/ config.features.enabled(Feature::ItemIds), /*attestation_provider*/ None, ), - code_mode_service: crate::tools::code_mode::CodeModeService::new(), + code_mode_service: crate::tools::code_mode::CodeModeService::new(Arc::new( + codex_code_mode::InProcessCodeModeSessionProvider, + )), tool_search_handler_cache: Default::default(), turn_environments: Arc::clone(&turn_environments), }; @@ -5480,7 +5511,6 @@ pub(crate) async fn make_session_and_context() -> (Session, TurnContext) { installation_id: "11111111-1111-4111-8111-111111111111".to_string(), tx_event, agent_status: agent_status_tx, - out_of_band_elicitation_paused: watch::channel(false).0, state: Mutex::new(state), managed_network_proxy_refresh_lock: Semaphore::new(/*permits*/ 1), features: config.features.clone(), @@ -5564,6 +5594,7 @@ async fn make_session_with_config_and_rx( app_server_client_name: None, app_server_client_version: None, session_source: SessionSource::Exec, + history_mode: Default::default(), forked_from_thread_id: None, parent_thread_id: None, thread_source: None, @@ -5597,6 +5628,7 @@ async fn make_session_with_config_and_rx( skills_service, plugins_manager, mcp_manager, + Arc::new(codex_code_mode::InProcessCodeModeSessionProvider), Arc::new(codex_extension_api::ExtensionRegistryBuilder::new().build()), codex_extension_api::ExtensionDataInit::default(), /*supports_openai_form_elicitation*/ false, @@ -5670,6 +5702,7 @@ async fn make_session_with_history_source_and_agent_control_and_rx( app_server_client_name: None, app_server_client_version: None, session_source: session_source.clone(), + history_mode: Default::default(), forked_from_thread_id: None, parent_thread_id: None, thread_source: None, @@ -5703,6 +5736,7 @@ async fn make_session_with_history_source_and_agent_control_and_rx( skills_service, plugins_manager, mcp_manager, + Arc::new(codex_code_mode::InProcessCodeModeSessionProvider), Arc::new(codex_extension_api::ExtensionRegistryBuilder::new().build()), codex_extension_api::ExtensionDataInit::default(), /*supports_openai_form_elicitation*/ false, @@ -6919,7 +6953,9 @@ async fn shutdown_complete_does_not_append_to_thread_store_after_shutdown() { originator: "test_originator".to_string(), base_instructions: BaseInstructions::default(), dynamic_tools: Vec::new(), + selected_capability_roots: Vec::new(), multi_agent_version: None, + history_mode: Default::default(), initial_window_id: Uuid::now_v7().to_string(), metadata: ThreadPersistenceMetadata { cwd: Some(config.cwd.to_path_buf()), @@ -6951,7 +6987,7 @@ async fn shutdown_complete_does_not_append_to_thread_store_after_shutdown() { } #[tokio::test] -async fn submission_loop_channel_close_emits_thread_stop_lifecycle() { +async fn submission_loop_channel_close_runs_full_thread_teardown() { struct SessionStopMarker; struct ThreadStopMarker; @@ -6978,6 +7014,41 @@ async fn submission_loop_channel_close_emits_thread_stop_lifecycle() { } let (mut session, turn_context) = make_session_and_context().await; + let store = Arc::new(codex_thread_store::InMemoryThreadStore::default()); + let thread_store: Arc = store.clone(); + let config = session.get_config().await; + let live_thread = LiveThread::create( + Arc::clone(&thread_store), + CreateThreadParams { + session_id: session.session_id(), + thread_id: session.thread_id, + extra_config: None, + forked_from_id: None, + parent_thread_id: None, + source: SessionSource::Exec, + thread_source: None, + originator: "test_originator".to_string(), + base_instructions: BaseInstructions::default(), + dynamic_tools: Vec::new(), + selected_capability_roots: Vec::new(), + multi_agent_version: None, + history_mode: Default::default(), + initial_window_id: Uuid::now_v7().to_string(), + metadata: ThreadPersistenceMetadata { + cwd: Some(config.cwd.to_path_buf()), + model_provider: config.model_provider_id.clone(), + memory_mode: if config.memories.generate_memories { + ThreadMemoryMode::Enabled + } else { + ThreadMemoryMode::Disabled + }, + }, + }, + ) + .await + .expect("create thread persistence"); + session.services.thread_store = thread_store; + session.services.live_thread = Some(live_thread); let calls = Arc::new(std::sync::atomic::AtomicUsize::new(0)); let mut builder = codex_extension_api::ExtensionRegistryBuilder::::new(); builder.thread_lifecycle_contributor(Arc::new(ThreadStopRecorder { @@ -7000,6 +7071,14 @@ async fn submission_loop_channel_close_emits_thread_stop_lifecycle() { submission_loop(session, Arc::clone(&turn_context.config), rx_sub).await; assert_eq!(1, calls.load(std::sync::atomic::Ordering::SeqCst)); + assert_eq!( + codex_thread_store::InMemoryThreadStoreCalls { + create_thread: 1, + shutdown_thread: 1, + ..Default::default() + }, + store.calls().await + ); } #[tokio::test] @@ -7393,6 +7472,7 @@ where app_server_client_name: None, app_server_client_version: None, session_source: SessionSource::Exec, + history_mode: Default::default(), forked_from_thread_id: None, parent_thread_id: None, thread_source: None, @@ -7436,18 +7516,17 @@ where /*bundled_skills_enabled*/ true, )); let network_approval = Arc::new(NetworkApprovalService::default()); + let mcp_runtime = + crate::session::McpRuntimeSnapshot::new_uninitialized_for_test(config.as_ref()); let services = SessionServices { - mcp_connection_manager: Arc::new(arc_swap::ArcSwap::from_pointee( - McpConnectionManager::new_uninitialized_with_permission_profile( - &config.permissions.approval_policy, - config.permissions.permission_profile(), - config.prefix_mcp_tool_names(), - ), - )), + mcp_connection_manager: Arc::new(arc_swap::ArcSwap::from(mcp_runtime.manager_arc())), + mcp_runtime: arc_swap::ArcSwapOption::from(Some(mcp_runtime)), + mcp_projection_lock: Mutex::new(()), mcp_startup_cancellation_token: Mutex::new(CancellationToken::new()), unified_exec_manager: UnifiedExecProcessManager::new( config.background_terminal_max_timeout, ), + elicitations: crate::elicitation::ElicitationService::new(), shell_zsh_path: None, main_execve_wrapper_exe: config.main_execve_wrapper_exe.clone(), analytics_events_client: AnalyticsEventsClient::new( @@ -7479,6 +7558,7 @@ where agent_control.session_id().to_string(), ), thread_extension_data: codex_extension_api::ExtensionData::new(thread_id.to_string()), + selected_capability_roots: Vec::new(), mcp_thread_init: codex_extension_api::ExtensionDataInit::default(), supports_openai_form_elicitation: std::sync::atomic::AtomicBool::new(false), agent_control, @@ -7508,7 +7588,9 @@ where /*item_ids_enabled*/ config.features.enabled(Feature::ItemIds), /*attestation_provider*/ None, ), - code_mode_service: crate::tools::code_mode::CodeModeService::new(), + code_mode_service: crate::tools::code_mode::CodeModeService::new(Arc::new( + codex_code_mode::InProcessCodeModeSessionProvider, + )), tool_search_handler_cache: Default::default(), turn_environments: Arc::clone(&turn_environments), }; @@ -7555,7 +7637,6 @@ where installation_id: "11111111-1111-4111-8111-111111111111".to_string(), tx_event, agent_status: agent_status_tx, - out_of_band_elicitation_paused: watch::channel(false).0, state: Mutex::new(state), managed_network_proxy_refresh_lock: Semaphore::new(/*permits*/ 1), features: config.features.clone(), @@ -7598,8 +7679,15 @@ pub(crate) async fn make_session_and_context_with_rx() -> ( } #[tokio::test] -async fn refresh_mcp_servers_is_deferred_until_next_turn() { +async fn refresh_mcp_servers_keeps_the_previous_runtime_alive() { let (session, turn_context) = make_session_and_context().await; + let session = Arc::new(session); + let turn_context = Arc::new(turn_context); + let old_runtime = session.services.latest_mcp_runtime(); + let step_context = session + .capture_step_context(Arc::clone(&turn_context)) + .await; + assert!(Arc::ptr_eq(&step_context.mcp, &old_runtime)); let old_token = session.mcp_startup_cancellation_token().await; assert!(!old_token.is_cancelled()); @@ -7607,8 +7695,16 @@ async fn refresh_mcp_servers_is_deferred_until_next_turn() { serde_json::to_value(OAuthCredentialsStoreMode::Auto).expect("serialize store mode"); let auth_keyring_backend_kind = serde_json::to_value(AuthKeyringBackendKind::Secrets).expect("serialize keyring backend"); + let refreshed_mcp_servers = serde_json::from_value::>(json!({ + "refreshed": { + "url": "https://refreshed.example/mcp", + "enabled": false + } + })) + .expect("parse refreshed MCP servers"); let refresh_config = McpServerRefreshConfig { - mcp_servers: json!({}), + mcp_servers: serde_json::to_value(&refreshed_mcp_servers) + .expect("serialize refreshed MCP servers"), mcp_oauth_credentials_store_mode, auth_keyring_backend_kind, }; @@ -7630,7 +7726,7 @@ async fn refresh_mcp_servers_is_deferred_until_next_turn() { .refresh_mcp_servers_if_requested(&turn_context, /*elicitation_reviewer*/ None) .await; - assert!(old_token.is_cancelled()); + assert!(!old_token.is_cancelled()); assert!( session .pending_mcp_server_refresh_config @@ -7640,6 +7736,156 @@ async fn refresh_mcp_servers_is_deferred_until_next_turn() { ); let new_token = session.mcp_startup_cancellation_token().await; assert!(!new_token.is_cancelled()); + let new_runtime = session.services.latest_mcp_runtime(); + assert!(!Arc::ptr_eq(&old_runtime, &new_runtime)); + assert!(Arc::ptr_eq(&step_context.mcp, &old_runtime)); + assert_eq!( + codex_mcp::configured_mcp_servers(new_runtime.config()), + refreshed_mcp_servers + ); +} + +#[tokio::test] +async fn plugin_availability_change_reuses_the_mcp_manager() { + struct ReadyPluginContributor; + + impl codex_extension_api::McpServerContributor for ReadyPluginContributor { + fn id(&self) -> &'static str { + "ready_plugin_test" + } + + fn contribute<'a>( + &'a self, + context: codex_extension_api::McpServerContributionContext<'a, Config>, + ) -> codex_extension_api::ExtensionFuture<'a, Vec> + { + Box::pin(async move { + let available = context + .available_environment_ids() + .is_some_and(|ids| ids.iter().any(|id| id == "executor")); + available + .then( + || codex_extension_api::McpServerContribution::SelectedPluginPackage { + plugin_id: "skill-only".to_string(), + plugin_display_name: "Skill Only".to_string(), + connector_ids: Vec::new(), + }, + ) + .into_iter() + .collect() + }) + } + } + + let (mut session, turn_context) = make_session_and_context().await; + let mut registry = codex_extension_api::ExtensionRegistryBuilder::new(); + registry.mcp_server_contributor(Arc::new(ReadyPluginContributor)); + let registry = Arc::new(registry.build()); + session.services.extensions = Arc::clone(®istry); + session.services.mcp_manager = Arc::new(McpManager::new_with_extensions( + Arc::clone(&session.services.plugins_manager), + registry, + )); + let session = Arc::new(session); + session + .refresh_mcp_servers_now( + &turn_context, + &turn_context.config, + /*elicitation_reviewer*/ None, + ) + .await; + let old_runtime = session.services.latest_mcp_runtime(); + let old_manager = old_runtime.manager_arc(); + + let selected_root = codex_protocol::capabilities::SelectedCapabilityRoot { + id: "skill-only".to_string(), + location: codex_protocol::capabilities::CapabilityRootLocation::Environment { + environment_id: "executor".to_string(), + path: PathUri::from_host_native_path(turn_context.config.cwd.as_path()) + .expect("selected capability root URI"), + }, + }; + let environment = Arc::clone( + &turn_context + .environments + .primary() + .expect("ready test environment") + .environment, + ); + let captured_environments = HashMap::from([("executor".to_string(), Some(environment))]); + let resolved_roots = session + .services + .turn_environments + .environment_manager() + .resolve_selected_capability_roots(&[selected_root], &captured_environments) + .await; + + let new_runtime = session + .mcp_runtime_for_step(&turn_context, &turn_context.environments, &resolved_roots) + .await; + + assert!(!old_runtime.plugins_available()); + assert!(new_runtime.plugins_available()); + assert!(!Arc::ptr_eq(&old_runtime, &new_runtime)); + assert!(Arc::ptr_eq(&old_manager, &new_runtime.manager_arc())); +} + +#[tokio::test] +async fn built_tools_uses_the_step_mcp_runtime() -> anyhow::Result<()> { + let (session, turn_context) = make_session_and_context().await; + let session = Arc::new(session); + let turn_context = Arc::new(turn_context); + let step_context = session.capture_step_context(turn_context).await; + + let mut refresh_config = step_context.turn.config.as_ref().clone(); + refresh_config.mcp_servers.set(HashMap::from([( + "newer".to_string(), + McpServerConfig { + auth: Default::default(), + transport: McpServerTransportConfig::Stdio { + command: "missing-test-mcp-server".to_string(), + args: Vec::new(), + env: None, + env_vars: Vec::new(), + cwd: None, + }, + environment_id: DEFAULT_MCP_SERVER_ENVIRONMENT_ID.to_string(), + enabled: true, + required: false, + supports_parallel_tool_calls: false, + disabled_reason: None, + startup_timeout_sec: None, + tool_timeout_sec: None, + default_tools_approval_mode: None, + enabled_tools: None, + disabled_tools: None, + scopes: None, + oauth: None, + oauth_resource: None, + tools: HashMap::new(), + }, + )]))?; + session + .refresh_mcp_servers_now( + step_context.turn.as_ref(), + &refresh_config, + /*elicitation_reviewer*/ None, + ) + .await; + + let router = crate::session::turn::built_tools( + session.as_ref(), + step_context.as_ref(), + &CancellationToken::new(), + ) + .await?; + assert!( + !router + .registered_tool_names_for_test() + .iter() + .any(|name| name.to_string() == "list_mcp_resources") + ); + Ok(()) } #[tokio::test] @@ -9068,6 +9314,91 @@ impl SessionTask for CompletingTask { } } +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum TerminalEventKind { + TurnComplete, + TurnAborted, +} + +async fn attach_in_memory_thread_store( + session: &mut Session, +) -> Arc { + let store = Arc::new(codex_thread_store::InMemoryThreadStore::default()); + let thread_store: Arc = store.clone(); + let config = session.get_config().await; + let live_thread = LiveThread::create( + Arc::clone(&thread_store), + CreateThreadParams { + session_id: session.session_id(), + thread_id: session.thread_id, + extra_config: None, + forked_from_id: None, + parent_thread_id: None, + source: SessionSource::Exec, + thread_source: None, + originator: "test_originator".to_string(), + base_instructions: BaseInstructions::default(), + dynamic_tools: Vec::new(), + selected_capability_roots: Vec::new(), + multi_agent_version: None, + history_mode: Default::default(), + initial_window_id: Uuid::now_v7().to_string(), + metadata: ThreadPersistenceMetadata { + cwd: Some(config.cwd.to_path_buf()), + model_provider: config.model_provider_id.clone(), + memory_mode: if config.memories.generate_memories { + ThreadMemoryMode::Enabled + } else { + ThreadMemoryMode::Disabled + }, + }, + }, + ) + .await + .expect("create thread persistence"); + session.services.thread_store = thread_store; + session.services.live_thread = Some(live_thread); + store +} + +async fn wait_for_flush_count( + store: &codex_thread_store::InMemoryThreadStore, + expected_flushes: usize, +) -> codex_thread_store::InMemoryThreadStoreCalls { + timeout(Duration::from_secs(2), async { + loop { + let calls = store.calls().await; + if calls.flush_thread >= expected_flushes { + return calls; + } + sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("store should observe expected flush count") +} + +async fn recv_terminal_event( + rx: &async_channel::Receiver, + expected: TerminalEventKind, +) -> Event { + timeout(Duration::from_secs(2), async { + loop { + let event = rx.recv().await.expect("event"); + match (&event.msg, expected) { + (EventMsg::TurnComplete(_), TerminalEventKind::TurnComplete) + | (EventMsg::TurnAborted(_), TerminalEventKind::TurnAborted) => return event, + (EventMsg::TurnComplete(_) | EventMsg::TurnAborted(_), _) => { + panic!("unexpected terminal event: {:?}", event.msg) + } + _ => {} + } + } + }) + .await + .expect("terminal event should be delivered") +} + #[derive(Clone, Copy)] struct NeverEndingTask { kind: TaskKind, @@ -9225,6 +9556,79 @@ async fn guardian_helper_review_interrupts_after_three_consecutive_denials() { assert_eq!(aborted.reason, TurnAbortReason::Interrupted); } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn turn_complete_flushes_terminal_event_after_delivery() { + let (mut sess, tc, rx) = make_session_and_context_with_rx().await; + let store = attach_in_memory_thread_store( + Arc::get_mut(&mut sess).expect("session should be uniquely owned"), + ) + .await; + + let input = vec![TurnInput::UserInput { + content: vec![UserInput::Text { + text: "complete normally".to_string(), + text_elements: Vec::new(), + }], + client_id: None, + }]; + sess.spawn_task(Arc::clone(&tc), input, CompletingTask) + .await; + + let event = recv_terminal_event(&rx, TerminalEventKind::TurnComplete).await; + assert!(matches!(event.msg, EventMsg::TurnComplete(_))); + // Expected flushes: + // 1. Task-runner flush after the task body finishes, before TurnComplete is emitted. + // 2. Terminal-event flush after TurnComplete is appended. + let calls = wait_for_flush_count(&store, /*expected_flushes*/ 2).await; + assert_eq!(2, calls.flush_thread); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn turn_aborted_flushes_terminal_event_after_delivery() { + let (mut sess, tc, rx) = make_session_and_context_with_rx().await; + let store = attach_in_memory_thread_store( + Arc::get_mut(&mut sess).expect("session should be uniquely owned"), + ) + .await; + + let input = vec![TurnInput::UserInput { + content: vec![UserInput::Text { + text: "interrupt me".to_string(), + text_elements: Vec::new(), + }], + client_id: None, + }]; + sess.spawn_task( + Arc::clone(&tc), + input, + NeverEndingTask { + kind: TaskKind::Regular, + listen_to_cancellation_token: true, + }, + ) + .await; + + let abort_task = tokio::spawn({ + let sess = Arc::clone(&sess); + async move { + sess.abort_all_tasks(TurnAbortReason::Interrupted).await; + } + }); + + let event = recv_terminal_event(&rx, TerminalEventKind::TurnAborted).await; + match event.msg { + EventMsg::TurnAborted(e) => assert_eq!(TurnAbortReason::Interrupted, e.reason), + other => panic!("unexpected event: {other:?}"), + } + abort_task.await.expect("abort task should finish"); + // Expected flushes: + // 1. Task-runner flush after the task body observes cancellation. + // 2. Interrupted-marker flush before TurnAborted so abort observers can reread it. + // 3. Terminal-event flush after TurnAborted is appended. + let calls = wait_for_flush_count(&store, /*expected_flushes*/ 3).await; + assert_eq!(3, calls.flush_thread); +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[test_log::test] async fn abort_regular_task_emits_marker_before_turn_aborted() { @@ -10156,8 +10560,8 @@ async fn fatal_tool_error_stops_turn_and_reports_error() { let tools = { session .services - .mcp_connection_manager - .load_full() + .latest_mcp_runtime() + .manager() .list_all_tools() .await }; @@ -10179,6 +10583,7 @@ async fn fatal_tool_error_stops_turn_and_reports_error() { status: None, call_id: "call-1".to_string(), name: "shell_command".to_string(), + namespace: None, input: "{}".to_string(), internal_chat_message_metadata_passthrough: None, }; diff --git a/codex-rs/core/src/session/tests/guardian_tests.rs b/codex-rs/core/src/session/tests/guardian_tests.rs index 64c4ba29f2..d13239d810 100644 --- a/codex-rs/core/src/session/tests/guardian_tests.rs +++ b/codex-rs/core/src/session/tests/guardian_tests.rs @@ -718,6 +718,7 @@ async fn guardian_subagent_does_not_inherit_parent_exec_policy_rules() { let CodexSpawnOk { codex, .. } = Codex::spawn(CodexSpawnArgs { config, + allow_provider_model_fallback: false, user_instructions: Default::default(), installation_id: "11111111-1111-4111-8111-111111111111".to_string(), auth_manager, @@ -726,8 +727,10 @@ async fn guardian_subagent_does_not_inherit_parent_exec_policy_rules() { skills_service, plugins_manager, mcp_manager, + code_mode_session_provider: Arc::new(codex_code_mode::InProcessCodeModeSessionProvider), extensions: codex_extension_api::empty_extension_registry(), conversation_history: InitialHistory::New, + requested_history_mode: None, session_source: SessionSource::SubAgent(SubAgentSource::Other( GUARDIAN_REVIEWER_NAME.to_string(), )), diff --git a/codex-rs/core/src/session/time_reminder.rs b/codex-rs/core/src/session/time_reminder.rs index 85831d2947..398ae58114 100644 --- a/codex-rs/core/src/session/time_reminder.rs +++ b/codex-rs/core/src/session/time_reminder.rs @@ -1,26 +1,57 @@ use chrono::DateTime; use chrono::Utc; +use codex_features::CurrentTimeReminderDeliveryMode; use codex_protocol::error::CodexErr; use codex_protocol::error::Result as CodexResult; +use codex_protocol::models::ResponseItem; use super::session::Session; use super::turn_context::TurnContext; use crate::context::ContextualUserFragment; +use crate::context_manager::is_user_turn_boundary; #[derive(Default)] pub(crate) struct CurrentTimeReminderState { last_delivery_time: Option>, last_window_id: Option, + pending_user_or_tool_output_boundary: bool, } impl CurrentTimeReminderState { + pub(super) fn note_recorded_items(&mut self, items: &[ResponseItem]) { + if items.iter().any(|item| { + is_user_turn_boundary(item) + || matches!( + item, + ResponseItem::FunctionCallOutput { .. } + | ResponseItem::CustomToolCallOutput { .. } + | ResponseItem::ToolSearchOutput { .. } + ) + }) { + self.pending_user_or_tool_output_boundary = true; + } + } + fn take_reminder_due( &mut self, window_id: &str, current_time: DateTime, interval_seconds: u64, + delivery_mode: CurrentTimeReminderDeliveryMode, ) -> bool { - let reminder_is_due = self.last_window_id.as_deref() != Some(window_id) + let is_new_window = self.last_window_id.as_deref() != Some(window_id); + // Consume the boundary for this inference even if the interval suppresses delivery. + let follows_user_or_tool_output = + std::mem::take(&mut self.pending_user_or_tool_output_boundary); + if delivery_mode == CurrentTimeReminderDeliveryMode::AfterUserOrToolOutput + && !is_new_window + && !follows_user_or_tool_output + { + return false; + } + + let reminder_is_due = is_new_window + || interval_seconds == 0 || self.last_delivery_time.is_none_or(|last_delivery_time| { current_time .signed_duration_since(last_delivery_time) @@ -59,6 +90,7 @@ pub(super) async fn maybe_record_current_time_reminder( window_id, current_time, config.reminder_interval_seconds, + config.delivery_mode, ) }; if !reminder_is_due { diff --git a/codex-rs/core/src/session/turn.rs b/codex-rs/core/src/session/turn.rs index 9b3588fc1e..f16b525002 100644 --- a/codex-rs/core/src/session/turn.rs +++ b/codex-rs/core/src/session/turn.rs @@ -167,12 +167,18 @@ pub(crate) async fn run_turn( // run_turn owns the step used to seed context and make the first sampling request. let first_step_context = sess.capture_step_context(Arc::clone(&turn_context)).await; // Keep the exact model-visible state used by this turn and its inline compactions. - let mut world_state = sess - .record_context_updates_and_set_reference_context_item(first_step_context.as_ref()) - .await; + let (mut world_state, display_roots) = tokio::join!( + sess.record_context_updates_and_set_reference_context_item(first_step_context.as_ref()), + turn_diff_display_roots(turn_context.as_ref()), + ); - let Some((injection_items, explicitly_enabled_connectors)) = - build_skills_and_plugins(&sess, turn_context.as_ref(), &input, &cancellation_token).await + let Some((injection_items, explicitly_enabled_connectors)) = build_skills_and_plugins( + &sess, + first_step_context.as_ref(), + &input, + &cancellation_token, + ) + .await else { return Ok(None); }; @@ -204,7 +210,6 @@ pub(crate) async fn run_turn( let mut stop_hook_active = false; // Although from the perspective of codex.rs, TurnDiffTracker has the lifecycle of a Task which contains // many turns, from the perspective of the user, it is a single turn. - let display_roots = turn_diff_display_roots(turn_context.as_ref()).await; let turn_diff_tracker = Arc::new(tokio::sync::Mutex::new( TurnDiffTracker::with_environment_display_roots(display_roots), )); @@ -504,10 +509,11 @@ async fn run_hooks_and_record_inputs( #[instrument(level = "trace", skip_all)] async fn build_skills_and_plugins( sess: &Arc, - turn_context: &TurnContext, + step_context: &StepContext, input: &[TurnInput], cancellation_token: &CancellationToken, ) -> Option<(Vec, HashSet)> { + let turn_context = step_context.turn.as_ref(); // Guardian input embeds the parent transcript as untrusted evidence. Do not interpret skill or // plugin mentions from that generated prompt as requests to inject additional instructions. if crate::guardian::is_guardian_reviewer_source(&turn_context.session_source) { @@ -538,17 +544,14 @@ async fn build_skills_and_plugins( // enabled plugins, then converted into turn-scoped guidance below. let mentioned_plugins = collect_explicit_plugin_mentions(&user_input, loaded_plugins.capability_summaries()); - let connector_snapshot = codex_connectors::ConnectorSnapshot::from_plugin_capability_summaries( - loaded_plugins.capability_summaries(), - ); + let connector_snapshot = step_context.mcp.config().connector_snapshot.clone(); let mcp_tools = if turn_context.apps_enabled() || !mentioned_plugins.is_empty() { // Plugin mentions need raw MCP/app inventory even when app tools // are normally hidden so we can describe the plugin's currently // usable capabilities for this turn. - match sess - .services - .mcp_connection_manager - .load_full() + match step_context + .mcp + .manager_arc() .list_all_tools() .or_cancel(cancellation_token) .await @@ -1177,7 +1180,7 @@ pub(crate) async fn built_tools( cancellation_token: &CancellationToken, ) -> CodexResult> { let turn_context = step_context.turn.as_ref(); - let mcp_connection_manager = sess.services.mcp_connection_manager.load_full(); + let mcp_connection_manager = step_context.mcp.manager(); let has_mcp_servers = mcp_connection_manager.has_servers(); let all_mcp_tools = mcp_connection_manager .list_all_tools() @@ -1189,9 +1192,7 @@ pub(crate) async fn built_tools( .plugins_for_config(&turn_context.config.plugins_config_input()) .instrument(trace_span!("built_tools.load_plugins")) .await; - let connector_snapshot = codex_connectors::ConnectorSnapshot::from_plugin_capability_summaries( - loaded_plugins.capability_summaries(), - ); + let connector_snapshot = step_context.mcp.config().connector_snapshot.clone(); let apps_enabled = turn_context.apps_enabled(); let accessible_connectors = @@ -2079,8 +2080,14 @@ async fn try_run_sampling_request( } } ResponseEvent::OutputItemAdded(item) => { - if let ResponseItem::CustomToolCall { call_id, name, .. } = &item { - let tool_name = ToolName::plain(name.as_str()); + if let ResponseItem::CustomToolCall { + call_id, + name, + namespace, + .. + } = &item + { + let tool_name = ToolName::new(namespace.clone(), name.as_str()); active_tool_argument_diff_consumer = tool_runtime .create_diff_consumer(&tool_name) .map(|consumer| (call_id.clone(), consumer)); diff --git a/codex-rs/core/src/session/turn_context.rs b/codex-rs/core/src/session/turn_context.rs index a5ae458957..b9bf42073b 100644 --- a/codex-rs/core/src/session/turn_context.rs +++ b/codex-rs/core/src/session/turn_context.rs @@ -686,7 +686,8 @@ impl Session { .unwrap_or_else(|| session_configuration.cwd().clone()); let per_turn_config = Self::build_per_turn_config(&session_configuration, cwd.clone()); { - let mcp_connection_manager = self.services.mcp_connection_manager.load_full(); + let mcp_runtime = self.services.latest_mcp_runtime(); + let mcp_connection_manager = mcp_runtime.manager(); mcp_connection_manager.set_approval_policy(&session_configuration.approval_policy); mcp_connection_manager .set_permission_profile(session_configuration.permission_profile()); @@ -700,6 +701,9 @@ impl Session { &per_turn_config.to_models_manager_config(), ) .await; + self.services + .thread_extension_data + .insert(model_info.clone()); let multi_agent_version = match multi_agent_runtime { TurnMultiAgentRuntime::ResolveAndStore => { diff --git a/codex-rs/core/src/session/world_state.rs b/codex-rs/core/src/session/world_state.rs index e19e8c2ff4..a0d93cac83 100644 --- a/codex-rs/core/src/session/world_state.rs +++ b/codex-rs/core/src/session/world_state.rs @@ -1,15 +1,24 @@ use super::session::Session; use super::step_context::StepContext; +use crate::connectors; use crate::context::world_state::AgentsMdState; +use crate::context::world_state::AppsInstructionsState; use crate::context::world_state::EnvironmentsState; +use crate::context::world_state::PluginsInstructionsState; use crate::context::world_state::WorldState; +use codex_extension_api::WorldStateContributionInput; impl Session { + #[tracing::instrument(name = "world_state.build", level = "info", skip_all)] pub(crate) async fn build_world_state_for_step( &self, step_context: &StepContext, ) -> WorldState { let turn_context = step_context.turn.as_ref(); + tracing::trace!( + selected_capability_root_count = step_context.selected_capability_roots.len(), + "building step world state" + ); let environment_subagents = if turn_context.config.include_environment_context { self.services .agent_control @@ -30,6 +39,44 @@ impl Session { .with_subagents(environment_subagents), ); } + let apps_available = + if turn_context.config.include_apps_instructions && turn_context.apps_enabled() { + let tools = step_context.mcp.manager().list_all_tools().await; + connectors::with_app_enabled_state( + connectors::accessible_connectors_from_mcp_tools(&tools), + &turn_context.config, + ) + .into_iter() + .any(|connector| connector.is_accessible && connector.is_enabled) + } else { + false + }; + world_state.add_section(AppsInstructionsState::new(apps_available)); + world_state.add_section(PluginsInstructionsState::new( + step_context.mcp.plugins_available(), + )); + let environments = step_context.environments.to_selections(); + let ready_selected_capability_roots = step_context + .selected_capability_roots + .iter() + .map(|root| root.selected_root().clone()) + .collect::>(); + for contributor in self.services.extensions.context_contributors() { + for section in contributor + .contribute_world_state(WorldStateContributionInput { + thread_id: self.thread_id(), + turn_id: turn_context.sub_id.as_str(), + environments: &environments, + ready_selected_capability_roots: &ready_selected_capability_roots, + session_store: &self.services.session_extension_data, + thread_store: &self.services.thread_extension_data, + turn_store: turn_context.extension_data.as_ref(), + }) + .await + { + world_state.add_extension_section(section); + } + } world_state } } diff --git a/codex-rs/core/src/session_rollout_init_error.rs b/codex-rs/core/src/session_rollout_init_error.rs index 57e1e8e57a..db71dfcd37 100644 --- a/codex-rs/core/src/session_rollout_init_error.rs +++ b/codex-rs/core/src/session_rollout_init_error.rs @@ -3,8 +3,16 @@ use std::path::Path; use crate::rollout::SESSIONS_SUBDIR; use codex_protocol::error::CodexErr; +use codex_thread_store::ThreadStoreError; pub(crate) fn map_session_init_error(err: &anyhow::Error, codex_home: &Path) -> CodexErr { + if let Some(ThreadStoreError::Unsupported { operation }) = err + .chain() + .find_map(|cause| cause.downcast_ref::()) + { + return CodexErr::UnsupportedOperation(format!("{operation} is not supported yet")); + } + if let Some(mapped) = err .chain() .filter_map(|cause| cause.downcast_ref::()) diff --git a/codex-rs/core/src/state/service.rs b/codex-rs/core/src/state/service.rs index cfed7d90d0..3104c84c94 100644 --- a/codex-rs/core/src/state/service.rs +++ b/codex-rs/core/src/state/service.rs @@ -10,11 +10,13 @@ use crate::client::ModelClient; use crate::config::NetworkProxyAuditMetadata; use crate::config::StartedNetworkProxy; use crate::current_time::TimeProvider; +use crate::elicitation::ElicitationService; use crate::environment_selection::ThreadEnvironments; use crate::exec_policy::ExecPolicyManager; use crate::guardian::GuardianRejection; use crate::guardian::GuardianRejectionCircuitBreaker; use crate::mcp::McpManager; +use crate::session::McpRuntimeSnapshot; use crate::tools::code_mode::CodeModeService; use crate::tools::handlers::ToolSearchHandlerCache; use crate::tools::network_approval::NetworkApprovalService; @@ -30,9 +32,12 @@ use codex_extension_api::ExtensionDataInit; use codex_extension_api::ExtensionRegistry; use codex_hooks::Hooks; use codex_login::AuthManager; +use codex_mcp::McpConfig; use codex_mcp::McpConnectionManager; +use codex_mcp::McpRuntimeContext; use codex_models_manager::manager::SharedModelsManager; use codex_otel::SessionTelemetry; +use codex_protocol::capabilities::SelectedCapabilityRoot; use codex_rollout::state_db::StateDbHandle; use codex_rollout_trace::ThreadTraceContext; use codex_thread_store::LiveThread; @@ -43,10 +48,15 @@ use tokio::sync::Mutex; use tokio_util::sync::CancellationToken; pub(crate) struct SessionServices { - /// The latest manager; callers retain an owned handle while performing MCP I/O. + /// Mirror of the latest manager for extension resource clients that predate runtime snapshots. pub(crate) mcp_connection_manager: Arc>, + /// The latest atomically published MCP config and manager pair. + pub(crate) mcp_runtime: ArcSwapOption, + /// Serializes environment-driven runtime rebuilds. + pub(crate) mcp_projection_lock: Mutex<()>, pub(crate) mcp_startup_cancellation_token: Mutex, pub(crate) unified_exec_manager: UnifiedExecProcessManager, + pub(crate) elicitations: ElicitationService, #[cfg_attr(not(unix), allow(dead_code))] pub(crate) shell_zsh_path: Option, #[cfg_attr(not(unix), allow(dead_code))] @@ -72,6 +82,9 @@ pub(crate) struct SessionServices { pub(crate) session_extension_data: ExtensionData, pub(crate) thread_extension_data: ExtensionData, pub(crate) supports_openai_form_elicitation: AtomicBool, + /// Raw capability selections for this thread. Each model step resolves them against its + /// current executor environments before using them. + pub(crate) selected_capability_roots: Vec, pub(crate) mcp_thread_init: ExtensionDataInit, pub(crate) agent_control: AgentControl, pub(crate) network_proxy: ArcSwapOption, @@ -95,12 +108,63 @@ impl SessionServices { /// resolve through the session's manager while validation waits. pub(crate) async fn install_mcp_connection_manager( &self, + config: Arc, + plugins_available: bool, + runtime_context: McpRuntimeContext, + available_environment_ids: Vec, manager: McpConnectionManager, ) -> Result<()> { - self.mcp_connection_manager.store(Arc::new(manager)); - self.mcp_connection_manager - .load_full() - .validate_required_servers() - .await + let runtime = self.publish_mcp_runtime( + config, + plugins_available, + runtime_context, + available_environment_ids, + manager, + ); + runtime.manager().validate_required_servers().await + } + + pub(crate) fn publish_mcp_runtime( + &self, + config: Arc, + plugins_available: bool, + runtime_context: McpRuntimeContext, + available_environment_ids: Vec, + manager: McpConnectionManager, + ) -> Arc { + let manager = Arc::new(manager); + let runtime = Arc::new(McpRuntimeSnapshot::new( + config, + plugins_available, + manager, + runtime_context, + available_environment_ids, + )); + self.store_mcp_runtime(Arc::clone(&runtime)); + runtime + } + + pub(crate) fn publish_replacement_mcp_manager( + &self, + current_runtime: &McpRuntimeSnapshot, + manager: Arc, + ) -> Arc { + let runtime = Arc::new(current_runtime.with_manager(manager)); + self.store_mcp_runtime(Arc::clone(&runtime)); + runtime + } + + fn store_mcp_runtime(&self, runtime: Arc) { + // Publish the manager for legacy resource clients first. Once the paired snapshot is + // visible, every model-scoped consumer observes this exact manager. + self.mcp_connection_manager.store(runtime.manager_arc()); + self.mcp_runtime.store(Some(runtime)); + } + + pub(crate) fn latest_mcp_runtime(&self) -> Arc { + let Some(runtime) = self.mcp_runtime.load_full() else { + unreachable!("MCP runtime must be installed before handling requests"); + }; + runtime } } diff --git a/codex-rs/core/src/stream_events_utils_tests.rs b/codex-rs/core/src/stream_events_utils_tests.rs index 78a219add6..b3312ad49d 100644 --- a/codex-rs/core/src/stream_events_utils_tests.rs +++ b/codex-rs/core/src/stream_events_utils_tests.rs @@ -116,6 +116,7 @@ fn external_context_pollution_items_exclude_local_tool_calls() { status: None, call_id: "custom-1".to_string(), name: "apply_patch".to_string(), + namespace: None, input: "*** Begin Patch\n*** End Patch\n".to_string(), internal_chat_message_metadata_passthrough: None, }, diff --git a/codex-rs/core/src/tasks/mod.rs b/codex-rs/core/src/tasks/mod.rs index cb532445fd..c6d9d2990c 100644 --- a/codex-rs/core/src/tasks/mod.rs +++ b/codex-rs/core/src/tasks/mod.rs @@ -793,10 +793,14 @@ impl Session { false } }; - if !cleared_active_turn { - return; + if cleared_active_turn { + self.emit_thread_idle_lifecycle_if_idle().await; + } + // Regular items were flushed before this terminal event was appended; buffering + // thread writers may not flush it without another explicit barrier. + if let Err(err) = self.flush_rollout().await { + warn!("failed to flush rollout after emitting terminal turn event: {err}"); } - self.emit_thread_idle_lifecycle_if_idle().await; } async fn take_active_turn(&self) -> Option { @@ -896,6 +900,11 @@ impl Session { .lock() .await .clear_turn(&task.turn_context.sub_id); + // Regular items were flushed before this terminal event was appended; buffering + // thread writers may not flush it without another explicit barrier. + if let Err(err) = self.flush_rollout().await { + warn!("failed to flush rollout after emitting terminal turn event: {err}"); + } } } diff --git a/codex-rs/core/src/thread_manager.rs b/codex-rs/core/src/thread_manager.rs index 98e13c5d67..4ea86cb33d 100644 --- a/codex-rs/core/src/thread_manager.rs +++ b/codex-rs/core/src/thread_manager.rs @@ -21,6 +21,9 @@ use codex_agent_graph_store::LocalAgentGraphStore; use codex_analytics::AnalyticsEventsClient; use codex_app_server_protocol::ThreadHistoryBuilder; use codex_app_server_protocol::TurnStatus; +use codex_code_mode::CodeModeSessionProvider; +use codex_code_mode::InProcessCodeModeSessionProvider; +use codex_code_mode::ProcessOwnedCodeModeSessionProvider; use codex_core_plugins::PluginsManager; use codex_exec_server::EnvironmentManager; use codex_extension_api::ExtensionDataInit; @@ -53,6 +56,7 @@ use codex_protocol::protocol::RolloutItem; use codex_protocol::protocol::SessionConfiguredEvent; use codex_protocol::protocol::SessionSource; use codex_protocol::protocol::SubAgentSource; +use codex_protocol::protocol::ThreadHistoryMode; use codex_protocol::protocol::ThreadSource; use codex_protocol::protocol::TurnAbortReason; use codex_protocol::protocol::TurnAbortedEvent; @@ -182,7 +186,9 @@ pub struct ThreadManager { pub struct StartThreadOptions { pub config: Config, + pub allow_provider_model_fallback: bool, pub initial_history: InitialHistory, + pub history_mode: Option, pub session_source: Option, pub thread_source: Option, pub dynamic_tools: Vec, @@ -195,8 +201,10 @@ pub struct StartThreadOptions { fn originator_from_service_name(service_name: Option<&str>) -> Option { let service_name = service_name?.trim(); - if service_name.eq_ignore_ascii_case("codex_work_desktop") { - return Some("codex_work_desktop".to_string()); + for originator in ["codex_work_desktop", "codex_work_web", "codex_work_mobile"] { + if service_name.eq_ignore_ascii_case(originator) { + return Some(originator.to_string()); + } } None } @@ -237,6 +245,7 @@ pub(crate) struct ThreadManagerState { skills_service: Arc, plugins_manager: Arc, mcp_manager: Arc, + code_mode_session_provider: Arc, extensions: Arc>, user_instructions_provider: Arc, thread_store: Arc, @@ -333,6 +342,11 @@ impl ThreadManager { skills_service, plugins_manager, mcp_manager, + code_mode_session_provider: if config.features.enabled(Feature::CodeModeHost) { + Arc::new(ProcessOwnedCodeModeSessionProvider::default()) + } else { + Arc::new(InProcessCodeModeSessionProvider) + }, extensions, user_instructions_provider, thread_store, @@ -438,6 +452,7 @@ impl ThreadManager { skills_service, plugins_manager, mcp_manager, + code_mode_session_provider: Arc::new(InProcessCodeModeSessionProvider), extensions: empty_extension_registry(), user_instructions_provider: Arc::new( crate::test_support::EmptyUserInstructionsProvider, @@ -632,7 +647,9 @@ impl ThreadManager { ); Box::pin(self.start_thread_with_options(StartThreadOptions { config, + allow_provider_model_fallback: false, initial_history: InitialHistory::New, + history_mode: None, session_source: None, thread_source: None, dynamic_tools, @@ -668,6 +685,8 @@ impl ThreadManager { Box::pin(self.state.spawn_thread_with_source( options.config, options.initial_history, + options.history_mode, + options.allow_provider_model_fallback, Arc::clone(&self.state.auth_manager), agent_control, session_source, @@ -763,6 +782,8 @@ impl ThreadManager { Box::pin(self.state.spawn_thread_with_source( config, initial_history, + /*history_mode*/ None, + /*allow_provider_model_fallback*/ false, auth_manager, agent_control, session_source, @@ -832,6 +853,8 @@ impl ThreadManager { Box::pin(self.state.spawn_thread_with_source( config, initial_history, + /*history_mode*/ None, + /*allow_provider_model_fallback*/ false, auth_manager, agent_control, session_source, @@ -1332,6 +1355,8 @@ impl ThreadManagerState { Box::pin(self.spawn_thread_with_source( config, InitialHistory::New, + /*history_mode*/ None, + /*allow_provider_model_fallback*/ false, Arc::clone(&self.auth_manager), agent_control, session_source, @@ -1370,6 +1395,8 @@ impl ThreadManagerState { Box::pin(self.spawn_thread_with_source( config, initial_history, + /*history_mode*/ None, + /*allow_provider_model_fallback*/ false, Arc::clone(&self.auth_manager), agent_control, session_source, @@ -1402,6 +1429,7 @@ impl ThreadManagerState { inherited_environments: Option, inherited_exec_policy: Option>, environments: Option>, + thread_extension_init: ExtensionDataInit, ) -> CodexResult { let environments = environments.unwrap_or_else(|| { default_thread_environment_selections(self.environment_manager.as_ref(), &config.cwd) @@ -1409,6 +1437,8 @@ impl ThreadManagerState { Box::pin(self.spawn_thread_with_source( config, initial_history, + /*history_mode*/ None, + /*allow_provider_model_fallback*/ false, Arc::clone(&self.auth_manager), agent_control, session_source, @@ -1421,7 +1451,7 @@ impl ThreadManagerState { inherited_exec_policy, /*parent_trace*/ None, environments, - /*thread_extension_init*/ ExtensionDataInit::default(), + thread_extension_init, /*supports_openai_form_elicitation*/ false, /*user_shell_override*/ None, )) @@ -1450,6 +1480,8 @@ impl ThreadManagerState { Box::pin(self.spawn_thread_with_source( config, initial_history, + /*history_mode*/ None, + /*allow_provider_model_fallback*/ false, auth_manager, agent_control, self.session_source.clone(), @@ -1474,6 +1506,8 @@ impl ThreadManagerState { &self, config: Config, initial_history: InitialHistory, + history_mode: Option, + allow_provider_model_fallback: bool, auth_manager: Arc, agent_control: AgentControl, session_source: SessionSource, @@ -1540,6 +1574,7 @@ impl ThreadManagerState { codex, thread_id, .. } = Box::pin(Codex::spawn(CodexSpawnArgs { config, + allow_provider_model_fallback, user_instructions, installation_id: self.installation_id.clone(), auth_manager, @@ -1548,8 +1583,10 @@ impl ThreadManagerState { skills_service: Arc::clone(&self.skills_service), plugins_manager: Arc::clone(&self.plugins_manager), mcp_manager: Arc::clone(&self.mcp_manager), + code_mode_session_provider: Arc::clone(&self.code_mode_session_provider), extensions: Arc::clone(&self.extensions), conversation_history: initial_history, + requested_history_mode: history_mode, session_source, forked_from_thread_id, parent_thread_id, diff --git a/codex-rs/core/src/thread_manager_tests.rs b/codex-rs/core/src/thread_manager_tests.rs index f1acc2d979..3c1d0146e7 100644 --- a/codex-rs/core/src/thread_manager_tests.rs +++ b/codex-rs/core/src/thread_manager_tests.rs @@ -20,6 +20,8 @@ use codex_protocol::protocol::AgentMessageEvent; use codex_protocol::protocol::InitialHistory; use codex_protocol::protocol::InternalSessionSource; use codex_protocol::protocol::ResumedHistory; +use codex_protocol::protocol::SessionMeta; +use codex_protocol::protocol::SessionMetaLine; use codex_protocol::protocol::SessionSource; use codex_protocol::protocol::ThreadSource; use codex_protocol::protocol::TurnStartedEvent; @@ -120,6 +122,18 @@ fn effective_originator_prefers_thread_scoped_sources_before_env_originator() { Some("inherited_originator"), "codex_work_desktop", ), + ( + Some("codex_work_web"), + Some("persisted_originator"), + Some("inherited_originator"), + "codex_work_web", + ), + ( + Some("codex_work_mobile"), + Some("persisted_originator"), + Some("inherited_originator"), + "codex_work_mobile", + ), ( None, Some("persisted_originator"), @@ -381,6 +395,64 @@ async fn shutdown_all_threads_bounded_submits_shutdown_to_every_thread() { assert!(manager.list_thread_ids().await.is_empty()); } +#[tokio::test] +async fn code_mode_session_provider_is_shared_across_threads() { + let temp_dir = tempdir().expect("tempdir"); + let mut config = test_config().await; + config.codex_home = temp_dir.path().join("codex-home").abs(); + config.cwd = config.codex_home.abs(); + std::fs::create_dir_all(&config.codex_home).expect("create codex home"); + + let manager = ThreadManager::with_models_provider_and_home_for_tests( + CodexAuth::from_api_key("dummy"), + config.model_provider.clone(), + config.codex_home.to_path_buf(), + Arc::new(codex_exec_server::EnvironmentManager::default_for_tests()), + ); + let first = manager + .start_thread(config.clone()) + .await + .expect("start first thread"); + let second = manager + .start_thread(config) + .await + .expect("start second thread"); + + let first_provider = first + .thread + .codex + .session + .services + .code_mode_service + .session_provider(); + let second_provider = second + .thread + .codex + .session + .services + .code_mode_service + .session_provider(); + assert!(Arc::ptr_eq(&first_provider, &second_provider)); + assert!(Arc::ptr_eq( + &first_provider, + &manager.state.code_mode_session_provider + )); + + let mut completed = vec![first.thread_id, second.thread_id]; + completed.sort_by_key(std::string::ToString::to_string); + let report = manager + .shutdown_all_threads_bounded(Duration::from_secs(10)) + .await; + assert_eq!( + report, + ThreadShutdownReport { + completed, + submit_failed: Vec::new(), + timed_out: Vec::new(), + } + ); +} + #[tokio::test] async fn start_thread_keeps_internal_threads_hidden_from_normal_lookups() { let temp_dir = tempdir().expect("tempdir"); @@ -398,7 +470,9 @@ async fn start_thread_keeps_internal_threads_hidden_from_normal_lookups() { let thread = manager .start_thread_with_options(StartThreadOptions { config, + allow_provider_model_fallback: false, initial_history: InitialHistory::New, + history_mode: None, session_source: Some(SessionSource::Internal( InternalSessionSource::MemoryConsolidation, )), @@ -540,7 +614,9 @@ async fn start_thread_seeds_extension_data_for_mcp_and_lifecycle_contributors() let first_thread = manager .start_thread_with_options(StartThreadOptions { config: config.clone(), + allow_provider_model_fallback: false, initial_history: InitialHistory::New, + history_mode: None, session_source: None, thread_source: None, dynamic_tools: Vec::new(), @@ -555,7 +631,9 @@ async fn start_thread_seeds_extension_data_for_mcp_and_lifecycle_contributors() let second_thread = manager .start_thread_with_options(StartThreadOptions { config: config.clone(), + allow_provider_model_fallback: false, initial_history: InitialHistory::New, + history_mode: None, session_source: None, thread_source: None, dynamic_tools: Vec::new(), @@ -567,8 +645,28 @@ async fn start_thread_seeds_extension_data_for_mcp_and_lifecycle_contributors() }) .await .expect("start second thread"); - let first_resolved = first_thread.thread.runtime_mcp_config(&config).await; - let second_resolved = second_thread.thread.runtime_mcp_config(&config).await; + let first_session = &first_thread.thread.codex.session; + let first_resolved = first_session + .services + .mcp_manager + .runtime_config_for_step( + &config, + &first_session.services.mcp_thread_init, + &first_session.services.thread_extension_data, + /*available_environment_ids*/ &[], + ) + .await; + let second_session = &second_thread.thread.codex.session; + let second_resolved = second_session + .services + .mcp_manager + .runtime_config_for_step( + &config, + &second_session.services.mcp_thread_init, + &second_session.services.thread_extension_data, + /*available_environment_ids*/ &[], + ) + .await; assert_eq!( *lifecycle_observed @@ -601,15 +699,82 @@ async fn start_thread_seeds_extension_data_for_mcp_and_lifecycle_contributors() .collect::>() }; assert_eq!( - selected_servers(&first_resolved), + selected_servers(&first_resolved.config), std::collections::BTreeMap::from([("selected-a".to_string(), "env-a".to_string())]) ); assert_eq!( - selected_servers(&second_resolved), + selected_servers(&second_resolved.config), std::collections::BTreeMap::from([("selected-b".to_string(), "env-b".to_string())]) ); } +#[tokio::test] +async fn selected_capability_roots_round_trip_through_fork() { + let temp_dir = tempdir().expect("tempdir"); + let mut config = test_config().await; + config.codex_home = temp_dir.path().join("codex-home").abs(); + config.cwd = config.codex_home.abs(); + std::fs::create_dir_all(&config.codex_home).expect("create codex home"); + + let manager = ThreadManager::with_models_provider_and_home_for_tests( + CodexAuth::from_api_key("dummy"), + config.model_provider.clone(), + config.codex_home.to_path_buf(), + Arc::new(codex_exec_server::EnvironmentManager::default_for_tests()), + ); + let selected_roots = vec![SelectedCapabilityRoot { + id: "demo@1".to_string(), + location: CapabilityRootLocation::Environment { + environment_id: "build".to_string(), + path: PathUri::parse("file:///plugins/demo").expect("plugin root URI"), + }, + }]; + let inherited = manager + .start_thread_with_options(StartThreadOptions { + config, + allow_provider_model_fallback: false, + initial_history: InitialHistory::Forked(vec![RolloutItem::SessionMeta( + SessionMetaLine { + meta: SessionMeta { + selected_capability_roots: selected_roots.clone(), + ..SessionMeta::default() + }, + git: None, + }, + )]), + history_mode: None, + session_source: None, + thread_source: None, + dynamic_tools: Vec::new(), + metrics_service_name: None, + parent_trace: None, + environments: Vec::new(), + thread_extension_init: Default::default(), + supports_openai_form_elicitation: false, + }) + .await + .expect("start inherited fork"); + inherited.thread.ensure_rollout_materialized().await; + inherited + .thread + .flush_rollout() + .await + .expect("flush inherited fork"); + let inherited_history = RolloutRecorder::get_rollout_history( + &inherited + .thread + .rollout_path() + .expect("inherited fork rollout path"), + ) + .await + .expect("read inherited fork rollout"); + + assert_eq!( + inherited_history.get_selected_capability_roots(), + selected_roots + ); +} + #[tokio::test] async fn resume_and_fork_do_not_restore_thread_environments_from_rollout() { let temp_dir = tempdir().expect("tempdir"); @@ -647,7 +812,9 @@ async fn resume_and_fork_do_not_restore_thread_environments_from_rollout() { let source = manager .start_thread_with_options(StartThreadOptions { config: source_config, + allow_provider_model_fallback: false, initial_history: InitialHistory::New, + history_mode: None, session_source: None, thread_source: None, dynamic_tools: Vec::new(), @@ -930,7 +1097,9 @@ async fn resume_stopped_thread_from_rollout_preserves_thread_source() { let source = manager .start_thread_with_options(StartThreadOptions { config: config.clone(), + allow_provider_model_fallback: false, initial_history: InitialHistory::New, + history_mode: None, session_source: None, thread_source: Some(ThreadSource::User), dynamic_tools: Vec::new(), diff --git a/codex-rs/core/src/thread_rollout_truncation.rs b/codex-rs/core/src/thread_rollout_truncation.rs index 25263b14b1..d7d6ad998c 100644 --- a/codex-rs/core/src/thread_rollout_truncation.rs +++ b/codex-rs/core/src/thread_rollout_truncation.rs @@ -5,6 +5,10 @@ use crate::context_manager::is_user_turn_boundary; use crate::event_mapping; +use codex_app_server_protocol::TurnStatus; +use codex_app_server_protocol::build_turns_from_rollout_items; +use codex_protocol::error::CodexErr; +use codex_protocol::error::Result as CodexResult; use codex_protocol::items::TurnItem; use codex_protocol::models::ResponseItem; use codex_protocol::protocol::EventMsg; @@ -149,6 +153,58 @@ pub(crate) fn truncate_rollout_before_nth_user_message_from_start( items[..cut_idx].to_vec() } +/// Return a rollout prefix ending after the requested persisted terminal turn. +/// +/// The turn must still be present in the effective post-rollback history and +/// must have an explicit persisted TurnStarted boundary. Synthetic IDs +/// generated while projecting legacy rollouts are intentionally unsupported +/// because they do not provide a stable raw rollout boundary for a fork. +pub fn truncate_rollout_after_turn_id( + items: &[RolloutItem], + last_turn_id: &str, +) -> CodexResult> { + let turns = build_turns_from_rollout_items(items); + let turn = turns + .iter() + .find(|turn| turn.id == last_turn_id) + .ok_or_else(|| { + CodexErr::InvalidRequest(format!( + "lastTurnId '{last_turn_id}' was not found in the source thread" + )) + })?; + + let target_start_index = items + .iter() + .position(|item| { + matches!( + item, + RolloutItem::EventMsg(EventMsg::TurnStarted(event)) + if event.turn_id == last_turn_id + ) + }) + .ok_or_else(|| { + CodexErr::InvalidRequest(format!( + "lastTurnId '{last_turn_id}' is not a persisted canonical turn in the source thread" + )) + })?; + + if matches!(turn.status, TurnStatus::InProgress) { + return Err(CodexErr::InvalidRequest(format!( + "lastTurnId '{last_turn_id}' identifies an in-progress turn" + ))); + } + + let cut_index = items + .iter() + .enumerate() + .skip(target_start_index.saturating_add(1)) + .find_map(|(index, item)| { + matches!(item, RolloutItem::EventMsg(EventMsg::TurnStarted(_))).then_some(index) + }) + .unwrap_or(items.len()); + Ok(items[..cut_index].to_vec()) +} + /// Return a suffix of `items` that keeps the last `n_from_end` fork turns. /// /// If fewer than or equal to `n_from_end` fork turns exist, this keeps from the first fork-turn diff --git a/codex-rs/core/src/thread_rollout_truncation_tests.rs b/codex-rs/core/src/thread_rollout_truncation_tests.rs index 0da9763fc7..16d10a8277 100644 --- a/codex-rs/core/src/thread_rollout_truncation_tests.rs +++ b/codex-rs/core/src/thread_rollout_truncation_tests.rs @@ -6,6 +6,9 @@ use codex_protocol::models::ContentItem; use codex_protocol::models::ReasoningItemReasoningSummary; use codex_protocol::protocol::InterAgentCommunication; use codex_protocol::protocol::ThreadRolledBackEvent; +use codex_protocol::protocol::TurnCompleteEvent; +use codex_protocol::protocol::TurnStartedEvent; +use codex_protocol::protocol::UserMessageEvent; use pretty_assertions::assert_eq; use std::sync::Arc; @@ -66,6 +69,104 @@ fn inter_agent_communication(text: &str, trigger_turn: bool) -> RolloutItem { )) } +fn turn_started(turn_id: &str) -> RolloutItem { + RolloutItem::EventMsg(EventMsg::TurnStarted(TurnStartedEvent { + turn_id: turn_id.to_string(), + trace_id: None, + started_at: None, + model_context_window: None, + collaboration_mode_kind: Default::default(), + })) +} + +fn turn_completed(turn_id: &str) -> RolloutItem { + RolloutItem::EventMsg(EventMsg::TurnComplete(TurnCompleteEvent { + turn_id: turn_id.to_string(), + last_agent_message: None, + completed_at: None, + duration_ms: None, + time_to_first_token_ms: None, + })) +} + +#[test] +fn truncates_rollout_after_terminal_canonical_turn_id() { + let rollout = vec![ + turn_started("turn-1"), + turn_completed("turn-1"), + turn_started("turn-2"), + turn_completed("turn-2"), + turn_started("turn-3"), + turn_completed("turn-3"), + ]; + + let truncated = + truncate_rollout_after_turn_id(&rollout, "turn-2").expect("truncate through turn-2"); + + assert_eq!( + serde_json::to_value(&truncated).unwrap(), + serde_json::to_value(&rollout[..4]).unwrap() + ); +} + +#[test] +fn truncate_rollout_after_turn_id_rejects_rolled_back_turn() { + let rollout = vec![ + turn_started("turn-1"), + turn_completed("turn-1"), + turn_started("turn-2"), + turn_completed("turn-2"), + RolloutItem::EventMsg(EventMsg::ThreadRolledBack(ThreadRolledBackEvent { + num_turns: 1, + })), + turn_started("turn-3"), + turn_completed("turn-3"), + ]; + + let err = truncate_rollout_after_turn_id(&rollout, "turn-2") + .expect_err("rolled-back turn should not be a fork anchor"); + + assert!(matches!( + err, + CodexErr::InvalidRequest(message) + if message == "lastTurnId 'turn-2' was not found in the source thread" + )); +} + +#[test] +fn truncate_rollout_after_turn_id_rejects_synthetic_legacy_turn_id() { + let rollout = vec![RolloutItem::EventMsg(EventMsg::UserMessage( + UserMessageEvent { + message: "legacy".to_string(), + ..Default::default() + }, + ))]; + + let err = truncate_rollout_after_turn_id(&rollout, "rollout-0") + .expect_err("synthetic turn should not be a fork anchor"); + + assert!(matches!( + err, + CodexErr::InvalidRequest(message) + if message + == "lastTurnId 'rollout-0' is not a persisted canonical turn in the source thread" + )); +} + +#[test] +fn truncate_rollout_after_turn_id_rejects_in_progress_turn() { + let rollout = vec![turn_started("turn-1")]; + + let err = truncate_rollout_after_turn_id(&rollout, "turn-1") + .expect_err("in-progress turn should not be a fork anchor"); + + assert!(matches!( + err, + CodexErr::InvalidRequest(message) + if message == "lastTurnId 'turn-1' identifies an in-progress turn" + )); +} + #[test] fn truncates_rollout_from_start_before_nth_user_only() { let items = [ diff --git a/codex-rs/core/src/tools/code_mode/execute_handler.rs b/codex-rs/core/src/tools/code_mode/execute_handler.rs index 2e46c1bd6b..8374ae1ca4 100644 --- a/codex-rs/core/src/tools/code_mode/execute_handler.rs +++ b/codex-rs/core/src/tools/code_mode/execute_handler.rs @@ -85,6 +85,7 @@ impl CodeModeExecuteHandler { .code_mode_service .finish_cell_dispatch(&cell_id); } + exec.session.services.elicitations.wait_until_clear().await; handle_runtime_response(&exec, response, args.max_output_tokens, started_at) .await .map_err(FunctionCallError::RespondToModel) diff --git a/codex-rs/core/src/tools/code_mode/mod.rs b/codex-rs/core/src/tools/code_mode/mod.rs index c28eef0727..ad746a1de2 100644 --- a/codex-rs/core/src/tools/code_mode/mod.rs +++ b/codex-rs/core/src/tools/code_mode/mod.rs @@ -6,16 +6,19 @@ mod wait_handler; pub(crate) mod wait_spec; use std::sync::Arc; +use std::sync::atomic::AtomicBool; +use std::sync::atomic::Ordering; use std::time::Duration; use codex_code_mode::CellId; use codex_code_mode::CodeModeNestedToolCall; use codex_code_mode::CodeModeSession; +use codex_code_mode::CodeModeSessionProvider; use codex_code_mode::CodeModeToolKind; -use codex_code_mode::InProcessCodeModeSession; use codex_code_mode::RuntimeResponse; use codex_protocol::models::FunctionCallOutputContentItem; use serde_json::Value as JsonValue; +use tokio::sync::OnceCell; use tokio_util::sync::CancellationToken; use crate::function_tool::FunctionCallError; @@ -61,46 +64,62 @@ pub(crate) struct ExecContext { } pub(crate) struct CodeModeService { - session: Option>, + session: OnceCell>, + session_provider: Arc, dispatch_broker: Arc, + shutting_down: AtomicBool, } impl CodeModeService { - pub(crate) fn new() -> Self { + pub(crate) fn new(session_provider: Arc) -> Self { let dispatch_broker = Arc::new(CodeModeDispatchBroker::new()); Self { - session: Some(Arc::new(InProcessCodeModeSession::with_delegate( - dispatch_broker.clone(), - ))), + session: OnceCell::new(), + session_provider, dispatch_broker, + shutting_down: AtomicBool::new(false), } } + pub(crate) fn session_provider(&self) -> Arc { + Arc::clone(&self.session_provider) + } + pub(crate) async fn execute( &self, request: codex_code_mode::ExecuteRequest, ) -> Result { - self.session()?.execute(request).await + self.session().await?.execute(request).await } pub(crate) async fn wait( &self, request: codex_code_mode::WaitRequest, ) -> Result { - self.session()?.wait(request).await + self.session().await?.wait(request).await } pub(crate) async fn terminate( &self, cell_id: CellId, ) -> Result { - self.session()?.terminate(cell_id).await + self.session().await?.terminate(cell_id).await } pub(crate) async fn shutdown(&self) -> Result<(), String> { - match &self.session { - Some(session) => session.shutdown().await, - None => Ok(()), + self.shutting_down.store(true, Ordering::Release); + // Join any initialization already in progress without initializing an unused service. + match self + .session + .get_or_try_init(|| async { + Err::, String>( + "code mode session is shutting down".to_string(), + ) + }) + .await + { + Ok(session) => session.shutdown().await, + Err(_) => Ok(()), } } @@ -121,9 +140,7 @@ impl CodeModeService { ) -> Option { let turn = &step_context.turn; let tool_mode = effective_tool_mode(turn); - if !matches!(tool_mode, ToolMode::CodeMode | ToolMode::CodeModeOnly) - || self.session.is_none() - { + if !matches!(tool_mode, ToolMode::CodeMode | ToolMode::CodeModeOnly) { return None; } @@ -137,10 +154,27 @@ impl CodeModeService { ) } - fn session(&self) -> Result<&Arc, String> { + async fn session(&self) -> Result, String> { + if self.shutting_down.load(Ordering::Acquire) { + return Err("code mode session is shutting down".to_string()); + } self.session - .as_ref() - .ok_or_else(|| "code mode is unavailable".to_string()) + .get_or_try_init(|| async { + if self.shutting_down.load(Ordering::Acquire) { + return Err("code mode session is shutting down".to_string()); + } + let session = self + .session_provider + .create_session(self.dispatch_broker.clone()) + .await?; + if self.shutting_down.load(Ordering::Acquire) { + let _ = session.shutdown().await; + return Err("code mode session is shutting down".to_string()); + } + Ok(session) + }) + .await + .map(Arc::clone) } } @@ -325,10 +359,15 @@ fn build_freeform_tool_payload( #[cfg(test)] mod tests { + use std::sync::Arc; + + use super::CodeModeService; use super::build_nested_tool_payload; use super::truncate_code_mode_result; use crate::tools::context::ToolPayload; use codex_code_mode::CodeModeToolKind; + use codex_code_mode::ExecuteRequest; + use codex_code_mode::ProcessOwnedCodeModeSessionProvider; use codex_protocol::models::FunctionCallOutputContentItem; use codex_tools::ToolName; use serde_json::json; @@ -385,4 +424,28 @@ mod tests { }] ); } + + #[tokio::test] + async fn missing_process_host_is_reported_without_failing_service_creation() { + let service = CodeModeService::new(Arc::new( + ProcessOwnedCodeModeSessionProvider::with_host_program( + "codex-code-mode-host-does-not-exist".into(), + ), + )); + + let error = service + .execute(ExecuteRequest { + tool_call_id: "call-1".to_string(), + enabled_tools: Vec::new(), + source: "text('unreachable')".to_string(), + yield_time_ms: None, + max_output_tokens: None, + }) + .await + .err() + .expect("missing host should reject execution"); + + assert!(error.contains("failed to spawn code-mode host")); + service.shutdown().await.expect("shutdown unused service"); + } } diff --git a/codex-rs/core/src/tools/code_mode/wait_handler.rs b/codex-rs/core/src/tools/code_mode/wait_handler.rs index 953391a573..c3c867f3ef 100644 --- a/codex-rs/core/src/tools/code_mode/wait_handler.rs +++ b/codex-rs/core/src/tools/code_mode/wait_handler.rs @@ -120,6 +120,7 @@ impl CodeModeWaitHandler { .code_mode_service .finish_cell_dispatch(runtime_cell_id); } + exec.session.services.elicitations.wait_until_clear().await; handle_runtime_response(&exec, wait_response.into(), args.max_tokens, started_at) .await .map(boxed_tool_output) diff --git a/codex-rs/core/src/tools/handlers/agent_jobs.rs b/codex-rs/core/src/tools/handlers/agent_jobs.rs index 360744973c..6405414731 100644 --- a/codex-rs/core/src/tools/handlers/agent_jobs.rs +++ b/codex-rs/core/src/tools/handlers/agent_jobs.rs @@ -203,7 +203,7 @@ async fn run_agent_job_loop( .agent_control .spawn_agent_with_metadata( options.spawn_config.clone(), - items.into(), + items, Some(SessionSource::SubAgent(SubAgentSource::Other(format!( "agent_job:{job_id}" )))), diff --git a/codex-rs/core/src/tools/handlers/mcp.rs b/codex-rs/core/src/tools/handlers/mcp.rs index b92a2a5655..c9533fd223 100644 --- a/codex-rs/core/src/tools/handlers/mcp.rs +++ b/codex-rs/core/src/tools/handlers/mcp.rs @@ -124,11 +124,12 @@ impl McpHandler { ) -> Result, FunctionCallError> { let ToolInvocation { session, - turn, + step_context, call_id, payload, .. } = invocation; + let turn = Arc::clone(&step_context.turn); let payload = match payload { ToolPayload::Function { arguments } => arguments, @@ -143,7 +144,7 @@ impl McpHandler { // TODO(sayan): Use StepContext for MCP file arguments when MCP follows dynamic environments. let result = handle_mcp_tool_call( Arc::clone(&session), - &turn, + &step_context, call_id.clone(), self.tool_info.server_name.clone(), self.tool_info.tool.name.to_string(), diff --git a/codex-rs/core/src/tools/handlers/mcp_resource.rs b/codex-rs/core/src/tools/handlers/mcp_resource.rs index 231bdf92a9..b073b5ee75 100644 --- a/codex-rs/core/src/tools/handlers/mcp_resource.rs +++ b/codex-rs/core/src/tools/handlers/mcp_resource.rs @@ -224,6 +224,9 @@ async fn emit_tool_call_begin( connector_id: None, mcp_app_resource_uri: None, link_id: None, + app_name: None, + template_id: None, + action_name: None, plugin_id: None, status: McpToolCallStatus::InProgress, result: None, @@ -265,6 +268,9 @@ async fn emit_tool_call_end( connector_id: None, mcp_app_resource_uri: None, link_id: None, + app_name: None, + template_id: None, + action_name: None, plugin_id: None, status, result, diff --git a/codex-rs/core/src/tools/handlers/mcp_resource/list_mcp_resource_templates.rs b/codex-rs/core/src/tools/handlers/mcp_resource/list_mcp_resource_templates.rs index ce762a0f02..8c7f21fd17 100644 --- a/codex-rs/core/src/tools/handlers/mcp_resource/list_mcp_resource_templates.rs +++ b/codex-rs/core/src/tools/handlers/mcp_resource/list_mcp_resource_templates.rs @@ -53,11 +53,13 @@ impl ListMcpResourceTemplatesHandler { ) -> Result, FunctionCallError> { let ToolInvocation { session, - turn, + step_context, call_id, payload, .. } = invocation; + let turn = std::sync::Arc::clone(&step_context.turn); + let manager = step_context.mcp.manager(); let arguments = match payload { ToolPayload::Function { arguments } => arguments, @@ -89,7 +91,7 @@ impl ListMcpResourceTemplatesHandler { let params = cursor .clone() .map(|value| PaginatedRequestParams::default().with_cursor(Some(value))); - let result = session + let result = manager .list_resource_templates(&server_name, params) .await .map_err(|err| { @@ -108,10 +110,7 @@ impl ListMcpResourceTemplatesHandler { )); } - let templates = session - .services - .mcp_connection_manager - .load_full() + let templates = manager .list_all_resource_templates(|server_name| { model_can_access_mcp_server(turn.as_ref(), server_name) }) diff --git a/codex-rs/core/src/tools/handlers/mcp_resource/list_mcp_resources.rs b/codex-rs/core/src/tools/handlers/mcp_resource/list_mcp_resources.rs index 564fb440fd..f5e9bae7e3 100644 --- a/codex-rs/core/src/tools/handlers/mcp_resource/list_mcp_resources.rs +++ b/codex-rs/core/src/tools/handlers/mcp_resource/list_mcp_resources.rs @@ -53,11 +53,13 @@ impl ListMcpResourcesHandler { ) -> Result, FunctionCallError> { let ToolInvocation { session, - turn, + step_context, call_id, payload, .. } = invocation; + let turn = std::sync::Arc::clone(&step_context.turn); + let manager = step_context.mcp.manager(); let arguments = match payload { ToolPayload::Function { arguments } => arguments, @@ -89,7 +91,7 @@ impl ListMcpResourcesHandler { let params = cursor .clone() .map(|value| PaginatedRequestParams::default().with_cursor(Some(value))); - let result = session + let result = manager .list_resources(&server_name, params) .await .map_err(|err| { @@ -106,10 +108,7 @@ impl ListMcpResourcesHandler { )); } - let resources = session - .services - .mcp_connection_manager - .load_full() + let resources = manager .list_all_resources(|server_name| { model_can_access_mcp_server(turn.as_ref(), server_name) }) diff --git a/codex-rs/core/src/tools/handlers/mcp_resource/read_mcp_resource.rs b/codex-rs/core/src/tools/handlers/mcp_resource/read_mcp_resource.rs index 3909011eb0..5e9b85241c 100644 --- a/codex-rs/core/src/tools/handlers/mcp_resource/read_mcp_resource.rs +++ b/codex-rs/core/src/tools/handlers/mcp_resource/read_mcp_resource.rs @@ -52,11 +52,13 @@ impl ReadMcpResourceHandler { ) -> Result, FunctionCallError> { let ToolInvocation { session, - turn, + step_context, call_id, payload, .. } = invocation; + let turn = std::sync::Arc::clone(&step_context.turn); + let manager = step_context.mcp.manager(); let arguments = match payload { ToolPayload::Function { arguments } => arguments, @@ -84,7 +86,7 @@ impl ReadMcpResourceHandler { let payload_result: Result = async { ensure_model_can_access_mcp_server(turn.as_ref(), &server)?; - let result = session + let result = manager .read_resource(&server, ReadResourceRequestParams::new(uri.clone())) .await .map_err(|err| { diff --git a/codex-rs/core/src/tools/handlers/multi_agents_common.rs b/codex-rs/core/src/tools/handlers/multi_agents_common.rs index 8f91ce5b8e..12ec89d651 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents_common.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents_common.rs @@ -18,7 +18,6 @@ use codex_protocol::openai_models::ReasoningEffort; use codex_protocol::openai_models::ReasoningEffortPreset; use codex_protocol::protocol::CollabAgentRef; use codex_protocol::protocol::CollabAgentStatusEntry; -use codex_protocol::protocol::Op; use codex_protocol::protocol::SessionSource; use codex_protocol::protocol::SubAgentSource; use codex_protocol::user_input::UserInput; @@ -163,7 +162,7 @@ pub(crate) fn thread_spawn_source( pub(crate) fn parse_collab_input( message: Option, items: Option>, -) -> Result { +) -> Result, FunctionCallError> { match (message, items) { (Some(_), Some(_)) => Err(FunctionCallError::RespondToModel( "Provide either message or items, but not both".to_string(), @@ -180,8 +179,7 @@ pub(crate) fn parse_collab_input( Ok(vec![UserInput::Text { text: message, text_elements: Vec::new(), - }] - .into()) + }]) } (None, Some(items)) => { if items.is_empty() { @@ -189,7 +187,7 @@ pub(crate) fn parse_collab_input( "Items can't be empty".to_string(), )); } - Ok(items.into()) + Ok(items) } } } diff --git a/codex-rs/core/src/tools/handlers/multi_agents_spec.rs b/codex-rs/core/src/tools/handlers/multi_agents_spec.rs index 08f8ca071f..23ac4e0bbc 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents_spec.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents_spec.rs @@ -673,9 +673,16 @@ fn spawn_agent_tool_description( {tool_description} This spawn_agent tool provides you access to sub-agents that inherit your current model by default. Do not set the `model` field unless the user explicitly asks for a different model or there is a clear task-specific reason. You should follow the rules and guidelines below to use this tool. -Do not spawn sub-agents unless the user explicitly asks for sub-agents, delegation, or parallel agent work. +Do not spawn sub-agents unless the user or applicable AGENTS.md/skill instructions explicitly ask for sub-agents, delegation, or parallel agent work. +Requests for depth, thoroughness, research, investigation, or detailed codebase analysis do not count as permission to spawn. {agent_role_usage_hint} +### When to delegate vs. do the subtask yourself +- First, quickly analyze the overall user task and form a succinct high-level plan. Identify which tasks are immediate blockers on the critical path, and which tasks are sidecar tasks that are needed but can run in parallel without blocking the next local step. As part of that plan, explicitly decide what immediate task you should do locally right now. Do this planning step before delegating to agents so you do not hand off the immediate blocking task to a submodel and then waste time waiting on it. +- Use a subagent when a subtask is easy enough for it to handle and can run in parallel with your local work. Prefer delegating concrete, bounded sidecar tasks that materially advance the main task without blocking your immediate next local step. +- Do not delegate urgent blocking work when your immediate next step depends on that result. If the very next action is blocked on that task, the main rollout should usually do it locally to keep the critical path moving. +- Keep work local when the subtask is too difficult to delegate well and when it is tightly coupled, urgent, or likely to block your immediate next step. + ### Designing delegated subtasks - Subtasks must be concrete, well-defined, and self-contained. - Delegated subtasks must materially advance the main task. diff --git a/codex-rs/core/src/tools/handlers/multi_agents_tests.rs b/codex-rs/core/src/tools/handlers/multi_agents_tests.rs index 8acb0a8c8d..f313fcf0d9 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents_tests.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents_tests.rs @@ -1377,8 +1377,7 @@ async fn multi_agent_v2_send_message_accepts_root_target_from_child() { vec![UserInput::Text { text: "inspect this repo".to_string(), text_elements: Vec::new(), - }] - .into(), + }], Some(SessionSource::SubAgent(SubAgentSource::ThreadSpawn { parent_thread_id: root.thread_id, depth: 1, @@ -1454,8 +1453,7 @@ async fn multi_agent_v2_followup_task_rejects_root_target_from_child() { vec![UserInput::Text { text: "inspect this repo".to_string(), text_elements: Vec::new(), - }] - .into(), + }], Some(SessionSource::SubAgent(SubAgentSource::ThreadSpawn { parent_thread_id: root.thread_id, depth: 1, @@ -1626,8 +1624,7 @@ async fn multi_agent_v2_list_agents_filters_by_relative_path_prefix() { vec![UserInput::Text { text: "research".to_string(), text_elements: Vec::new(), - }] - .into(), + }], Some(SessionSource::SubAgent(SubAgentSource::ThreadSpawn { parent_thread_id: root.thread_id, depth: 1, @@ -1647,8 +1644,7 @@ async fn multi_agent_v2_list_agents_filters_by_relative_path_prefix() { vec![UserInput::Text { text: "build".to_string(), text_elements: Vec::new(), - }] - .into(), + }], Some(SessionSource::SubAgent(SubAgentSource::ThreadSpawn { parent_thread_id: root.thread_id, depth: 2, @@ -4097,8 +4093,7 @@ async fn multi_agent_v2_interrupt_agent_rejects_self_target_by_id() { vec![UserInput::Text { text: "inspect this repo".to_string(), text_elements: Vec::new(), - }] - .into(), + }], Some(SessionSource::SubAgent(SubAgentSource::ThreadSpawn { parent_thread_id: root.thread_id, depth: 1, @@ -4165,8 +4160,7 @@ async fn multi_agent_v2_interrupt_agent_rejects_self_target_by_task_name() { vec![UserInput::Text { text: "inspect this repo".to_string(), text_elements: Vec::new(), - }] - .into(), + }], Some(SessionSource::SubAgent(SubAgentSource::ThreadSpawn { parent_thread_id: root.thread_id, depth: 1, diff --git a/codex-rs/core/src/tools/handlers/multi_agents_v2.rs b/codex-rs/core/src/tools/handlers/multi_agents_v2.rs index cae3497a13..8e7f73a762 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents_v2.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents_v2.rs @@ -19,7 +19,6 @@ use codex_protocol::protocol::CollabWaitingEndEvent; use codex_protocol::protocol::InterAgentCommunication; use codex_protocol::protocol::SubAgentActivityEvent; use codex_protocol::protocol::SubAgentActivityKind; -use codex_protocol::user_input::UserInput; use codex_tools::ToolName; use serde::Deserialize; use serde::Serialize; diff --git a/codex-rs/core/src/tools/handlers/multi_agents_v2/message_tool.rs b/codex-rs/core/src/tools/handlers/multi_agents_v2/message_tool.rs index 7fdb80e8bc..9501b4355a 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents_v2/message_tool.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents_v2/message_tool.rs @@ -4,6 +4,8 @@ //! resulting `InterAgentCommunication` should wake the target immediately. use super::*; +use crate::agent_communication::AgentCommunicationContext; +use crate::agent_communication::AgentCommunicationKind; use crate::tools::context::FunctionToolOutput; use crate::turn_timing::now_unix_timestamp_ms; use codex_protocol::protocol::InterAgentCommunication; @@ -46,7 +48,7 @@ pub(crate) struct FollowupTaskArgs { pub(crate) message: String, } -fn message_content(message: String) -> Result { +pub(super) fn message_content(message: String) -> Result { if message.trim().is_empty() { return Err(FunctionCallError::RespondToModel( "Empty message can't be sent to an agent".to_string(), @@ -101,10 +103,15 @@ pub(crate) async fn handle_message_string_tool( .unwrap_or_else(AgentPath::root); let communication = communication_from_tool_message(author, receiver_agent_path.clone(), message); + let kind = match mode { + MessageDeliveryMode::QueueOnly => AgentCommunicationKind::Message, + MessageDeliveryMode::TriggerTurn => AgentCommunicationKind::Followup, + }; + let context = AgentCommunicationContext::new(kind, session.thread_id); let result = session .services .agent_control - .send_inter_agent_communication(receiver_thread_id, mode.apply(communication)) + .send_inter_agent_communication(receiver_thread_id, mode.apply(communication), context) .await .map_err(|err| collab_agent_error(receiver_thread_id, err)); result?; diff --git a/codex-rs/core/src/tools/handlers/multi_agents_v2/spawn.rs b/codex-rs/core/src/tools/handlers/multi_agents_v2/spawn.rs index 2186a4a6d6..8eeb6ddd1b 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents_v2/spawn.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents_v2/spawn.rs @@ -4,11 +4,13 @@ use crate::agent::control::SpawnAgentOptions; use crate::agent::next_thread_spawn_depth; use crate::agent::role::DEFAULT_ROLE_NAME; use crate::agent::role::apply_role_to_config; +use crate::agent_communication::AgentCommunicationContext; +use crate::agent_communication::AgentCommunicationKind; use crate::tools::handlers::multi_agents_spec::SpawnAgentToolOptions; use crate::tools::handlers::multi_agents_spec::create_spawn_agent_tool_v2; +use crate::tools::handlers::multi_agents_v2::message_tool::message_content; use crate::turn_timing::now_unix_timestamp_ms; use codex_protocol::AgentPath; -use codex_protocol::protocol::Op; use codex_tools::ToolSpec; #[derive(Default)] @@ -55,8 +57,7 @@ async fn handle_spawn_agent( .map(str::trim) .filter(|role| !role.is_empty()); - let message = args.message.clone(); - let initial_operation = parse_collab_input(Some(args.message), /*items*/ None)?; + let message = message_content(args.message)?; let session_source = turn.session_source.clone(); let child_depth = next_thread_spawn_depth(&session_source); let mut config = @@ -104,33 +105,28 @@ async fn handle_spawn_agent( "spawned agent is missing a canonical task name".to_string(), ) })?; + let author = turn + .session_source + .get_agent_path() + .unwrap_or_else(AgentPath::root); + let communication = communication_from_tool_message(author, new_agent_path.clone(), message); + let context = AgentCommunicationContext::new(AgentCommunicationKind::Spawn, session.thread_id); let spawned_agent = Box::pin( - session.services.agent_control.spawn_agent_with_metadata( - config, - match initial_operation { - Op::UserInput { items, .. } - if items - .iter() - .all(|item| matches!(item, UserInput::Text { .. })) => - { - let author = turn - .session_source - .get_agent_path() - .unwrap_or_else(AgentPath::root); - let communication = - communication_from_tool_message(author, new_agent_path.clone(), message); - Op::InterAgentCommunication { communication } - } - initial_operation => initial_operation, - }, - Some(spawn_source), - SpawnAgentOptions { - fork_parent_spawn_call_id: fork_mode.as_ref().map(|_| call_id.clone()), - fork_mode, - parent_thread_id: Some(session.thread_id), - environments: Some(turn.environments.to_selections()), - }, - ), + session + .services + .agent_control + .spawn_agent_with_communication( + config, + communication, + context, + Some(spawn_source), + SpawnAgentOptions { + fork_parent_spawn_call_id: fork_mode.as_ref().map(|_| call_id.clone()), + fork_mode, + parent_thread_id: Some(session.thread_id), + environments: Some(turn.environments.to_selections()), + }, + ), ) .await .map_err(collab_spawn_error)?; diff --git a/codex-rs/core/src/tools/handlers/request_plugin_install.rs b/codex-rs/core/src/tools/handlers/request_plugin_install.rs index ce7ce10da8..e44ed41fa4 100644 --- a/codex-rs/core/src/tools/handlers/request_plugin_install.rs +++ b/codex-rs/core/src/tools/handlers/request_plugin_install.rs @@ -1,4 +1,5 @@ use std::collections::HashSet; +use std::sync::Arc; use codex_analytics::PluginInstallRequestSource; use codex_analytics::PluginInstallRequested; @@ -94,10 +95,12 @@ impl RequestPluginInstallHandler { let ToolInvocation { payload, session, - turn, + step_context, call_id, .. } = invocation; + let turn = Arc::clone(&step_context.turn); + let manager = step_context.mcp.manager(); let arguments = match payload { ToolPayload::Function { arguments } => arguments, @@ -227,7 +230,8 @@ impl RequestPluginInstallHandler { let auth = session.services.auth_manager.auth().await; let completed = if user_confirmed { - verify_request_plugin_install_completed(&session, &turn, &tool, auth.as_ref()).await + verify_request_plugin_install_completed(&session, &turn, manager, &tool, auth.as_ref()) + .await } else { false }; @@ -345,13 +349,14 @@ fn disabled_install_request(tool: &DiscoverableTool) -> ToolSuggestDisabledTool async fn verify_request_plugin_install_completed( session: &crate::session::session::Session, turn: &crate::session::turn_context::TurnContext, + manager: &codex_mcp::McpConnectionManager, tool: &DiscoverableTool, auth: Option<&codex_login::CodexAuth>, ) -> bool { match tool { DiscoverableTool::Connector(connector) => refresh_missing_requested_connectors( - session, turn, + manager, auth, std::slice::from_ref(&connector.id), connector.id.as_str(), @@ -370,8 +375,8 @@ async fn verify_request_plugin_install_completed( plugin.id.as_str(), ), refresh_missing_requested_connectors( - session, turn, + manager, auth, &plugin.app_connector_ids, plugin.id.as_str(), @@ -393,8 +398,8 @@ async fn verify_request_plugin_install_completed( session.services.plugins_manager.as_ref(), ); let _ = refresh_missing_requested_connectors( - session, turn, + manager, auth, &plugin.app_connector_ids, plugin.id.as_str(), @@ -435,8 +440,8 @@ fn is_remote_plugin_install_suggestion(plugin_id: &str) -> bool { } async fn refresh_missing_requested_connectors( - session: &crate::session::session::Session, turn: &crate::session::turn_context::TurnContext, + manager: &codex_mcp::McpConnectionManager, auth: Option<&codex_login::CodexAuth>, expected_connector_ids: &[String], tool_id: &str, @@ -445,7 +450,6 @@ async fn refresh_missing_requested_connectors( return Some(Vec::new()); } - let manager = session.services.mcp_connection_manager.load_full(); let mcp_tools = manager.list_all_tools().await; let accessible_connectors = connectors::with_app_enabled_state( connectors::accessible_connectors_from_mcp_tools(&mcp_tools), diff --git a/codex-rs/core/src/tools/handlers/shell/shell_command.rs b/codex-rs/core/src/tools/handlers/shell/shell_command.rs index 1faa89b31f..62bcee8bdf 100644 --- a/codex-rs/core/src/tools/handlers/shell/shell_command.rs +++ b/codex-rs/core/src/tools/handlers/shell/shell_command.rs @@ -6,6 +6,7 @@ use codex_utils_absolute_path::AbsolutePathBuf; use crate::exec::ExecCapturePolicy; use crate::exec::ExecParams; use crate::exec_env::create_env; +use crate::exec_env::inject_permission_profile_env; use crate::function_tool::FunctionCallError; use crate::maybe_emit_implicit_skill_invocation; use crate::session::turn_context::TurnContext; @@ -99,15 +100,19 @@ impl ShellCommandHandler { let use_login_shell = Self::resolve_use_login_shell(params.login, allow_login_shell)?; let command = Self::base_command(shell, ¶ms.command, use_login_shell); + let mut env = create_env( + &turn_context.config.permissions.shell_environment_policy, + Some(session.thread_id), + ); + let active_permission_profile = turn_context.config.permissions.active_permission_profile(); + inject_permission_profile_env(&mut env, active_permission_profile.as_ref()); + Ok(ExecParams { command, cwd, expiration: params.timeout_ms.into(), capture_policy: ExecCapturePolicy::ShellTool, - env: create_env( - &turn_context.config.permissions.shell_environment_policy, - Some(session.thread_id), - ), + env, network: turn_context.network.clone(), network_environment_id: Some(turn_environment.environment_id.clone()), sandbox_permissions: params.sandbox_permissions.unwrap_or_default(), diff --git a/codex-rs/core/src/tools/handlers/shell_tests.rs b/codex-rs/core/src/tools/handlers/shell_tests.rs index ded7733022..ebfbf9127c 100644 --- a/codex-rs/core/src/tools/handlers/shell_tests.rs +++ b/codex-rs/core/src/tools/handlers/shell_tests.rs @@ -1,10 +1,14 @@ use std::path::PathBuf; use std::sync::Arc; +use codex_protocol::models::ActivePermissionProfile; use codex_protocol::models::ShellCommandToolCallParams; use pretty_assertions::assert_eq; +use crate::config::PermissionProfileSnapshot; +use crate::exec_env::CODEX_PERMISSION_PROFILE_ENV_VAR; use crate::exec_env::create_env; +use crate::exec_env::inject_permission_profile_env; use crate::sandboxing::SandboxPermissions; use crate::session::step_context::StepContext; use crate::session::tests::make_session_and_context; @@ -71,7 +75,15 @@ fn assert_safe(shell: &Shell, command: &str) { #[tokio::test] async fn shell_command_handler_to_exec_params_uses_selected_environment() { - let (session, turn_context) = make_session_and_context().await; + let (session, mut turn_context) = make_session_and_context().await; + let permission_profile = turn_context.config.permissions.permission_profile().clone(); + Arc::make_mut(&mut turn_context.config) + .permissions + .set_permission_profile_from_session_snapshot(PermissionProfileSnapshot::active( + permission_profile, + ActivePermissionProfile::new("test-profile"), + )) + .expect("set active permission profile"); let command = "echo hello".to_string(); let workdir = Some("subdir".to_string()); @@ -99,10 +111,12 @@ async fn shell_command_handler_to_exec_params_uses_selected_environment() { PathUri::from_abs_path(&selected_cwd), Some(selected_shell), ); - let expected_env = create_env( + let mut expected_env = create_env( &turn_context.config.permissions.shell_environment_policy, Some(session.thread_id), ); + let active_permission_profile = turn_context.config.permissions.active_permission_profile(); + inject_permission_profile_env(&mut expected_env, active_permission_profile.as_ref()); let params = ShellCommandToolCallParams { command, @@ -129,6 +143,12 @@ async fn shell_command_handler_to_exec_params_uses_selected_environment() { assert_eq!(exec_params.command, expected_command); assert_eq!(exec_params.cwd, expected_cwd); assert_eq!(exec_params.env, expected_env); + assert_eq!( + exec_params.env.get(CODEX_PERMISSION_PROFILE_ENV_VAR), + active_permission_profile + .as_ref() + .map(|profile| &profile.id) + ); assert_eq!(exec_params.network, turn_context.network); assert_eq!( exec_params.network_environment_id.as_deref(), diff --git a/codex-rs/core/src/tools/network_approval.rs b/codex-rs/core/src/tools/network_approval.rs index 2b63164b4d..747ae80a6f 100644 --- a/codex-rs/core/src/tools/network_approval.rs +++ b/codex-rs/core/src/tools/network_approval.rs @@ -27,9 +27,11 @@ use codex_protocol::protocol::Event; use codex_protocol::protocol::EventMsg; use codex_protocol::protocol::ReviewDecision; use codex_protocol::protocol::WarningEvent; +use codex_sandboxing::SandboxType; use indexmap::IndexMap; use std::collections::HashMap; use std::collections::HashSet; +use std::io; use std::sync::Arc; use tokio::sync::Mutex; use tokio::sync::Notify; @@ -59,6 +61,7 @@ pub(crate) struct DeferredNetworkApproval { registration_id: String, cancellation_token: CancellationToken, finish_outcome: Arc>>, + _execution_proxy: Option, } impl DeferredNetworkApproval { @@ -89,6 +92,7 @@ pub(crate) struct ActiveNetworkApproval { registration_id: Option, mode: NetworkApprovalMode, cancellation_token: CancellationToken, + execution_proxy: NetworkProxy, } impl ActiveNetworkApproval { @@ -100,11 +104,16 @@ impl ActiveNetworkApproval { self.cancellation_token.clone() } + pub(crate) fn execution_proxy(&self) -> &NetworkProxy { + &self.execution_proxy + } + pub(crate) fn into_deferred(self) -> Option { let ActiveNetworkApproval { registration_id, mode, cancellation_token, + execution_proxy, } = self; match (mode, registration_id) { (NetworkApprovalMode::Deferred, Some(registration_id)) => { @@ -112,6 +121,7 @@ impl ActiveNetworkApproval { registration_id, cancellation_token, finish_outcome: Arc::new(OnceCell::new()), + _execution_proxy: Some(execution_proxy), }) } _ => None, @@ -241,6 +251,11 @@ enum ActiveNetworkApprovalAttribution { Ambiguous, } +struct NetworkRequestAttribution { + owner_call: Option>, + environment_id: Option, +} + #[derive(Default)] struct NetworkApprovalCallState { active_calls: IndexMap>, @@ -305,9 +320,8 @@ impl NetworkApprovalService { async fn resolve_single_active_call(&self) -> Option> { let calls = self.calls.lock().await; - // Blocked proxy requests are not attributed to a specific tool call. Only pick an owner - // when there is exactly one candidate; with concurrent calls, canceling one would be a guess. - // TODO: Carry blocked-request attribution so concurrent active calls can be handled safely. + // Shared proxy requests can still arrive without an execution ID. Only pick an owner when + // there is exactly one candidate; with concurrent calls, canceling one would be a guess. if calls.active_calls.len() == 1 { return calls.active_calls.values().next().cloned(); } @@ -315,6 +329,18 @@ impl NetworkApprovalService { None } + async fn resolve_active_call_by_execution_id( + &self, + execution_id: &str, + ) -> Option> { + self.calls + .lock() + .await + .active_calls + .get(execution_id) + .cloned() + } + async fn resolve_active_call_attribution(&self) -> ActiveNetworkApprovalAttribution { let calls = self.calls.lock().await; match calls.active_calls.len() { @@ -327,6 +353,54 @@ impl NetworkApprovalService { } } + async fn resolve_request_attribution( + &self, + request: &NetworkPolicyRequest, + ) -> Option { + if let Some(execution_id) = request.execution_id.as_deref() { + let call = self + .resolve_active_call_by_execution_id(execution_id) + .await?; + let environment_id = request + .environment_id + .clone() + .unwrap_or_else(|| call.environment_id.clone()); + return (call.environment_id == environment_id).then_some(NetworkRequestAttribution { + owner_call: Some(call), + environment_id: Some(environment_id), + }); + } + + if let Some(environment_id) = request.environment_id.clone() { + let owner_call = match self.resolve_active_call_attribution().await { + ActiveNetworkApprovalAttribution::Single(call) => { + (call.environment_id == environment_id).then_some(call) + } + ActiveNetworkApprovalAttribution::None + | ActiveNetworkApprovalAttribution::Ambiguous => None, + }; + return Some(NetworkRequestAttribution { + owner_call, + environment_id: Some(environment_id), + }); + } + + match self.resolve_active_call_attribution().await { + ActiveNetworkApprovalAttribution::None => Some(NetworkRequestAttribution { + owner_call: None, + environment_id: None, + }), + ActiveNetworkApprovalAttribution::Single(call) => { + let environment_id = call.environment_id.clone(); + Some(NetworkRequestAttribution { + owner_call: Some(call), + environment_id: Some(environment_id), + }) + } + ActiveNetworkApprovalAttribution::Ambiguous => None, + } + } + async fn get_or_create_pending_approval( &self, key: HostApprovalKey, @@ -393,8 +467,12 @@ impl NetworkApprovalService { return; }; - self.record_outcome_for_single_active_call(NetworkApprovalOutcome::DeniedByPolicy(message)) - .await; + let outcome = NetworkApprovalOutcome::DeniedByPolicy(message); + if let Some(execution_id) = blocked.execution_id.as_deref() { + self.record_call_outcome(execution_id, outcome).await; + } else { + self.record_outcome_for_single_active_call(outcome).await; + } } async fn active_turn_context( @@ -431,28 +509,13 @@ impl NetworkApprovalService { NetworkProtocol::Socks5Tcp => NetworkApprovalProtocol::Socks5Tcp, NetworkProtocol::Socks5Udp => NetworkApprovalProtocol::Socks5Udp, }; - let (owner_call, active_environment_id) = - if let Some(environment_id) = request.environment_id.clone() { - let owner_call = match self.resolve_active_call_attribution().await { - ActiveNetworkApprovalAttribution::Single(call) => { - (call.environment_id == environment_id).then_some(call) - } - ActiveNetworkApprovalAttribution::None - | ActiveNetworkApprovalAttribution::Ambiguous => None, - }; - (owner_call, Some(environment_id)) - } else { - match self.resolve_active_call_attribution().await { - ActiveNetworkApprovalAttribution::None => (None, None), - ActiveNetworkApprovalAttribution::Single(call) => { - let environment_id = call.environment_id.clone(); - (Some(call), Some(environment_id)) - } - ActiveNetworkApprovalAttribution::Ambiguous => { - return NetworkDecision::deny(REASON_NOT_ALLOWED); - } - } - }; + let Some(NetworkRequestAttribution { + owner_call, + environment_id: active_environment_id, + }) = self.resolve_request_attribution(&request).await + else { + return NetworkDecision::deny(REASON_NOT_ALLOWED); + }; let turn_context = Self::active_turn_context(session.as_ref()).await; let Some(environment_id) = active_environment_id.or_else(|| { turn_context @@ -788,20 +851,39 @@ pub(crate) async fn begin_network_approval( session: &Session, turn_id: &str, managed_network_active: bool, + selected_sandbox: SandboxType, spec: Option, -) -> Option { +) -> Result, ToolError> { let NetworkApprovalSpec { network, mode, trigger, command, environment_id, - } = spec?; - if !managed_network_active || network.is_none() { - return None; + } = match spec { + Some(spec) => spec, + None => return Ok(None), + }; + let Some(network) = network else { + return Ok(None); + }; + if !managed_network_active { + return Ok(None); } let registration_id = Uuid::new_v4().to_string(); + let execution_proxy = if selected_sandbox == SandboxType::LinuxSeccomp { + let attribution_token = Uuid::new_v4().to_string(); + network + .for_execution(&environment_id, ®istration_id, attribution_token) + .map_err(|err| { + ToolError::Codex(codex_protocol::error::CodexErr::Io(io::Error::other( + format!("failed to create execution-scoped network proxy: {err}"), + ))) + })? + } else { + network.clone() + }; let cancellation_token = CancellationToken::new(); session .services @@ -816,11 +898,12 @@ pub(crate) async fn begin_network_approval( ) .await; - Some(ActiveNetworkApproval { + Ok(Some(ActiveNetworkApproval { registration_id: Some(registration_id), mode, cancellation_token, - }) + execution_proxy, + })) } pub(crate) async fn finish_immediate_network_approval( diff --git a/codex-rs/core/src/tools/network_approval_tests.rs b/codex-rs/core/src/tools/network_approval_tests.rs index 7a9ba86cef..4fdeb50ed7 100644 --- a/codex-rs/core/src/tools/network_approval_tests.rs +++ b/codex-rs/core/src/tools/network_approval_tests.rs @@ -285,6 +285,12 @@ fn denied_blocked_request(host: &str) -> BlockedRequest { }) } +fn denied_blocked_request_for_execution(host: &str, execution_id: &str) -> BlockedRequest { + let mut blocked = denied_blocked_request(host); + blocked.execution_id = Some(execution_id.to_string()); + blocked +} + async fn register_call_with_default_shell_trigger( service: &NetworkApprovalService, registration_id: &str, @@ -429,6 +435,7 @@ async fn deferred_finish_reuses_denial_result_after_first_consumer() { registration_id: "registration-1".to_string(), cancellation_token, finish_outcome: Arc::new(OnceCell::new()), + _execution_proxy: None, }; service .record_call_outcome( @@ -481,3 +488,27 @@ async fn record_blocked_request_ignores_ambiguous_unattributed_blocked_requests( assert_eq!(service.take_call_outcome("registration-1").await, None); assert_eq!(service.take_call_outcome("registration-2").await, None); } + +#[tokio::test] +async fn attributed_blocked_request_targets_one_of_multiple_active_calls() { + let service = NetworkApprovalService::default(); + let first = register_call_with_default_shell_trigger(&service, "registration-1").await; + let second = register_call_with_default_shell_trigger(&service, "registration-2").await; + + service + .record_blocked_request(denied_blocked_request_for_execution( + "example.com", + "registration-2", + )) + .await; + + assert!(!first.is_cancelled()); + assert!(second.is_cancelled()); + assert_eq!(service.take_call_outcome("registration-1").await, None); + assert_eq!( + service.take_call_outcome("registration-2").await, + Some(NetworkApprovalOutcome::DeniedByPolicy( + "Network access to \"example.com\" was blocked: domain is not on the allowlist for the current sandbox mode.".to_string() + )) + ); +} diff --git a/codex-rs/core/src/tools/orchestrator.rs b/codex-rs/core/src/tools/orchestrator.rs index aaa0dfe9d1..f82ad5bb70 100644 --- a/codex-rs/core/src/tools/orchestrator.rs +++ b/codex-rs/core/src/tools/orchestrator.rs @@ -9,6 +9,7 @@ caching). use crate::guardian::guardian_rejection_message; use crate::guardian::guardian_timeout_message; use crate::guardian::new_guardian_review_id; +use crate::guardian::review_approval_request; use crate::guardian::routes_approval_to_guardian; use crate::hook_runtime::run_permission_request_hooks; use crate::network_policy_decision::network_approval_context_from_payload; @@ -68,13 +69,18 @@ impl ToolOrchestrator { where T: ToolRuntime, { - let network_approval = begin_network_approval( + let network_approval = match begin_network_approval( &tool_ctx.session, &tool_ctx.turn.sub_id, managed_network_active, + attempt.sandbox, tool.network_approval_spec(req, tool_ctx), ) - .await; + .await + { + Ok(network_approval) => network_approval, + Err(err) => return (Err(err), None), + }; let attempt_tool_ctx = ToolCtx { session: tool_ctx.session.clone(), @@ -98,6 +104,9 @@ impl ToolOrchestrator { network_denial_cancellation_token: network_approval .as_ref() .map(ActiveNetworkApproval::cancellation_token), + network_proxy: network_approval + .as_ref() + .map(ActiveNetworkApproval::execution_proxy), }; let run_result = tool .run(req, &attempt_with_network_approval, &attempt_tool_ctx) @@ -274,6 +283,7 @@ impl ToolOrchestrator { .permissions .windows_sandbox_private_desktop, network_denial_cancellation_token: None, + network_proxy: None, }; let initial_attempt_start = Instant::now(); @@ -456,6 +466,7 @@ impl ToolOrchestrator { .permissions .windows_sandbox_private_desktop, network_denial_cancellation_token: None, + network_proxy: None, }; // Second attempt. @@ -563,7 +574,26 @@ impl ToolOrchestrator { } else { ToolDecisionSource::User }; - let decision = tool.start_approval_async(req, approval_ctx).await; + let decision = if let Some(review_id) = approval_ctx.guardian_review_id.clone() { + match tool.approval_action(req, &approval_ctx) { + Ok(action) => { + review_approval_request( + approval_ctx.session, + approval_ctx.turn, + review_id, + action, + approval_ctx.retry_reason.clone(), + ) + .await + } + Err(err) => { + tracing::error!(%err, "failed to build guardian approval action"); + ReviewDecision::Abort + } + } + } else { + tool.start_approval_async(req, approval_ctx).await + }; let tool_name = flat_tool_name(&tool_ctx.tool_name); otel.tool_decision( tool_name.as_ref(), diff --git a/codex-rs/core/src/tools/parallel.rs b/codex-rs/core/src/tools/parallel.rs index a2ee89ab83..e201f0ffce 100644 --- a/codex-rs/core/src/tools/parallel.rs +++ b/codex-rs/core/src/tools/parallel.rs @@ -1,4 +1,5 @@ use std::sync::Arc; +use std::sync::OnceLock; use std::sync::atomic::AtomicBool; use std::sync::atomic::Ordering; use std::time::Instant; @@ -9,6 +10,7 @@ use tokio_util::either::Either; use tokio_util::sync::CancellationToken; use tokio_util::task::AbortOnDropHandle; use tracing::Instrument; +use tracing::info; use tracing::instrument; use tracing::trace_span; @@ -27,6 +29,15 @@ use crate::tools::router::ToolRouter; use codex_protocol::error::CodexErr; use codex_protocol::models::ResponseInputItem; +struct ToolCallTimingGuard { + started_at: Instant, + execution_started_at: Arc>, + conversation_id: String, + turn_id: String, + call_id: String, + tool_name: codex_tools::ToolName, +} + #[derive(Clone)] pub(crate) struct ToolCallRuntime { router: Arc, @@ -96,6 +107,11 @@ impl ToolCallRuntime { let invocation_cancellation_token = cancellation_token.clone(); let wait_for_runtime_cancellation = self.router.tool_waits_for_runtime_cancellation(&call); let started = Instant::now(); + let tool_call_timing_guard = + ToolCallTimingGuard::capture(started, &session.thread_id, &turn.sub_id, &call, &source); + let execution_started_at = tool_call_timing_guard + .as_ref() + .map(|timing| Arc::clone(&timing.execution_started_at)); let abort_session = Arc::clone(&session); let abort_source = source.clone(); let abort_turn = Arc::clone(&turn); @@ -112,13 +128,18 @@ impl ToolCallRuntime { ); let abort_dispatch_span = dispatch_span.clone(); - let mut handle: AbortOnDropHandle> = + let mut dispatch_handle: AbortOnDropHandle> = AbortOnDropHandle::new(tokio::spawn(async move { let _guard = if supports_parallel { Either::Left(lock.read().await) } else { Either::Right(lock.write().await) }; + // Admission through the parallel-execution gate marks the end + // of dispatch waiting and the start of handler execution. + if let Some(execution_started_at) = execution_started_at { + let _ = execution_started_at.set(Instant::now()); + } router .dispatch_tool_call_with_terminal_outcome( @@ -135,28 +156,29 @@ impl ToolCallRuntime { })); async move { + let _tool_call_timing_guard = tool_call_timing_guard; tokio::select! { - res = &mut handle => res.map_err(Self::tool_task_join_error)?, + res = &mut dispatch_handle => res.map_err(Self::tool_task_join_error)?, _ = cancellation_token.cancelled() => { - if terminal_outcome_reached.load(Ordering::Acquire) || handle.is_finished() { - handle.await.map_err(Self::tool_task_join_error)? + if terminal_outcome_reached.load(Ordering::Acquire) || dispatch_handle.is_finished() { + dispatch_handle.await.map_err(Self::tool_task_join_error)? } else { let secs = started.elapsed().as_secs_f32().max(0.1); abort_dispatch_span.record("aborted", true); if wait_for_runtime_cancellation { if terminal_outcome_reached.swap(true, Ordering::AcqRel) { - return handle.await.map_err(Self::tool_task_join_error)?; + return dispatch_handle.await.map_err(Self::tool_task_join_error)?; } // The abort owns the terminal outcome; await only so // the runtime can finish process teardown. - match handle.await { + match dispatch_handle.await { Ok(_) => {} Err(err) if err.is_cancelled() => {} Err(err) => return Err(Self::tool_task_join_error(err)), } } else { - handle.abort(); - match handle.await { + dispatch_handle.abort(); + match dispatch_handle.await { Ok(result) => return result, Err(err) if err.is_cancelled() => {} Err(err) => return Err(Self::tool_task_join_error(err)), @@ -237,6 +259,88 @@ impl ToolCallRuntime { } } +impl ToolCallTimingGuard { + fn capture( + started_at: Instant, + conversation_id: &impl std::fmt::Display, + turn_id: &str, + call: &ToolCall, + source: &ToolCallSource, + ) -> Option { + // Code-mode calls are nested within a direct code-mode tool call whose + // timing already includes them. Suppress nested guards so consumers do + // not mistake overlapping events for independent tool-call latency. + if !matches!(source, ToolCallSource::Direct) || !tracing::enabled!(tracing::Level::INFO) { + return None; + } + + Some(Self { + started_at, + execution_started_at: Arc::new(OnceLock::new()), + conversation_id: conversation_id.to_string(), + turn_id: turn_id.to_string(), + call_id: call.call_id.clone(), + tool_name: call.tool_name.clone(), + }) + } +} + +impl Drop for ToolCallTimingGuard { + fn drop(&mut self) { + let completed_at = Instant::now(); + // Snapshot once so a concurrently-starting dispatch cannot make one + // event internally inconsistent. + let execution_started_at = self + .execution_started_at + .get() + .copied() + .filter(|execution_started_at| *execution_started_at <= completed_at); + let duration_ms = |duration: std::time::Duration| u64::try_from(duration.as_millis()).ok(); + let total_duration_ms = duration_ms(completed_at.duration_since(self.started_at)); + let dispatch_duration_ms = execution_started_at.map_or_else( + || total_duration_ms, + |execution_started_at| { + duration_ms(execution_started_at.duration_since(self.started_at)) + }, + ); + let handler_duration_ms = execution_started_at.map_or(Some(0), |execution_started_at| { + duration_ms(completed_at.duration_since(execution_started_at)) + }); + + macro_rules! log_tool_call { + ($dispatch_duration_ms:expr, $handler_duration_ms:expr, $total_duration_ms:expr) => { + info!( + event.name = "codex.tool_call", + trace_id = %codex_otel::current_span_trace_id().unwrap_or_default(), + conversation.id = %self.conversation_id, + turn_id = %self.turn_id, + tool_name = %self.tool_name, + call_id = %self.call_id, + tool_source = "direct", + execution_started = execution_started_at.is_some(), + dispatch_duration_ms = $dispatch_duration_ms, + handler_duration_ms = $handler_duration_ms, + total_duration_ms = $total_duration_ms, + "tool call completed" + ); + }; + } + + match (dispatch_duration_ms, handler_duration_ms, total_duration_ms) { + (Some(dispatch_duration_ms), Some(handler_duration_ms), Some(total_duration_ms)) => { + log_tool_call!(dispatch_duration_ms, handler_duration_ms, total_duration_ms); + } + _ => { + log_tool_call!( + tracing::field::Empty, + tracing::field::Empty, + tracing::field::Empty + ); + } + } + } +} + #[cfg(test)] mod tests { use super::*; @@ -255,6 +359,154 @@ mod tests { use pretty_assertions::assert_eq; use tokio::sync::Notify; use tokio::sync::oneshot; + use tracing_test::internal::MockWriter; + + #[test] + fn tool_call_timing_guard_ignores_code_mode_source() { + let subscriber = tracing_subscriber::fmt() + .with_max_level(tracing::Level::INFO) + .finish(); + tracing::subscriber::with_default(subscriber, || { + let call = ToolCall { + tool_name: codex_tools::ToolName::plain("test_tool"), + call_id: "call-1".to_string(), + payload: ToolPayload::Function { + arguments: "{}".to_string(), + }, + }; + let direct_guard = ToolCallTimingGuard::capture( + Instant::now(), + &"conversation-id", + "turn-id", + &call, + &ToolCallSource::Direct, + ); + assert!( + direct_guard.is_some(), + "direct tool calls should create a timing guard" + ); + drop(direct_guard); + + let code_mode_guard = ToolCallTimingGuard::capture( + Instant::now(), + &"conversation-id", + "turn-id", + &call, + &ToolCallSource::CodeMode { + cell_id: "cell-1".to_string(), + runtime_tool_call_id: "runtime-call-1".to_string(), + }, + ); + assert!( + code_mode_guard.is_none(), + "nested code-mode calls should not create overlapping timing events" + ); + }); + } + + #[tokio::test] + async fn cancellation_before_dispatch_admission_logs_dispatch_only_timing() -> anyhow::Result<()> + { + let (session, turn_context) = crate::session::tests::make_session_and_context().await; + let session = Arc::new(session); + let turn_context = Arc::new(turn_context); + let tool_name = codex_tools::ToolName::plain("test_tool"); + let handler = Arc::new(ImmediateHandler { + tool_name: tool_name.clone(), + }) as Arc; + let step_context = StepContext::for_test(Arc::clone(&turn_context)); + let router = Arc::new(ToolRouter::from_parts( + ToolRegistry::from_tools([handler]), + Vec::new(), + )); + let tracker = Arc::new(tokio::sync::Mutex::new(TurnDiffTracker::new())); + let runtime = ToolCallRuntime::new(router, session, step_context, tracker); + let execution_gate = Arc::clone(&runtime.parallel_execution); + let execution_gate_guard = execution_gate + .try_write_owned() + .expect("execution gate should be available before dispatch starts"); + let (release_execution_gate_tx, release_execution_gate_rx) = std::sync::mpsc::channel(); + let execution_gate_task = tokio::task::spawn_blocking(move || { + let _execution_gate_guard = execution_gate_guard; + release_execution_gate_rx + .recv() + .expect("test should release the execution gate"); + }); + + let buffer: &'static std::sync::Mutex> = + Box::leak(Box::new(std::sync::Mutex::new(Vec::new()))); + let subscriber = tracing_subscriber::fmt() + .with_ansi(false) + .with_max_level(tracing::Level::INFO) + .with_writer(MockWriter::new(buffer)) + .finish(); + let _subscriber_guard = tracing::subscriber::set_default(subscriber); + + let cancellation_token = CancellationToken::new(); + let call = ToolCall { + tool_name, + call_id: "call-1".to_string(), + payload: ToolPayload::Function { + arguments: "{}".to_string(), + }, + }; + let response_task = + tokio::spawn(runtime.handle_tool_call(call, cancellation_token.clone())); + cancellation_token.cancel(); + tokio::time::timeout(Duration::from_secs(1), response_task) + .await + .expect("timed out waiting for cancelled tool response") + .expect("cancelled tool response task should join") + .expect("cancelled tool call should produce a response"); + + let logs = String::from_utf8( + buffer + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clone(), + )?; + let timing_events = logs + .lines() + .filter(|line| line.contains("event.name=\"codex.tool_call\"")) + .collect::>(); + assert_eq!( + timing_events.len(), + 1, + "cancelled tool call should emit exactly one timing event; logs:\n{logs}" + ); + let timing_event = timing_events[0]; + assert!( + timing_event.contains("execution_started=false"), + "tool cancelled before admission should not report execution started: {timing_event}" + ); + assert!( + timing_event.contains("handler_duration_ms=0"), + "tool cancelled before admission should report zero handler duration: {timing_event}" + ); + let duration_field = |name: &str| { + timing_event.split_whitespace().find_map(|field| { + field + .strip_prefix(&format!("{name}=")) + .and_then(|value| value.parse::().ok()) + }) + }; + let dispatch_duration_ms = duration_field("dispatch_duration_ms") + .expect("timing event should include dispatch_duration_ms"); + let total_duration_ms = duration_field("total_duration_ms") + .expect("timing event should include total_duration_ms"); + assert_eq!( + dispatch_duration_ms, total_duration_ms, + "tool cancelled before admission should attribute all elapsed time to dispatch: {timing_event}" + ); + release_execution_gate_tx + .send(()) + .expect("execution gate task should remain available"); + execution_gate_task + .await + .expect("execution gate task should join"); + + Ok(()) + } struct ImmediateHandler { tool_name: codex_tools::ToolName, diff --git a/codex-rs/core/src/tools/router.rs b/codex-rs/core/src/tools/router.rs index 4bdd121f6e..aca86b124b 100644 --- a/codex-rs/core/src/tools/router.rs +++ b/codex-rs/core/src/tools/router.rs @@ -25,7 +25,7 @@ use tracing::instrument; pub use crate::tools::context::ToolCallSource; -#[derive(Clone, Debug)] +#[derive(Clone, Debug, PartialEq)] pub struct ToolCall { pub tool_name: ToolName, pub call_id: String, @@ -147,11 +147,12 @@ impl ToolRouter { ResponseItem::ToolSearchCall { .. } => Ok(None), ResponseItem::CustomToolCall { name, + namespace, input, call_id, .. } => Ok(Some(ToolCall { - tool_name: ToolName::plain(name), + tool_name: ToolName::new(namespace, name), call_id, payload: ToolPayload::Custom { input }, })), diff --git a/codex-rs/core/src/tools/router_tests.rs b/codex-rs/core/src/tools/router_tests.rs index db1209cc03..62391395f8 100644 --- a/codex-rs/core/src/tools/router_tests.rs +++ b/codex-rs/core/src/tools/router_tests.rs @@ -110,8 +110,8 @@ async fn parallel_support_does_not_match_namespaced_local_tool_names() -> anyhow let step_context = StepContext::for_test(Arc::clone(&turn)); let mcp_tools = session .services - .mcp_connection_manager - .load_full() + .latest_mcp_runtime() + .manager() .list_all_tools() .await; let router = ToolRouter::from_context( @@ -179,6 +179,35 @@ async fn build_tool_call_uses_namespace_for_registry_name() -> anyhow::Result<() Ok(()) } +#[tokio::test] +async fn build_custom_tool_call_uses_namespace_for_registry_name() -> anyhow::Result<()> { + let tool_name = "exec".to_string(); + + let call = ToolRouter::build_tool_call(ResponseItem::CustomToolCall { + id: None, + status: None, + call_id: "call-namespace".to_string(), + name: tool_name.clone(), + namespace: Some("mcp__python".to_string()), + input: "print('hello')".to_string(), + internal_chat_message_metadata_passthrough: None, + })? + .expect("custom_tool_call should produce a tool call"); + + assert_eq!( + call, + ToolCall { + tool_name: ToolName::namespaced("mcp__python", tool_name), + call_id: "call-namespace".to_string(), + payload: ToolPayload::Custom { + input: "print('hello')".to_string(), + }, + } + ); + + Ok(()) +} + #[tokio::test] async fn mcp_parallel_support_uses_handler_data() -> anyhow::Result<()> { let (_, turn) = make_session_and_context().await; diff --git a/codex-rs/core/src/tools/runtimes/apply_patch.rs b/codex-rs/core/src/tools/runtimes/apply_patch.rs index 840a0dd01e..b839b05c44 100644 --- a/codex-rs/core/src/tools/runtimes/apply_patch.rs +++ b/codex-rs/core/src/tools/runtimes/apply_patch.rs @@ -4,11 +4,10 @@ //! selected turn environment filesystem for both local and remote turns, with //! sandboxing enforced by the explicit filesystem sandbox context. use crate::exec::is_likely_sandbox_denied; -use crate::guardian::GuardianApprovalRequest; -use crate::guardian::review_approval_request; use crate::session::turn_context::TurnEnvironment; use crate::tools::hook_names::HookToolName; use crate::tools::sandboxing::Approvable; +use crate::tools::sandboxing::ApprovalAction; use crate::tools::sandboxing::ApprovalCtx; use crate::tools::sandboxing::ExecApprovalRequirement; use crate::tools::sandboxing::PermissionRequestPayload; @@ -77,7 +76,7 @@ impl ApplyPatchRuntime { fn build_guardian_review_request( req: &ApplyPatchRequest, call_id: &str, - ) -> std::io::Result { + ) -> std::io::Result { // TODO(anp): Remove this conversion once the guardian API supports PathUri. let cwd = req.action.cwd.to_abs_path()?; let files = req @@ -85,7 +84,7 @@ impl ApplyPatchRuntime { .iter() .map(PathUri::to_abs_path) .collect::>>()?; - Ok(GuardianApprovalRequest::ApplyPatch { + Ok(ApprovalAction::ApplyPatch { id: call_id.to_string(), cwd, files, @@ -152,22 +151,7 @@ impl Approvable for ApplyPatchRuntime { let retry_reason = ctx.retry_reason.clone(); let approval_keys = self.approval_keys(req); let changes = req.changes.clone(); - let guardian_review_id = ctx.guardian_review_id.clone(); Box::pin(async move { - if let Some(review_id) = guardian_review_id { - let action = match ApplyPatchRuntime::build_guardian_review_request( - req, - ctx.call_id, - ) { - Ok(action) => action, - Err(err) => { - tracing::error!(cwd = %req.action.cwd, %err, "guardian apply_patch cwd is not host-native"); - return ReviewDecision::Abort; - } - }; - return review_approval_request(session, turn, review_id, action, retry_reason) - .await; - } if req.permissions_preapproved && retry_reason.is_none() { return ReviewDecision::Approved; } @@ -201,6 +185,14 @@ impl Approvable for ApplyPatchRuntime { }) } + fn approval_action( + &self, + req: &ApplyPatchRequest, + ctx: &ApprovalCtx<'_>, + ) -> std::io::Result { + ApplyPatchRuntime::build_guardian_review_request(req, ctx.call_id) + } + fn wants_no_sandbox_approval(&self, policy: AskForApproval) -> bool { match policy { AskForApproval::Never => false, diff --git a/codex-rs/core/src/tools/runtimes/apply_patch_tests.rs b/codex-rs/core/src/tools/runtimes/apply_patch_tests.rs index f1c6f43aa3..c70002e62f 100644 --- a/codex-rs/core/src/tools/runtimes/apply_patch_tests.rs +++ b/codex-rs/core/src/tools/runtimes/apply_patch_tests.rs @@ -80,7 +80,7 @@ async fn guardian_review_request_includes_patch_context() { assert_eq!( guardian_request, - GuardianApprovalRequest::ApplyPatch { + ApprovalAction::ApplyPatch { id: "call-1".to_string(), cwd: expected_cwd, files: vec![path], @@ -232,6 +232,7 @@ async fn file_system_sandbox_context_uses_active_attempt() { windows_sandbox_level: WindowsSandboxLevel::RestrictedToken, windows_sandbox_private_desktop: true, network_denial_cancellation_token: None, + network_proxy: None, }; let sandbox = ApplyPatchRuntime::file_system_sandbox_context_for_attempt(&req, &attempt) @@ -300,6 +301,7 @@ async fn no_sandbox_attempt_has_no_file_system_context() { windows_sandbox_level: WindowsSandboxLevel::Disabled, windows_sandbox_private_desktop: false, network_denial_cancellation_token: None, + network_proxy: None, }; assert_eq!( diff --git a/codex-rs/core/src/tools/runtimes/mod.rs b/codex-rs/core/src/tools/runtimes/mod.rs index db55cc2bbc..1a83ef5657 100644 --- a/codex-rs/core/src/tools/runtimes/mod.rs +++ b/codex-rs/core/src/tools/runtimes/mod.rs @@ -4,6 +4,7 @@ Module: runtimes Concrete ToolRuntime implementations for specific tools. Each runtime stays small and focused and reuses the orchestrator for approvals + sandbox + retry. */ +use crate::exec_env::CODEX_PERMISSION_PROFILE_ENV_VAR; use crate::exec_env::CODEX_THREAD_ID_ENV_VAR; use crate::sandboxing::SandboxPermissions; use crate::shell::Shell; @@ -287,10 +288,14 @@ pub(crate) fn maybe_wrap_shell_lc_with_snapshot( .map(|arg| format!(" '{}'", shell_single_quote(arg))) .collect::(); let mut override_env = explicit_env_overrides.clone(); - if let Some(thread_id) = env.get(CODEX_THREAD_ID_ENV_VAR) { - override_env.insert(CODEX_THREAD_ID_ENV_VAR.to_string(), thread_id.clone()); + for key in [CODEX_THREAD_ID_ENV_VAR, CODEX_PERMISSION_PROFILE_ENV_VAR] { + if let Some(value) = env.get(key) { + override_env.insert(key.to_string(), value.clone()); + } } - let (override_captures, override_exports) = build_override_exports(&override_env); + // Do not let a snapshot resurrect a stale profile when no named profile is active. + let (override_captures, override_exports) = + build_override_exports(&override_env, &[CODEX_PERMISSION_PROFILE_ENV_VAR]); let (proxy_captures, proxy_exports) = build_proxy_env_exports(); let runtime_path_prepend_exports = runtime_path_prepends.shell_exports_after_snapshot(explicit_env_overrides); @@ -313,13 +318,18 @@ pub(crate) fn maybe_wrap_shell_lc_with_snapshot( vec![shell_path.to_string(), "-c".to_string(), rewritten_script] } -fn build_override_exports(explicit_env_overrides: &HashMap) -> (String, String) { +fn build_override_exports( + explicit_env_overrides: &HashMap, + restore_even_when_absent: &[&str], +) -> (String, String) { let mut keys = explicit_env_overrides .keys() .map(String::as_str) + .chain(restore_even_when_absent.iter().copied()) .filter(|key| is_valid_shell_variable_name(key)) .collect::>(); keys.sort_unstable(); + keys.dedup(); build_override_exports_for_keys("__CODEX_SNAPSHOT_OVERRIDE", &keys) } diff --git a/codex-rs/core/src/tools/runtimes/mod_tests.rs b/codex-rs/core/src/tools/runtimes/mod_tests.rs index 9b485dbe58..f6ec38f6c2 100644 --- a/codex-rs/core/src/tools/runtimes/mod_tests.rs +++ b/codex-rs/core/src/tools/runtimes/mod_tests.rs @@ -118,6 +118,7 @@ async fn explicit_escalation_prepares_exec_without_managed_network() -> anyhow:: windows_sandbox_level: WindowsSandboxLevel::Disabled, windows_sandbox_private_desktop: false, network_denial_cancellation_token: None, + network_proxy: None, }; let exec_request = attempt @@ -534,6 +535,78 @@ fn maybe_wrap_shell_lc_with_snapshot_restores_codex_thread_id_from_env() { assert_eq!(String::from_utf8_lossy(&output.stdout), "nested-thread"); } +#[test] +fn maybe_wrap_shell_lc_with_snapshot_restores_permission_profile_from_env() { + let dir = tempdir().expect("create temp dir"); + let snapshot_path = dir.path().join("snapshot.sh"); + std::fs::write( + &snapshot_path, + "# Snapshot file\nexport CODEX_PERMISSION_PROFILE='parent-profile'\n", + ) + .expect("write snapshot"); + let (session_shell, shell_snapshot) = + shell_with_snapshot(ShellType::Bash, "/bin/bash", snapshot_path.abs()); + let command = vec![ + "/bin/bash".to_string(), + "-lc".to_string(), + "printenv CODEX_PERMISSION_PROFILE".to_string(), + ]; + let env = HashMap::from([( + CODEX_PERMISSION_PROFILE_ENV_VAR.to_string(), + "current-profile".to_string(), + )]); + let rewritten = maybe_wrap_shell_lc_with_snapshot( + &command, + &session_shell, + Some(&shell_snapshot), + &HashMap::new(), + &env, + &RuntimePathPrepends::default(), + ); + let output = Command::new(&rewritten[0]) + .args(&rewritten[1..]) + .env(CODEX_PERMISSION_PROFILE_ENV_VAR, "current-profile") + .output() + .expect("run rewritten command"); + + assert!(output.status.success(), "command failed: {output:?}"); + assert_eq!(String::from_utf8_lossy(&output.stdout), "current-profile\n"); +} + +#[test] +fn maybe_wrap_shell_lc_with_snapshot_unsets_absent_permission_profile() { + let dir = tempdir().expect("create temp dir"); + let snapshot_path = dir.path().join("snapshot.sh"); + std::fs::write( + &snapshot_path, + "# Snapshot file\nexport CODEX_PERMISSION_PROFILE='stale-profile'\n", + ) + .expect("write snapshot"); + let (session_shell, shell_snapshot) = + shell_with_snapshot(ShellType::Bash, "/bin/bash", snapshot_path.abs()); + let command = vec![ + "/bin/bash".to_string(), + "-lc".to_string(), + "printenv CODEX_PERMISSION_PROFILE".to_string(), + ]; + let rewritten = maybe_wrap_shell_lc_with_snapshot( + &command, + &session_shell, + Some(&shell_snapshot), + &HashMap::new(), + &HashMap::new(), + &RuntimePathPrepends::default(), + ); + let output = Command::new(&rewritten[0]) + .args(&rewritten[1..]) + .env_remove(CODEX_PERMISSION_PROFILE_ENV_VAR) + .output() + .expect("run rewritten command"); + + assert_eq!(output.status.code(), Some(1)); + assert_eq!(output.stdout, b""); +} + #[test] fn maybe_wrap_shell_lc_with_snapshot_restores_proxy_env_from_process_env() { let dir = tempdir().expect("create temp dir"); diff --git a/codex-rs/core/src/tools/runtimes/shell.rs b/codex-rs/core/src/tools/runtimes/shell.rs index 0d386103db..3ff50f3f51 100644 --- a/codex-rs/core/src/tools/runtimes/shell.rs +++ b/codex-rs/core/src/tools/runtimes/shell.rs @@ -10,9 +10,7 @@ pub(crate) mod zsh_fork_backend; use crate::command_canonicalization::canonicalize_command_for_approval; use crate::exec::ExecCapturePolicy; -use crate::guardian::GuardianApprovalRequest; use crate::guardian::GuardianNetworkAccessTrigger; -use crate::guardian::review_approval_request; use crate::sandboxing::ExecOptions; use crate::sandboxing::SandboxPermissions; use crate::sandboxing::execute_env; @@ -29,6 +27,7 @@ use crate::tools::runtimes::disable_powershell_profile_for_elevated_windows_sand use crate::tools::runtimes::exec_env_for_sandbox_permissions; use crate::tools::runtimes::maybe_wrap_shell_lc_with_snapshot; use crate::tools::sandboxing::Approvable; +use crate::tools::sandboxing::ApprovalAction; use crate::tools::sandboxing::ApprovalCtx; use crate::tools::sandboxing::ExecApprovalRequirement; use crate::tools::sandboxing::PermissionRequestPayload; @@ -145,30 +144,14 @@ impl Approvable for ShellRuntime { let command = req.command.clone(); let cwd = req.cwd.clone(); let environment_id = Some(req.turn_environment.environment_id.clone()); - let retry_reason = ctx.retry_reason.clone(); - let reason = retry_reason.clone().or_else(|| req.justification.clone()); + let reason = ctx + .retry_reason + .clone() + .or_else(|| req.justification.clone()); let session = ctx.session; let turn = ctx.turn; let call_id = ctx.call_id.to_string(); - let guardian_review_id = ctx.guardian_review_id.clone(); Box::pin(async move { - if let Some(review_id) = guardian_review_id { - return review_approval_request( - session, - turn, - review_id, - GuardianApprovalRequest::Shell { - id: call_id, - command, - cwd: cwd.clone(), - sandbox_permissions: req.sandbox_permissions, - additional_permissions: req.additional_permissions.clone(), - justification: req.justification.clone(), - }, - retry_reason, - ) - .await; - } with_cached_approval(&session.services, "shell", keys, move || async move { let available_decisions = None; session @@ -193,6 +176,21 @@ impl Approvable for ShellRuntime { }) } + fn approval_action( + &self, + req: &ShellRequest, + ctx: &ApprovalCtx<'_>, + ) -> std::io::Result { + Ok(ApprovalAction::Shell { + id: ctx.call_id.to_string(), + command: req.command.clone(), + cwd: req.cwd.clone(), + sandbox_permissions: req.sandbox_permissions, + additional_permissions: req.additional_permissions.clone(), + justification: req.justification.clone(), + }) + } + fn exec_approval_requirement(&self, req: &ShellRequest) -> Option { Some(req.exec_approval_requirement.clone()) } diff --git a/codex-rs/core/src/tools/runtimes/unified_exec.rs b/codex-rs/core/src/tools/runtimes/unified_exec.rs index 5df42cc979..a1b9b33c6d 100644 --- a/codex-rs/core/src/tools/runtimes/unified_exec.rs +++ b/codex-rs/core/src/tools/runtimes/unified_exec.rs @@ -7,9 +7,7 @@ the process manager to spawn PTYs once an ExecRequest is prepared. use crate::command_canonicalization::canonicalize_command_for_approval; use crate::exec::ExecCapturePolicy; use crate::exec::ExecExpiration; -use crate::guardian::GuardianApprovalRequest; use crate::guardian::GuardianNetworkAccessTrigger; -use crate::guardian::review_approval_request; use crate::sandboxing::ExecOptions; use crate::sandboxing::ExecServerEnvConfig; use crate::sandboxing::SandboxPermissions; @@ -26,6 +24,7 @@ use crate::tools::runtimes::exec_env_for_sandbox_permissions; use crate::tools::runtimes::maybe_wrap_shell_lc_with_snapshot; use crate::tools::runtimes::shell::zsh_fork_backend; use crate::tools::sandboxing::Approvable; +use crate::tools::sandboxing::ApprovalAction; use crate::tools::sandboxing::ApprovalCtx; use crate::tools::sandboxing::ExecApprovalRequirement; use crate::tools::sandboxing::PermissionRequestPayload; @@ -179,9 +178,10 @@ impl Approvable for UnifiedExecRuntime<'_> { let call_id = ctx.call_id.to_string(); let command = req.command.clone(); let environment_id = Some(req.turn_environment.environment_id.clone()); - let retry_reason = ctx.retry_reason.clone(); - let reason = retry_reason.clone().or_else(|| req.justification.clone()); - let guardian_review_id = ctx.guardian_review_id.clone(); + let reason = ctx + .retry_reason + .clone() + .or_else(|| req.justification.clone()); Box::pin(async move { let native_cwd = match req.cwd.to_abs_path() { Ok(c) => c, @@ -192,24 +192,6 @@ impl Approvable for UnifiedExecRuntime<'_> { return ReviewDecision::Abort; } }; - if let Some(review_id) = guardian_review_id { - return review_approval_request( - session, - turn, - review_id, - GuardianApprovalRequest::ExecCommand { - id: call_id, - command, - cwd: native_cwd.clone(), - sandbox_permissions: req.sandbox_permissions, - additional_permissions: req.additional_permissions.clone(), - justification: req.justification.clone(), - tty: req.tty, - }, - retry_reason, - ) - .await; - } with_cached_approval(&session.services, "unified_exec", keys, || async move { let available_decisions = None; session @@ -234,6 +216,22 @@ impl Approvable for UnifiedExecRuntime<'_> { }) } + fn approval_action( + &self, + req: &UnifiedExecRequest, + ctx: &ApprovalCtx<'_>, + ) -> std::io::Result { + Ok(ApprovalAction::ExecCommand { + id: ctx.call_id.to_string(), + command: req.command.clone(), + cwd: req.cwd.to_abs_path()?, + sandbox_permissions: req.sandbox_permissions, + additional_permissions: req.additional_permissions.clone(), + justification: req.justification.clone(), + tty: req.tty, + }) + } + fn exec_approval_requirement( &self, req: &UnifiedExecRequest, @@ -320,10 +318,10 @@ impl<'a> ToolRuntime for UnifiedExecRunt req.sandbox_permissions, &file_system_sandbox_policy, ); - let managed_network = managed_network_for_sandbox_permissions( + let managed_network = attempt.network_proxy(managed_network_for_sandbox_permissions( req.network.as_ref(), launch_sandbox_permissions, - ); + )); let env = exec_env_for_sandbox_permissions(&req.env, launch_sandbox_permissions); let (env, managed_network_context) = match managed_network { Some(network) => { diff --git a/codex-rs/core/src/tools/sandboxing.rs b/codex-rs/core/src/tools/sandboxing.rs index 8e9e274f24..571c5ce6df 100644 --- a/codex-rs/core/src/tools/sandboxing.rs +++ b/codex-rs/core/src/tools/sandboxing.rs @@ -133,6 +133,8 @@ pub(crate) struct ApprovalCtx<'a> { pub network_approval_context: Option, } +pub(crate) type ApprovalAction = crate::guardian::GuardianApprovalRequest; + #[derive(Clone, Debug, PartialEq, Eq)] pub(crate) struct PermissionRequestPayload { pub tool_name: HookToolName, @@ -368,6 +370,8 @@ pub(crate) trait Approvable { req: &'a Req, ctx: ApprovalCtx<'a>, ) -> BoxFuture<'a, ReviewDecision>; + + fn approval_action(&self, req: &Req, ctx: &ApprovalCtx<'_>) -> std::io::Result; } pub(crate) trait Sandboxable { @@ -423,9 +427,17 @@ pub(crate) struct SandboxAttempt<'a> { pub windows_sandbox_level: codex_protocol::config_types::WindowsSandboxLevel, pub windows_sandbox_private_desktop: bool, pub network_denial_cancellation_token: Option, + pub(crate) network_proxy: Option<&'a NetworkProxy>, } impl<'a> SandboxAttempt<'a> { + pub(crate) fn network_proxy<'b>( + &'b self, + fallback: Option<&'b NetworkProxy>, + ) -> Option<&'b NetworkProxy> { + fallback.map(|fallback| self.network_proxy.unwrap_or(fallback)) + } + pub fn env_for( &self, command: SandboxCommand, @@ -433,6 +445,7 @@ impl<'a> SandboxAttempt<'a> { network: Option<&NetworkProxy>, environment_id: Option<&str>, ) -> Result { + let network = self.network_proxy(network); let request = self .manager .transform(SandboxTransformRequest { @@ -465,6 +478,7 @@ impl<'a> SandboxAttempt<'a> { network: Option<&NetworkProxy>, environment_id: Option<&str>, ) -> Result { + let network = self.network_proxy(network); let managed_network = command.managed_network.clone(); let exec_server_permissions = effective_permission_profile( self.exec_server_permissions, diff --git a/codex-rs/core/src/tools/sandboxing_tests.rs b/codex-rs/core/src/tools/sandboxing_tests.rs index 28b9de9ce1..e52186f772 100644 --- a/codex-rs/core/src/tools/sandboxing_tests.rs +++ b/codex-rs/core/src/tools/sandboxing_tests.rs @@ -227,6 +227,7 @@ fn exec_server_env_keeps_command_native_and_carries_sandbox_context() { windows_sandbox_level: codex_protocol::config_types::WindowsSandboxLevel::Disabled, windows_sandbox_private_desktop: false, network_denial_cancellation_token: None, + network_proxy: None, }; let managed_network = ManagedNetworkSandboxContext { loopback_ports: vec![43123], diff --git a/codex-rs/core/src/tools/spec_plan.rs b/codex-rs/core/src/tools/spec_plan.rs index b873f87e27..8ca5eec898 100644 --- a/codex-rs/core/src/tools/spec_plan.rs +++ b/codex-rs/core/src/tools/spec_plan.rs @@ -98,7 +98,6 @@ use tracing::warn; const MULTI_AGENT_V2_NAMESPACE_DESCRIPTION: &str = "Tools for spawning and managing sub-agents."; const IMAGE_GEN_NAMESPACE: &str = "image_gen"; const IMAGEGEN_TOOL_NAME: &str = "imagegen"; -const ACTOR_AUTHORIZATION_HEADER: &str = "x-openai-actor-authorization"; type PlannedRuntime = Arc; @@ -387,7 +386,10 @@ fn image_generation_tool_enabled(turn_context: &TurnContext) -> bool { } fn image_generation_runtime_enabled(turn_context: &TurnContext) -> bool { - (provider_uses_actor_authorization(turn_context) + (turn_context + .provider + .info() + .uses_openai_actor_authorization() || (turn_context.provider.info().requires_openai_auth && turn_context .auth_manager @@ -400,16 +402,6 @@ fn image_generation_runtime_enabled(turn_context: &TurnContext) -> bool { .contains(&InputModality::Image) } -fn provider_uses_actor_authorization(turn_context: &TurnContext) -> bool { - let provider_info = turn_context.provider.info(); - !provider_info.requires_openai_auth - && provider_info.http_headers.as_ref().is_some_and(|headers| { - headers.iter().any(|(name, value)| { - name.eq_ignore_ascii_case(ACTOR_AUTHORIZATION_HEADER) && !value.trim().is_empty() - }) - }) -} - fn standalone_image_generation_model_visible(turn_context: &TurnContext) -> bool { if !image_generation_runtime_enabled(turn_context) || !namespace_tools_enabled(turn_context) { return false; diff --git a/codex-rs/core/src/tools/spec_plan_tests.rs b/codex-rs/core/src/tools/spec_plan_tests.rs index c422cc749e..d95cb5ff44 100644 --- a/codex-rs/core/src/tools/spec_plan_tests.rs +++ b/codex-rs/core/src/tools/spec_plan_tests.rs @@ -275,22 +275,19 @@ fn use_bedrock_provider(turn: &mut TurnContext) { turn.provider = create_model_provider(provider_info, turn.auth_manager.clone()); } -fn use_provider_auth( - turn: &mut TurnContext, - requires_openai_auth: bool, - actor_header: Option<(&str, &str)>, -) { +fn use_actor_authorized_provider(turn: &mut TurnContext) { let mut provider_info = turn.config.model_provider.clone(); - provider_info.requires_openai_auth = requires_openai_auth; - provider_info.http_headers = actor_header.map(|(name, value)| { - HashMap::from([ - (name.to_string(), value.to_string()), - ( - "ChatGPT-Account-ID".to_string(), - "test-account-id".to_string(), - ), - ]) - }); + provider_info.requires_openai_auth = false; + provider_info.http_headers = Some(HashMap::from([ + ( + "x-openai-actor-authorization".to_string(), + "test-actor-authorization".to_string(), + ), + ( + "ChatGPT-Account-ID".to_string(), + "test-account-id".to_string(), + ), + ])); turn.auth_manager = None; update_config(turn, |config| { config.model_provider = provider_info.clone(); @@ -686,9 +683,12 @@ async fn environment_tools_follow_the_step_context() { let environments = turn.environments.clone(); turn.environments.turn_environments.clear(); + let turn = Arc::new(turn); let step_context = Arc::new(StepContext::new( - Arc::new(turn), + Arc::clone(&turn), environments, + Vec::new(), + crate::session::McpRuntimeSnapshot::new_uninitialized_for_test(&turn.config), /*loaded_agents_md*/ None, )); @@ -1549,47 +1549,17 @@ async fn hosted_tools_follow_provider_auth_model_and_config_gates() { .await; unrelated_chatgpt_auth.assert_visible_lacks(&["image_generation"]); - let provider_without_actor_auth = probe(|turn| { + let actor_authorized_provider = probe(|turn| { set_feature(turn, Feature::ImageGeneration, /*enabled*/ true); - use_provider_auth( - turn, /*requires_openai_auth*/ false, /*actor_header*/ None, - ); + use_actor_authorized_provider(turn); turn.model_info.input_modalities = vec![InputModality::Image]; }) .await; - provider_without_actor_auth.assert_visible_lacks(&["image_generation"]); - - let provider_authenticated = probe(|turn| { - set_feature(turn, Feature::ImageGeneration, /*enabled*/ true); - use_provider_auth( - turn, - /*requires_openai_auth*/ false, - Some(("X-OpenAI-Actor-Authorization", "test-actor-authorization")), - ); - turn.model_info.input_modalities = vec![InputModality::Image]; - }) - .await; - provider_authenticated.assert_visible_contains(&["image_generation"]); - - let empty_actor_auth = probe(|turn| { - set_feature(turn, Feature::ImageGeneration, /*enabled*/ true); - use_provider_auth( - turn, - /*requires_openai_auth*/ false, - Some(("x-openai-actor-authorization", " ")), - ); - turn.model_info.input_modalities = vec![InputModality::Image]; - }) - .await; - empty_actor_auth.assert_visible_lacks(&["image_generation"]); + actor_authorized_provider.assert_visible_contains(&["image_generation"]); let feature_disabled = probe(|turn| { set_feature(turn, Feature::ImageGeneration, /*enabled*/ false); - use_provider_auth( - turn, - /*requires_openai_auth*/ false, - Some(("x-openai-actor-authorization", "test-actor-authorization")), - ); + use_actor_authorized_provider(turn); turn.model_info.input_modalities = vec![InputModality::Image]; }) .await; @@ -1597,11 +1567,7 @@ async fn hosted_tools_follow_provider_auth_model_and_config_gates() { let text_only_model = probe(|turn| { set_feature(turn, Feature::ImageGeneration, /*enabled*/ true); - use_provider_auth( - turn, - /*requires_openai_auth*/ false, - Some(("x-openai-actor-authorization", "test-actor-authorization")), - ); + use_actor_authorized_provider(turn); turn.model_info.input_modalities = vec![]; }) .await; @@ -1610,34 +1576,12 @@ async fn hosted_tools_follow_provider_auth_model_and_config_gates() { let unsupported_image_generation_provider = probe(|turn| { set_feature(turn, Feature::ImageGeneration, /*enabled*/ true); use_bedrock_provider(turn); - let mut provider_info = turn.provider.info().clone(); - provider_info.requires_openai_auth = false; - provider_info.http_headers = Some(HashMap::from([( - "x-openai-actor-authorization".to_string(), - "test-actor-authorization".to_string(), - )])); - turn.auth_manager = None; - update_config(turn, |config| { - config.model_provider = provider_info.clone(); - }); - turn.provider = create_model_provider(provider_info, /*auth_manager*/ None); + use_actor_authorized_provider(turn); turn.model_info.input_modalities = vec![InputModality::Image]; }) .await; unsupported_image_generation_provider.assert_visible_lacks(&["image_generation"]); - let codex_managed_auth_provider = probe(|turn| { - set_feature(turn, Feature::ImageGeneration, /*enabled*/ true); - use_provider_auth( - turn, - /*requires_openai_auth*/ true, - Some(("x-openai-actor-authorization", "test-actor-authorization")), - ); - turn.model_info.input_modalities = vec![InputModality::Image]; - }) - .await; - codex_managed_auth_provider.assert_visible_lacks(&["image_generation"]); - let image_generation = probe(|turn| { use_chatgpt_auth(turn); set_feature(turn, Feature::ImageGeneration, /*enabled*/ true); diff --git a/codex-rs/core/src/turn_timing_tests.rs b/codex-rs/core/src/turn_timing_tests.rs index 956468da23..fcb34b4742 100644 --- a/codex-rs/core/src/turn_timing_tests.rs +++ b/codex-rs/core/src/turn_timing_tests.rs @@ -121,6 +121,7 @@ fn response_item_records_turn_ttft_for_first_output_signals() { status: None, call_id: "call-2".to_string(), name: "custom".to_string(), + namespace: None, input: "echo hi".to_string(), internal_chat_message_metadata_passthrough: None, } diff --git a/codex-rs/core/src/unified_exec/mod_tests.rs b/codex-rs/core/src/unified_exec/mod_tests.rs index 54ee788e3b..81403c8e52 100644 --- a/codex-rs/core/src/unified_exec/mod_tests.rs +++ b/codex-rs/core/src/unified_exec/mod_tests.rs @@ -160,7 +160,7 @@ async fn exec_command_with_tty( &output_closed, &output_closed_notify, &cancellation_token, - Some(session.subscribe_out_of_band_elicitation_pause_state()), + Some(session.subscribe_elicitation_pause_state()), deadline, ) .await; @@ -515,12 +515,11 @@ async fn unified_exec_pause_blocks_yield_timeout() -> anyhow::Result<()> { skip_if_sandbox!(Ok(())); let (session, turn) = test_session_and_turn().await; - session.set_out_of_band_elicitation_pause_state(/*paused*/ true); + let elicitation = session.services.elicitations.register(); - let paused_session = Arc::clone(&session); tokio::spawn(async move { tokio::time::sleep(Duration::from_secs(2)).await; - paused_session.set_out_of_band_elicitation_pause_state(/*paused*/ false); + drop(elicitation); }); let started = tokio::time::Instant::now(); diff --git a/codex-rs/core/src/unified_exec/process.rs b/codex-rs/core/src/unified_exec/process.rs index 5f785adfa7..7bedfd43e3 100644 --- a/codex-rs/core/src/unified_exec/process.rs +++ b/codex-rs/core/src/unified_exec/process.rs @@ -14,6 +14,7 @@ use tokio_util::sync::CancellationToken; use crate::exec::is_likely_sandbox_denied; use codex_exec_server::ExecProcess; +use codex_exec_server::ExecProcessEvent; use codex_exec_server::ProcessSignal as ExecServerProcessSignal; use codex_exec_server::ReadResponse as ExecReadResponse; use codex_exec_server::StartedExecProcess; @@ -425,83 +426,150 @@ impl UnifiedExecProcess { cancellation_token, } = output_handles; let process = started.process; - let mut wake_rx = process.subscribe_wake(); + let mut events = process.subscribe_events(); tokio::spawn(async move { - let mut after_seq = None; + let mut last_seq: u64 = 0; loop { - match process - .read(after_seq, /*max_bytes*/ None, /*wait_ms*/ Some(0)) - .await - { - Ok(response) => { - let ExecReadResponse { - chunks, - next_seq, - exited, - exit_code, - closed, - failure, - sandbox_denied, - } = response; - - for chunk in chunks { - let bytes = chunk.chunk.into_inner(); - let mut guard = output_buffer.lock().await; - guard.push_chunk(bytes.clone()); - drop(guard); - let _ = output_tx.send(bytes); - output_notify.notify_waiters(); - } - - if let Some(message) = failure { - let state = state_tx.borrow().clone(); - let _ = state_tx.send_replace(state.failed(message)); - output_closed.store(true, Ordering::Release); - output_closed_notify.notify_waiters(); - cancellation_token.cancel(); - break; - } - - if sandbox_denied { - let mut state = state_tx.borrow().clone(); - state.sandbox_denied = true; - let _ = state_tx.send_replace(state); - } - - if exited { - let state = state_tx.borrow().clone(); - let _ = state_tx.send_replace(state.exited(exit_code)); - } - - if closed { - output_closed.store(true, Ordering::Release); - output_closed_notify.notify_waiters(); - cancellation_token.cancel(); - } - - after_seq = next_seq.checked_sub(1); - if output_closed.load(Ordering::Acquire) { - break; - } - } - Err(err) => { + let event = match events.recv().await { + Ok(event) => Some(event), + Err(broadcast::error::RecvError::Lagged(_)) => None, + Err(broadcast::error::RecvError::Closed) => { let state = state_tx.borrow().clone(); - let _ = state_tx.send_replace(state.failed(err.to_string())); + let _ = state_tx.send_replace( + state.failed("exec-server process event stream closed".to_string()), + ); output_closed.store(true, Ordering::Release); output_closed_notify.notify_waiters(); cancellation_token.cancel(); break; } + }; + let event_seq = event.as_ref().and_then(|event| match event { + ExecProcessEvent::Output(chunk) => Some(chunk.seq), + ExecProcessEvent::Exited { seq, .. } | ExecProcessEvent::Closed { seq } => { + Some(*seq) + } + ExecProcessEvent::Failed(_) => None, + }); + let missing_sandbox_denial = matches!( + event.as_ref(), + Some(ExecProcessEvent::Exited { + sandbox_denied: None, + .. + }) + ); + if event.is_none() + || event_seq.is_some_and(|seq| seq > last_seq.saturating_add(1)) + || missing_sandbox_denial + { + let response = match process + .read( + Some(last_seq), + /*max_bytes*/ None, + /*wait_ms*/ Some(0), + ) + .await + { + Ok(response) => response, + Err(err) => { + let state = state_tx.borrow().clone(); + let _ = state_tx.send_replace(state.failed(err.to_string())); + output_closed.store(true, Ordering::Release); + output_closed_notify.notify_waiters(); + cancellation_token.cancel(); + break; + } + }; + let ExecReadResponse { + chunks, + next_seq, + exited, + exit_code, + closed, + failure, + sandbox_denied, + } = response; + for chunk in chunks.into_iter().filter(|chunk| chunk.seq > last_seq) { + let bytes = chunk.chunk.into_inner(); + let mut guard = output_buffer.lock().await; + guard.push_chunk(bytes.clone()); + drop(guard); + let _ = output_tx.send(bytes); + output_notify.notify_waiters(); + } + last_seq = last_seq.max(next_seq.saturating_sub(1)); + if let Some(message) = failure { + let state = state_tx.borrow().clone(); + let _ = state_tx.send_replace(state.failed(message)); + output_closed.store(true, Ordering::Release); + output_closed_notify.notify_waiters(); + cancellation_token.cancel(); + break; + } + if sandbox_denied || exited { + let mut state = state_tx.borrow().clone(); + state.sandbox_denied |= sandbox_denied; + let _ = state_tx.send_replace(if exited { + state.exited(exit_code) + } else { + state + }); + } + if closed { + output_closed.store(true, Ordering::Release); + output_closed_notify.notify_waiters(); + cancellation_token.cancel(); + break; + } + continue; } - if wake_rx.changed().await.is_err() { - let state = state_tx.borrow().clone(); - let _ = state_tx - .send_replace(state.failed("exec-server wake channel closed".to_string())); - output_closed.store(true, Ordering::Release); - output_closed_notify.notify_waiters(); - cancellation_token.cancel(); - break; + let Some(event) = event else { + continue; + }; + match event { + ExecProcessEvent::Output(chunk) => { + if chunk.seq <= last_seq { + continue; + } + last_seq = chunk.seq; + let bytes = chunk.chunk.into_inner(); + let mut guard = output_buffer.lock().await; + guard.push_chunk(bytes.clone()); + drop(guard); + let _ = output_tx.send(bytes); + output_notify.notify_waiters(); + } + ExecProcessEvent::Exited { + seq, + exit_code, + sandbox_denied, + } => { + if seq <= last_seq { + continue; + } + last_seq = seq; + let mut state = state_tx.borrow().clone(); + state.sandbox_denied |= sandbox_denied.unwrap_or(false); + let _ = state_tx.send_replace(state.exited(Some(exit_code))); + } + ExecProcessEvent::Closed { seq } => { + if seq <= last_seq { + continue; + } + output_closed.store(true, Ordering::Release); + output_closed_notify.notify_waiters(); + cancellation_token.cancel(); + break; + } + ExecProcessEvent::Failed(message) => { + let state = state_tx.borrow().clone(); + let _ = state_tx.send_replace(state.failed(message)); + output_closed.store(true, Ordering::Release); + output_closed_notify.notify_waiters(); + cancellation_token.cancel(); + break; + } } } }) diff --git a/codex-rs/core/src/unified_exec/process_manager.rs b/codex-rs/core/src/unified_exec/process_manager.rs index 5e89445765..a895708f79 100644 --- a/codex-rs/core/src/unified_exec/process_manager.rs +++ b/codex-rs/core/src/unified_exec/process_manager.rs @@ -13,8 +13,10 @@ use tokio_util::sync::CancellationToken; use uuid::Uuid; use crate::codex_thread::BackgroundTerminalInfo; +use crate::exec_env::CODEX_PERMISSION_PROFILE_ENV_VAR; use crate::exec_env::CODEX_THREAD_ID_ENV_VAR; use crate::exec_env::create_env; +use crate::exec_env::inject_permission_profile_env; use crate::exec_policy::ExecApprovalRequest; use crate::sandboxing::ExecOptions; use crate::sandboxing::ExecRequest; @@ -109,15 +111,19 @@ fn apply_unified_exec_env(mut env: HashMap) -> HashMap codex_exec_server::ExecEnvPolicy { + let mut exclude = policy + .exclude + .iter() + .map(std::string::ToString::to_string) + .collect::>(); + exclude.push(CODEX_PERMISSION_PROFILE_ENV_VAR.to_string()); + let mut r#set = policy.r#set.clone(); + r#set.retain(|key, _| !key.eq_ignore_ascii_case(CODEX_PERMISSION_PROFILE_ENV_VAR)); codex_exec_server::ExecEnvPolicy { inherit: policy.inherit.clone(), ignore_default_excludes: policy.ignore_default_excludes, - exclude: policy - .exclude - .iter() - .map(std::string::ToString::to_string) - .collect(), - r#set: policy.r#set.clone(), + exclude, + r#set, include_only: policy .include_only .iter() @@ -132,7 +138,10 @@ fn env_overlay_for_exec_server( ) -> HashMap { request_env .iter() - .filter(|(key, value)| local_policy_env.get(*key) != Some(*value)) + .filter(|(key, value)| { + key.as_str() == CODEX_PERMISSION_PROFILE_ENV_VAR + || local_policy_env.get(*key) != Some(*value) + }) .map(|(key, value)| (key.clone(), value.clone())) .collect() } @@ -484,11 +493,7 @@ impl UnifiedExecProcessManager { &output_closed, &output_closed_notify, &cancellation_token, - Some( - context - .session - .subscribe_out_of_band_elicitation_pause_state(), - ), + Some(context.session.subscribe_elicitation_pause_state()), deadline, ) .await; @@ -830,7 +835,7 @@ impl UnifiedExecProcessManager { let pause_state = entry .session .upgrade() - .map(|session| session.subscribe_out_of_band_elicitation_pause_state()); + .map(|session| session.subscribe_elicitation_pause_state()); let session = entry.session.upgrade(); Ok(PreparedProcessHandles { @@ -1110,6 +1115,8 @@ impl UnifiedExecProcessManager { CODEX_THREAD_ID_ENV_VAR.to_string(), context.session.thread_id.to_string(), ); + let active_permission_profile = context.turn.config.permissions.active_permission_profile(); + inject_permission_profile_env(&mut env, active_permission_profile.as_ref()); let env = apply_unified_exec_env(env); let exec_server_env_config = ExecServerEnvConfig { policy: exec_env_policy_from_shell_policy( diff --git a/codex-rs/core/src/unified_exec/process_manager_tests.rs b/codex-rs/core/src/unified_exec/process_manager_tests.rs index 93ffd0cedb..096cbe0a13 100644 --- a/codex-rs/core/src/unified_exec/process_manager_tests.rs +++ b/codex-rs/core/src/unified_exec/process_manager_tests.rs @@ -42,12 +42,20 @@ fn env_overlay_for_exec_server_keeps_runtime_changes_only() { ("HOME".to_string(), "/client-home".to_string()), ("PATH".to_string(), "/client-path".to_string()), ("SHELL_SET".to_string(), "policy".to_string()), + ( + CODEX_PERMISSION_PROFILE_ENV_VAR.to_string(), + "current-profile".to_string(), + ), ]); let request_env = HashMap::from([ ("HOME".to_string(), "/client-home".to_string()), ("PATH".to_string(), "/sandbox-path".to_string()), ("SHELL_SET".to_string(), "policy".to_string()), ("CODEX_THREAD_ID".to_string(), "thread-1".to_string()), + ( + CODEX_PERMISSION_PROFILE_ENV_VAR.to_string(), + "current-profile".to_string(), + ), ( "CODEX_SANDBOX_NETWORK_DISABLED".to_string(), "1".to_string(), @@ -59,6 +67,10 @@ fn env_overlay_for_exec_server_keeps_runtime_changes_only() { HashMap::from([ ("PATH".to_string(), "/sandbox-path".to_string()), ("CODEX_THREAD_ID".to_string(), "thread-1".to_string()), + ( + CODEX_PERMISSION_PROFILE_ENV_VAR.to_string(), + "current-profile".to_string(), + ), ( "CODEX_SANDBOX_NETWORK_DISABLED".to_string(), "1".to_string() @@ -67,6 +79,31 @@ fn env_overlay_for_exec_server_keeps_runtime_changes_only() { ); } +#[test] +fn exec_env_policy_excludes_runtime_permission_profile() { + let policy = ShellEnvironmentPolicy { + r#set: HashMap::from([ + ( + "codex_permission_profile".to_string(), + "stale-profile".to_string(), + ), + ("KEEP".to_string(), "value".to_string()), + ]), + ..Default::default() + }; + + assert_eq!( + exec_env_policy_from_shell_policy(&policy), + codex_exec_server::ExecEnvPolicy { + inherit: policy.inherit, + ignore_default_excludes: policy.ignore_default_excludes, + exclude: vec![CODEX_PERMISSION_PROFILE_ENV_VAR.to_string()], + r#set: HashMap::from([("KEEP".to_string(), "value".to_string())]), + include_only: Vec::new(), + } + ); +} + #[test] fn exec_server_params_use_path_uri_and_env_policy_overlay_contract() { let cwd: codex_utils_absolute_path::AbsolutePathBuf = std::env::current_dir() diff --git a/codex-rs/core/src/unified_exec/process_tests.rs b/codex-rs/core/src/unified_exec/process_tests.rs index db64d461ec..af0cb62741 100644 --- a/codex-rs/core/src/unified_exec/process_tests.rs +++ b/codex-rs/core/src/unified_exec/process_tests.rs @@ -15,7 +15,6 @@ use std::collections::VecDeque; use std::sync::Arc; use tokio::sync::Mutex; use tokio::sync::watch; -use tokio::time::Duration; struct MockExecProcess { process_id: ProcessId, @@ -173,39 +172,3 @@ async fn remote_terminate_confirmed_updates_state_on_success_only() { assert!(process.has_exited()); } - -#[tokio::test] -async fn remote_process_waits_for_early_exit_event() { - let (wake_tx, _wake_rx) = watch::channel(0); - let started = StartedExecProcess { - process: Arc::new(MockExecProcess { - process_id: "test-process".to_string().into(), - write_response: WriteResponse { - status: WriteStatus::Accepted, - }, - read_responses: Mutex::new(VecDeque::from([ReadResponse { - chunks: Vec::new(), - next_seq: 2, - exited: true, - exit_code: Some(17), - closed: true, - failure: None, - sandbox_denied: false, - }])), - terminate_error: None, - wake_tx: wake_tx.clone(), - }), - }; - - tokio::spawn(async move { - tokio::time::sleep(Duration::from_millis(10)).await; - let _ = wake_tx.send(1); - }); - - let process = UnifiedExecProcess::from_exec_server_started(started) - .await - .expect("remote process should observe early exit"); - - assert!(process.has_exited()); - assert_eq!(process.exit_code(), Some(17)); -} diff --git a/codex-rs/core/tests/common/apps_test_server.rs b/codex-rs/core/tests/common/apps_test_server.rs index 0849087d0b..3daaada6c5 100644 --- a/codex-rs/core/tests/common/apps_test_server.rs +++ b/codex-rs/core/tests/common/apps_test_server.rs @@ -7,6 +7,7 @@ use codex_login::CodexAuth; use codex_models_manager::bundled_models_response; use serde_json::Value; use serde_json::json; +use std::sync::Arc; use std::sync::atomic::AtomicUsize; use std::sync::atomic::Ordering; use wiremock::Mock; @@ -55,6 +56,23 @@ pub struct AppsTestServer { pub chatgpt_base_url: String, } +#[derive(Clone)] +pub struct AppsTestServerStartupControl { + initialize_attempts: Arc, + remaining_initialize_failures: Arc, +} + +impl AppsTestServerStartupControl { + pub fn fail_next_initialize_attempts(&self, attempts: usize) { + self.remaining_initialize_failures + .store(attempts, Ordering::SeqCst); + } + + pub fn initialize_attempts(&self) -> usize { + self.initialize_attempts.load(Ordering::SeqCst) + } +} + #[derive(Clone, Copy)] pub enum AppsTestToolLoading { Direct, @@ -130,6 +148,34 @@ impl AppsTestServer { }) } + pub async fn mount_with_startup_control( + server: &MockServer, + ) -> Result<(Self, AppsTestServerStartupControl)> { + mount_oauth_metadata(server).await; + mount_connectors_directory(server).await; + let control = AppsTestServerStartupControl { + initialize_attempts: Arc::new(AtomicUsize::new(0)), + remaining_initialize_failures: Arc::new(AtomicUsize::new(0)), + }; + mount_streamable_http_json_rpc_with_startup_control( + server, + CONNECTOR_NAME.to_string(), + CONNECTOR_DESCRIPTION.to_string(), + /*searchable*/ true, + /*include_app_only_tool*/ false, + AppsTestToolsListBehavior::AlwaysAvailable, + Some(Arc::clone(&control.initialize_attempts)), + Some(Arc::clone(&control.remaining_initialize_failures)), + ) + .await; + Ok(( + Self { + chatgpt_base_url: server.uri(), + }, + control, + )) + } + pub async fn mount_with_tools_available_after_initial_list( server: &MockServer, ) -> Result { @@ -312,6 +358,30 @@ async fn mount_streamable_http_json_rpc( searchable: bool, include_app_only_tool: bool, tools_list_behavior: AppsTestToolsListBehavior, +) { + mount_streamable_http_json_rpc_with_startup_control( + server, + connector_name, + connector_description, + searchable, + include_app_only_tool, + tools_list_behavior, + /*initialize_attempts*/ None, + /*remaining_initialize_failures*/ None, + ) + .await; +} + +#[allow(clippy::too_many_arguments)] +async fn mount_streamable_http_json_rpc_with_startup_control( + server: &MockServer, + connector_name: String, + connector_description: String, + searchable: bool, + include_app_only_tool: bool, + tools_list_behavior: AppsTestToolsListBehavior, + initialize_attempts: Option>, + remaining_initialize_failures: Option>, ) { Mock::given(method("POST")) .and(path_regex("^/api/codex/apps/?$")) @@ -322,6 +392,8 @@ async fn mount_streamable_http_json_rpc( include_app_only_tool, tools_list_behavior, tools_list_calls: AtomicUsize::new(0), + initialize_attempts, + remaining_initialize_failures, }) .mount(server) .await; @@ -334,6 +406,8 @@ struct CodexAppsJsonRpcResponder { include_app_only_tool: bool, tools_list_behavior: AppsTestToolsListBehavior, tools_list_calls: AtomicUsize, + initialize_attempts: Option>, + remaining_initialize_failures: Option>, } impl Respond for CodexAppsJsonRpcResponder { @@ -355,6 +429,24 @@ impl Respond for CodexAppsJsonRpcResponder { match method { "initialize" => { + if let Some(initialize_attempts) = &self.initialize_attempts { + initialize_attempts.fetch_add(1, Ordering::SeqCst); + } + if self + .remaining_initialize_failures + .as_ref() + .is_some_and(|remaining| { + remaining + .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |remaining| { + remaining.checked_sub(1) + }) + .is_ok() + }) + { + return ResponseTemplate::new(400).set_body_json(json!({ + "error": "simulated non-retryable Apps MCP startup failure", + })); + } let id = body.get("id").cloned().unwrap_or(Value::Null); let protocol_version = body .pointer("/params/protocolVersion") diff --git a/codex-rs/core/tests/common/responses.rs b/codex-rs/core/tests/common/responses.rs index 6a605a80b3..9569df1c90 100644 --- a/codex-rs/core/tests/common/responses.rs +++ b/codex-rs/core/tests/common/responses.rs @@ -895,6 +895,24 @@ pub fn ev_custom_tool_call(call_id: &str, name: &str, input: &str) -> Value { }) } +pub fn ev_custom_tool_call_with_namespace( + call_id: &str, + namespace: &str, + name: &str, + input: &str, +) -> Value { + serde_json::json!({ + "type": "response.output_item.done", + "item": { + "type": "custom_tool_call", + "call_id": call_id, + "namespace": namespace, + "name": name, + "input": input + } + }) +} + pub fn ev_local_shell_call(call_id: &str, status: &str, command: Vec<&str>) -> Value { serde_json::json!({ "type": "response.output_item.done", diff --git a/codex-rs/core/tests/common/test_codex.rs b/codex-rs/core/tests/common/test_codex.rs index eb4fa3b39f..e0e82c37fd 100644 --- a/codex-rs/core/tests/common/test_codex.rs +++ b/codex-rs/core/tests/common/test_codex.rs @@ -658,7 +658,9 @@ impl TestCodexBuilder { Box::pin( thread_manager.start_thread_with_options(StartThreadOptions { config: config.clone(), + allow_provider_model_fallback: false, initial_history: InitialHistory::New, + history_mode: None, session_source: None, thread_source: None, dynamic_tools: Vec::new(), diff --git a/codex-rs/core/tests/common/zsh_fork.rs b/codex-rs/core/tests/common/zsh_fork.rs index 5abfc753b9..8736da72c3 100644 --- a/codex-rs/core/tests/common/zsh_fork.rs +++ b/codex-rs/core/tests/common/zsh_fork.rs @@ -10,6 +10,7 @@ use codex_protocol::permissions::NetworkSandboxPolicy; use codex_protocol::protocol::AskForApproval; use crate::test_codex::TestCodex; +use crate::test_codex::TestCodexBuilder; use crate::test_codex::test_codex; #[derive(Clone)] @@ -19,12 +20,7 @@ pub struct ZshForkRuntime { } impl ZshForkRuntime { - fn apply_to_config( - &self, - config: &mut Config, - approval_policy: AskForApproval, - permission_profile: PermissionProfile, - ) { + fn apply_to_config(&self, config: &mut Config, approval_policy: AskForApproval) { config .features .enable(Feature::ShellTool) @@ -37,10 +33,6 @@ impl ZshForkRuntime { config.main_execve_wrapper_exe = Some(self.main_execve_wrapper_exe.clone()); config.permissions.allow_login_shell = false; config.permissions.approval_policy = Constrained::allow_any(approval_policy); - config - .permissions - .set_permission_profile(permission_profile) - .expect("set permission profile"); } } @@ -86,10 +78,13 @@ pub async fn build_zsh_fork_test( where F: FnOnce(&Path) + Send + 'static, { - let mut builder = test_codex() + let mut builder = zsh_fork_test_builder(runtime, approval_policy) .with_pre_build_hook(pre_build_hook) .with_config(move |config| { - runtime.apply_to_config(config, approval_policy, permission_profile); + config + .permissions + .set_permission_profile(permission_profile) + .expect("set permission profile"); }); builder.build(server).await } @@ -104,10 +99,13 @@ pub async fn build_unified_exec_zsh_fork_test( where F: FnOnce(&Path) + Send + 'static, { - let mut builder = test_codex() + let mut builder = zsh_fork_test_builder(runtime, approval_policy) .with_pre_build_hook(pre_build_hook) .with_config(move |config| { - runtime.apply_to_config(config, approval_policy, permission_profile); + config + .permissions + .set_permission_profile(permission_profile) + .expect("set permission profile"); config.use_experimental_unified_exec_tool = true; config .features @@ -121,6 +119,15 @@ where builder.build(server).await } +pub fn zsh_fork_test_builder( + runtime: ZshForkRuntime, + approval_policy: AskForApproval, +) -> TestCodexBuilder { + test_codex().with_config(move |config| { + runtime.apply_to_config(config, approval_policy); + }) +} + fn find_test_zsh_path() -> Result> { let repo_root = codex_utils_cargo_bin::repo_root()?; let dotslash_zsh = repo_root.join("codex-rs/app-server/tests/suite/zsh"); diff --git a/codex-rs/core/tests/suite/agents_md.rs b/codex-rs/core/tests/suite/agents_md.rs index 894b760270..7c73b90603 100644 --- a/codex-rs/core/tests/suite/agents_md.rs +++ b/codex-rs/core/tests/suite/agents_md.rs @@ -10,6 +10,8 @@ use codex_home::CodexHomeUserInstructionsProvider; use codex_protocol::protocol::EventMsg; use codex_protocol::protocol::InitialHistory; use codex_protocol::protocol::Op; +use codex_protocol::protocol::RolloutItem; +use codex_protocol::protocol::RolloutLine; use codex_protocol::protocol::TurnEnvironmentSelection; use codex_protocol::user_input::UserInput; use codex_utils_absolute_path::AbsolutePathBuf; @@ -31,6 +33,7 @@ use core_test_support::test_codex::test_codex; use core_test_support::wait_for_event; use pretty_assertions::assert_eq; use serde_json::json; +use std::path::Path; use std::sync::Arc; use std::time::Duration; use tempfile::TempDir; @@ -78,6 +81,32 @@ fn write_global_file( Ok(path.abs()) } +fn remove_agents_md_world_state_section(rollout_path: &Path) -> Result<()> { + let rollout = std::fs::read_to_string(rollout_path)?; + let mut removed_section = false; + let retained = rollout + .lines() + .map(serde_json::from_str::) + .collect::, _>>()? + .into_iter() + .map(|mut line| { + if let RolloutItem::WorldState(world_state) = &mut line.item + && let Some(state) = world_state.state.as_object_mut() + { + removed_section |= state.remove("agents_md").is_some(); + } + serde_json::to_string(&line) + }) + .collect::, _>>()? + .join("\n"); + anyhow::ensure!( + removed_section, + "rollout did not contain a persisted AGENTS.md WorldState section" + ); + std::fs::write(rollout_path, format!("{retained}\n"))?; + Ok(()) +} + fn instruction_fragments(request: &responses::ResponsesRequest) -> Vec { request .message_input_texts("user") @@ -457,7 +486,9 @@ async fn loads_user_instructions_without_a_primary_environment() -> Result<()> { .thread_manager .start_thread_with_options(StartThreadOptions { config: test.config.clone(), + allow_provider_model_fallback: false, initial_history: InitialHistory::New, + history_mode: None, session_source: None, thread_source: None, dynamic_tools: Vec::new(), @@ -662,7 +693,9 @@ async fn multi_environment_thread_loads_every_project_and_keeps_creation_snapsho .thread_manager .start_thread_with_options(StartThreadOptions { config: test.config.clone(), + allow_provider_model_fallback: false, initial_history: InitialHistory::New, + history_mode: None, session_source: None, thread_source: None, dynamic_tools: Vec::new(), @@ -771,7 +804,7 @@ async fn invalid_utf8_global_instructions_are_lossy() -> Result<()> { } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn cold_resume_injects_changed_agents_md_once() -> Result<()> { +async fn cold_resume_invalidates_deleted_legacy_agents_md_once() -> Result<()> { // Set up an initial turn and a later cold-resumed turn against the same rollout. let server = responses::start_mock_server().await; let response_mock = responses::mount_sse_sequence( @@ -821,23 +854,20 @@ async fn cold_resume_injects_changed_agents_md_once() -> Result<()> { }) .await; - // Add a preferred override source, then cold-resume with freshly loaded configuration. - let new_source = write_global_file( - home.as_ref(), - GLOBAL_AGENTS_OVERRIDE_FILENAME, - NEW_GLOBAL_INSTRUCTIONS, - )?; - assert_ne!(old_source, new_source); + // Simulate a rollout written before AGENTS.md had a persisted WorldState section. + remove_agents_md_world_state_section(&rollout_path)?; + + std::fs::remove_file(old_source.as_path())?; let mut resume_builder = test_codex().with_home(Arc::clone(&home)); let resumed = resume_builder .resume(&server, Arc::clone(&home), rollout_path) .await?; - // Assert the API reports the new source while model history replays the old structured prefix. + // Model history still contains the old fragment, but the source no longer exists. assert_eq!( resumed.codex.instruction_sources().await, - vec![PathUri::from_abs_path(&new_source)], - "resume reports sources from the newly loaded config" + Vec::::new(), + "resume reports no deleted instruction source" ); resumed.submit_turn("continue resumed thread").await?; @@ -852,11 +882,16 @@ async fn cold_resume_injects_changed_agents_md_once() -> Result<()> { Some(initial_input.as_slice()), "cold resume should replay the original structured input prefix" ); - assert_instruction_replacement_once( - &requests, - OLD_GLOBAL_INSTRUCTIONS, - NEW_GLOBAL_INSTRUCTIONS, + let initial = expected_provider_only_instruction_fragment(OLD_GLOBAL_INSTRUCTIONS); + let removal = expected_provider_only_instruction_fragment( + "The previously provided AGENTS.md instructions no longer apply.", ); + assert_eq!(instruction_fragments(&requests[0]), vec![initial.clone()]); + assert_eq!( + instruction_fragments(&requests[1]), + vec![initial.clone(), removal.clone()] + ); + assert_eq!(instruction_fragments(&requests[2]), vec![initial, removal]); Ok(()) } diff --git a/codex-rs/core/tests/suite/approvals.rs b/codex-rs/core/tests/suite/approvals.rs index f0ab48c5e1..54205f7e8b 100644 --- a/codex-rs/core/tests/suite/approvals.rs +++ b/codex-rs/core/tests/suite/approvals.rs @@ -10,6 +10,7 @@ use codex_features::Feature; use codex_protocol::approvals::NetworkApprovalProtocol; use codex_protocol::approvals::NetworkPolicyAmendment; use codex_protocol::approvals::NetworkPolicyRuleAction; +use codex_protocol::models::BUILT_IN_PERMISSION_PROFILE_WORKSPACE; use codex_protocol::models::PermissionProfile; use codex_protocol::permissions::FileSystemAccessMode; use codex_protocol::permissions::FileSystemPath; @@ -47,6 +48,7 @@ use core_test_support::wait_for_event_with_timeout; use core_test_support::zsh_fork::build_zsh_fork_test; use core_test_support::zsh_fork::restrictive_workspace_write_profile; use core_test_support::zsh_fork::zsh_fork_runtime; +use core_test_support::zsh_fork::zsh_fork_test_builder; use pretty_assertions::assert_eq; use regex_lite::Regex; use serde_json::Value; @@ -131,6 +133,8 @@ enum ActionKind { const DEFAULT_UNIFIED_EXEC_JUSTIFICATION: &str = "Requires escalated permissions to bypass the sandbox in tests."; +const WORKSPACE_PERMISSION_PROFILE_CONFIG: &str = r#"default_permissions = ":workspace" +"#; impl ActionKind { fn policy_src(&self) -> Option<&'static str> { @@ -640,6 +644,7 @@ enum ScenarioGroup { UnifiedExec, } +#[derive(Debug, Eq, PartialEq)] struct CommandResult { exit_code: Option, stdout: String, @@ -683,6 +688,52 @@ async fn submit_turn( Ok(()) } +async fn submit_turn_preserving_active_permission_profile( + test: &TestCodex, + prompt: &str, + approval_policy: AskForApproval, +) -> Result<()> { + let session_model = test.session_configured.model.clone(); + + test.codex + .submit(Op::UserInput { + items: vec![UserInput::Text { + text: prompt.into(), + text_elements: Vec::new(), + }], + final_output_json_schema: None, + responsesapi_client_metadata: None, + additional_context: Default::default(), + thread_settings: codex_protocol::protocol::ThreadSettingsOverrides { + environments: Some(local_selections(test.config.cwd.clone())), + approval_policy: Some(approval_policy), + approvals_reviewer: Some(ApprovalsReviewer::User), + collaboration_mode: Some(codex_protocol::config_types::CollaborationMode { + mode: codex_protocol::config_types::ModeKind::Default, + settings: codex_protocol::config_types::Settings { + model: session_model, + reasoning_effort: None, + developer_instructions: None, + }, + }), + ..Default::default() + }, + }) + .await?; + + Ok(()) +} + +fn assert_active_workspace_permission_profile(test: &TestCodex) { + assert_eq!( + test.session_configured + .active_permission_profile + .as_ref() + .map(|profile| profile.id.as_str()), + Some(BUILT_IN_PERMISSION_PROFILE_WORKSPACE) + ); +} + fn parse_result(item: &Value) -> CommandResult { let output_str = item .get("output") @@ -2696,6 +2747,325 @@ async fn matched_prefix_rule_runs_unsandboxed_under_zsh_fork() -> Result<()> { Ok(()) } +/// Verifies that an allowlisted script retains the originating named profile +/// when its shell tool call requests escalated, unsandboxed execution. +/// +/// Tool owners use this pattern when a trusted wrapper must run outside the +/// current sandbox, but then needs to launch child commands back inside the +/// same sandbox with `codex sandbox -P`. The nested invocation must also pass +/// `--include-managed-config` so it continues to honor enterprise requirements. +/// The test proves both halves of that contract: the wrapper writes outside the +/// `:workspace` sandbox, while its inherited profile name remains `:workspace`. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[cfg(unix)] +async fn allowed_escalated_shell_command_inherits_active_permission_profile() -> Result<()> { + skip_if_no_network!(Ok(())); + + let server = start_mock_server().await; + let home = Arc::new(TempDir::new()?); + fs::write( + home.path().join("config.toml"), + WORKSPACE_PERMISSION_PROFILE_CONFIG, + )?; + + let script_dir = tempfile::tempdir_in(std::env::current_dir()?)?; + let script_path = script_dir.path().join("print-permission-profile.sh"); + let outside_path = script_dir.path().join("unsandboxed-marker"); + fs::write( + &script_path, + format!( + r#"#!/bin/sh +# Print the inherited profile so the test can verify that it reached this script. +printenv CODEX_PERMISSION_PROFILE +touch {outside_path:?} +"# + ), + )?; + + let rules_dir = home.path().join("rules"); + fs::create_dir_all(&rules_dir)?; + let script_pattern = serde_json::to_string(&script_path.to_string_lossy())?; + fs::write( + rules_dir.join("default.rules"), + format!(r#"prefix_rule(pattern=["/bin/sh", {script_pattern}], decision="allow")"#), + )?; + + let approval_policy = AskForApproval::OnRequest; + let mut builder = test_codex().with_home(home).with_config(move |config| { + config.permissions.approval_policy = Constrained::allow_any(approval_policy); + }); + let test = builder.build(&server).await?; + assert!(!outside_path.starts_with(test.config.cwd.as_path())); + assert_active_workspace_permission_profile(&test); + + let call_id = "allowed-escalated-shell-inherits-permission-profile"; + let command = format!("/bin/sh {script_path:?}"); + let event = shell_event( + call_id, + &command, + /*timeout_ms*/ 5_000, + SandboxPermissions::RequireEscalated, + )?; + let _ = mount_sse_once( + &server, + sse(vec![ + ev_response_created("resp-escalated-profile-1"), + event, + ev_completed("resp-escalated-profile-1"), + ]), + ) + .await; + let results = mount_sse_once( + &server, + sse(vec![ + ev_assistant_message("msg-escalated-profile-1", "done"), + ev_completed("resp-escalated-profile-2"), + ]), + ) + .await; + + submit_turn_preserving_active_permission_profile( + &test, + "run the allowed script with escalated permissions", + approval_policy, + ) + .await?; + + wait_for_completion_without_approval(&test).await; + + let result = parse_result(&results.single_request().function_call_output(call_id)); + assert_eq!( + result, + CommandResult { + exit_code: Some(0), + stdout: format!("{BUILT_IN_PERMISSION_PROFILE_WORKSPACE}\n"), + }, + "the unsandboxed script should inherit CODEX_PERMISSION_PROFILE from the shell command" + ); + assert!( + outside_path.exists(), + "allowed escalated script should run outside the :workspace sandbox" + ); + + Ok(()) +} + +/// Verifies that zsh-fork applies an inner script's allow rule even when the +/// model invokes an outer wrapper, and that the escalated script retains the +/// named profile needed to reconstruct the original sandbox remotely without +/// dropping managed enterprise requirements. The script treats the inherited +/// environment value as untrusted and accepts only explicitly allowlisted +/// profile names before passing one to `codex sandbox -P`. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[cfg(unix)] +async fn zsh_fork_inner_allowed_script_inherits_active_permission_profile() -> Result<()> { + skip_if_no_network!(Ok(())); + + let Some(runtime) = zsh_fork_runtime("zsh-fork remote sandbox wrapper test")? else { + return Ok(()); + }; + + const HOST: &str = "builder.example.com"; + let approval_policy = AskForApproval::OnRequest; + let script_dir = tempfile::tempdir_in(std::env::current_dir()?)?; + let wrapper_path = script_dir.path().join("remote-bash"); + let remote_bash_path = script_dir.path().join("remote_bash.py"); + let outside_path = script_dir.path().join("remote-bash-unsandboxed-marker"); + let outside_path_literal = serde_json::to_string(&outside_path.to_string_lossy())?; + fs::write( + &remote_bash_path, + format!( + r#"#!/usr/bin/env python3 +import argparse +import os +from pathlib import Path +import re +import shlex +import sys + +ALLOWED_HOSTS = ("builder.example.com",) +ALLOWED_PROFILES = (":workspace",) + + +def parse_args(): + parser = argparse.ArgumentParser( + description="Print an ssh command that recreates the current Codex sandbox remotely." + ) + parser.add_argument("--host", required=True) + try: + separator = sys.argv.index("--") + except ValueError: + parser.error("the remote command must follow --") + args = parser.parse_args(sys.argv[1:separator]) + args.command = sys.argv[separator + 1:] + if not args.command: + parser.error("the remote command must not be empty") + if not re.fullmatch(r"[a-z0-9.-]+", args.host) or args.host not in ALLOWED_HOSTS: + parser.error("host is not allowlisted") + return args + + +def main(): + args = parse_args() + profile_name = os.environ.get("CODEX_PERMISSION_PROFILE") + if not profile_name: + raise SystemExit("CODEX_PERMISSION_PROFILE must not be empty") + if profile_name not in ALLOWED_PROFILES: + raise SystemExit("CODEX_PERMISSION_PROFILE is not allowlisted") + + shell_command = shlex.join(args.command) + sandbox_command = shlex.join( + [ + "codex", + "sandbox", + "-P", + profile_name, + "--include-managed-config", + "--", + "bash", + "-lc", + shell_command, + ] + ) + print(shlex.join(["ssh", args.host, sandbox_command])) + + # Test-only proof that this inner script was allowed to run unsandboxed. + Path({outside_path_literal}).write_text("unsandboxed", encoding="utf-8") + + +if __name__ == "__main__": + main() +"# + ), + )?; + let remote_bash_exec = shlex::try_join([remote_bash_path.to_string_lossy().as_ref()])?; + fs::write( + &wrapper_path, + format!( + r#"#!/usr/bin/env zsh +exec {remote_bash_exec} "$@" +"# + ), + )?; + for path in [&wrapper_path, &remote_bash_path] { + let mut permissions = fs::metadata(path)?.permissions(); + permissions.set_mode(0o755); + fs::set_permissions(path, permissions)?; + } + + let remote_bash_pattern = serde_json::to_string(&remote_bash_path.to_string_lossy())?; + let rules = format!(r#"prefix_rule(pattern=[{remote_bash_pattern}], decision="allow")"#); + let server = start_mock_server().await; + let mut builder = + zsh_fork_test_builder(runtime, approval_policy).with_pre_build_hook(move |home| { + fs::write( + home.join("config.toml"), + WORKSPACE_PERMISSION_PROFILE_CONFIG, + ) + .expect("write config"); + let rules_dir = home.join("rules"); + fs::create_dir_all(&rules_dir).expect("create rules dir"); + fs::write(rules_dir.join("default.rules"), rules).expect("write rules"); + }); + let test = builder.build(&server).await?; + assert!(!outside_path.starts_with(test.config.cwd.as_path())); + assert_active_workspace_permission_profile(&test); + + let command = shlex::try_join([ + wrapper_path.to_string_lossy().as_ref(), + "--host", + HOST, + "--", + "printf", + "%s", + "hello world", + ])?; + let call_id = "zsh-fork-remote-sandbox-wrapper"; + let event = shell_event( + call_id, + &command, + /*timeout_ms*/ 30_000, + SandboxPermissions::UseDefault, + )?; + let _ = mount_sse_once( + &server, + sse(vec![ + ev_response_created("resp-zsh-fork-remote-wrapper-1"), + event, + ev_completed("resp-zsh-fork-remote-wrapper-1"), + ]), + ) + .await; + let results = mount_sse_once( + &server, + sse(vec![ + ev_assistant_message("msg-zsh-fork-remote-wrapper-1", "done"), + ev_completed("resp-zsh-fork-remote-wrapper-2"), + ]), + ) + .await; + + submit_turn_preserving_active_permission_profile( + &test, + "run the remote sandbox wrapper", + approval_policy, + ) + .await?; + wait_for_completion_without_approval(&test).await; + + let result = parse_result(&results.single_request().function_call_output(call_id)); + assert_eq!( + result.exit_code.unwrap_or(0), + 0, + "the inner remote_bash.py script should run successfully: {}", + result.stdout + ); + let ssh_argv = shlex::split(result.stdout.trim()).context("parse printed ssh command")?; + assert_eq!( + ssh_argv.len(), + 3, + "expected ssh HOST LONG_COMMAND, got: {}", + result.stdout + ); + assert_eq!( + ssh_argv[..2], + ["ssh", HOST], + "remote_bash.py should target only the allowlisted host" + ); + let sandbox_argv = shlex::split(&ssh_argv[2]).context("parse remote sandbox command")?; + assert_eq!( + sandbox_argv.len(), + 9, + "expected codex sandbox ... bash -lc CMD" + ); + assert_eq!( + sandbox_argv[..8], + [ + "codex", + "sandbox", + "-P", + BUILT_IN_PERMISSION_PROFILE_WORKSPACE, + "--include-managed-config", + "--", + "bash", + "-lc", + ], + "remote_bash.py should use the allowlisted inherited profile and managed configuration to reconstruct the Codex sandbox" + ); + let command_argv = shlex::split(&sandbox_argv[8]).context("parse remote bash command")?; + assert_eq!( + command_argv, + ["printf", "%s", "hello world"], + "remote_bash.py should preserve every argument after --" + ); + assert!( + outside_path.exists(), + "the inner allowlisted script should run outside the :workspace sandbox" + ); + + Ok(()) +} + #[tokio::test(flavor = "current_thread")] #[cfg(unix)] async fn invalid_requested_prefix_rule_falls_back_for_compound_command() -> Result<()> { diff --git a/codex-rs/core/tests/suite/auto_review.rs b/codex-rs/core/tests/suite/auto_review.rs index 4dd3effe49..dd39713ef2 100644 --- a/codex-rs/core/tests/suite/auto_review.rs +++ b/codex-rs/core/tests/suite/auto_review.rs @@ -241,6 +241,7 @@ fn remote_model_with_auto_review_override(slug: &str, review_model: &str) -> Mod upgrade: None, base_instructions: "base instructions".to_string(), model_messages: None, + include_skills_usage_instructions: false, supports_reasoning_summaries: false, default_reasoning_summary: ReasoningSummary::Auto, support_verbosity: false, diff --git a/codex-rs/core/tests/suite/client.rs b/codex-rs/core/tests/suite/client.rs index 4df2bca1c1..c87539f940 100644 --- a/codex-rs/core/tests/suite/client.rs +++ b/codex-rs/core/tests/suite/client.rs @@ -363,6 +363,107 @@ async fn response_item_ids_persist_across_resume_and_preserve_server_ids() -> an Ok(()) } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn synthetic_call_output_id_is_stable_across_resumes() -> anyhow::Result<()> { + let function_call_id = "missing-output-call"; + let thread_id = ThreadId::default(); + let rollout = vec![ + RolloutLine { + timestamp: "2024-01-01T00:00:00.000Z".to_string(), + item: RolloutItem::SessionMeta(SessionMetaLine { + meta: SessionMeta { + session_id: thread_id.into(), + id: thread_id, + parent_thread_id: None, + timestamp: "2024-01-01T00:00:00Z".to_string(), + cwd: ".".into(), + originator: "test_originator".to_string(), + cli_version: "test_version".to_string(), + model_provider: Some("test-provider".to_string()), + ..Default::default() + }, + git: None, + }), + }, + RolloutLine { + timestamp: "2024-01-01T00:00:01.000Z".to_string(), + item: RolloutItem::ResponseItem(ResponseItem::FunctionCall { + id: Some("fc_existing".to_string()), + name: "do_it".to_string(), + namespace: None, + arguments: "{}".to_string(), + call_id: function_call_id.to_string(), + internal_chat_message_metadata_passthrough: None, + }), + }, + ]; + let tmpdir = TempDir::new()?; + let session_path = tmpdir.path().join("normalized-call-output-item-id.jsonl"); + let mut file = std::fs::File::create(&session_path)?; + for line in rollout { + writeln!(file, "{}", serde_json::to_string(&line)?)?; + } + + let server = MockServer::start().await; + let response_mock = mount_sse_sequence( + &server, + vec![ + sse(vec![ev_response_created("resp-1"), ev_completed("resp-1")]), + sse(vec![ev_response_created("resp-2"), ev_completed("resp-2")]), + ], + ) + .await; + let codex_home = Arc::new(TempDir::new()?); + let mut builder = test_codex().with_config(|config| { + let _ = config.features.enable(Feature::ItemIds); + }); + let first = builder + .resume(&server, Arc::clone(&codex_home), session_path.clone()) + .await?; + + first.submit_turn("first resume").await?; + first.codex.submit(Op::Shutdown).await?; + wait_for_event(&first.codex, |event| { + matches!(event, EventMsg::ShutdownComplete) + }) + .await; + assert!( + !std::fs::read_to_string(&session_path)?.contains("\"type\":\"function_call_output\""), + "prompt-only repair should not be persisted to the rollout" + ); + + builder = builder.with_config(|config| { + let _ = config.features.enable(Feature::ItemIds); + }); + let second = builder.resume(&server, codex_home, session_path).await?; + second.submit_turn("second resume").await?; + + let requests = response_mock.requests(); + assert_eq!(requests.len(), 2); + let first_output = requests[0].function_call_output(function_call_id); + let first_output_id = first_output + .get("id") + .and_then(serde_json::Value::as_str) + .expect("reconstructed output should have an item ID") + .to_string(); + let first_output_uuid = first_output_id + .strip_prefix("fco_") + .expect("synthetic output should use the Responses API prefix"); + assert_eq!( + Uuid::parse_str(first_output_uuid)?.get_version(), + Some(uuid::Version::Sha1) + ); + assert_eq!( + requests[1] + .function_call_output(function_call_id) + .get("id") + .and_then(serde_json::Value::as_str), + Some(first_output_id.as_str()) + ); + + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn response_item_ids_are_sent_for_all_remote_v2_compaction_requests() -> anyhow::Result<()> { let server = MockServer::start().await; @@ -785,6 +886,7 @@ async fn resume_replays_legacy_js_repl_image_rollout_shapes() { status: None, call_id: "legacy-js-call".to_string(), name: "js_repl".to_string(), + namespace: None, input: "console.log('legacy image flow')".to_string(), internal_chat_message_metadata_passthrough: None, }; @@ -976,6 +1078,7 @@ async fn resume_replays_image_tool_outputs_with_detail() { status: Some("completed".to_string()), call_id: custom_call_id.to_string(), name: "js_repl".to_string(), + namespace: None, input: "console.log('image flow')".to_string(), internal_chat_message_metadata_passthrough: None, }), @@ -2068,7 +2171,7 @@ async fn skills_use_aliases_in_developer_message_under_budget_pressure() { } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn includes_configured_effort_in_request() -> anyhow::Result<()> { +async fn includes_configured_max_effort_in_request() -> anyhow::Result<()> { skip_if_no_network!(Ok(())); let server = MockServer::start().await; @@ -2080,7 +2183,7 @@ async fn includes_configured_effort_in_request() -> anyhow::Result<()> { let TestCodex { codex, .. } = test_codex() .with_model("gpt-5.4") .with_config(|config| { - config.model_reasoning_effort = Some(ReasoningEffort::Medium); + config.model_reasoning_effort = Some(ReasoningEffort::Max); }) .build(&server) .await?; @@ -2109,7 +2212,7 @@ async fn includes_configured_effort_in_request() -> anyhow::Result<()> { .get("reasoning") .and_then(|t| t.get("effort")) .and_then(|v| v.as_str()), - Some("medium") + Some("max") ); Ok(()) @@ -2912,6 +3015,7 @@ async fn azure_responses_request_includes_store_and_reasoning_ids() { status: Some("completed".into()), call_id: "custom-tool-call-id".into(), name: "custom_tool".into(), + namespace: None, input: "{}".into(), internal_chat_message_metadata_passthrough: None, }); diff --git a/codex-rs/core/tests/suite/client_websockets.rs b/codex-rs/core/tests/suite/client_websockets.rs index ed569805dc..d5218cdcab 100755 --- a/codex-rs/core/tests/suite/client_websockets.rs +++ b/codex-rs/core/tests/suite/client_websockets.rs @@ -966,35 +966,77 @@ async fn responses_websocket_v2_requests_use_v2_when_provider_supports_websocket async fn responses_websocket_v2_incremental_requests_are_reused_across_turns() { skip_if_no_network!(); + let mut assistant_output_with_turn_id = ev_assistant_message("msg-1", "assistant output"); + assistant_output_with_turn_id["item"]["internal_chat_message_metadata_passthrough"] = + json!({ "turn_id": "turn-1" }); + + let assistant_output_without_turn_id = ev_assistant_message("msg-2", "second assistant output"); + let server = start_websocket_server(vec![vec![ vec![ ev_response_created("resp-1"), - ev_assistant_message("msg-1", "assistant output"), + assistant_output_with_turn_id, ev_completed("resp-1"), ], - vec![ev_response_created("resp-2"), ev_completed("resp-2")], + vec![ + ev_response_created("resp-2"), + assistant_output_without_turn_id, + ev_completed("resp-2"), + ], + vec![ev_response_created("resp-3"), ev_completed("resp-3")], ]]) .await; - let harness = websocket_harness_with_options(&server, /*runtime_metrics_enabled*/ false).await; - let prompt_one = prompt_with_input(vec![message_item("hello")]); - let prompt_two = prompt_with_input(vec![ - message_item("hello"), - assistant_message_item("msg-1", "assistant output"), - message_item("second"), - ]); + // use OpenAI provider to check metadata logic + let mut provider = websocket_provider(&server); + provider.name = ModelProviderInfo::create_openai_provider(/*base_url*/ None).name; + let harness = + websocket_harness_with_provider_options(provider, /*runtime_metrics_enabled*/ false).await; + // Turn one: initiate + let prompt_one = prompt_with_input(vec![message_item("hello")]); { let mut client_session = harness.client.new_session(); stream_until_complete(&mut client_session, &harness, &prompt_one).await; } - let mut client_session = harness.client.new_session(); - stream_until_complete(&mut client_session, &harness, &prompt_two).await; + // Turn two: the response and reconstructed history have matching metadata. + let mut first_assistant_output = assistant_message_item("msg-1", "assistant output"); + first_assistant_output.set_turn_id_if_missing("turn-1"); + + let prompt_two = prompt_with_input(vec![ + message_item("hello"), + first_assistant_output.clone(), + message_item("second"), + ]); + + { + let mut client_session = harness.client.new_session(); + stream_until_complete(&mut client_session, &harness, &prompt_two).await; + } + + // Turn three: the reconstructed history has metadata that the response omitted. + let mut second_assistant_output = assistant_message_item("msg-2", "second assistant output"); + second_assistant_output.set_turn_id_if_missing("turn-2"); + + let prompt_three = prompt_with_input(vec![ + message_item("hello"), + first_assistant_output, + message_item("second"), + second_assistant_output, + message_item("third"), + ]); + + { + let mut client_session = harness.client.new_session(); + stream_until_complete(&mut client_session, &harness, &prompt_three).await; + } assert_eq!(server.handshakes().len(), 1); let connection = server.single_connection(); - assert_eq!(connection.len(), 2); + assert_eq!(connection.len(), 3); + + // validate second turn let second = connection.get(1).expect("missing request").body_json(); assert_eq!(second["type"].as_str(), Some("response.create")); assert_eq!(second["previous_response_id"].as_str(), Some("resp-1")); @@ -1003,6 +1045,15 @@ async fn responses_websocket_v2_incremental_requests_are_reused_across_turns() { serde_json::to_value(&prompt_two.input[2..]).unwrap() ); + // validate third turn + let third = connection.get(2).expect("missing request").body_json(); + assert_eq!(third["type"].as_str(), Some("response.create")); + assert_eq!(third["previous_response_id"].as_str(), Some("resp-2")); + assert_eq!( + third["input"], + serde_json::to_value(&prompt_three.input[4..]).unwrap() + ); + server.shutdown().await; } diff --git a/codex-rs/core/tests/suite/code_mode.rs b/codex-rs/core/tests/suite/code_mode.rs index 92b6c9d473..9293b506ef 100644 --- a/codex-rs/core/tests/suite/code_mode.rs +++ b/codex-rs/core/tests/suite/code_mode.rs @@ -219,6 +219,32 @@ async fn run_code_mode_turn_with_model_and_config( Ok((test, second_mock)) } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn missing_process_host_returns_a_tool_error() -> Result<()> { + skip_if_no_network!(Ok(())); + + let server = responses::start_mock_server().await; + let (_test, follow_up_mock) = + run_code_mode_turn_with_config(&server, "Run code mode", "text('unreachable')", |config| { + config + .features + .enable(Feature::CodeModeHost) + .expect("code mode host should be enabled"); + }) + .await?; + + let output = follow_up_mock + .single_request() + .custom_tool_call_output("call-1"); + assert!( + output["output"] + .as_str() + .is_some_and(|output| output.contains("failed to spawn code-mode host")) + ); + + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn code_mode_can_call_standalone_web_search() -> Result<()> { assert_code_mode_standalone_web_search(WebSearchMode::Live, serde_json::json!(true)).await @@ -495,6 +521,76 @@ text(JSON.stringify(await tools.exec_command({ cmd: "printf code_mode_exec_marke Ok(()) } +#[tokio::test] +async fn code_mode_exec_holds_captured_result_during_elicitation() -> Result<()> { + let server = responses::start_mock_server().await; + let mut builder = test_codex() + .with_model("test-gpt-5.1-codex") + .with_config(|config| { + let _ = config.features.enable(Feature::CodeMode); + }); + let test = builder.build(&server).await?; + + let first_mock = responses::mount_sse_once( + &server, + sse(vec![ + ev_response_created("resp-1"), + ev_custom_tool_call("call-1", "exec", "text('captured');"), + ev_completed("resp-1"), + ]), + ) + .await; + let second_mock = responses::mount_sse_once( + &server, + sse(vec![ + ev_assistant_message("msg-1", "done"), + ev_completed("resp-2"), + ]), + ) + .await; + + assert_eq!( + test.codex.increment_out_of_band_elicitation_count().await?, + 1 + ); + assert_eq!( + test.codex.increment_out_of_band_elicitation_count().await?, + 2 + ); + let release_elicitation = async { + tokio::time::timeout(Duration::from_secs(5), async { + while first_mock.requests().is_empty() { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("initial response request should arrive"); + tokio::time::sleep(Duration::from_millis(250)).await; + assert!( + second_mock.requests().is_empty(), + "captured exec result should not return during an elicitation" + ); + assert_eq!( + test.codex.decrement_out_of_band_elicitation_count().await?, + 1 + ); + tokio::time::sleep(Duration::from_millis(100)).await; + assert!( + second_mock.requests().is_empty(), + "captured exec result should wait for every elicitation" + ); + assert_eq!( + test.codex.decrement_out_of_band_elicitation_count().await?, + 0 + ); + Ok::<(), anyhow::Error>(()) + }; + + tokio::try_join!(test.submit_turn("run code mode"), release_elicitation)?; + second_mock.single_request(); + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn code_mode_only_restricts_prompt_tools() -> Result<()> { skip_if_no_network!(Ok(())); diff --git a/codex-rs/core/tests/suite/compact_remote.rs b/codex-rs/core/tests/suite/compact_remote.rs index d0e69d6569..e22f70b052 100644 --- a/codex-rs/core/tests/suite/compact_remote.rs +++ b/codex-rs/core/tests/suite/compact_remote.rs @@ -209,6 +209,7 @@ async fn start_realtime_conversation(codex: &codex_core::CodexThread) -> Result< codex .submit(Op::RealtimeConversationStart(ConversationStartParams { client_managed_handoffs: false, + flush_transcript_tail_on_session_end: false, codex_responses_as_items: false, codex_response_item_prefix: None, codex_response_handoff_prefix: None, diff --git a/codex-rs/core/tests/suite/current_time_reminder.rs b/codex-rs/core/tests/suite/current_time_reminder.rs index 8ebf5d6b16..9a9440fb33 100644 --- a/codex-rs/core/tests/suite/current_time_reminder.rs +++ b/codex-rs/core/tests/suite/current_time_reminder.rs @@ -12,6 +12,7 @@ use codex_core::SleepFuture; use codex_core::TimeFuture; use codex_core::TimeProvider; use codex_core::config::CurrentTimeReminderConfig; +use codex_features::CurrentTimeReminderDeliveryMode; use codex_features::CurrentTimeSource; use codex_features::Feature; use codex_model_provider_info::built_in_model_providers; @@ -39,6 +40,7 @@ use pretty_assertions::assert_eq; use serde_json::json; const FIRST_REMINDER: &str = "It is 2026-06-17 17:34:15 UTC."; +const EARLIER_REMINDER: &str = "It is 2026-06-17 17:33:15 UTC."; const SECOND_REMINDER: &str = "It is 2026-06-17 17:35:15 UTC."; const THIRD_REMINDER: &str = "It is 2026-06-17 17:36:15 UTC."; const FIRST_TIME_UNIX_SECONDS: i64 = 1_781_717_655; @@ -162,6 +164,107 @@ async fn current_time_reminders_follow_time_interval_and_persist_in_history() -> Ok(()) } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn zero_current_time_reminder_interval_delivers_when_time_moves_backward() -> Result<()> { + skip_if_no_network!(Ok(())); + + let server = start_mock_server().await; + let responses = mount_sse_sequence( + &server, + vec![ + sse(vec![ev_response_created("resp-1"), ev_completed("resp-1")]), + sse(vec![ev_response_created("resp-2"), ev_completed("resp-2")]), + ], + ) + .await; + let time_provider = Arc::new(TestTimeProvider::default()); + let test = test_codex() + .with_config(|config| { + enable_current_time_reminder(config, /*interval*/ 0, CurrentTimeSource::External) + }) + .with_external_time_provider(time_provider.clone()) + .build(&server) + .await?; + + test.submit_turn("first turn").await?; + time_provider + .current_time + .store(FIRST_TIME_UNIX_SECONDS - 60, Ordering::Relaxed); + test.submit_turn("second turn").await?; + + let requests = responses.requests(); + assert_eq!(requests.len(), 2); + assert_eq!(current_time_reminders(&requests[0]), vec![FIRST_REMINDER]); + assert_eq!( + current_time_reminders(&requests[1]), + vec![FIRST_REMINDER, EARLIER_REMINDER] + ); + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn current_time_reminders_can_follow_only_user_or_tool_outputs() -> Result<()> { + skip_if_no_network!(Ok(())); + + let server = start_mock_server().await; + let tool_args = json!({ + "command": "echo current time", + "timeout_ms": 1_000, + }); + let mut continue_response = ev_completed("resp-2"); + // Ask for another inference without recording a new user message or tool output. + continue_response["response"]["end_turn"] = json!(false); + let responses = mount_sse_sequence( + &server, + vec![ + sse(vec![ + ev_response_created("resp-1"), + ev_function_call( + "current-time-tool-call", + "shell_command", + &serde_json::to_string(&tool_args)?, + ), + ev_completed("resp-1"), + ]), + sse(vec![ + ev_response_created("resp-2"), + ev_assistant_message("msg-2", "continue"), + continue_response, + ]), + sse(vec![ev_response_created("resp-3"), ev_completed("resp-3")]), + ], + ) + .await; + let test = test_codex() + .with_config(|config| { + enable_current_time_reminder(config, /*interval*/ 0, CurrentTimeSource::External); + config + .current_time_reminder + .as_mut() + .expect("current-time reminder should be configured") + .delivery_mode = CurrentTimeReminderDeliveryMode::AfterUserOrToolOutput; + }) + .with_external_time_provider(Arc::new(TestTimeProvider::default())) + .build(&server) + .await?; + + test.submit_turn_with_permission_profile("first turn", PermissionProfile::Disabled) + .await?; + + let requests = responses.requests(); + assert_eq!(requests.len(), 3); + assert_eq!(current_time_reminders(&requests[0]), vec![FIRST_REMINDER]); + assert_eq!( + current_time_reminders(&requests[1]), + vec![FIRST_REMINDER, SECOND_REMINDER] + ); + assert_eq!( + current_time_reminders(&requests[2]), + vec![FIRST_REMINDER, SECOND_REMINDER] + ); + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn system_time_source_adds_current_time_reminder() -> Result<()> { skip_if_no_network!(Ok(())); @@ -221,6 +324,11 @@ async fn current_time_reminder_is_refreshed_after_compaction() -> Result<()> { /*interval*/ 3_000, CurrentTimeSource::External, ); + config + .current_time_reminder + .as_mut() + .expect("current-time reminder should be configured") + .delivery_mode = CurrentTimeReminderDeliveryMode::AfterUserOrToolOutput; }) .with_external_time_provider(Arc::new(TestTimeProvider::default())) .build(&server) diff --git a/codex-rs/core/tests/suite/guardian_review.rs b/codex-rs/core/tests/suite/guardian_review.rs index c1da262033..383954b816 100644 --- a/codex-rs/core/tests/suite/guardian_review.rs +++ b/codex-rs/core/tests/suite/guardian_review.rs @@ -20,6 +20,7 @@ use core_test_support::responses::start_mock_server; use core_test_support::responses::start_websocket_server; use core_test_support::skip_if_no_network; use core_test_support::skip_if_sandbox; +use core_test_support::skip_if_wine_exec; use core_test_support::test_codex::local_selections; use core_test_support::test_codex::test_codex; use core_test_support::wait_for_event; @@ -110,6 +111,121 @@ async fn guardian_session_prewarms_and_is_reused_for_first_review() -> Result<() Ok(()) } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn guardian_denial_rejects_tool_call_with_rationale() -> Result<()> { + skip_if_no_network!(Ok(())); + skip_if_sandbox!(Ok(())); + skip_if_wine_exec!( + Ok(()), + "Guardian approval actions require host-native paths" + ); + + let server = start_mock_server().await; + let approval_policy = AskForApproval::OnRequest; + let sandbox_policy = SandboxPolicy::WorkspaceWrite { + writable_roots: vec![], + network_access: false, + exclude_tmpdir_env_var: true, + exclude_slash_tmp: true, + }; + let sandbox_policy_for_config = sandbox_policy.clone(); + + let mut builder = test_codex().with_config(move |config| { + config.permissions.approval_policy = Constrained::allow_any(approval_policy); + config + .set_legacy_sandbox_policy(sandbox_policy_for_config) + .expect("set sandbox policy"); + }); + let test = builder.build_with_auto_env(&server).await?; + + let output_file = test.cwd.path().join("guardian-denied.txt"); + let command = format!("printf should-not-run > {}", output_file.display()); + let tool_args = json!({ + "cmd": command, + "yield_time_ms": 1_000_u64, + "sandbox_permissions": SandboxPermissions::RequireEscalated, + "justification": "Exercise Guardian denial routing.", + }); + let responses = mount_sse_sequence( + &server, + vec![ + sse(vec![ + ev_response_created("resp-parent-tool-denied"), + ev_function_call( + "exec-call-denied", + "exec_command", + &serde_json::to_string(&tool_args)?, + ), + ev_completed("resp-parent-tool-denied"), + ]), + sse(vec![ + ev_response_created("resp-guardian-denied"), + ev_assistant_message( + "msg-guardian-denied", + &json!({ + "risk_level": "high", + "user_authorization": "low", + "outcome": "deny", + "rationale": "The requested write has unacceptable test risk.", + }) + .to_string(), + ), + ev_completed("resp-guardian-denied"), + ]), + sse(vec![ + ev_response_created("resp-parent-after-denial"), + ev_assistant_message("msg-parent-after-denial", "denied"), + ev_completed("resp-parent-after-denial"), + ]), + ], + ) + .await; + + test.codex + .submit(Op::UserInput { + items: vec![UserInput::Text { + text: "run a command that Guardian should deny".into(), + text_elements: Vec::new(), + }], + final_output_json_schema: None, + responsesapi_client_metadata: None, + additional_context: Default::default(), + thread_settings: codex_protocol::protocol::ThreadSettingsOverrides { + approval_policy: Some(approval_policy), + approvals_reviewer: Some(ApprovalsReviewer::AutoReview), + sandbox_policy: Some(sandbox_policy), + ..Default::default() + }, + }) + .await?; + wait_for_event(&test.codex, |event| { + matches!(event, EventMsg::TurnComplete(_)) + }) + .await; + + let requests = responses.requests(); + let guardian_request = requests + .iter() + .find(|request| request.body_contains_text("Exercise Guardian denial routing.")) + .expect("expected Guardian review request"); + assert!(guardian_request.body_contains_text(&command)); + + let tool_output = requests + .iter() + .find_map(|request| request.function_call_output_text("exec-call-denied")) + .expect("expected rejected tool output to be returned to the parent model"); + assert!( + tool_output.contains("The requested write has unacceptable test risk."), + "Guardian rationale missing from rejected tool output: {tool_output}" + ); + assert!( + !output_file.exists(), + "Guardian-denied command unexpectedly executed" + ); + + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn guardian_review_session_does_not_inherit_legacy_notify() -> Result<()> { skip_if_no_network!(Ok(())); diff --git a/codex-rs/core/tests/suite/mcp_refresh_cleanup.rs b/codex-rs/core/tests/suite/mcp_refresh_cleanup.rs index 7df18727b9..620b6c734c 100644 --- a/codex-rs/core/tests/suite/mcp_refresh_cleanup.rs +++ b/codex-rs/core/tests/suite/mcp_refresh_cleanup.rs @@ -16,7 +16,7 @@ use core_test_support::test_codex::test_codex; use core_test_support::wait_for_mcp_server; #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn refresh_shuts_down_superseded_mcp_stdio_server() -> anyhow::Result<()> { +async fn refresh_keeps_superseded_mcp_server_alive_for_in_flight_calls() -> anyhow::Result<()> { skip_if_no_network!(Ok(())); let server = responses::start_mock_server().await; @@ -84,7 +84,7 @@ async fn refresh_shuts_down_superseded_mcp_stdio_server() -> anyhow::Result<()> "sync", Some(serde_json::json!({ "barrier": barrier, - "sleep_after_ms": 30_000 + "sleep_after_ms": 300_000 })), /*meta*/ None, ) @@ -119,9 +119,16 @@ async fn refresh_shuts_down_superseded_mcp_stdio_server() -> anyhow::Result<()> let replacement_pid = wait_for_pid_file(&pid_file).await?; assert_ne!(replacement_pid, superseded_pid); + assert!(process_is_alive(&superseded_pid)?); + long_call.abort(); + assert!( + long_call + .await + .expect_err("call should be aborted") + .is_cancelled() + ); wait_for_process_exit(&superseded_pid).await?; assert!(process_is_alive(&replacement_pid)?); - assert!(long_call.await?.is_err()); fixture.codex.shutdown_and_wait().await?; wait_for_process_exit(&replacement_pid).await diff --git a/codex-rs/core/tests/suite/mcp_tool_exposure.rs b/codex-rs/core/tests/suite/mcp_tool_exposure.rs index b86862edc2..51ab7842ee 100644 --- a/codex-rs/core/tests/suite/mcp_tool_exposure.rs +++ b/codex-rs/core/tests/suite/mcp_tool_exposure.rs @@ -1,17 +1,33 @@ use anyhow::Result; use codex_features::Feature; +use codex_mcp::CODEX_APPS_MCP_SERVER_NAME; +use codex_protocol::protocol::EventMsg; +use codex_protocol::protocol::McpServerRefreshConfig; +use codex_protocol::protocol::Op; +use codex_protocol::request_user_input::RequestUserInputAnswer; +use codex_protocol::request_user_input::RequestUserInputResponse; +use codex_protocol::user_input::UserInput; use core_test_support::apps_test_server::AppsTestServer; use core_test_support::apps_test_server::SEARCH_CALENDAR_CREATE_TOOL; use core_test_support::apps_test_server::SEARCH_CALENDAR_NAMESPACE; +use core_test_support::apps_test_server::apps_enabled_builder; use core_test_support::apps_test_server::search_capable_apps_builder; use core_test_support::responses; use core_test_support::responses::ev_assistant_message; use core_test_support::responses::ev_completed; +use core_test_support::responses::ev_function_call; use core_test_support::responses::ev_response_created; +use core_test_support::responses::mount_sse_sequence; use core_test_support::responses::namespace_child_tool; use core_test_support::responses::sse; use core_test_support::skip_if_no_network; +use core_test_support::wait_for_event; +use core_test_support::wait_for_event_match; +use core_test_support::wait_for_mcp_server; use serde_json::Value; +use serde_json::json; +use std::collections::HashMap; +use std::time::Duration; #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn code_mode_only_exposes_direct_model_only_mcp_namespaces() -> Result<()> { @@ -79,3 +95,234 @@ async fn code_mode_only_exposes_direct_model_only_mcp_namespaces() -> Result<()> Ok(()) } + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn apps_guidance_appears_after_background_recovery_within_a_turn() -> Result<()> { + skip_if_no_network!(Ok(())); + + let server = responses::start_mock_server().await; + let (apps_server, startup_control) = + AppsTestServer::mount_with_startup_control(&server).await?; + // Initial context is rendered twice before the first request, so keep both reads unavailable. + startup_control.fail_next_initialize_attempts(/*attempts*/ 2); + let call_id = "pause-for-apps"; + let response = mount_sse_sequence( + &server, + vec![ + sse(vec![ + ev_response_created("resp-1"), + ev_function_call( + call_id, + "request_user_input", + &json!({ + "questions": [{ + "id": "continue", + "header": "Continue", + "question": "Continue after Apps recovers?", + "options": [{ + "label": "Yes (Recommended)", + "description": "Continue the test." + }, { + "label": "No", + "description": "Stop the test." + }] + }] + }) + .to_string(), + ), + ev_completed("resp-1"), + ]), + sse(vec![ + ev_response_created("resp-2"), + ev_assistant_message("msg-2", "done"), + ev_completed("resp-2"), + ]), + ], + ) + .await; + let mut builder = + apps_enabled_builder(apps_server.chatgpt_base_url.clone()).with_config(|config| { + config + .features + .enable(Feature::DeferredExecutor) + .expect("test config should allow feature update"); + config + .features + .enable(Feature::DefaultModeRequestUserInput) + .expect("test config should allow feature update"); + }); + let test = builder.build(&server).await?; + let mcp_runtime = test.codex.current_mcp_runtime().await; + + test.codex + .submit(Op::UserInput { + items: vec![UserInput::Text { + text: "use an app after it recovers".into(), + text_elements: Vec::new(), + }], + final_output_json_schema: None, + responsesapi_client_metadata: None, + additional_context: Default::default(), + thread_settings: Default::default(), + }) + .await?; + let request = wait_for_event_match(&test.codex, |event| match event { + EventMsg::RequestUserInput(request) => Some(request.clone()), + _ => None, + }) + .await; + + let initial_requests = response.requests(); + assert_eq!(initial_requests.len(), 1); + let initial_request = &initial_requests[0]; + assert_eq!( + initial_request + .message_input_texts("developer") + .iter() + .filter(|text| text.contains("")) + .count(), + 0 + ); + + tokio::time::timeout(Duration::from_secs(3), async { + loop { + if !mcp_runtime.manager().list_all_tools().await.is_empty() { + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("Apps MCP should recover while the turn is paused"); + test.codex + .submit(Op::UserInputAnswer { + id: request.turn_id, + response: RequestUserInputResponse { + answers: HashMap::from([( + "continue".to_string(), + RequestUserInputAnswer { + answers: vec!["Yes (Recommended)".to_string()], + }, + )]), + }, + }) + .await?; + wait_for_event(&test.codex, |event| { + matches!(event, EventMsg::TurnComplete(_)) + }) + .await; + + let requests = response.requests(); + assert_eq!(requests.len(), 2); + assert_eq!( + requests[1] + .message_input_texts("developer") + .iter() + .filter(|text| text.contains("")) + .count(), + 1 + ); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn later_follow_up_uses_background_recovered_apps_after_mid_thread_startup_failures() +-> Result<()> { + skip_if_no_network!(Ok(())); + + let server = responses::start_mock_server().await; + let (apps_server, startup_control) = + AppsTestServer::mount_with_startup_control(&server).await?; + let response = mount_sse_sequence( + &server, + vec![ + sse(vec![ + ev_response_created("resp-1"), + ev_assistant_message("msg-1", "initial turn"), + ev_completed("resp-1"), + ]), + sse(vec![ + ev_response_created("resp-2"), + ev_assistant_message("msg-2", "recovery-trigger turn"), + ev_completed("resp-2"), + ]), + sse(vec![ + ev_response_created("resp-3"), + ev_assistant_message("msg-3", "recovered follow-up turn"), + ev_completed("resp-3"), + ]), + ], + ) + .await; + + let mut builder = search_capable_apps_builder(apps_server.chatgpt_base_url.clone()) + .with_config(move |config| { + config + .features + .enable(Feature::CodeModeOnly) + .expect("test config should allow feature update"); + config.code_mode.direct_only_tool_namespaces = + vec![SEARCH_CALENDAR_NAMESPACE.to_string()]; + }); + let test = builder.build(&server).await?; + wait_for_mcp_server(&test.codex, CODEX_APPS_MCP_SERVER_NAME).await?; + test.submit_turn("use Calendar before refreshing MCP") + .await?; + + let initial_request = response.requests()[0].body_json(); + assert!( + namespace_child_tool( + &initial_request, + SEARCH_CALENDAR_NAMESPACE, + SEARCH_CALENDAR_CREATE_TOOL, + ) + .is_some(), + "Calendar should be available before the MCP refresh: {initial_request}" + ); + + tokio::fs::remove_dir_all(test.codex_home_path().join("cache/codex_apps_tools")).await?; + startup_control.fail_next_initialize_attempts(/*attempts*/ 1); + let runtime_mcp_config = test.codex.runtime_mcp_config(&test.config).await; + let refresh_config = McpServerRefreshConfig { + mcp_servers: serde_json::to_value(codex_mcp::configured_mcp_servers(&runtime_mcp_config))?, + mcp_oauth_credentials_store_mode: serde_json::to_value( + runtime_mcp_config.mcp_oauth_credentials_store_mode, + )?, + auth_keyring_backend_kind: serde_json::to_value( + runtime_mcp_config.auth_keyring_backend_kind, + )?, + }; + test.codex + .submit(Op::RefreshMcpServers { + config: refresh_config, + }) + .await?; + test.submit_turn("use Calendar after transient Apps startup failures") + .await?; + tokio::time::timeout(Duration::from_secs(1), async { + while startup_control.initialize_attempts() < 3 { + tokio::time::sleep(Duration::from_millis(1)).await; + } + }) + .await + .expect("background Apps reconnect should complete"); + test.submit_turn("use Calendar after background Apps recovery") + .await?; + + let requests = response.requests(); + assert_eq!(requests.len(), 3); + let recovered_request = requests[2].body_json(); + assert!( + namespace_child_tool( + &recovered_request, + SEARCH_CALENDAR_NAMESPACE, + SEARCH_CALENDAR_CREATE_TOOL, + ) + .is_some(), + "Calendar should recover on the follow-up turn: {recovered_request}", + ); + assert_eq!(startup_control.initialize_attempts(), 3); + + Ok(()) +} diff --git a/codex-rs/core/tests/suite/mod.rs b/codex-rs/core/tests/suite/mod.rs index 3d4a191f12..3e27463370 100644 --- a/codex-rs/core/tests/suite/mod.rs +++ b/codex-rs/core/tests/suite/mod.rs @@ -126,6 +126,7 @@ mod tools; mod truncation; mod turn_state; mod unified_exec; +mod unified_exec_process_events; #[cfg(unix)] mod unified_exec_zsh_fork_approvals; mod unstable_features_warning; diff --git a/codex-rs/core/tests/suite/model_switching.rs b/codex-rs/core/tests/suite/model_switching.rs index 5b7a3dde5a..b08ca7cc8a 100644 --- a/codex-rs/core/tests/suite/model_switching.rs +++ b/codex-rs/core/tests/suite/model_switching.rs @@ -124,6 +124,7 @@ fn test_model_info( upgrade: None, base_instructions: "base instructions".to_string(), model_messages: None, + include_skills_usage_instructions: false, supports_reasoning_summaries: false, default_reasoning_summary: ReasoningSummary::Auto, support_verbosity: false, @@ -948,6 +949,7 @@ async fn model_switch_to_smaller_model_updates_token_context_window() -> Result< upgrade: None, base_instructions: "base instructions".to_string(), model_messages: None, + include_skills_usage_instructions: false, supports_reasoning_summaries: false, default_reasoning_summary: ReasoningSummary::Auto, support_verbosity: false, diff --git a/codex-rs/core/tests/suite/models_cache_ttl.rs b/codex-rs/core/tests/suite/models_cache_ttl.rs index 5103f82123..4044ba8efa 100644 --- a/codex-rs/core/tests/suite/models_cache_ttl.rs +++ b/codex-rs/core/tests/suite/models_cache_ttl.rs @@ -352,6 +352,7 @@ fn test_remote_model(slug: &str, priority: i32) -> ModelInfo { upgrade: None, base_instructions: "base instructions".to_string(), model_messages: None, + include_skills_usage_instructions: false, supports_reasoning_summaries: false, default_reasoning_summary: ReasoningSummary::Auto, support_verbosity: false, diff --git a/codex-rs/core/tests/suite/multi_agent_mode.rs b/codex-rs/core/tests/suite/multi_agent_mode.rs index 451b9cd38c..283ed6c5cc 100644 --- a/codex-rs/core/tests/suite/multi_agent_mode.rs +++ b/codex-rs/core/tests/suite/multi_agent_mode.rs @@ -20,9 +20,11 @@ use core_test_support::test_codex::test_codex; use core_test_support::wait_for_event; use pretty_assertions::assert_eq; use serde_json::Value; +use serde_json::json; -const NO_SPAWN_TEXT: &str = "Do not spawn sub-agents unless the user explicitly asks for sub-agents, delegation, or parallel agent work."; +const NO_SPAWN_TEXT: &str = "Do not spawn sub-agents unless the user or applicable AGENTS.md/skill instructions explicitly ask for sub-agents, delegation, or parallel agent work."; const PROACTIVE_TEXT: &str = "Proactive multi-agent delegation is active."; +const CUSTOM_MODE_HINT_TEXT: &str = "Use the configured delegation policy."; fn add_ultra_reasoning(model_info: &mut ModelInfo) { model_info.supports_reasoning_summaries = true; @@ -41,6 +43,12 @@ fn configure_multi_agent_v2(config: &mut Config) { .expect("test config should allow feature update"); } +// Configuring a custom mode hint also enables multi-agent V2 for the test. +fn configure_custom_mode_hint(config: &mut Config) { + configure_multi_agent_v2(config); + config.multi_agent_v2.multi_agent_mode_hint_text = Some(CUSTOM_MODE_HINT_TEXT.to_string()); +} + fn configure_ultra(config: &mut Config) { configure_multi_agent_v2(config); config.model_reasoning_effort = Some(ReasoningEffort::Ultra); @@ -120,6 +128,105 @@ async fn ultra_reasoning_uses_max_and_proactive_mode() -> Result<()> { Ok(()) } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn configured_mode_hint_uses_custom_mode_across_reasoning_efforts() -> Result<()> { + skip_if_no_network!(Ok(())); + + let server = start_mock_server().await; + let responses = mount_sse_sequence( + &server, + (1..=2) + .map(|index| { + sse(vec![ + ev_response_created(&format!("resp-{index}")), + ev_completed(&format!("resp-{index}")), + ]) + }) + .collect(), + ) + .await; + let test = test_codex() + .with_model_info_override("gpt-5.4", add_ultra_reasoning) + .with_config(configure_custom_mode_hint) + .build(&server) + .await?; + let rollout_path = test + .session_configured + .rollout_path + .clone() + .expect("rollout path"); + + submit_turn(&test.codex, "explicit", Some(ReasoningEffort::High)).await?; + submit_turn(&test.codex, "proactive", Some(ReasoningEffort::Ultra)).await?; + + let requests = responses.requests(); + let first_input = requests[0].input(); + let first_texts = developer_texts(&first_input); + let second_input = requests[1].input(); + let second_texts = developer_texts(&second_input); + let instruction_counts = |texts: &[&str]| { + ( + count_containing(texts, CUSTOM_MODE_HINT_TEXT), + count_containing(texts, NO_SPAWN_TEXT), + count_containing(texts, PROACTIVE_TEXT), + ) + }; + assert_eq!(instruction_counts(&first_texts), (1, 0, 0)); + assert_eq!(instruction_counts(&second_texts), (1, 0, 0)); + let rollout_values = std::fs::read_to_string(rollout_path)? + .lines() + .map(serde_json::from_str::) + .collect::>>()?; + let recorded_modes = rollout_values + .iter() + .filter(|value| value.get("type").and_then(Value::as_str) == Some("turn_context")) + .filter_map(|value| value.pointer("/payload/multi_agent_mode").cloned()) + .collect::>(); + assert_eq!( + recorded_modes, + [ + json!({"custom": CUSTOM_MODE_HINT_TEXT}), + json!({"custom": CUSTOM_MODE_HINT_TEXT}), + ] + ); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn empty_configured_mode_hint_suppresses_builtin_text() -> Result<()> { + skip_if_no_network!(Ok(())); + + let server = start_mock_server().await; + let response = mount_sse_once( + &server, + sse(vec![ev_response_created("resp-1"), ev_completed("resp-1")]), + ) + .await; + let test = test_codex() + .with_config(|config| { + configure_multi_agent_v2(config); + config.multi_agent_v2.multi_agent_mode_hint_text = Some(String::new()); + }) + .build(&server) + .await?; + + submit_turn(&test.codex, "hello", Some(ReasoningEffort::High)).await?; + + let input = response.single_request().input(); + let texts = developer_texts(&input); + assert_eq!( + ( + count_containing(&texts, MULTI_AGENT_MODE_OPEN_TAG), + count_containing(&texts, NO_SPAWN_TEXT), + count_containing(&texts, PROACTIVE_TEXT), + ), + (1, 0, 0) + ); + + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn leaving_ultra_after_cold_resume_emits_explicit_mode() -> Result<()> { skip_if_no_network!(Ok(())); diff --git a/codex-rs/core/tests/suite/network_approval.rs b/codex-rs/core/tests/suite/network_approval.rs index 6b96c0d635..437155f325 100644 --- a/codex-rs/core/tests/suite/network_approval.rs +++ b/codex-rs/core/tests/suite/network_approval.rs @@ -28,6 +28,7 @@ use core_test_support::responses::ev_assistant_message; use core_test_support::responses::ev_completed; use core_test_support::responses::ev_function_call; use core_test_support::responses::ev_response_created; +use core_test_support::responses::mount_sse_once_match; use core_test_support::responses::mount_sse_sequence; use core_test_support::responses::sse; use core_test_support::responses::start_mock_server; @@ -56,6 +57,181 @@ use tempfile::TempDir; const NETWORK_TEST_HOST: &str = "codex-network-test.invalid"; const NETWORK_TEST_TARGET: &str = "http://codex-network-test.invalid:80"; +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[cfg_attr( + not(target_os = "linux"), + ignore = "requires the trusted Linux proxy bridge" +)] +async fn guardian_receives_exact_triggers_for_concurrent_network_requests() -> Result<()> { + skip_if_target_windows!(Ok(()), "uses the POSIX/Python network fixture"); + skip_if_host_windows!(Ok(())); + skip_if_no_network!(Ok(())); + skip_if_sandbox!(Ok(())); + + let server = start_mock_server().await; + let test = managed_network_unified_exec_test(&server).await?; + let barrier_dir = TempDir::new_in(test.cwd.path())?; + let first_marker = barrier_dir.path().join("first"); + let second_marker = barrier_dir.path().join("second"); + let network_command = |marker: &PathBuf, peer_marker: &PathBuf, host: &str| { + format!( + "touch '{}' && while [ ! -e '{}' ]; do sleep 0.01; done && python3 -c \"import urllib.request; urllib.request.build_opener(urllib.request.ProxyHandler()).open('http://{host}', timeout=10).read()\"", + marker.display(), + peer_marker.display(), + ) + }; + let first_command = network_command(&first_marker, &second_marker, "1.1.1.1"); + let second_command = network_command(&second_marker, &first_marker, "8.8.8.8"); + mount_sse_once_match( + &server, + |request: &wiremock::Request| { + !is_guardian_request(request) + && request_body_contains(request, "run both network requests") + && !request_body_contains(request, "exec-network-first") + }, + sse(vec![ + ev_response_created("resp-network-concurrent"), + ev_function_call( + "exec-network-first", + "exec_command", + &serde_json::to_string(&network_exec_args(&first_command))?, + ), + ev_function_call( + "exec-network-second", + "exec_command", + &serde_json::to_string(&network_exec_args(&second_command))?, + ), + ev_completed("resp-network-concurrent"), + ]), + ) + .await; + let first_guardian = mount_sse_once_match( + &server, + |request: &wiremock::Request| guardian_request_is_for(request, "exec-network-first"), + sse(vec![ + ev_response_created("resp-network-guardian-1"), + ev_assistant_message("msg-network-guardian-1", r#"{"outcome":"deny"}"#), + ev_completed("resp-network-guardian-1"), + ]), + ) + .await; + let second_guardian = mount_sse_once_match( + &server, + |request: &wiremock::Request| guardian_request_is_for(request, "exec-network-second"), + sse(vec![ + ev_response_created("resp-network-guardian-2"), + ev_assistant_message("msg-network-guardian-2", r#"{"outcome":"deny"}"#), + ev_completed("resp-network-guardian-2"), + ]), + ) + .await; + mount_sse_once_match( + &server, + |request: &wiremock::Request| { + !is_guardian_request(request) + && request_body_contains(request, "exec-network-first") + && request_body_contains(request, "exec-network-second") + }, + sse(vec![ + ev_response_created("resp-network-done"), + ev_assistant_message("msg-network-done", "done"), + ev_completed("resp-network-done"), + ]), + ) + .await; + + submit_managed_network_turn( + &test, + "run both network requests", + vec![local(test.config.cwd.clone())], + ApprovalsReviewer::AutoReview, + AskForApproval::OnRequest, + ) + .await?; + let deadline = tokio::time::Instant::now() + Duration::from_secs(10); + let actual_triggers = loop { + let mut actual_triggers = guardian_network_triggers(&[&first_guardian, &second_guardian])?; + actual_triggers.sort_unstable(); + actual_triggers.dedup(); + if actual_triggers.len() == 2 { + break actual_triggers; + } + if tokio::time::Instant::now() >= deadline { + anyhow::bail!("timed out waiting for both Guardian network reviews"); + } + tokio::time::sleep(Duration::from_millis(10)).await; + }; + wait_for_turn_complete(&test).await; + + assert_eq!( + actual_triggers, + vec![ + ("exec-network-first".to_string(), first_command), + ("exec-network-second".to_string(), second_command), + ] + ); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[cfg_attr( + not(target_os = "linux"), + ignore = "requires the trusted Linux proxy bridge" +)] +async fn guardian_receives_exact_trigger_for_single_network_request() -> Result<()> { + skip_if_target_windows!(Ok(()), "uses the POSIX/Python network fixture"); + skip_if_host_windows!(Ok(())); + skip_if_no_network!(Ok(())); + skip_if_sandbox!(Ok(())); + + let server = start_mock_server().await; + let test = managed_network_unified_exec_test(&server).await?; + let command = "python3 -c \"import urllib.request; opener = urllib.request.build_opener(urllib.request.ProxyHandler()); print('OK:' + opener.open('http://1.1.1.1', timeout=10).read().decode(errors='replace'))\"".to_string(); + let responses = mount_sse_sequence( + &server, + vec![ + sse(vec![ + ev_response_created("resp-network-single"), + ev_function_call( + "exec-network-single", + "exec_command", + &serde_json::to_string(&network_exec_args(&command))?, + ), + ev_completed("resp-network-single"), + ]), + sse(vec![ + ev_response_created("resp-network-guardian"), + ev_assistant_message("msg-network-guardian", r#"{"outcome":"deny"}"#), + ev_completed("resp-network-guardian"), + ]), + sse(vec![ + ev_response_created("resp-network-done"), + ev_assistant_message("msg-network-done", "done"), + ev_completed("resp-network-done"), + ]), + ], + ) + .await; + + submit_managed_network_turn( + &test, + "run one network request", + vec![local(test.config.cwd.clone())], + ApprovalsReviewer::AutoReview, + AskForApproval::OnRequest, + ) + .await?; + wait_for_turn_complete(&test).await; + + assert_eq!( + guardian_network_triggers(&[&responses])?, + vec![("exec-network-single".to_string(), command)] + ); + + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn approved_network_host_for_one_environment_still_prompts_in_another() -> Result<()> { skip_if_target_windows!(Ok(()), "uses the POSIX/Python network fixture"); @@ -99,6 +275,8 @@ async fn approved_network_host_for_one_environment_still_prompts_in_another() -> &test, "fetch from the local environment", environments.clone(), + ApprovalsReviewer::User, + AskForApproval::UnlessTrusted, ) .await?; let approval = expect_network_approval(&test, LOCAL_ENVIRONMENT_ID).await?; @@ -122,6 +300,8 @@ async fn approved_network_host_for_one_environment_still_prompts_in_another() -> &test, "fetch from the remote environment", environments.clone(), + ApprovalsReviewer::User, + AskForApproval::UnlessTrusted, ) .await?; let approval = expect_network_approval(&test, REMOTE_ENVIRONMENT_ID).await?; @@ -225,12 +405,20 @@ async fn mount_exec_network_turn( } fn network_fetch_args(environment_id: &str) -> Value { + let command = format!( + "python3 -c \"import urllib.request; opener = urllib.request.build_opener(urllib.request.ProxyHandler()); print('OK:' + opener.open('http://{NETWORK_TEST_HOST}', timeout=2).read().decode(errors='replace'))\"" + ); + let mut args = network_exec_args(&command); + args["environment_id"] = json!(environment_id); + args +} + +fn network_exec_args(command: &str) -> Value { json!({ "shell": "/bin/sh", - "cmd": format!("python3 -c \"import urllib.request; opener = urllib.request.build_opener(urllib.request.ProxyHandler()); print('OK:' + opener.open('http://{NETWORK_TEST_HOST}', timeout=2).read().decode(errors='replace'))\""), + "cmd": command, "login": false, "yield_time_ms": 1_000, - "environment_id": environment_id, }) } @@ -238,6 +426,8 @@ async fn submit_managed_network_turn( test: &TestCodex, prompt: &str, environments: Vec, + approvals_reviewer: ApprovalsReviewer, + approval_policy: AskForApproval, ) -> Result<()> { let permission_profile = PermissionProfile::workspace_write_with( &[], @@ -261,8 +451,8 @@ async fn submit_managed_network_turn( additional_context: Default::default(), thread_settings: codex_protocol::protocol::ThreadSettingsOverrides { environments: Some(turn_environment_selections), - approval_policy: Some(AskForApproval::OnRequest), - approvals_reviewer: Some(ApprovalsReviewer::User), + approval_policy: Some(approval_policy), + approvals_reviewer: Some(approvals_reviewer), sandbox_policy: Some(sandbox_policy), permission_profile, collaboration_mode: Some(codex_protocol::config_types::CollaborationMode { @@ -281,6 +471,93 @@ async fn submit_managed_network_turn( Ok(()) } +fn decoded_request_body(request: &wiremock::Request) -> Option> { + let is_zstd = request + .headers + .get("content-encoding") + .and_then(|value| value.to_str().ok()) + .is_some_and(|value| { + value + .split(',') + .any(|entry| entry.trim().eq_ignore_ascii_case("zstd")) + }); + if is_zstd { + zstd::stream::decode_all(std::io::Cursor::new(&request.body)).ok() + } else { + Some(request.body.clone()) + } +} + +fn request_body_contains(request: &wiremock::Request, text: &str) -> bool { + decoded_request_body(request) + .and_then(|body| String::from_utf8(body).ok()) + .is_some_and(|body| body.contains(text)) +} + +fn is_guardian_request(request: &wiremock::Request) -> bool { + decoded_request_body(request) + .and_then(|body| serde_json::from_slice::(&body).ok()) + .is_some_and(|body| { + body.pointer("/client_metadata/x-openai-subagent") + .and_then(Value::as_str) + == Some("guardian") + }) +} + +fn guardian_request_is_for(request: &wiremock::Request, call_id: &str) -> bool { + decoded_request_body(request) + .and_then(|body| serde_json::from_slice::(&body).ok()) + .filter(|body| { + body.pointer("/client_metadata/x-openai-subagent") + .and_then(Value::as_str) + == Some("guardian") + }) + .and_then(|body| { + body.get("input") + .and_then(Value::as_array) + .and_then(|input| { + input + .iter() + .rev() + .find(|item| item.get("role").and_then(Value::as_str) == Some("user")) + }) + .cloned() + }) + .is_some_and(|latest_user_message| latest_user_message.to_string().contains(call_id)) +} + +fn guardian_network_triggers(responses: &[&ResponseMock]) -> Result> { + responses + .iter() + .flat_map(|responses| responses.requests()) + .filter(|request| { + request.body_json()["client_metadata"]["x-openai-subagent"].as_str() == Some("guardian") + }) + .map(|request| { + let user_texts = request.message_input_texts("user"); + let action: Value = serde_json::from_str( + user_texts + .iter() + .find(|text| text.contains("\"tool\": \"network_access\"")) + .context("expected network access JSON in Guardian request")? + .trim(), + )?; + Ok(( + action + .pointer("/trigger/callId") + .and_then(Value::as_str) + .context("expected exact trigger call id")? + .to_string(), + action + .pointer("/trigger/command/2") + .and_then(Value::as_str) + .context("expected exact trigger command")? + .to_string(), + )) + }) + .collect() +} + async fn expect_network_approval( test: &TestCodex, expected_environment_id: &str, diff --git a/codex-rs/core/tests/suite/otel.rs b/codex-rs/core/tests/suite/otel.rs index e3415d945b..4a2b2993ad 100644 --- a/codex-rs/core/tests/suite/otel.rs +++ b/codex-rs/core/tests/suite/otel.rs @@ -567,19 +567,22 @@ async fn process_sse_emits_completed_telemetry() { mount_sse_once( &server, - sse(vec![serde_json::json!({ - "type": "response.completed", - "response": { - "id": "resp1", - "usage": { - "input_tokens": 3, - "input_tokens_details": { "cached_tokens": 1 }, - "output_tokens": 5, - "output_tokens_details": { "reasoning_tokens": 2 }, - "total_tokens": 9 + sse(vec![ + ev_reasoning_item_added("reasoning-1", &[]), + serde_json::json!({ + "type": "response.completed", + "response": { + "id": "resp1", + "usage": { + "input_tokens": 3, + "input_tokens_details": { "cached_tokens": 1 }, + "output_tokens": 5, + "output_tokens_details": { "reasoning_tokens": 2 }, + "total_tokens": 9 + } } - } - })]), + }), + ]), ) .await; @@ -612,6 +615,9 @@ async fn process_sse_emits_completed_telemetry() { && line.contains("cached_token_count=1") && line.contains("reasoning_token_count=2") && line.contains("tool_token_count=9") + && extract_log_field(line, "ttft_ms") + .and_then(|ttft_ms| ttft_ms.parse::().ok()) + .is_some() }) .map(|_| Ok(())) .unwrap_or(Err("missing response.completed telemetry".to_string())) diff --git a/codex-rs/core/tests/suite/personality.rs b/codex-rs/core/tests/suite/personality.rs index a58e5f20fe..e74695b59d 100644 --- a/codex-rs/core/tests/suite/personality.rs +++ b/codex-rs/core/tests/suite/personality.rs @@ -574,6 +574,7 @@ async fn remote_model_friendly_personality_instructions_with_feature() -> anyhow personality_pragmatic: Some("Pragmatic variant".to_string()), }), }), + include_skills_usage_instructions: false, supports_reasoning_summaries: false, default_reasoning_summary: ReasoningSummary::Auto, support_verbosity: false, @@ -689,6 +690,7 @@ async fn user_turn_personality_remote_model_template_includes_update_message() - personality_pragmatic: Some(remote_pragmatic_message.to_string()), }), }), + include_skills_usage_instructions: false, supports_reasoning_summaries: false, default_reasoning_summary: ReasoningSummary::Auto, support_verbosity: false, diff --git a/codex-rs/core/tests/suite/personality_migration.rs b/codex-rs/core/tests/suite/personality_migration.rs index ca1d148728..0ca3e1b2a9 100644 --- a/codex-rs/core/tests/suite/personality_migration.rs +++ b/codex-rs/core/tests/suite/personality_migration.rs @@ -75,7 +75,9 @@ async fn write_rollout_with_user_event(dir: &Path, thread_id: ThreadId) -> io::R model_provider: None, base_instructions: None, dynamic_tools: None, + selected_capability_roots: Vec::new(), memory_mode: None, + history_mode: Default::default(), multi_agent_version: None, context_window: None, }, @@ -127,7 +129,9 @@ async fn write_rollout_with_meta_only(dir: &Path, thread_id: ThreadId) -> io::Re model_provider: None, base_instructions: None, dynamic_tools: None, + selected_capability_roots: Vec::new(), memory_mode: None, + history_mode: Default::default(), multi_agent_version: None, context_window: None, }, diff --git a/codex-rs/core/tests/suite/plugins.rs b/codex-rs/core/tests/suite/plugins.rs index b3420feabe..aaf65baad0 100644 --- a/codex-rs/core/tests/suite/plugins.rs +++ b/codex-rs/core/tests/suite/plugins.rs @@ -255,8 +255,8 @@ async fn capability_sections_render_in_developer_message_in_order() -> Result<() .find("## Plugins") .expect("expected plugins section in developer message"); assert!( - apps_pos < skills_pos && skills_pos < plugins_pos, - "expected Apps -> Skills -> Plugins order: {developer_messages:?}" + skills_pos < apps_pos && apps_pos < plugins_pos, + "expected Skills -> Apps -> Plugins order: {developer_messages:?}" ); assert!( !developer_text.contains("`sample`: inspect sample data"), diff --git a/codex-rs/core/tests/suite/realtime_conversation.rs b/codex-rs/core/tests/suite/realtime_conversation.rs index 1a95223bff..f4262e8e40 100644 --- a/codex-rs/core/tests/suite/realtime_conversation.rs +++ b/codex-rs/core/tests/suite/realtime_conversation.rs @@ -285,6 +285,7 @@ async fn conversation_start_audio_text_close_round_trip() -> Result<()> { test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { client_managed_handoffs: false, + flush_transcript_tail_on_session_end: false, codex_responses_as_items: false, codex_response_item_prefix: None, codex_response_handoff_prefix: None, @@ -431,6 +432,7 @@ async fn conversation_start_defaults_to_v2_and_gpt_realtime_1_5() -> Result<()> test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { client_managed_handoffs: false, + flush_transcript_tail_on_session_end: false, codex_responses_as_items: false, codex_response_item_prefix: None, codex_response_handoff_prefix: None, @@ -526,6 +528,7 @@ async fn conversation_webrtc_start_posts_generated_session() -> Result<()> { test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { client_managed_handoffs: false, + flush_transcript_tail_on_session_end: false, codex_responses_as_items: false, codex_response_item_prefix: None, codex_response_handoff_prefix: None, @@ -713,6 +716,7 @@ async fn conversation_webrtc_start_uses_avas_query() -> Result<()> { test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { client_managed_handoffs: false, + flush_transcript_tail_on_session_end: false, codex_responses_as_items: false, codex_response_item_prefix: None, codex_response_handoff_prefix: None, @@ -811,6 +815,7 @@ async fn conversation_webrtc_default_v1_ignores_configured_v2_voice() -> Result< test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { client_managed_handoffs: false, + flush_transcript_tail_on_session_end: false, codex_responses_as_items: false, codex_response_item_prefix: None, codex_response_handoff_prefix: None, @@ -871,6 +876,7 @@ async fn conversation_webrtc_default_v1_rejects_explicit_v2_voice() -> Result<() test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { client_managed_handoffs: false, + flush_transcript_tail_on_session_end: false, codex_responses_as_items: false, codex_response_item_prefix: None, codex_response_handoff_prefix: None, @@ -941,6 +947,7 @@ async fn conversation_webrtc_start_uses_configured_call_base_url_for_avas() -> R test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { client_managed_handoffs: false, + flush_transcript_tail_on_session_end: false, codex_responses_as_items: false, codex_response_item_prefix: None, codex_response_handoff_prefix: None, @@ -1035,6 +1042,7 @@ async fn conversation_webrtc_close_while_sideband_connecting_drops_pending_join( test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { client_managed_handoffs: false, + flush_transcript_tail_on_session_end: false, codex_responses_as_items: false, codex_response_item_prefix: None, codex_response_handoff_prefix: None, @@ -1126,6 +1134,7 @@ async fn conversation_webrtc_sideband_connect_failure_closes_with_error() -> Res test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { client_managed_handoffs: false, + flush_transcript_tail_on_session_end: false, codex_responses_as_items: false, codex_response_item_prefix: None, codex_response_handoff_prefix: None, @@ -1219,6 +1228,7 @@ async fn conversation_start_uses_openai_env_key_fallback_with_chatgpt_auth() -> test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { client_managed_handoffs: false, + flush_transcript_tail_on_session_end: false, codex_responses_as_items: false, codex_response_item_prefix: None, codex_response_handoff_prefix: None, @@ -1271,27 +1281,46 @@ async fn conversation_start_uses_openai_env_key_fallback_with_chatgpt_auth() -> Ok(()) } -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn conversation_transport_close_emits_closed_event() -> Result<()> { +async fn assert_transport_close_tail_flush( + flush_transcript_tail_on_session_end: bool, +) -> Result<()> { skip_if_no_network!(Ok(())); - let session_updated = vec![json!({ - "type": "session.updated", - "session": { "id": "sess_1", "instructions": "backend prompt" } - })]; - let server = start_websocket_server(vec![vec![], vec![session_updated]]).await; + let api_server = start_mock_server().await; + let response_mock = responses::mount_sse_once( + &api_server, + responses::sse(vec![ + responses::ev_response_created("resp-1"), + responses::ev_assistant_message("msg-1", "ok"), + responses::ev_completed("resp-1"), + ]), + ) + .await; + let realtime_server = start_websocket_server(vec![vec![vec![ + json!({ + "type": "session.updated", + "session": { "id": "sess_1", "instructions": "backend prompt" } + }), + json!({ + "type": "conversation.input_transcript.delta", + "delta": "transport tail" + }), + ]]]) + .await; - let mut builder = test_codex(); - let test = builder.build_with_websocket_server(&server).await?; - assert!( - server - .wait_for_handshakes(/*expected*/ 1, Duration::from_secs(2)) - .await - ); + let mut builder = test_codex().with_config({ + let realtime_base_url = realtime_server.uri().to_string(); + move |config| { + config.experimental_realtime_ws_base_url = Some(realtime_base_url); + config.realtime.version = RealtimeWsVersion::V1; + } + }); + let test = builder.build(&api_server).await?; test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { client_managed_handoffs: false, + flush_transcript_tail_on_session_end, codex_responses_as_items: false, codex_response_item_prefix: None, codex_response_handoff_prefix: None, @@ -1334,8 +1363,27 @@ async fn conversation_transport_close_emits_closed_event() -> Result<()> { }) .await; assert_eq!(closed.reason.as_deref(), Some("transport_closed")); + if flush_transcript_tail_on_session_end { + let deadline = tokio::time::Instant::now() + Duration::from_secs(2); + while response_mock.requests().is_empty() { + assert!(tokio::time::Instant::now() < deadline); + tokio::time::sleep(Duration::from_millis(10)).await; + } + assert!(response_mock.single_request().message_input_texts("user").iter().any(|text| text + == "\n The user just ended their realtime session. Here is the remaining handoff/transcript tail. You probably do not have to do anything; acknowledge the handoff unless the transcript itself asks for something.\n user: transport tail\n")); + } else { + tokio::time::sleep(Duration::from_millis(200)).await; + assert!(response_mock.requests().is_empty()); + } - server.shutdown().await; + realtime_server.shutdown().await; + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn conversation_transport_close_tail_flush_is_opt_in() -> Result<()> { + assert_transport_close_tail_flush(/*flush_transcript_tail_on_session_end*/ false).await?; + assert_transport_close_tail_flush(/*flush_transcript_tail_on_session_end*/ true).await?; Ok(()) } @@ -1389,6 +1437,7 @@ async fn conversation_start_preflight_failure_emits_realtime_error_only() -> Res test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { client_managed_handoffs: false, + flush_transcript_tail_on_session_end: false, codex_responses_as_items: false, codex_response_item_prefix: None, codex_response_handoff_prefix: None, @@ -1440,6 +1489,7 @@ async fn conversation_start_connect_failure_emits_realtime_error_only() -> Resul test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { client_managed_handoffs: false, + flush_transcript_tail_on_session_end: false, codex_responses_as_items: false, codex_response_item_prefix: None, codex_response_handoff_prefix: None, @@ -1539,6 +1589,7 @@ async fn conversation_second_start_replaces_runtime() -> Result<()> { test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { client_managed_handoffs: false, + flush_transcript_tail_on_session_end: false, codex_responses_as_items: false, codex_response_item_prefix: None, codex_response_handoff_prefix: None, @@ -1569,6 +1620,7 @@ async fn conversation_second_start_replaces_runtime() -> Result<()> { test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { client_managed_handoffs: false, + flush_transcript_tail_on_session_end: false, codex_responses_as_items: false, codex_response_item_prefix: None, codex_response_handoff_prefix: None, @@ -1670,6 +1722,7 @@ async fn conversation_uses_experimental_realtime_ws_base_url_override() -> Resul test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { client_managed_handoffs: false, + flush_transcript_tail_on_session_end: false, codex_responses_as_items: false, codex_response_item_prefix: None, codex_response_handoff_prefix: None, @@ -1739,6 +1792,7 @@ async fn conversation_uses_default_realtime_backend_prompt() -> Result<()> { test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { client_managed_handoffs: false, + flush_transcript_tail_on_session_end: false, codex_responses_as_items: false, codex_response_item_prefix: None, codex_response_handoff_prefix: None, @@ -1816,6 +1870,7 @@ async fn conversation_uses_empty_instructions_for_null_or_empty_prompt() -> Resu test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { client_managed_handoffs: false, + flush_transcript_tail_on_session_end: false, codex_responses_as_items: false, codex_response_item_prefix: None, codex_response_handoff_prefix: None, @@ -1886,6 +1941,7 @@ async fn conversation_uses_explicit_start_voice() -> Result<()> { test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { client_managed_handoffs: false, + flush_transcript_tail_on_session_end: false, codex_responses_as_items: false, codex_response_item_prefix: None, codex_response_handoff_prefix: None, @@ -1948,6 +2004,7 @@ async fn conversation_uses_configured_realtime_voice() -> Result<()> { test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { client_managed_handoffs: false, + flush_transcript_tail_on_session_end: false, codex_responses_as_items: false, codex_response_item_prefix: None, codex_response_handoff_prefix: None, @@ -1998,6 +2055,7 @@ async fn conversation_rejects_voice_for_wrong_realtime_version() -> Result<()> { test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { client_managed_handoffs: false, + flush_transcript_tail_on_session_end: false, codex_responses_as_items: false, codex_response_item_prefix: None, codex_response_handoff_prefix: None, @@ -2049,6 +2107,7 @@ async fn conversation_uses_experimental_realtime_ws_backend_prompt_override() -> test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { client_managed_handoffs: false, + flush_transcript_tail_on_session_end: false, codex_responses_as_items: false, codex_response_item_prefix: None, codex_response_handoff_prefix: None, @@ -2126,6 +2185,7 @@ async fn conversation_uses_experimental_realtime_ws_startup_context_override() - test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { client_managed_handoffs: false, + flush_transcript_tail_on_session_end: false, codex_responses_as_items: false, codex_response_item_prefix: None, codex_response_handoff_prefix: None, @@ -2197,6 +2257,7 @@ async fn conversation_disables_realtime_startup_context_with_empty_override() -> test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { client_managed_handoffs: false, + flush_transcript_tail_on_session_end: false, codex_responses_as_items: false, codex_response_item_prefix: None, codex_response_handoff_prefix: None, @@ -2261,6 +2322,7 @@ async fn conversation_start_injects_startup_context_from_thread_history() -> Res test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { client_managed_handoffs: false, + flush_transcript_tail_on_session_end: false, codex_responses_as_items: false, codex_response_item_prefix: None, codex_response_handoff_prefix: None, @@ -2380,6 +2442,7 @@ async fn conversation_startup_context_current_thread_selects_many_turns_by_budge codex .submit(Op::RealtimeConversationStart(ConversationStartParams { client_managed_handoffs: false, + flush_transcript_tail_on_session_end: false, codex_responses_as_items: false, codex_response_item_prefix: None, codex_response_handoff_prefix: None, @@ -2492,6 +2555,7 @@ async fn conversation_startup_context_falls_back_to_workspace_map() -> Result<() test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { client_managed_handoffs: false, + flush_transcript_tail_on_session_end: false, codex_responses_as_items: false, codex_response_item_prefix: None, codex_response_handoff_prefix: None, @@ -2556,6 +2620,7 @@ async fn conversation_startup_context_is_truncated_and_sent_once_per_start() -> test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { client_managed_handoffs: false, + flush_transcript_tail_on_session_end: false, codex_responses_as_items: false, codex_response_item_prefix: None, codex_response_handoff_prefix: None, @@ -2641,6 +2706,7 @@ async fn conversation_user_text_turn_is_not_sent_to_realtime() -> Result<()> { test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { client_managed_handoffs: false, + flush_transcript_tail_on_session_end: false, codex_responses_as_items: false, codex_response_item_prefix: None, codex_response_handoff_prefix: None, @@ -2742,6 +2808,7 @@ async fn realtime_v2_noop_tool_call_returns_empty_function_output_without_respon test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { client_managed_handoffs: false, + flush_transcript_tail_on_session_end: false, codex_responses_as_items: false, codex_response_item_prefix: None, codex_response_handoff_prefix: None, @@ -2845,6 +2912,7 @@ async fn conversation_mirrors_assistant_message_text_to_realtime_handoff() -> Re test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { client_managed_handoffs: false, + flush_transcript_tail_on_session_end: false, codex_responses_as_items: false, codex_response_item_prefix: None, codex_response_handoff_prefix: None, @@ -2984,6 +3052,7 @@ async fn conversation_handoff_persists_across_item_done_until_turn_complete() -> test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { client_managed_handoffs: false, + flush_transcript_tail_on_session_end: false, codex_responses_as_items: false, codex_response_item_prefix: None, codex_response_handoff_prefix: Some(SILENT_CONTEXT_PREFIX.to_string()), @@ -3140,6 +3209,7 @@ async fn inbound_handoff_request_starts_turn() -> Result<()> { test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { client_managed_handoffs: false, + flush_transcript_tail_on_session_end: false, codex_responses_as_items: false, codex_response_item_prefix: None, codex_response_handoff_prefix: None, @@ -3254,6 +3324,7 @@ async fn inbound_handoff_request_uses_active_transcript() -> Result<()> { test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { client_managed_handoffs: false, + flush_transcript_tail_on_session_end: false, codex_responses_as_items: false, codex_response_item_prefix: None, codex_response_handoff_prefix: None, @@ -3361,6 +3432,7 @@ async fn inbound_handoff_request_sends_transcript_delta_after_each_handoff() -> test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { client_managed_handoffs: false, + flush_transcript_tail_on_session_end: false, codex_responses_as_items: false, codex_response_item_prefix: None, codex_response_handoff_prefix: None, @@ -3426,6 +3498,115 @@ async fn inbound_handoff_request_sends_transcript_delta_after_each_handoff() -> Ok(()) } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn conversation_close_routes_only_remaining_transcript_tail_once() -> Result<()> { + skip_if_no_network!(Ok(())); + + let api_server = start_mock_server().await; + let response_mock = responses::mount_sse_sequence( + &api_server, + vec![ + responses::sse(vec![ + responses::ev_response_created("resp-1"), + responses::ev_assistant_message("msg-1", "first ok"), + responses::ev_completed("resp-1"), + ]), + responses::sse(vec![ + responses::ev_response_created("resp-2"), + responses::ev_assistant_message("msg-2", "tail ok"), + responses::ev_completed("resp-2"), + ]), + ], + ) + .await; + let realtime_server = start_websocket_server(vec![vec![ + vec![ + json!({ + "type": "session.updated", + "session": { "id": "sess_tail", "instructions": "backend prompt" } + }), + json!({ + "type": "conversation.input_transcript.delta", + "delta": "already handed off" + }), + json!({ + "type": "conversation.handoff.requested", + "handoff_id": "handoff_tail", + "item_id": "item_tail", + "input_transcript": "already handed off" + }), + json!({ + "type": "conversation.output_transcript.delta", + "delta": "remaining answer" + }), + json!({ + "type": "conversation.input_transcript.delta", + "delta": "remaining question" + }), + ], + vec![], + ]]) + .await; + let mut builder = test_codex().with_config({ + let realtime_base_url = realtime_server.uri().to_string(); + move |config| { + config.experimental_realtime_ws_base_url = Some(realtime_base_url); + config.realtime.version = RealtimeWsVersion::V1; + } + }); + let test = builder.build(&api_server).await?; + + test.codex + .submit(Op::RealtimeConversationStart(ConversationStartParams { + client_managed_handoffs: false, + flush_transcript_tail_on_session_end: true, + codex_responses_as_items: false, + codex_response_item_prefix: None, + codex_response_handoff_prefix: None, + model: None, + output_modality: RealtimeOutputModality::Audio, + include_startup_context: true, + prompt: Some(Some("backend prompt".to_string())), + realtime_session_id: None, + transport: None, + version: None, + voice: None, + })) + .await?; + + wait_for_event(&test.codex, |event| { + matches!(event, EventMsg::TurnComplete(_)) + }) + .await; + test.codex.submit(Op::RealtimeConversationClose).await?; + + let closed = wait_for_event_match(&test.codex, |msg| match msg { + EventMsg::RealtimeConversationClosed(closed) => Some(closed.clone()), + _ => None, + }) + .await; + assert_eq!(closed.reason.as_deref(), Some("requested")); + + let deadline = tokio::time::Instant::now() + Duration::from_secs(2); + while response_mock.requests().len() < 2 { + assert!(tokio::time::Instant::now() < deadline); + tokio::time::sleep(Duration::from_millis(10)).await; + } + + test.codex.submit(Op::RealtimeConversationClose).await?; + tokio::time::sleep(Duration::from_millis(200)).await; + + let requests = response_mock.requests(); + assert_eq!(requests.len(), 2); + assert!(requests[0].message_input_texts("user").iter().any(|text| text + == "\n already handed off\n user: already handed off\n")); + assert!(requests[1].message_input_texts("user").iter().any(|text| text + == "\n The user just ended their realtime session. Here is the remaining handoff/transcript tail. You probably do not have to do anything; acknowledge the handoff unless the transcript itself asks for something.\n assistant: remaining answer\nuser: remaining question\n")); + + realtime_server.shutdown().await; + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn inbound_conversation_item_does_not_start_turn_and_still_forwards_audio() -> Result<()> { skip_if_no_network!(Ok(())); @@ -3466,6 +3647,7 @@ async fn inbound_conversation_item_does_not_start_turn_and_still_forwards_audio( test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { client_managed_handoffs: false, + flush_transcript_tail_on_session_end: false, codex_responses_as_items: false, codex_response_item_prefix: None, codex_response_handoff_prefix: None, @@ -3593,6 +3775,7 @@ async fn delegated_turn_user_role_echo_does_not_redelegate_and_still_forwards_au test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { client_managed_handoffs: false, + flush_transcript_tail_on_session_end: false, codex_responses_as_items: false, codex_response_item_prefix: None, codex_response_handoff_prefix: None, @@ -3750,6 +3933,7 @@ async fn inbound_handoff_request_does_not_block_realtime_event_forwarding() -> R test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { client_managed_handoffs: false, + flush_transcript_tail_on_session_end: false, codex_responses_as_items: false, codex_response_item_prefix: None, codex_response_handoff_prefix: None, @@ -3896,6 +4080,7 @@ async fn inbound_handoff_request_steers_active_turn() -> Result<()> { test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { client_managed_handoffs: false, + flush_transcript_tail_on_session_end: false, codex_responses_as_items: false, codex_response_item_prefix: None, codex_response_handoff_prefix: None, @@ -4053,6 +4238,7 @@ async fn inbound_handoff_request_starts_turn_and_does_not_block_realtime_audio() test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { client_managed_handoffs: false, + flush_transcript_tail_on_session_end: false, codex_responses_as_items: false, codex_response_item_prefix: None, codex_response_handoff_prefix: None, diff --git a/codex-rs/core/tests/suite/remote_models.rs b/codex-rs/core/tests/suite/remote_models.rs index b63aff5400..16e31e5082 100644 --- a/codex-rs/core/tests/suite/remote_models.rs +++ b/codex-rs/core/tests/suite/remote_models.rs @@ -329,7 +329,7 @@ async fn remote_models_long_model_slug_is_sent_with_custom_reasoning() -> Result /*priority*/ 1_000, TruncationPolicyConfig::bytes(/*limit*/ 10_000), ); - let custom_reasoning_effort = ReasoningEffort::Custom("max".to_string()); + let custom_reasoning_effort = ReasoningEffort::Custom("future".to_string()); remote_model.default_reasoning_level = Some(custom_reasoning_effort.clone()); remote_model.supported_reasoning_levels = vec![ ReasoningEffortPreset { @@ -391,7 +391,7 @@ async fn remote_models_long_model_slug_is_sent_with_custom_reasoning() -> Result .and_then(|reasoning| reasoning.get("summary")) .and_then(|value| value.as_str()); assert_eq!(body["model"].as_str(), Some(requested_model)); - assert_eq!(reasoning_effort, Some("max")); + assert_eq!(reasoning_effort, Some("future")); assert_eq!(reasoning_summary, Some("detailed")); Ok(()) @@ -485,6 +485,7 @@ async fn remote_models_remote_model_uses_unified_exec() -> Result<()> { upgrade: None, base_instructions: "base instructions".to_string(), model_messages: None, + include_skills_usage_instructions: false, supports_reasoning_summaries: false, default_reasoning_summary: ReasoningSummary::Auto, support_verbosity: false, @@ -738,6 +739,7 @@ async fn remote_models_apply_remote_base_instructions() -> Result<()> { upgrade: None, base_instructions: remote_base.to_string(), model_messages: None, + include_skills_usage_instructions: false, supports_reasoning_summaries: false, default_reasoning_summary: ReasoningSummary::Auto, support_verbosity: false, @@ -1059,7 +1061,11 @@ async fn remote_models_request_times_out_after_5s() -> Result<()> { let start = Instant::now(); let model = timeout( Duration::from_secs(7), - manager.get_default_model(&None, RefreshStrategy::OnlineIfUncached), + manager.get_default_model( + &None, + /*allow_provider_model_fallback*/ false, + RefreshStrategy::OnlineIfUncached, + ), ) .await; let elapsed = start.elapsed(); @@ -1127,7 +1133,11 @@ async fn remote_models_hide_picker_only_models() -> Result<()> { ); let selected = manager - .get_default_model(&None, RefreshStrategy::OnlineIfUncached) + .get_default_model( + &None, + /*allow_provider_model_fallback*/ false, + RefreshStrategy::OnlineIfUncached, + ) .await; assert_eq!(selected, bundled_default_model_slug()); @@ -1224,6 +1234,7 @@ fn test_remote_model_with_policy( upgrade: None, base_instructions: "base instructions".to_string(), model_messages: None, + include_skills_usage_instructions: false, supports_reasoning_summaries: false, default_reasoning_summary: ReasoningSummary::Auto, support_verbosity: false, diff --git a/codex-rs/core/tests/suite/responses_lite.rs b/codex-rs/core/tests/suite/responses_lite.rs index 8074344111..5f623bd4c6 100644 --- a/codex-rs/core/tests/suite/responses_lite.rs +++ b/codex-rs/core/tests/suite/responses_lite.rs @@ -1,3 +1,4 @@ +use std::collections::HashMap; use std::sync::Arc; use anyhow::Context; @@ -56,6 +57,18 @@ fn has_hosted_tool(tools: &[Value], tool_type: &str) -> bool { .any(|tool| tool.get("type").and_then(Value::as_str) == Some(tool_type)) } +fn has_namespaced_tool(tools: &[Value], namespace: &str, tool_name: &str) -> bool { + tools.iter().any(|tool| { + tool.get("type").and_then(Value::as_str) == Some("namespace") + && tool.get("name").and_then(Value::as_str) == Some(namespace) + && tool["tools"].as_array().is_some_and(|tools| { + tools + .iter() + .any(|tool| tool.get("name").and_then(Value::as_str) == Some(tool_name)) + }) + }) +} + fn additional_tools(body: &Value) -> Result<&[Value]> { body["input"] .as_array() @@ -227,13 +240,58 @@ async fn responses_lite_uses_standalone_web_search_and_image_generation() -> Res let body = request.body_json(); assert!(body.get("tools").is_none()); let tools = additional_tools(&body)?; - assert!(!tools.is_empty()); + assert!(has_namespaced_tool(tools, "web", "run")); + assert!(has_namespaced_tool(tools, "image_gen", "imagegen")); assert!(!has_hosted_tool(tools, "web_search")); assert!(!has_hosted_tool(tools, "image_generation")); Ok(()) } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn responses_lite_exposes_standalone_tools_for_actor_authorized_provider() -> Result<()> { + skip_if_no_network!(Ok(())); + + let server = responses::start_mock_server().await; + let response_mock = responses::mount_sse_once( + &server, + responses::sse(vec![ + responses::ev_response_created("resp-1"), + responses::ev_completed("resp-1"), + ]), + ) + .await; + + let auth = CodexAuth::from_api_key("dummy"); + let extensions = responses_extensions(&auth); + let mut builder = test_codex() + .with_auth(auth) + .with_extensions(extensions) + .with_model_info_override("gpt-5.4", |model_info| { + model_info.use_responses_lite = true; + configure_image_capable_model(model_info); + }) + .with_config(|config| { + configure_responses_tools(config); + config.model_provider.name = "local".to_string(); + config.model_provider.requires_openai_auth = false; + config.model_provider.http_headers = Some(HashMap::from([( + "x-openai-actor-authorization".to_string(), + "test-actor-authorization".to_string(), + )])); + }); + let test = builder.build(&server).await?; + + test.submit_turn("Use standalone tools").await?; + + let body = response_mock.single_request().body_json(); + let tools = additional_tools(&body)?; + assert!(has_namespaced_tool(tools, "web", "run")); + assert!(has_namespaced_tool(tools, "image_gen", "imagegen")); + + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn responses_lite_compact_request_uses_lite_transport_contract() -> Result<()> { skip_if_no_network!(Ok(())); diff --git a/codex-rs/core/tests/suite/review.rs b/codex-rs/core/tests/suite/review.rs index af44baf6ca..e403b7e1ff 100644 --- a/codex-rs/core/tests/suite/review.rs +++ b/codex-rs/core/tests/suite/review.rs @@ -29,6 +29,7 @@ use core_test_support::wait_for_event; use pretty_assertions::assert_eq; use std::path::PathBuf; use std::sync::Arc; +use std::time::Duration; use tempfile::TempDir; use tokio::io::AsyncWriteExt as _; use uuid::Uuid; @@ -200,6 +201,95 @@ async fn review_op_emits_lifecycle_and_review_output() { server.verify().await; } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn cancelled_review_does_not_forward_delegate_mcp_startup() { + skip_if_no_network!(); + + let server = start_mock_server().await; + let request_log = responses::mount_response_once( + &server, + responses::sse_response(responses::sse(vec![responses::ev_response_created( + "resp-1", + )])) + .set_delay(Duration::from_secs(30)), + ) + .await; + let codex_home = Arc::new(TempDir::new().unwrap()); + let codex = new_conversation_for_server(&server, codex_home.clone(), |_| {}).await; + + // Consume the parent session's own empty startup round before starting the review. + wait_for_event(&codex, |event| { + matches!(event, EventMsg::McpStartupComplete(_)) + }) + .await; + + codex + .submit(Op::Review { + review_request: ReviewRequest { + target: ReviewTarget::Custom { + instructions: "Cancel this review".to_string(), + }, + user_facing_hint: None, + }, + }) + .await + .unwrap(); + + tokio::time::timeout(Duration::from_secs(5), async { + loop { + match codex.next_event().await.expect("review event").msg { + event @ (EventMsg::McpStartupUpdate(_) | EventMsg::McpStartupComplete(_)) => { + panic!("review forwarded delegate MCP startup: {event:?}") + } + EventMsg::EnteredReviewMode(_) => break, + _ => {} + } + } + }) + .await + .expect("timed out waiting for review entry"); + + tokio::time::timeout(Duration::from_secs(5), async { + while request_log.requests().is_empty() { + tokio::task::yield_now().await; + } + }) + .await + .expect("review request did not reach the server"); + + codex.submit(Op::Interrupt).await.unwrap(); + + let mut exited_review = false; + tokio::time::timeout(Duration::from_secs(5), async { + loop { + match codex + .next_event() + .await + .expect("review cancellation event") + .msg + { + event @ (EventMsg::McpStartupUpdate(_) | EventMsg::McpStartupComplete(_)) => { + panic!("cancelled review forwarded delegate MCP startup: {event:?}") + } + EventMsg::ExitedReviewMode(ExitedReviewModeEvent { review_output }) => { + assert_eq!(review_output, None); + exited_review = true; + } + EventMsg::TurnAborted(_) if exited_review => break, + EventMsg::TurnAborted(_) => panic!("review turn aborted before review mode exited"), + _ => {} + } + } + }) + .await + .expect("timed out waiting for review cancellation"); + + assert_eq!(request_log.requests().len(), 1); + + let _codex_home_guard = codex_home; + server.verify().await; +} + /// When the model returns plain text that is not JSON, ensure the child /// lifecycle still occurs and the plain text is surfaced via /// ExitedReviewMode(Some(..)) as the overall_explanation. diff --git a/codex-rs/core/tests/suite/rmcp_client.rs b/codex-rs/core/tests/suite/rmcp_client.rs index a4d56f8cca..e5bc6a34b6 100644 --- a/codex-rs/core/tests/suite/rmcp_client.rs +++ b/codex-rs/core/tests/suite/rmcp_client.rs @@ -24,6 +24,7 @@ use codex_config::types::McpServerTransportConfig; use codex_core::config::Config; use codex_exec_server::CreateDirectoryOptions; use codex_exec_server::Environment; +use codex_exec_server::HttpRedirectPolicy; use codex_exec_server::HttpRequestParams; use codex_login::CodexAuth; use codex_mcp::MCP_SANDBOX_STATE_META_CAPABILITY; @@ -1358,6 +1359,9 @@ async fn stdio_image_responses_round_trip() -> anyhow::Result<()> { connector_id: None, mcp_app_resource_uri: None, link_id: None, + app_name: None, + template_id: None, + action_name: None, plugin_id: None, }, ); @@ -1647,6 +1651,7 @@ async fn stdio_image_responses_are_sanitized_for_text_only_model() -> anyhow::Re upgrade: None, base_instructions: "base instructions".to_string(), model_messages: None, + include_skills_usage_instructions: false, supports_reasoning_summaries: false, default_reasoning_summary: ReasoningSummary::Auto, support_verbosity: false, @@ -2934,6 +2939,7 @@ async fn wait_for_remote_streamable_http_server( headers: Vec::new(), body: None, timeout_ms: Some(remaining.as_millis().clamp(1, 1_000) as u64), + redirect_policy: HttpRedirectPolicy::Follow, request_id: "buffered-request".to_string(), stream_response: false, }; diff --git a/codex-rs/core/tests/suite/safety_buffering.rs b/codex-rs/core/tests/suite/safety_buffering.rs index 6e41b23766..473cc18ced 100644 --- a/codex-rs/core/tests/suite/safety_buffering.rs +++ b/codex-rs/core/tests/suite/safety_buffering.rs @@ -19,7 +19,7 @@ use serde_json::json; const FASTER_MODEL: &str = "faster-model"; #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn emits_safety_buffering_with_the_requested_model() -> anyhow::Result<()> { +async fn emits_safety_buffering_with_the_header_fallback_model() -> anyhow::Result<()> { skip_if_no_network!(Ok(())); let server = start_mock_server().await; @@ -72,3 +72,58 @@ async fn emits_safety_buffering_with_the_requested_model() -> anyhow::Result<()> Ok(()) } + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn emits_safety_buffering_with_the_responses_api_model_without_header_gating() +-> anyhow::Result<()> { + skip_if_no_network!(Ok(())); + + let server = start_mock_server().await; + let mut created = ev_response_created("resp-1"); + created["safety_buffering"] = json!({ + "use_cases": ["cyber"], + "reasons": ["policy-check"], + "retry_model": FASTER_MODEL, + }); + mount_response_once( + &server, + sse_response(sse(vec![created, ev_completed("resp-1")])), + ) + .await; + + let test = test_codex().build(&server).await?; + test.codex + .submit(Op::UserInput { + items: vec![UserInput::Text { + text: "Check this request".into(), + text_elements: Vec::new(), + }], + final_output_json_schema: None, + responsesapi_client_metadata: None, + additional_context: Default::default(), + thread_settings: Default::default(), + }) + .await?; + + let event = wait_for_event_match(&test.codex, |event| match event { + EventMsg::SafetyBuffering(event) => Some(event.clone()), + _ => None, + }) + .await; + assert_eq!( + event, + SafetyBufferingEvent { + model: test.session_configured.model.clone(), + use_cases: vec!["cyber".to_string()], + reasons: vec!["policy-check".to_string()], + show_buffering_ui: true, + faster_model: Some(FASTER_MODEL.to_string()), + } + ); + wait_for_event(&test.codex, |event| { + matches!(event, EventMsg::TurnComplete(_)) + }) + .await; + + Ok(()) +} diff --git a/codex-rs/core/tests/suite/search_tool.rs b/codex-rs/core/tests/suite/search_tool.rs index edfc53fdc6..50ccea0d87 100644 --- a/codex-rs/core/tests/suite/search_tool.rs +++ b/codex-rs/core/tests/suite/search_tool.rs @@ -545,6 +545,8 @@ async fn tool_search_returns_deferred_tools_without_follow_up_tool_injection() - unreachable!("event guard guarantees McpToolCallBegin"); }; assert_eq!(begin.call_id, "calendar-call-1"); + assert_eq!(begin.app_name.as_deref(), Some("Calendar")); + assert_eq!(begin.action_name.as_deref(), Some("calendar_create_event")); assert_eq!( begin.mcp_app_resource_uri.as_deref(), Some(CALENDAR_CREATE_EVENT_MCP_APP_RESOURCE_URI) @@ -559,6 +561,8 @@ async fn tool_search_returns_deferred_tools_without_follow_up_tool_injection() - }; assert_eq!(end.call_id, "calendar-call-1"); assert_eq!(end.connector_id.as_deref(), Some("calendar")); + assert_eq!(end.app_name.as_deref(), Some("Calendar")); + assert_eq!(end.action_name.as_deref(), Some("calendar_create_event")); assert_eq!( end.mcp_app_resource_uri.as_deref(), Some(CALENDAR_CREATE_EVENT_MCP_APP_RESOURCE_URI) @@ -823,7 +827,7 @@ async fn tool_search_returns_deferred_v1_multi_agent_tools() -> Result<()> { assert!( !first_request_body .to_string() - .contains("Only use `spawn_agent` if and only if"), + .contains("### When to delegate vs. do the subtask yourself"), "deferred v1 multi-agent guidance should stay out of initial developer context" ); @@ -854,10 +858,10 @@ async fn tool_search_returns_deferred_v1_multi_agent_tools() -> Result<()> { .and_then(Value::as_str) .expect("spawn_agent description should be present"); assert!(description.contains( - "Do not spawn sub-agents unless the user explicitly asks for sub-agents, delegation, or parallel agent work." + "Do not spawn sub-agents unless the user or applicable AGENTS.md/skill instructions explicitly ask for sub-agents, delegation, or parallel agent work." )); assert!(description.contains("### Designing delegated subtasks")); - assert!(!description.contains("### When to delegate vs. do the subtask yourself")); + assert!(description.contains("### When to delegate vs. do the subtask yourself")); Ok(()) } diff --git a/codex-rs/core/tests/suite/spawn_agent_description.rs b/codex-rs/core/tests/suite/spawn_agent_description.rs index fb493c1b14..e7076e7465 100644 --- a/codex-rs/core/tests/suite/spawn_agent_description.rs +++ b/codex-rs/core/tests/suite/spawn_agent_description.rs @@ -71,6 +71,7 @@ fn test_model_info( upgrade: None, base_instructions: "base instructions".to_string(), model_messages: None, + include_skills_usage_instructions: false, supports_reasoning_summaries: false, default_reasoning_summary: ReasoningSummary::Auto, support_verbosity: false, @@ -213,13 +214,15 @@ async fn spawn_agent_description_lists_visible_models_and_reasoning_efforts() -> ); assert!( description.contains( - "Do not spawn sub-agents unless the user explicitly asks for sub-agents, delegation, or parallel agent work." + "Do not spawn sub-agents unless the user or applicable AGENTS.md/skill instructions explicitly ask for sub-agents, delegation, or parallel agent work." ), "expected explicit authorization rule in spawn_agent description: {description:?}" ); assert!( - !description.contains("### When to delegate vs. do the subtask yourself"), - "spawn_agent description should not include extra when-to-use delegation guidance: {description:?}" + description.contains( + "Requests for depth, thoroughness, research, investigation, or detailed codebase analysis do not count as permission to spawn." + ) && description.contains("### When to delegate vs. do the subtask yourself"), + "expected delegation decision guidance in spawn_agent description: {description:?}" ); assert!( description.contains( diff --git a/codex-rs/core/tests/suite/sqlite_state.rs b/codex-rs/core/tests/suite/sqlite_state.rs index 5eea5a09e7..726d74ebfd 100644 --- a/codex-rs/core/tests/suite/sqlite_state.rs +++ b/codex-rs/core/tests/suite/sqlite_state.rs @@ -372,7 +372,9 @@ async fn backfill_scans_existing_rollouts() -> Result<()> { model_provider: None, base_instructions: None, dynamic_tools: None, + selected_capability_roots: Vec::new(), memory_mode: None, + history_mode: Default::default(), multi_agent_version: None, context_window: None, }, diff --git a/codex-rs/core/tests/suite/subagent_notifications.rs b/codex-rs/core/tests/suite/subagent_notifications.rs index aad8fc494b..7424d3f360 100644 --- a/codex-rs/core/tests/suite/subagent_notifications.rs +++ b/codex-rs/core/tests/suite/subagent_notifications.rs @@ -39,10 +39,13 @@ use serde_json::Value; use serde_json::json; use std::fs; use std::path::Path; +use std::sync::Mutex; use std::time::Duration; use test_case::test_case; use tokio::time::Instant; use tokio::time::sleep; +use tracing::Level; +use tracing_test::internal::MockWriter; use wiremock::MockServer; const SPAWN_CALL_ID: &str = "spawn-call-1"; @@ -96,6 +99,13 @@ fn decoded_body(req: &wiremock::Request) -> Option> { } } +fn log_field<'a>(line: &'a str, name: &str) -> Option<&'a str> { + let prefix = format!("{name}="); + line.split_ascii_whitespace() + .find_map(|field| field.strip_prefix(&prefix)) + .map(|value| value.trim_matches('"')) +} + fn has_subagent_notification(req: &ResponsesRequest) -> bool { req.message_input_texts("user") .iter() @@ -761,7 +771,9 @@ async fn subagent_stop_replaces_stop_and_skips_internal_subagents() -> Result<() .thread_manager .start_thread_with_options(StartThreadOptions { config: test.config.clone(), + allow_provider_model_fallback: false, initial_history: InitialHistory::New, + history_mode: None, session_source: Some(SessionSource::SubAgent(SubAgentSource::Review)), thread_source: None, dynamic_tools: Vec::new(), @@ -1040,8 +1052,16 @@ async fn spawned_multi_agent_v2_child_inherits_parent_developer_context() -> Res Ok(()) } -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[tokio::test] async fn encrypted_multi_agent_v2_spawn_sends_agent_message_to_child() -> Result<()> { + let output: &'static Mutex> = Box::leak(Box::new(Mutex::new(Vec::new()))); + let subscriber = tracing_subscriber::fmt() + .with_ansi(false) + .with_max_level(Level::INFO) + .with_writer(MockWriter::new(output)) + .finish(); + let _guard = tracing::subscriber::set_default(subscriber); + let server = start_mock_server().await; let encrypted_message = "opaque-encrypted-message"; let spawn_args = serde_json::to_string(&json!({ @@ -1096,6 +1116,7 @@ async fn encrypted_multi_agent_v2_spawn_sends_agent_message_to_child() -> Result .expect("test config should allow feature update"); }); let test = builder.build(&server).await?; + let root_thread_id = test.session_configured.thread_id; test.submit_turn(TURN_1_PROMPT).await?; @@ -1122,6 +1143,41 @@ async fn encrypted_multi_agent_v2_spawn_sends_agent_message_to_child() -> Result })]) ); + let child_thread_id = test + .thread_manager + .list_thread_ids() + .await + .into_iter() + .find(|thread_id| *thread_id != root_thread_id) + .expect("child thread ID"); + let logs = tokio::time::timeout(Duration::from_secs(5), async { + loop { + let logs = String::from_utf8(output.lock().expect("buffer lock").clone()) + .expect("logs should be UTF-8"); + if logs.contains("kind=\"spawn\"") && logs.contains("state=\"receive\"") { + break logs; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("spawn communication logs should be emitted"); + let send = logs + .lines() + .find(|line| line.contains("kind=\"spawn\"") && line.contains("state=\"send\"")) + .expect("spawn send event"); + assert!(send.contains(&format!("sender_thread_id={root_thread_id}"))); + assert!(send.contains(&format!("receiver_thread_id={child_thread_id}"))); + assert!(send.contains(&format!("content=\"{encrypted_message}\""))); + + let communication_id = log_field(send, "communication_id").expect("communication ID"); + logs.lines() + .find(|line| { + line.contains("state=\"receive\"") + && log_field(line, "communication_id") == Some(communication_id) + }) + .expect("correlated receive event"); + Ok(()) } diff --git a/codex-rs/core/tests/suite/tools.rs b/codex-rs/core/tests/suite/tools.rs index 83584b8ec0..5e47eae52e 100644 --- a/codex-rs/core/tests/suite/tools.rs +++ b/codex-rs/core/tests/suite/tools.rs @@ -20,6 +20,7 @@ use core_test_support::assert_regex_match; use core_test_support::responses::ev_assistant_message; use core_test_support::responses::ev_completed; use core_test_support::responses::ev_custom_tool_call; +use core_test_support::responses::ev_custom_tool_call_with_namespace; use core_test_support::responses::ev_function_call; use core_test_support::responses::ev_response_created; use core_test_support::responses::mount_sse_once; @@ -177,6 +178,80 @@ async fn custom_tool_unknown_returns_custom_output_error() -> Result<()> { Ok(()) } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn namespaced_custom_tool_call_preserves_namespace_through_dispatch_and_replay() -> Result<()> +{ + skip_if_no_network!(Ok(())); + + let server = start_mock_server().await; + let mut builder = test_codex(); + let test = builder.build(&server).await?; + + let call_id = "custom-namespaced"; + let namespace = "test_namespace::"; + let tool_name = "unsupported_tool"; + let input = "\"payload\""; + + mount_sse_once( + &server, + sse(vec![ + ev_response_created("resp-1"), + ev_custom_tool_call_with_namespace(call_id, namespace, tool_name, input), + ev_completed("resp-1"), + ]), + ) + .await; + let mock = mount_sse_once( + &server, + sse(vec![ + ev_assistant_message("msg-1", "done"), + ev_completed("resp-2"), + ]), + ) + .await; + + test.submit_turn_with_approval_and_permission_profile( + "invoke namespaced custom tool", + AskForApproval::Never, + PermissionProfile::Disabled, + ) + .await?; + + let request = mock.single_request(); + let custom_tool_calls = request.inputs_of_type("custom_tool_call"); + let turn_id = custom_tool_calls + .first() + .and_then(|item| item.pointer("/internal_chat_message_metadata_passthrough/turn_id")) + .and_then(Value::as_str) + .map(str::to_string) + .expect("custom tool call should include turn metadata"); + assert_eq!( + (custom_tool_calls, request.custom_tool_call_output(call_id),), + ( + vec![json!({ + "type": "custom_tool_call", + "call_id": call_id, + "namespace": namespace, + "name": tool_name, + "input": input, + "internal_chat_message_metadata_passthrough": { + "turn_id": turn_id, + }, + })], + json!({ + "type": "custom_tool_call_output", + "call_id": call_id, + "output": format!("unsupported custom tool call: {namespace}{tool_name}"), + "internal_chat_message_metadata_passthrough": { + "turn_id": turn_id, + }, + }), + ) + ); + + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn shell_command_escalated_permissions_rejected_then_ok() -> Result<()> { skip_if_no_network!(Ok(())); diff --git a/codex-rs/core/tests/suite/unified_exec_process_events.rs b/codex-rs/core/tests/suite/unified_exec_process_events.rs new file mode 100644 index 0000000000..8ebd332ea2 --- /dev/null +++ b/codex-rs/core/tests/suite/unified_exec_process_events.rs @@ -0,0 +1,489 @@ +use anyhow::Context; +use anyhow::Result; +use base64::Engine; +use base64::engine::general_purpose::STANDARD as BASE64_STANDARD; +use codex_features::Feature; +use codex_protocol::models::PermissionProfile; +use codex_protocol::protocol::AskForApproval; +use codex_protocol::protocol::EventMsg; +use codex_protocol::protocol::Op; +use codex_protocol::user_input::UserInput; +use core_test_support::responses::ev_assistant_message; +use core_test_support::responses::ev_completed; +use core_test_support::responses::ev_function_call; +use core_test_support::responses::ev_response_created; +use core_test_support::responses::mount_sse_sequence; +use core_test_support::responses::sse; +use core_test_support::responses::start_mock_server; +use core_test_support::test_codex::test_codex; +use core_test_support::test_codex::turn_permission_fields; +use futures::SinkExt; +use futures::StreamExt; +use pretty_assertions::assert_eq; +use serde_json::Value; +use serde_json::json; +use std::time::Duration; +use test_case::test_case; +use tokio::net::TcpListener; +use tokio::net::TcpStream; +use tokio::time::timeout; +use tokio_tungstenite::WebSocketStream; +use tokio_tungstenite::accept_async; +use tokio_tungstenite::tungstenite::Message; + +const CALL_ID: &str = "pushed-remote-process-events"; +const COMPLETE_OUTPUT: &str = "pushed remote output\n"; +const RECOVERED_OUTPUT: &str = "recovered missing output\n"; +const RETAINED_OUTPUT: &str = "retained output\n"; +const REPLAY_OUTPUT_EVENT_COUNT: u64 = 1024; +const REPLAY_RETAINED_OUTPUT_SEQ: u64 = 800; + +#[derive(Debug, Clone, Copy)] +enum PushedExecScenario { + Complete, + DirectDenied, + LegacyExit, + ReplayGap, +} + +async fn read_exec_server_json(websocket: &mut WebSocketStream) -> Value { + loop { + match timeout(Duration::from_secs(5), websocket.next()) + .await + .expect("websocket read should not time out") + .expect("websocket should stay open") + .expect("websocket frame should read") + { + Message::Text(text) => { + return serde_json::from_str(text.as_ref()).expect("valid JSON-RPC message"); + } + Message::Binary(bytes) => { + return serde_json::from_slice(bytes.as_ref()).expect("valid JSON-RPC message"); + } + Message::Ping(_) | Message::Pong(_) => {} + other => panic!("expected JSON-RPC message, got {other:?}"), + } + } +} + +async fn send_exec_server_json(websocket: &mut WebSocketStream, message: Value) { + websocket + .send(Message::Text(message.to_string().into())) + .await + .expect("exec-server message should send"); +} + +async fn accept_initialized_exec_server(listener: TcpListener) -> WebSocketStream { + let (stream, _) = listener.accept().await.expect("connection"); + let mut websocket = accept_async(stream).await.expect("websocket handshake"); + + let initialize = read_exec_server_json(&mut websocket).await; + assert_eq!(initialize["method"], "initialize"); + send_exec_server_json( + &mut websocket, + json!({ + "id": initialize["id"], + "result": { "sessionId": "test-session" } + }), + ) + .await; + let initialized = read_exec_server_json(&mut websocket).await; + assert_eq!(initialized["method"], "initialized"); + + websocket +} + +async fn send_environment_info(websocket: &mut WebSocketStream) { + let info = read_exec_server_json(websocket).await; + assert_eq!(info["method"], "environment/info"); + send_exec_server_json( + websocket, + json!({ + "id": info["id"], + "result": { "shell": { "name": "zsh", "path": "/bin/zsh" } } + }), + ) + .await; +} + +async fn serve_exec_with_pushed_events( + listener: TcpListener, + scenario: PushedExecScenario, +) -> usize { + let mut websocket = accept_initialized_exec_server(listener).await; + send_environment_info(&mut websocket).await; + + let process_start = loop { + let request = read_exec_server_json(&mut websocket).await; + match request["method"].as_str() { + Some("process/start") => break request, + Some("fs/getMetadata") => { + send_exec_server_json( + &mut websocket, + json!({ + "id": request["id"], + "error": { "code": -32004, "message": "not found" } + }), + ) + .await; + } + Some("fs/canonicalize") => { + send_exec_server_json( + &mut websocket, + json!({ + "id": request["id"], + "result": { "path": request["params"]["path"] } + }), + ) + .await; + } + method => panic!("unexpected exec-server request before process/start: {method:?}"), + } + }; + let process_id = process_start["params"]["processId"] + .as_str() + .expect("process/start should include processId") + .to_string(); + + let replay_output = |seq| -> &'static [u8] { + match seq { + 1 => RECOVERED_OUTPUT.as_bytes(), + REPLAY_RETAINED_OUTPUT_SEQ => RETAINED_OUTPUT.as_bytes(), + _ => b"x", + } + }; + if matches!(scenario, PushedExecScenario::ReplayGap) { + // The process replay log retains 256 events. This burst is much larger + // than both that log and the JSON-RPC event queue, so the reader must + // apply enough notifications to evict seq 1 before it can read the + // start response. The total output stays well below the server's 1 MiB + // retained-output limit, making the subsequent read genuinely able to + // recover every missing chunk. + for seq in 1..=REPLAY_OUTPUT_EVENT_COUNT { + send_exec_server_json( + &mut websocket, + json!({ + "method": "process/output", + "params": { + "processId": &process_id, + "seq": seq, + "stream": "stdout", + "chunk": BASE64_STANDARD.encode(replay_output(seq)), + } + }), + ) + .await; + } + send_exec_server_json( + &mut websocket, + json!({ + "method": "process/exited", + "params": { + "processId": &process_id, + "seq": REPLAY_OUTPUT_EVENT_COUNT + 1, + "exitCode": 0, + "sandboxDenied": false, + } + }), + ) + .await; + } + + send_exec_server_json( + &mut websocket, + json!({ + "id": process_start["id"], + "result": { "processId": &process_id } + }), + ) + .await; + + match scenario { + PushedExecScenario::Complete => { + let encoded_output = BASE64_STANDARD.encode(COMPLETE_OUTPUT); + for message in [ + json!({ + "method": "process/output", + "params": { + "processId": &process_id, + "seq": 1, + "stream": "stdout", + "chunk": encoded_output, + } + }), + json!({ + "method": "process/exited", + "params": { + "processId": &process_id, + "seq": 2, + "exitCode": 0, + "sandboxDenied": false, + } + }), + json!({ + "method": "process/closed", + "params": { "processId": &process_id, "seq": 3 } + }), + ] { + send_exec_server_json(&mut websocket, message).await; + } + } + PushedExecScenario::DirectDenied => { + send_exec_server_json( + &mut websocket, + json!({ + "method": "process/exited", + "params": { + "processId": &process_id, + "seq": 1, + "exitCode": 1, + "sandboxDenied": true, + } + }), + ) + .await; + } + PushedExecScenario::LegacyExit => { + send_exec_server_json( + &mut websocket, + json!({ + "method": "process/exited", + "params": { + "processId": &process_id, + "seq": 1, + "exitCode": 1, + } + }), + ) + .await; + } + PushedExecScenario::ReplayGap => {} + } + + let mut process_read_requests = 0; + loop { + let request = read_exec_server_json(&mut websocket).await; + match request["method"].as_str() { + Some("process/read") => { + process_read_requests += 1; + let result = match scenario { + PushedExecScenario::Complete => json!({ + "chunks": [{ + "seq": 1, + "stream": "stdout", + "chunk": BASE64_STANDARD.encode(COMPLETE_OUTPUT), + }], + "nextSeq": 4, + "exited": true, + "exitCode": 0, + "closed": true, + "failure": null, + "sandboxDenied": false, + }), + PushedExecScenario::DirectDenied => json!({ + "chunks": [], + "nextSeq": 2, + "exited": true, + "exitCode": 1, + "closed": false, + "failure": null, + "sandboxDenied": true, + }), + PushedExecScenario::LegacyExit => json!({ + "chunks": [], + "nextSeq": 3, + "exited": true, + "exitCode": 1, + "closed": true, + "failure": null, + "sandboxDenied": true, + }), + PushedExecScenario::ReplayGap => { + let chunks = (1..=REPLAY_OUTPUT_EVENT_COUNT) + .map(|seq| { + json!({ + "seq": seq, + "stream": "stdout", + "chunk": BASE64_STANDARD.encode(replay_output(seq)), + }) + }) + .collect::>(); + json!({ + "chunks": chunks, + "nextSeq": REPLAY_OUTPUT_EVENT_COUNT + 2, + "exited": true, + "exitCode": 0, + "closed": false, + "failure": null, + "sandboxDenied": false, + }) + } + }; + send_exec_server_json( + &mut websocket, + json!({ + "id": request["id"], + "result": result, + }), + ) + .await; + if matches!(scenario, PushedExecScenario::ReplayGap) && process_read_requests == 1 { + send_exec_server_json( + &mut websocket, + json!({ + "method": "process/closed", + "params": { + "processId": &process_id, + "seq": REPLAY_OUTPUT_EVENT_COUNT + 2, + } + }), + ) + .await; + } + } + Some("process/terminate") => { + send_exec_server_json( + &mut websocket, + json!({ + "id": request["id"], + "result": { "running": false } + }), + ) + .await; + return process_read_requests; + } + method => panic!("unexpected exec-server request: {method:?}"), + } + } +} + +#[test_case(PushedExecScenario::Complete ; "complete_event_stream")] +#[test_case(PushedExecScenario::DirectDenied ; "direct_sandbox_denial")] +#[test_case(PushedExecScenario::LegacyExit ; "legacy_exit_metadata")] +#[test_case(PushedExecScenario::ReplayGap ; "truncated_event_replay")] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn exec_command_consumes_pushed_remote_process_events( + scenario: PushedExecScenario, +) -> Result<()> { + let listener = TcpListener::bind("127.0.0.1:0").await?; + let server = start_mock_server().await; + let response_mock = mount_sse_sequence( + &server, + vec![ + sse(vec![ + ev_response_created("resp-1"), + ev_function_call( + CALL_ID, + "exec_command", + &json!({ + "cmd": "ignored by fake exec-server", + "yield_time_ms": 1_000, + }) + .to_string(), + ), + ev_completed("resp-1"), + ]), + sse(vec![ + ev_response_created("resp-2"), + ev_assistant_message("msg-2", "done"), + ev_completed("resp-2"), + ]), + ], + ) + .await; + let exec_server_url = format!("ws://{}", listener.local_addr()?); + let exec_server = tokio::spawn(serve_exec_with_pushed_events(listener, scenario)); + let mut builder = test_codex() + .with_exec_server_url(exec_server_url) + .with_config(|config| { + config.project_doc_max_bytes = 0; + config.use_experimental_unified_exec_tool = true; + config + .features + .enable(Feature::UnifiedExec) + .expect("test config should allow feature update"); + }); + let test = timeout(Duration::from_secs(5), builder.build(&server)) + .await + .context("thread startup should connect to the fake exec-server")??; + + let (sandbox_policy, permission_profile) = + turn_permission_fields(PermissionProfile::Disabled, test.config.cwd.as_path()); + test.codex + .submit(Op::UserInput { + items: vec![UserInput::Text { + text: "run a one-shot remote command".into(), + text_elements: Vec::new(), + }], + final_output_json_schema: None, + responsesapi_client_metadata: None, + additional_context: Default::default(), + thread_settings: codex_protocol::protocol::ThreadSettingsOverrides { + approval_policy: Some(AskForApproval::Never), + sandbox_policy: Some(sandbox_policy), + permission_profile, + collaboration_mode: Some(codex_protocol::config_types::CollaborationMode { + mode: codex_protocol::config_types::ModeKind::Default, + settings: codex_protocol::config_types::Settings { + model: test.session_configured.model.clone(), + reasoning_effort: None, + developer_instructions: None, + }, + }), + ..Default::default() + }, + }) + .await?; + let mut saw_exec_command_begin = false; + loop { + let event = timeout(Duration::from_secs(5), test.codex.next_event()) + .await + .context("turn should complete")?? + .msg; + match event { + EventMsg::ExecCommandBegin(event) if event.call_id == CALL_ID => { + saw_exec_command_begin = true; + } + EventMsg::TurnComplete(_) => break, + _ => {} + } + } + let process_read_requests = timeout(Duration::from_secs(5), exec_server) + .await + .context("fake exec-server should observe process cleanup")??; + let request = response_mock + .last_request() + .context("model should receive the exec_command output")?; + let (output, success) = request + .function_call_output_content_and_success(CALL_ID) + .context("exec_command output should be model visible")?; + let output = output.context("exec_command output should contain text")?; + match scenario { + PushedExecScenario::Complete => { + assert_ne!(success, Some(false)); + assert!(saw_exec_command_begin); + assert!(output.contains("Process exited with code 0")); + assert!(output.contains(COMPLETE_OUTPUT)); + assert_eq!(process_read_requests, 0, "unexpected compatibility read"); + } + PushedExecScenario::DirectDenied => { + assert!(!saw_exec_command_begin); + assert!(output.contains("Process exited with code 1")); + assert_eq!(process_read_requests, 0, "unexpected compatibility read"); + } + PushedExecScenario::LegacyExit => { + assert!(!saw_exec_command_begin); + assert!(output.contains("Process exited with code 1")); + assert_eq!(process_read_requests, 1, "expected compatibility read"); + } + PushedExecScenario::ReplayGap => { + assert_ne!(success, Some(false)); + assert!(saw_exec_command_begin); + assert_eq!(output.matches(RECOVERED_OUTPUT).count(), 1); + assert_eq!(output.matches(RETAINED_OUTPUT).count(), 1); + assert_eq!(process_read_requests, 1, "expected replay recovery read"); + } + } + + Ok(()) +} diff --git a/codex-rs/core/tests/suite/view_image.rs b/codex-rs/core/tests/suite/view_image.rs index c11c2747d7..a40f2ae105 100644 --- a/codex-rs/core/tests/suite/view_image.rs +++ b/codex-rs/core/tests/suite/view_image.rs @@ -1374,6 +1374,7 @@ async fn view_image_tool_returns_unsupported_message_for_text_only_model() -> an upgrade: None, base_instructions: "base instructions".to_string(), model_messages: None, + include_skills_usage_instructions: false, supports_reasoning_summaries: false, default_reasoning_summary: ReasoningSummary::Auto, support_verbosity: false, diff --git a/codex-rs/deny.toml b/codex-rs/deny.toml index 9d06651e58..4e52422f39 100644 --- a/codex-rs/deny.toml +++ b/codex-rs/deny.toml @@ -70,7 +70,7 @@ feature-depth = 1 # A list of advisory IDs to ignore. Note that ignored advisories will still # output a note when they are encountered. ignore = [ - # Reviewed 2026-06-07. Keep this list in sync with .cargo/audit.toml. + # Reviewed 2026-07-02. Keep this list in sync with .cargo/audit.toml. # Each exception must identify the dependency path and removal condition. { id = "RUSTSEC-2024-0388", reason = "derivative is unmaintained; pulled in via starlark/starlark_syntax v0.14.2 used by execpolicy/cli/core; no fixed starlark release yet" }, { id = "RUSTSEC-2025-0057", reason = "fxhash is unmaintained; pulled in via starlark_map under starlark v0.14.2 used by execpolicy/cli/core and bm25 used by core; remove when upstream dependencies drop fxhash" }, @@ -82,6 +82,8 @@ ignore = [ { id = "RUSTSEC-2026-0118", reason = "hickory-proto v0.25.2 is pulled in via rama-dns/rama-tcp used by codex-network-proxy; DNSSEC features are not enabled; remove when rama updates to hickory 0.26.1 or hickory-net" }, { id = "RUSTSEC-2026-0119", reason = "hickory-proto v0.25.2 is pulled in via rama-dns/rama-tcp used by codex-network-proxy; no fixed rama release is available yet; remove when rama updates to hickory 0.26.1 or hickory-net" }, { id = "RUSTSEC-2026-0173", reason = "proc-macro-error2 is unmaintained; pulled in via i18n-embed-fl/age used by codex-secrets local storage; remove when codex-secrets migrates off age or age drops i18n-embed-fl" }, + { id = "RUSTSEC-2026-0194", reason = "quick-xml v0.39.4 remains via plist/syntect and wayland-scanner; plist does not exercise the affected APIs and wayland-scanner parses trusted build-time XML; remove when rust-plist#191 and wayland-rs#938 are released" }, + { id = "RUSTSEC-2026-0195", reason = "quick-xml v0.39.4 remains via plist/syntect and wayland-scanner; plist does not exercise the affected APIs and wayland-scanner parses trusted build-time XML; remove when rust-plist#191 and wayland-rs#938 are released" }, ] # If this is true, then cargo deny will use the git executable to fetch advisory database. # If this is false, then it uses a built-in git library. diff --git a/codex-rs/exec-server-protocol/src/protocol.rs b/codex-rs/exec-server-protocol/src/protocol.rs index 133654dc0a..a265aaed0c 100644 --- a/codex-rs/exec-server-protocol/src/protocol.rs +++ b/codex-rs/exec-server-protocol/src/protocol.rs @@ -419,6 +419,17 @@ pub struct HttpHeader { pub value: String, } +/// Redirect behavior for an executor-side HTTP request. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum HttpRedirectPolicy { + /// Follow redirects using the HTTP client's normal limits. + #[default] + Follow, + /// Return the redirect response without following its location. + Stop, +} + /// Executor-side HTTP request envelope. /// /// This intentionally stays transport-shaped rather than MCP-shaped so callers @@ -443,6 +454,9 @@ pub struct HttpRequestParams { /// millisecond deadline. #[serde(default, skip_serializing_if = "Option::is_none")] pub timeout_ms: Option, + /// Whether the executor should follow HTTP redirects. + #[serde(default)] + pub redirect_policy: HttpRedirectPolicy, /// Caller-chosen stream id for `http/request/bodyDelta` notifications. /// /// The id must remain unique on a connection until the terminal body delta @@ -513,6 +527,8 @@ pub struct ExecExitedNotification { pub process_id: ProcessId, pub seq: u64, pub exit_code: i32, + #[serde(default)] + pub sandbox_denied: Option, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -550,6 +566,7 @@ mod base64_bytes { #[cfg(test)] mod tests { use super::EnvironmentInfo; + use super::ExecExitedNotification; use super::ExecParams; use super::FsReadFileParams; use super::HttpRequestParams; @@ -694,4 +711,16 @@ mod tests { ("req-explicit-timeout", Some(1234)) ); } + + #[test] + fn exited_notification_accepts_legacy_payload_without_sandbox_denied() { + let notification: ExecExitedNotification = serde_json::from_value(serde_json::json!({ + "processId": "proc-1", + "seq": 3, + "exitCode": 1, + })) + .expect("legacy exited notification should deserialize"); + + assert_eq!(notification.sandbox_denied, None); + } } diff --git a/codex-rs/exec-server/Cargo.toml b/codex-rs/exec-server/Cargo.toml index a8c1133098..35fa411021 100644 --- a/codex-rs/exec-server/Cargo.toml +++ b/codex-rs/exec-server/Cargo.toml @@ -69,6 +69,8 @@ http = { workspace = true } opentelemetry = { workspace = true } opentelemetry_sdk = { workspace = true } pretty_assertions = { workspace = true } +rcgen = { workspace = true } +rustls = { workspace = true } serial_test = { workspace = true } tempfile = { workspace = true } test-case = "3.3.1" diff --git a/codex-rs/exec-server/README.md b/codex-rs/exec-server/README.md index b1f9279186..a250ba5886 100644 --- a/codex-rs/exec-server/README.md +++ b/codex-rs/exec-server/README.md @@ -321,10 +321,15 @@ Params: { "processId": "proc-1", "seq": 2, - "exitCode": 0 + "exitCode": 0, + "sandboxDenied": false } ``` +`sandboxDenied` lets streaming clients preserve executor-side sandbox denial +detection without issuing a final `process/read` request. Clients recover it +with `process/read` when an older server omits the field. + ### `process/closed` Notification emitted after process output is closed and the process handle is @@ -334,7 +339,8 @@ Params: ```json { - "processId": "proc-1" + "processId": "proc-1", + "seq": 3 } ``` @@ -431,6 +437,6 @@ Terminate it: ```json {"id":4,"method":"process/terminate","params":{"processId":"proc-1"}} {"id":4,"result":{"running":true}} -{"method":"process/exited","params":{"processId":"proc-1","seq":3,"exitCode":0}} -{"method":"process/closed","params":{"processId":"proc-1"}} +{"method":"process/exited","params":{"processId":"proc-1","seq":3,"exitCode":0,"sandboxDenied":false}} +{"method":"process/closed","params":{"processId":"proc-1","seq":4}} ``` diff --git a/codex-rs/exec-server/src/client.rs b/codex-rs/exec-server/src/client.rs index e2429d8f14..5590fd3b5e 100644 --- a/codex-rs/exec-server/src/client.rs +++ b/codex-rs/exec-server/src/client.rs @@ -110,6 +110,7 @@ mod recovery; const CONNECT_TIMEOUT: Duration = Duration::from_secs(10); const INITIALIZE_TIMEOUT: Duration = Duration::from_secs(10); +const ENVIRONMENT_INFO_TIMEOUT: Duration = Duration::from_secs(30); const PROCESS_EVENT_CHANNEL_CAPACITY: usize = 256; const PROCESS_EVENT_RETAINED_BYTES: usize = 1024 * 1024; @@ -510,7 +511,12 @@ impl ExecServerClient { } pub async fn environment_info(&self) -> Result { - self.call(ENVIRONMENT_INFO_METHOD, &()).await + let rpc_client = self.inner.rpc_client().await?; + map_rpc_call_result( + rpc_client + .call_with_timeout(ENVIRONMENT_INFO_METHOD, &(), ENVIRONMENT_INFO_TIMEOUT) + .await, + ) } pub async fn read(&self, params: ReadParams) -> Result { @@ -780,22 +786,21 @@ impl ExecServerClient { P: serde::Serialize, T: serde::de::DeserializeOwned, { - match rpc_client.call(method, params).await { - Ok(response) => Ok(response), - Err(error) => { - let error = ExecServerError::from(error); - if is_transport_closed_error(&error) { - Err(ExecServerError::Disconnected(disconnected_message( - /*reason*/ None, - ))) - } else { - Err(error) - } - } - } + map_rpc_call_result(rpc_client.call(method, params).await) } } +fn map_rpc_call_result(result: Result) -> Result { + result.map_err(|error| { + let error = ExecServerError::from(error); + if is_transport_closed_error(&error) { + ExecServerError::Disconnected(disconnected_message(/*reason*/ None)) + } else { + error + } + }) +} + async fn cleanup_process_start( client: &ExecServerClient, process_id: &ProcessId, @@ -822,6 +827,9 @@ impl From for ExecServerError { code: error.code, message: error.message, }, + RpcCallError::TimedOut { method, timeout } => Self::Protocol(format!( + "timed out waiting for exec-server `{method}` response after {timeout:?}" + )), } } } @@ -1161,6 +1169,7 @@ async fn handle_server_notification( let published_closed = session.publish_ordered_event(ExecProcessEvent::Exited { seq: params.seq, exit_code: params.exit_code, + sandbox_denied: params.sandbox_denied, }); if published_closed { inner.remove_session_if(¶ms.process_id, &session); @@ -1385,7 +1394,16 @@ mod tests { assert_eq!(session.process_id(), &process_id); let trace = server.await.expect("server task").expect("trace context"); - assert_eq!(trace, expected_trace); + let expected_traceparent = expected_trace + .traceparent + .as_deref() + .expect("parent traceparent"); + let traceparent = trace.traceparent.as_deref().expect("request traceparent"); + let expected_parts = expected_traceparent.split('-').collect::>(); + let parts = traceparent.split('-').collect::>(); + assert_eq!(parts[1], expected_parts[1]); + assert_ne!(parts[2], expected_parts[2]); + assert_eq!(trace.tracestate, expected_trace.tracestate); } async fn accept_websocket(listener: &TcpListener) -> WebSocketStream { @@ -1737,6 +1755,7 @@ mod tests { process_id: process_id.clone(), seq: 3, exit_code: 0, + sandbox_denied: Some(true), }) .expect("exit notification should serialize"), ), @@ -1786,6 +1805,7 @@ mod tests { ExecProcessEvent::Exited { seq: 3, exit_code: 0, + sandbox_denied: Some(true), }, ExecProcessEvent::Closed { seq: 4 }, ] @@ -2391,6 +2411,7 @@ mod tests { process_id: quiet_process_id, seq: 1, exit_code: 17, + sandbox_denied: Some(false), }) .expect("exit notification should serialize"), ), diff --git a/codex-rs/exec-server/src/client/reqwest_http_client.rs b/codex-rs/exec-server/src/client/reqwest_http_client.rs index 9de35315e1..7ead01c9ba 100644 --- a/codex-rs/exec-server/src/client/reqwest_http_client.rs +++ b/codex-rs/exec-server/src/client/reqwest_http_client.rs @@ -9,6 +9,7 @@ use std::error::Error as StdError; use std::time::Duration; use codex_client::build_reqwest_client_with_custom_ca; +use codex_client::with_chatgpt_cloudflare_cookie_store; use codex_exec_server_protocol::JSONRPCErrorError; use futures::FutureExt; use futures::StreamExt; @@ -18,12 +19,14 @@ use reqwest::Url; use reqwest::header::HeaderMap; use reqwest::header::HeaderName; use reqwest::header::HeaderValue; +use tracing::Instrument; use super::HttpResponseBodyStream; use super::response_body_stream::send_body_delta; use crate::HttpClient; use crate::client::ExecServerError; use crate::protocol::HttpHeader; +use crate::protocol::HttpRedirectPolicy; use crate::protocol::HttpRequestBodyDeltaNotification; use crate::protocol::HttpRequestParams; use crate::protocol::HttpRequestResponse; @@ -50,14 +53,21 @@ pub(crate) struct ReqwestHttpRequestRunner { } impl ReqwestHttpClient { - fn build_client(timeout_ms: Option) -> Result { + fn build_client( + timeout_ms: Option, + redirect_policy: HttpRedirectPolicy, + ) -> Result { let builder = match timeout_ms { None => reqwest::Client::builder(), Some(timeout_ms) => { reqwest::Client::builder().timeout(Duration::from_millis(timeout_ms)) } }; - build_reqwest_client_with_custom_ca(builder) + let builder = match redirect_policy { + HttpRedirectPolicy::Follow => builder, + HttpRedirectPolicy::Stop => builder.redirect(reqwest::redirect::Policy::none()), + }; + build_reqwest_client_with_custom_ca(with_chatgpt_cloudflare_cookie_store(builder)) .map_err(|error| ExecServerError::HttpRequest(error.to_string())) } } @@ -68,7 +78,7 @@ impl HttpClient for ReqwestHttpClient { params: HttpRequestParams, ) -> BoxFuture<'_, Result> { async move { - let runner = ReqwestHttpRequestRunner::new(params.timeout_ms) + let runner = ReqwestHttpRequestRunner::new(params.timeout_ms, params.redirect_policy) .map_err(|error| ExecServerError::HttpRequest(error.message))?; let (response, _) = runner .run(HttpRequestParams { @@ -87,7 +97,7 @@ impl HttpClient for ReqwestHttpClient { params: HttpRequestParams, ) -> BoxFuture<'_, Result<(HttpRequestResponse, HttpResponseBodyStream), ExecServerError>> { async move { - let runner = ReqwestHttpRequestRunner::new(params.timeout_ms) + let runner = ReqwestHttpRequestRunner::new(params.timeout_ms, params.redirect_policy) .map_err(|error| ExecServerError::HttpRequest(error.message))?; let (response, pending_stream) = runner .run(HttpRequestParams { @@ -111,8 +121,11 @@ impl HttpClient for ReqwestHttpClient { } impl ReqwestHttpRequestRunner { - pub(crate) fn new(timeout_ms: Option) -> Result { - let client = ReqwestHttpClient::build_client(timeout_ms) + pub(crate) fn new( + timeout_ms: Option, + redirect_policy: HttpRedirectPolicy, + ) -> Result { + let client = ReqwestHttpClient::build_client(timeout_ms, redirect_policy) .map_err(|error| internal_error(error.to_string()))?; Ok(Self { client }) } @@ -135,15 +148,26 @@ impl ReqwestHttpRequestRunner { } } - let headers = Self::build_headers(params.headers)?; + let request_span = tracing::info_span!( + "codex.exec_server.http_request", + otel.kind = "client", + http.request.method = method.as_str(), + server.address = url.host_str().unwrap_or_default(), + server.port = u64::from(url.port_or_known_default().unwrap_or_default()), + http.response.status_code = tracing::field::Empty, + error.type = tracing::field::Empty, + ); + let mut headers = Self::build_headers(params.headers)?; + codex_otel::inject_span_w3c_trace_headers(&request_span, &mut headers); let mut request = self.client.request(method.clone(), url).headers(headers); if let Some(body) = params.body { request = request.body(body.into_inner()); } - let response = match request.send().await { + let response = match request.send().instrument(request_span.clone()).await { Ok(response) => response, Err(error) => { + request_span.record("error.type", "request"); let error_message = error.to_string(); log_send_error(&method, error); return Err(internal_error(format!( @@ -152,6 +176,7 @@ impl ReqwestHttpRequestRunner { } }; let status = response.status().as_u16(); + request_span.record("http.response.status_code", u64::from(status)); let headers = Self::response_headers(response.headers()); if params.stream_response { diff --git a/codex-rs/exec-server/src/client_recovery.rs b/codex-rs/exec-server/src/client_recovery.rs index f58e365728..252948cfc1 100644 --- a/codex-rs/exec-server/src/client_recovery.rs +++ b/codex-rs/exec-server/src/client_recovery.rs @@ -58,7 +58,7 @@ impl SessionState { exit_code, closed, failure, - sandbox_denied: _, + sandbox_denied, } = response; if let Some(message) = failure { return Err(ExecServerError::Protocol(format!( @@ -102,11 +102,21 @@ impl SessionState { } } - let exit_known = ordered_events.exit_published - || ordered_events - .pending - .range(..=target_seq) - .any(|(_, event)| matches!(event, ExecProcessEvent::Exited { .. })); + let pending_exit = ordered_events.pending.range_mut(..=target_seq).find_map( + |(_, event)| match event { + ExecProcessEvent::Exited { + sandbox_denied: pending_sandbox_denied, + .. + } => Some(pending_sandbox_denied), + _ => None, + }, + ); + let exit_pending = pending_exit.is_some(); + if let Some(pending_sandbox_denied) = pending_exit { + *pending_sandbox_denied = + Some(pending_sandbox_denied.unwrap_or(false) || sandbox_denied); + } + let exit_known = ordered_events.exit_published || exit_pending; let event_count = target_seq - ordered_events.last_published_seq; let retained_count = ordered_events .pending @@ -123,9 +133,14 @@ impl SessionState { "recovering exited process did not include its exit code".to_string(), ) })?; - ordered_events - .pending - .insert(seq, ExecProcessEvent::Exited { seq, exit_code }); + ordered_events.pending.insert( + seq, + ExecProcessEvent::Exited { + seq, + exit_code, + sandbox_denied: Some(sandbox_denied), + }, + ); } else if missing_count != 0 { return Err(recovery_gap_error(target_seq)); } @@ -530,10 +545,15 @@ fn is_retryable_registry_error(error: &ExecServerError) -> bool { if error.is_connect() || error.is_timeout() ) || matches!( error, - ExecServerError::EnvironmentRegistryHttp { status, .. } + ExecServerError::EnvironmentRegistryHttp { status, code, .. } if status.is_server_error() || *status == reqwest::StatusCode::REQUEST_TIMEOUT || *status == reqwest::StatusCode::TOO_MANY_REQUESTS + // TODO: Replace this coarse retry with an explicit registry/presence + // recovery FSM so `environment_offline` is retried only while the + // executor is expected to reconnect. + || (*status == reqwest::StatusCode::CONFLICT + && code.as_deref() == Some("environment_offline")) ) } diff --git a/codex-rs/exec-server/src/client_recovery_tests.rs b/codex-rs/exec-server/src/client_recovery_tests.rs index a6ff91fc9c..d2126fd70b 100644 --- a/codex-rs/exec-server/src/client_recovery_tests.rs +++ b/codex-rs/exec-server/src/client_recovery_tests.rs @@ -1,11 +1,13 @@ use std::time::Duration; use super::*; +use crate::protocol::ExecOutputStream; +use crate::protocol::ProcessOutputChunk; -fn registry_error() -> ExecServerError { +fn registry_error(status: reqwest::StatusCode, code: Option<&str>) -> ExecServerError { ExecServerError::EnvironmentRegistryHttp { - status: reqwest::StatusCode::TOO_MANY_REQUESTS, - code: None, + status, + code: code.map(str::to_string), message: "registry unavailable".to_string(), } } @@ -33,8 +35,64 @@ fn registry_recovery_retry_delay_exponentially_backs_off_and_caps() { #[test] fn recovery_retries_transient_registry_errors() { - let error = registry_error(); + let error = registry_error(reqwest::StatusCode::TOO_MANY_REQUESTS, /*code*/ None); assert!(is_retryable_registry_error(&error)); assert!(is_retryable_recovery_error(&error)); } + +#[test] +fn recovery_retries_environment_offline_conflicts() { + let error = registry_error(reqwest::StatusCode::CONFLICT, Some("environment_offline")); + + assert!(is_retryable_registry_error(&error)); + assert!(is_retryable_recovery_error(&error)); +} + +#[test] +fn recovery_does_not_retry_other_registry_conflicts() { + let error = registry_error(reqwest::StatusCode::CONFLICT, Some("registration_conflict")); + + assert!(!is_retryable_registry_error(&error)); + assert!(!is_retryable_recovery_error(&error)); +} + +#[tokio::test] +async fn recovery_adds_sandbox_denial_to_pending_exit_event() { + let state = SessionState::new(/*recoverable*/ true); + assert!(!state.publish_ordered_event(ExecProcessEvent::Exited { + seq: 2, + exit_code: 1, + sandbox_denied: None, + })); + + state + .recover_events(ReadResponse { + chunks: vec![ProcessOutputChunk { + seq: 1, + stream: ExecOutputStream::Stderr, + chunk: b"sandbox denied".to_vec().into(), + }], + next_seq: 3, + exited: true, + exit_code: Some(1), + closed: false, + failure: None, + sandbox_denied: true, + }) + .expect("recovery should publish the pending exit"); + + let mut events = state.subscribe_events(); + assert!(matches!( + events.recv().await, + Ok(ExecProcessEvent::Output(_)) + )); + assert_eq!( + events.recv().await, + Ok(ExecProcessEvent::Exited { + seq: 2, + exit_code: 1, + sandbox_denied: Some(true), + }) + ); +} diff --git a/codex-rs/exec-server/src/client_transport.rs b/codex-rs/exec-server/src/client_transport.rs index c6cff4ed30..1ccfd69152 100644 --- a/codex-rs/exec-server/src/client_transport.rs +++ b/codex-rs/exec-server/src/client_transport.rs @@ -8,6 +8,7 @@ use tokio::process::Command; use tokio::time::timeout; use tokio_tungstenite::connect_async; use tokio_tungstenite::connect_async_with_config; +use tokio_tungstenite::tungstenite::client::IntoClientRequest; use tracing::debug; use tracing::warn; @@ -30,6 +31,7 @@ use crate::noise_relay::NoiseHarnessConnectionArgs; use crate::noise_relay::noise_harness_connection_from_websocket; use crate::noise_relay::noise_relay_websocket_config; use crate::relay::harness_connection_from_websocket; +use crate::trace_context::current_trace_context_headers; const ENVIRONMENT_CLIENT_NAME: &str = "codex-environment"; @@ -216,6 +218,14 @@ impl ExecServerClient { /// only ciphertext after that. Environment-managed connections use a /// retained [`NoiseRendezvousConnectProvider`] so recovery can fetch a fresh /// bundle for each reconnect. + #[tracing::instrument( + name = "codex.exec_server.remote.harness.connect", + skip_all, + fields( + otel.kind = "client", + otel.name = "codex.exec_server.remote.harness.connect", + ) + )] pub async fn connect_noise_rendezvous( args: NoiseRendezvousConnectArgs, ) -> Result { @@ -249,12 +259,24 @@ impl ExecServerClient { .next() .unwrap_or(websocket_url.as_str()) .to_string(); + let mut request = websocket_url + .as_str() + .into_client_request() + .map_err(|source| ExecServerError::WebSocketConnect { + url: diagnostic_url.clone(), + source, + })?; + request + .headers_mut() + .extend(current_trace_context_headers()); let (stream, _) = timeout( connect_timeout, connect_async_with_config( - websocket_url.as_str(), + request, Some(noise_relay_websocket_config()), - /*disable_nagle*/ false, + // Rendezvous sends small, latency-sensitive frames, so avoid Nagle's coalescing delay. + /*disable_nagle*/ + true, ), ) .await diff --git a/codex-rs/exec-server/src/environment.rs b/codex-rs/exec-server/src/environment.rs index 664ef730d0..66f7381103 100644 --- a/codex-rs/exec-server/src/environment.rs +++ b/codex-rs/exec-server/src/environment.rs @@ -54,7 +54,7 @@ pub const CODEX_EXEC_SERVER_NOISE_CHATGPT_ACCOUNT_ID_ENV_VAR: &str = #[derive(Debug)] pub struct EnvironmentManager { default_environment: Option, - environments: RwLock>>, + pub(super) environments: RwLock>>, local_environment: Option>, local_runtime_paths: Option, } @@ -574,6 +574,25 @@ impl Environment { } } + /// Starts the initial connection after an environment is actually selected for use. + pub(crate) fn start_connecting_for_use(environment: &Arc) { + if environment.remote_client.is_none() { + return; + } + let mut startup_task = environment + .startup_task + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if startup_task.is_none() { + let environment = Arc::clone(environment); + *startup_task = Some(AbortOnDropHandle::new(tokio::spawn(async move { + if let Err(error) = environment.wait_until_ready().await { + tracing::debug!(%error, "exec-server environment startup failed"); + } + }))); + } + } + /// Returns whether initial startup has either succeeded or permanently failed. pub fn startup_finished(&self) -> bool { self.remote_client diff --git a/codex-rs/exec-server/src/lib.rs b/codex-rs/exec-server/src/lib.rs index 69d4c07dfb..882c83230c 100644 --- a/codex-rs/exec-server/src/lib.rs +++ b/codex-rs/exec-server/src/lib.rs @@ -22,10 +22,14 @@ mod relay_proto; mod remote; mod remote_file_system; mod remote_process; +mod resolved_capability; mod rpc; mod runtime_paths; mod sandboxed_file_system; mod server; +mod telemetry; +mod trace_context; +mod websocket_pong_watchdog; use codex_exec_server_protocol as protocol; @@ -123,6 +127,7 @@ pub use protocol::FsWalkResponse; pub use protocol::FsWriteFileParams; pub use protocol::FsWriteFileResponse; pub use protocol::HttpHeader; +pub use protocol::HttpRedirectPolicy; pub use protocol::HttpRequestBodyDeltaNotification; pub use protocol::HttpRequestParams; pub use protocol::HttpRequestResponse; @@ -142,7 +147,10 @@ pub use protocol::WriteResponse; pub use protocol::WriteStatus; pub use remote::RemoteEnvironmentConfig; pub use remote::run_remote_environment; +pub use resolved_capability::ResolvedSelectedCapabilityRoot; pub use runtime_paths::ExecServerRuntimePaths; pub use server::DEFAULT_LISTEN_URL; pub use server::ExecServerListenUrlParseError; pub use server::run_main; +pub use server::run_main_with_telemetry; +pub use telemetry::ExecServerTelemetry; diff --git a/codex-rs/exec-server/src/local_process.rs b/codex-rs/exec-server/src/local_process.rs index 973019808a..3c488f0ad7 100644 --- a/codex-rs/exec-server/src/local_process.rs +++ b/codex-rs/exec-server/src/local_process.rs @@ -59,6 +59,8 @@ use crate::rpc::RpcServerOutboundMessage; use crate::rpc::internal_error; use crate::rpc::invalid_params; use crate::rpc::invalid_request; +use crate::telemetry::ExecServerTelemetry; +use crate::telemetry::ProcessMetricGuard; const RETAINED_OUTPUT_BYTES_PER_PROCESS: usize = 1024 * 1024; const NOTIFICATION_CHANNEL_CAPACITY: usize = 256; @@ -91,6 +93,8 @@ struct RunningProcess { output_notify: Arc, open_streams: usize, closed: bool, + metrics: Option, + termination_requested: bool, sandbox: SandboxType, sandbox_denied: bool, } @@ -136,6 +140,7 @@ enum ProcessEntry { struct Inner { notifications: std::sync::RwLock>, processes: Mutex>, + telemetry: ExecServerTelemetry, } #[derive(Clone)] @@ -166,24 +171,31 @@ impl LocalProcess { let (outgoing_tx, mut outgoing_rx) = mpsc::channel::(NOTIFICATION_CHANNEL_CAPACITY); tokio::spawn(async move { while outgoing_rx.recv().await.is_some() {} }); - Self::with_runtime_paths(RpcNotificationSender::new(outgoing_tx), runtime_paths) + Self::with_runtime_paths( + RpcNotificationSender::new(outgoing_tx), + ExecServerTelemetry::default(), + runtime_paths, + ) } pub(crate) fn new( notifications: RpcNotificationSender, + telemetry: ExecServerTelemetry, runtime_paths: ExecServerRuntimePaths, ) -> Self { - Self::with_runtime_paths(notifications, Some(runtime_paths)) + Self::with_runtime_paths(notifications, telemetry, Some(runtime_paths)) } fn with_runtime_paths( notifications: RpcNotificationSender, + telemetry: ExecServerTelemetry, runtime_paths: Option, ) -> Self { Self { inner: Arc::new(Inner { notifications: std::sync::RwLock::new(Some(notifications)), processes: Mutex::new(HashMap::new()), + telemetry, }), runtime_paths, } @@ -200,7 +212,10 @@ impl LocalProcess { }) .collect::>() }; - for process in remaining { + for mut process in remaining { + if let Some(metrics) = process.metrics.take() { + metrics.finish("terminated"); + } process.session.terminate(); } } @@ -319,12 +334,13 @@ impl LocalProcess { output_notify: Arc::clone(&output_notify), open_streams: 2, closed: false, + metrics: Some(self.inner.telemetry.process_started()), + termination_requested: false, sandbox: prepared.sandbox, sandbox_denied: false, })), ); } - tokio::spawn(stream_output( process_id.clone(), if params.tty { @@ -536,11 +552,12 @@ impl LocalProcess { ) -> Result { let running = { let mut process_map = self.inner.processes.lock().await; - match process_map.get(¶ms.process_id) { + match process_map.get_mut(¶ms.process_id) { Some(ProcessEntry::Running(process)) => { if process.exit_code.is_some() { return Ok(TerminateResponse { running: false }); } + process.termination_requested = true; process.session.terminate(); true } @@ -807,11 +824,23 @@ async fn watch_exit( ) { let exit_code = exit_rx.await.unwrap_or(-1); let sandboxed = { - let processes = inner.processes.lock().await; - matches!( - processes.get(&process_id), - Some(ProcessEntry::Running(process)) if process.sandbox != SandboxType::None - ) + let mut processes = inner.processes.lock().await; + match processes.get_mut(&process_id) { + Some(ProcessEntry::Running(process)) => { + let sandboxed = process.sandbox != SandboxType::None; + if let Some(metrics) = process.metrics.take() { + metrics.finish(if process.termination_requested { + "terminated" + } else if exit_code == 0 { + "success" + } else { + "error" + }); + } + sandboxed + } + Some(ProcessEntry::Starting(_)) | None => false, + } }; if sandboxed { let _ = tokio::time::timeout(Duration::from_millis(20), output_notify.notified()).await; @@ -847,13 +876,16 @@ async fn watch_exit( process.sandbox_denied = is_likely_sandbox_denied(process.sandbox, &exec_output); } let _ = process.wake_tx.send(seq); - process - .events - .publish(ExecProcessEvent::Exited { seq, exit_code }); + process.events.publish(ExecProcessEvent::Exited { + seq, + exit_code, + sandbox_denied: Some(process.sandbox_denied), + }); Some(ExecExitedNotification { process_id: process_id.clone(), seq, exit_code, + sandbox_denied: Some(process.sandbox_denied), }) } else { None @@ -945,9 +977,13 @@ fn notification_sender(inner: &Inner) -> Option { #[cfg(test)] mod tests { use super::*; + use codex_otel::MetricsConfig; use codex_protocol::config_types::ShellEnvironmentPolicyInherit; use codex_utils_path_uri::PathUri; use codex_utils_pty::ProcessDriver; + use opentelemetry_sdk::metrics::InMemoryMetricExporter; + use opentelemetry_sdk::metrics::data::AggregatedMetrics; + use opentelemetry_sdk::metrics::data::MetricData; use pretty_assertions::assert_eq; use tokio::sync::oneshot; use tokio::time::timeout; @@ -969,6 +1005,62 @@ mod tests { } } + fn telemetry_backend() -> ( + LocalProcess, + codex_otel::MetricsClient, + InMemoryMetricExporter, + ) { + let exporter = InMemoryMetricExporter::default(); + let metrics = codex_otel::MetricsClient::new(MetricsConfig::in_memory( + "test", + "codex-exec-server", + env!("CARGO_PKG_VERSION"), + exporter.clone(), + )) + .expect("metrics"); + let telemetry = ExecServerTelemetry::new(metrics.clone()); + let (outgoing_tx, mut outgoing_rx) = mpsc::channel(NOTIFICATION_CHANNEL_CAPACITY); + tokio::spawn(async move { while outgoing_rx.recv().await.is_some() {} }); + ( + LocalProcess::with_runtime_paths( + RpcNotificationSender::new(outgoing_tx), + telemetry, + /*runtime_paths*/ None, + ), + metrics, + exporter, + ) + } + + fn assert_finished_process_result( + metrics: codex_otel::MetricsClient, + exporter: &InMemoryMetricExporter, + expected: &str, + ) { + metrics.shutdown().expect("shutdown metrics"); + let resource_metrics = exporter + .get_finished_metrics() + .expect("finished metrics") + .into_iter() + .last() + .expect("metrics export"); + let finished_processes = resource_metrics + .scope_metrics() + .flat_map(opentelemetry_sdk::metrics::data::ScopeMetrics::metrics) + .find(|metric| metric.name() == "exec_server_processes_finished_total") + .expect("finished process metric"); + let AggregatedMetrics::U64(MetricData::Sum(sum)) = finished_processes.data() else { + panic!("finished process metric should be a u64 sum"); + }; + let results = sum + .data_points() + .flat_map(opentelemetry_sdk::metrics::data::SumDataPoint::attributes) + .filter(|attribute| attribute.key.as_str() == "result") + .map(|attribute| attribute.value.as_str().into_owned()) + .collect::>(); + assert_eq!(results, vec![expected.to_string()]); + } + #[tokio::test] async fn start_process_rejects_non_native_cwd_before_launch() { #[cfg(unix)] @@ -1028,6 +1120,50 @@ mod tests { assert_eq!(child_env(¶ms), expected); } + #[tokio::test] + async fn exit_before_shutdown_records_success() { + let (backend, metrics, exporter) = telemetry_backend(); + let mut process = spawn_test_process(&backend, "exit-before-shutdown").await; + + process.exit(/*exit_code*/ 0); + let _ = read_process_until_change(&backend, &process.process_id, /*after_seq*/ None).await; + backend.shutdown().await; + + assert_finished_process_result(metrics, &exporter, "success"); + } + + #[tokio::test] + async fn termination_request_before_exit_records_terminated() { + let (backend, metrics, exporter) = telemetry_backend(); + let mut process = spawn_test_process(&backend, "terminate-before-exit").await; + + assert_eq!( + backend + .terminate_process(TerminateParams { + process_id: process.process_id.clone(), + }) + .await + .expect("terminate process"), + TerminateResponse { running: true }, + ); + process.exit(/*exit_code*/ 0); + let _ = read_process_until_change(&backend, &process.process_id, /*after_seq*/ None).await; + backend.shutdown().await; + + assert_finished_process_result(metrics, &exporter, "terminated"); + } + + #[tokio::test] + async fn shutdown_before_exit_records_terminated() { + let (backend, metrics, exporter) = telemetry_backend(); + let mut process = spawn_test_process(&backend, "shutdown-before-exit").await; + + backend.shutdown().await; + process.exit(/*exit_code*/ 0); + + assert_finished_process_result(metrics, &exporter, "terminated"); + } + #[tokio::test] async fn exited_process_retains_late_output_past_retention() { let backend = LocalProcess::default(); @@ -1170,6 +1306,8 @@ mod tests { output_notify: Arc::clone(&output_notify), open_streams: 2, closed: false, + metrics: Some(backend.inner.telemetry.process_started()), + termination_requested: false, sandbox: SandboxType::None, sandbox_denied: false, })), diff --git a/codex-rs/exec-server/src/noise_relay/executor_stream.rs b/codex-rs/exec-server/src/noise_relay/executor_stream.rs index 57a05579dc..89f33d938a 100644 --- a/codex-rs/exec-server/src/noise_relay/executor_stream.rs +++ b/codex-rs/exec-server/src/noise_relay/executor_stream.rs @@ -27,6 +27,7 @@ use crate::relay::encode_relay_message_frame; use crate::relay_proto::RelayData; use crate::relay_proto::RelayMessageFrame; use crate::server::ConnectionProcessor; +use crate::telemetry::ConnectionTransport; /// Identifies one completed virtual-stream instance. /// @@ -169,7 +170,9 @@ pub(crate) fn spawn_noise_virtual_stream( transport: JsonRpcTransport::Plain, }; tokio::spawn(async move { - processor.run_connection(connection).await; + processor + .run_connection(connection, ConnectionTransport::Relay) + .await; let _ = processor_closed_stream_tx .send(ClosedNoiseVirtualStream { stream_id: processor_stream_id, diff --git a/codex-rs/exec-server/src/noise_relay/harness.rs b/codex-rs/exec-server/src/noise_relay/harness.rs index 1989525224..9969a3e130 100644 --- a/codex-rs/exec-server/src/noise_relay/harness.rs +++ b/codex-rs/exec-server/src/noise_relay/harness.rs @@ -6,6 +6,7 @@ //! normal `JsonRpcConnection`. Outbound JSON-RPC is framed and split into Noise //! records; inbound records are reordered before decryption and reassembly. +use futures::FutureExt; use futures::Sink; use futures::SinkExt; use futures::Stream; @@ -24,6 +25,7 @@ use crate::connection::CHANNEL_CAPACITY; use crate::connection::JsonRpcConnection; use crate::connection::JsonRpcConnectionEvent; use crate::connection::JsonRpcTransport; +use crate::connection::WEBSOCKET_KEEPALIVE_INTERVAL; use crate::noise_channel::InitiatorHandshake; use crate::noise_channel::NoiseChannelIdentity; use crate::noise_channel::NoiseChannelPublicKey; @@ -39,6 +41,9 @@ use crate::relay::decode_relay_message_frame; use crate::relay::encode_relay_message_frame; use crate::relay_proto::RelayData; use crate::relay_proto::RelayMessageFrame; +use crate::websocket_pong_watchdog::WEBSOCKET_PONG_TIMEOUT; +use crate::websocket_pong_watchdog::WEBSOCKET_PONG_TIMEOUT_REASON; +use crate::websocket_pong_watchdog::WebSocketPongWatchdog; /// Values that bind one harness websocket to the intended executor registration. /// @@ -58,6 +63,8 @@ pub(crate) struct NoiseHarnessConnectionArgs { // Preserve the availability signal while replacing attacker-controlled reason // text before it reaches disconnect diagnostics. const NOISE_RELAY_RESET_DISCONNECT_REASON: &str = "Noise relay stream reset"; +// Give a Pong already queued behind data a bounded chance to reach the reader. +const MAX_FRAMES_DRAINED_AFTER_PONG_DEADLINE: usize = 32; /// Adapt one harness rendezvous websocket into an authenticated JSON-RPC connection. /// @@ -112,8 +119,7 @@ where &incoming_tx, &disconnected_tx, format!("failed to start Noise relay handshake: {error}"), - ) - .await; + ); return; } }; @@ -147,8 +153,7 @@ where &incoming_tx, &disconnected_tx, "Noise relay websocket ended during handshake".to_string(), - ) - .await; + ); return; }; let message = match incoming_message { @@ -158,8 +163,7 @@ where &incoming_tx, &disconnected_tx, "Noise relay websocket received close frame during handshake".to_string(), - ) - .await; + ); return; } Ok(Message::Ping(_) | Message::Pong(_) | Message::Frame(_)) => continue, @@ -168,8 +172,7 @@ where &incoming_tx, &disconnected_tx, "Noise relay transport expects binary protobuf frames".to_string(), - ) - .await; + ); return; } Err(error) => { @@ -179,8 +182,7 @@ where format!( "failed to read Noise relay websocket from {connection_label}: {error}" ), - ) - .await; + ); return; } }; @@ -191,8 +193,7 @@ where &incoming_tx, &disconnected_tx, format!("failed to parse Noise relay frame: {error}"), - ) - .await; + ); return; } }; @@ -209,8 +210,7 @@ where &incoming_tx, &disconnected_tx, format!("invalid Noise relay handshake response: {error}"), - ) - .await; + ); return; } }; @@ -228,8 +228,7 @@ where &incoming_tx, &disconnected_tx, format!("Noise relay handshake failed: {error}"), - ) - .await; + ); return; } } @@ -239,8 +238,7 @@ where &incoming_tx, &disconnected_tx, NOISE_RELAY_RESET_DISCONNECT_REASON.to_string(), - ) - .await; + ); return; } Ok( @@ -253,8 +251,7 @@ where &incoming_tx, &disconnected_tx, "Noise relay received data before handshake completion".to_string(), - ) - .await; + ); return; } } @@ -263,57 +260,177 @@ where // After the handshake, each relay sequence maps to exactly one Noise // transport record. Outbound records are encrypted once; inbound // records are reordered and deduplicated before decryption. + let mut websocket = websocket.peekable(); let mut next_outbound_seq = 0u32; let mut inbound_ciphertexts = OrderedCiphertextFrames::default(); let mut inbound_decoder = JsonRpcMessageDecoder::default(); + let mut keepalive = tokio::time::interval_at( + tokio::time::Instant::now() + WEBSOCKET_KEEPALIVE_INTERVAL, + WEBSOCKET_KEEPALIVE_INTERVAL, + ); + keepalive.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + let mut pong_watchdog = WebSocketPongWatchdog::new(WEBSOCKET_PONG_TIMEOUT); + let pong_deadline = tokio::time::sleep(WEBSOCKET_PONG_TIMEOUT); + tokio::pin!(pong_deadline); + // Keep one framed message as a cursor. Sending one Noise record per loop + // creates a scheduling point for keepalive and inbound control frames + // without splitting the WebSocket reader and writer. + let mut pending_outbound: Option<(Vec, usize)> = None; + let mut force_incoming = false; + let mut frames_drained_after_pong_deadline = 0usize; 'relay: loop { + // Consume a due tick before the always-ready record arm below can win + // another select iteration and postpone the keepalive. + if pong_watchdog.deadline().is_none() + && keepalive.tick().now_or_never().is_some() + { + if let Err(error) = send_keepalive_ping( + &mut websocket, + &mut pong_watchdog, + pong_deadline.as_mut(), + ) + .await + { + warn!("failed to write Noise relay keepalive ping: {error}"); + break; + } + frames_drained_after_pong_deadline = 0; + continue; + } + + let pong_deadline_expired = pong_watchdog + .deadline() + .is_some_and(|deadline| tokio::time::Instant::now() >= deadline); + // After expiry, inspect only frames already queued. Forcing the peeked + // item through next() makes the 32-frame grace deterministic. + if pong_deadline_expired && !force_incoming { + if frames_drained_after_pong_deadline + < MAX_FRAMES_DRAINED_AFTER_PONG_DEADLINE + && std::pin::Pin::new(&mut websocket) + .peek() + .now_or_never() + .is_some() + { + force_incoming = true; + } else { + warn!( + noise_reason = WEBSOCKET_PONG_TIMEOUT_REASON, + "Noise harness rendezvous websocket disconnected" + ); + send_disconnected( + &incoming_tx, + &disconnected_tx, + WEBSOCKET_PONG_TIMEOUT_REASON.to_string(), + ); + return; + } + } + + // While a Pong is outstanding, drain already-queued inbound traffic + // before the next fragment so a queued Pong cannot sit behind writes. + if !force_incoming + && pong_watchdog.deadline().is_some() + && pending_outbound.is_some() + && std::pin::Pin::new(&mut websocket) + .peek() + .now_or_never() + .is_some() + { + force_incoming = true; + } + tokio::select! { - maybe_message = outgoing_rx.recv() => { + maybe_message = outgoing_rx.recv(), if pending_outbound.is_none() && !force_incoming && !pong_deadline_expired => { let Some(message) = maybe_message else { break; }; - let framed = match frame_jsonrpc_message(&message) { - Ok(framed) => framed, + pending_outbound = Some(match frame_jsonrpc_message(&message) { + Ok(framed) => (framed, 0), Err(error) => { warn!("failed to frame JSON-RPC payload for Noise relay: {error}"); break; } + }); + } + _ = std::future::ready(()), if pending_outbound.is_some() && !force_incoming && !pong_deadline_expired => { + let seq = match take_next_sequence(&mut next_outbound_seq) { + Ok(seq) => seq, + Err(error) => { + warn!("Noise relay sequence exhausted: {error}"); + break 'relay; + } }; - for plaintext_record in framed.chunks(NOISE_RECORD_PLAINTEXT_LEN) { - let seq = match take_next_sequence(&mut next_outbound_seq) { - Ok(seq) => seq, - Err(error) => { - warn!("Noise relay sequence exhausted: {error}"); - break 'relay; - } + let (ciphertext, next_offset, message_complete) = { + let Some((framed, offset)) = pending_outbound.as_ref() else { + continue; }; - let ciphertext = match transport.encrypt(plaintext_record) { + let next_offset = (*offset + NOISE_RECORD_PLAINTEXT_LEN).min(framed.len()); + let ciphertext = match transport.encrypt(&framed[*offset..next_offset]) { Ok(ciphertext) => ciphertext, Err(error) => { warn!("failed to encrypt JSON-RPC payload for Noise relay: {error}"); break 'relay; } }; - let frame = RelayMessageFrame::data(stream_id.clone(), seq, ciphertext); - if let Err(error) = websocket - .send(Message::Binary(encode_relay_message_frame(&frame).into())) - .await - { - warn!("failed to write Noise relay websocket: {error}"); - break 'relay; - } + (ciphertext, next_offset, next_offset == framed.len()) + }; + let frame = RelayMessageFrame::data(stream_id.clone(), seq, ciphertext); + // A Pong can arrive after the readiness check while this write owns the + // combined sink and stream. A single bounded record can therefore hit the + // deadline and disconnect with that Pong queued. Treat that as write + // backpressure; this loop yields only between records. + if let Err(error) = send_websocket_message( + &mut websocket, + Message::Binary(encode_relay_message_frame(&frame).into()), + pong_watchdog.write_deadline(tokio::time::Instant::now()), + ) + .await + { + warn!("failed to write Noise relay websocket: {error}"); + break 'relay; + } + if message_complete { + pending_outbound = None; + } else if let Some((_framed, offset)) = pending_outbound.as_mut() { + *offset = next_offset; } } + _ = &mut pong_deadline, if pong_watchdog.deadline().is_some() && !force_incoming => { + continue; + } + _ = keepalive.tick(), if pong_watchdog.deadline().is_none() => { + if let Err(error) = send_keepalive_ping( + &mut websocket, + &mut pong_watchdog, + pong_deadline.as_mut(), + ) + .await + { + warn!("failed to write Noise relay keepalive ping: {error}"); + break; + } + frames_drained_after_pong_deadline = 0; + } incoming_message = websocket.next() => { + force_incoming = false; let Some(incoming_message) = incoming_message else { break; }; + // Count each completed read after expiry. If only the deadline arm + // advanced this counter, reads won by a simultaneously ready incoming + // arm would not count toward the 32-frame cap. + if pong_watchdog + .deadline() + .is_some_and(|deadline| tokio::time::Instant::now() >= deadline) + { + frames_drained_after_pong_deadline += 1; + } match incoming_message { Ok(Message::Binary(payload)) => { let frame = match decode_relay_message_frame(payload.as_ref()) { Ok(frame) => frame, Err(error) => { - send_malformed(&incoming_tx, error.to_string()).await; + send_malformed(&incoming_tx, error.to_string()); break; } }; @@ -325,7 +442,7 @@ where let data = match frame.into_data() { Ok(data) => data, Err(error) => { - send_malformed(&incoming_tx, error.to_string()).await; + send_malformed(&incoming_tx, error.to_string()); break; } }; @@ -334,22 +451,26 @@ where &mut transport, &mut inbound_decoder, data, + pong_watchdog.write_deadline(tokio::time::Instant::now()), &incoming_tx, ) .await { - send_malformed(&incoming_tx, error.to_string()).await; + if matches!(error, ExecServerError::Closed) { + break; + } + send_malformed(&incoming_tx, error.to_string()); break; } } Ok(RelayFrameBodyKind::Reset) => { - let _ = incoming_tx - .send(JsonRpcConnectionEvent::Disconnected { + let _ = incoming_tx.try_send( + JsonRpcConnectionEvent::Disconnected { reason: Some( NOISE_RELAY_RESET_DISCONNECT_REASON.to_string(), ), - }) - .await; + }, + ); break; } Ok( @@ -361,20 +482,22 @@ where send_malformed( &incoming_tx, "Noise relay received invalid post-handshake frame".to_string(), - ) - .await; + ); break; } } } Ok(Message::Close(_)) => break, - Ok(Message::Ping(_) | Message::Pong(_) | Message::Frame(_)) => {} + Ok(Message::Pong(_)) => { + pong_watchdog.received_pong(); + frames_drained_after_pong_deadline = 0; + } + Ok(Message::Ping(_) | Message::Frame(_)) => {} Ok(Message::Text(_)) => { send_malformed( &incoming_tx, "Noise relay transport expects binary protobuf frames".to_string(), - ) - .await; + ); break; } Err(error) => { @@ -398,6 +521,46 @@ where } } +async fn send_websocket_message( + websocket: &mut T, + message: Message, + deadline: tokio::time::Instant, +) -> Result<(), String> +where + T: Sink + Unpin, + E: std::fmt::Display, +{ + match tokio::time::timeout_at(deadline, websocket.send(message)).await { + Ok(Ok(())) => Ok(()), + Ok(Err(error)) => Err(error.to_string()), + Err(_) => Err("websocket write timed out".to_string()), + } +} + +async fn send_keepalive_ping( + websocket: &mut T, + pong_watchdog: &mut WebSocketPongWatchdog, + pong_deadline: std::pin::Pin<&mut tokio::time::Sleep>, +) -> Result<(), String> +where + T: Sink + Unpin, + E: std::fmt::Display, +{ + send_websocket_message( + websocket, + Message::Ping(Vec::new().into()), + pong_watchdog.write_deadline(tokio::time::Instant::now()), + ) + .await?; + // Start the response clock after the Ping flushes; waiting for sink capacity + // is governed by the write deadline above. + pong_watchdog.ping_sent(tokio::time::Instant::now()); + if let Some(deadline) = pong_watchdog.deadline() { + pong_deadline.reset(deadline); + } + Ok(()) +} + /// Order and decrypt one relay frame, then emit any complete JSON-RPC messages. /// Relay records and JSON-RPC messages do not share boundaries, so reassembly /// happens after decryption. @@ -406,6 +569,7 @@ async fn receive_data( transport: &mut NoiseTransport, decoder: &mut JsonRpcMessageDecoder, data: RelayData, + delivery_deadline: tokio::time::Instant, incoming_tx: &mpsc::Sender, ) -> Result<(), ExecServerError> { // Ordering must happen before decryption because Noise transport nonces are @@ -419,30 +583,50 @@ async fn receive_data( // The authenticated byte stream can carry partial or multiple JSON-RPC // messages; emit only complete, successfully parsed messages. for message in decoder.push(&plaintext)? { - incoming_tx - .send(JsonRpcConnectionEvent::Message(message)) - .await - .map_err(|_| ExecServerError::Closed)?; + send_incoming_event( + incoming_tx, + JsonRpcConnectionEvent::Message(message), + delivery_deadline, + ) + .await?; } } Ok(()) } -async fn send_malformed(incoming_tx: &mpsc::Sender, reason: String) { - let _ = incoming_tx - .send(JsonRpcConnectionEvent::MalformedMessage { reason }) - .await; +async fn send_incoming_event( + incoming_tx: &mpsc::Sender, + event: JsonRpcConnectionEvent, + deadline: tokio::time::Instant, +) -> Result<(), ExecServerError> { + match tokio::time::timeout_at(deadline, incoming_tx.send(event)).await { + Ok(Ok(())) => Ok(()), + Ok(Err(_)) => Err(ExecServerError::Closed), + Err(_) => { + warn!( + noise_reason = "application_backpressure", + "Noise harness application event delivery timed out" + ); + Err(ExecServerError::Closed) + } + } } -async fn send_disconnected( +fn send_malformed(incoming_tx: &mpsc::Sender, reason: String) { + let _ = incoming_tx.try_send(JsonRpcConnectionEvent::MalformedMessage { reason }); +} + +fn send_disconnected( incoming_tx: &mpsc::Sender, disconnected_tx: &watch::Sender, reason: String, ) { let _ = disconnected_tx.send(true); - let _ = incoming_tx - .send(JsonRpcConnectionEvent::Disconnected { - reason: Some(reason), - }) - .await; + let _ = incoming_tx.try_send(JsonRpcConnectionEvent::Disconnected { + reason: Some(reason), + }); } + +#[cfg(test)] +#[path = "harness_tests.rs"] +mod tests; diff --git a/codex-rs/exec-server/src/noise_relay/harness_tests.rs b/codex-rs/exec-server/src/noise_relay/harness_tests.rs new file mode 100644 index 0000000000..ce26a8ca8c --- /dev/null +++ b/codex-rs/exec-server/src/noise_relay/harness_tests.rs @@ -0,0 +1,442 @@ +use std::pin::Pin; +use std::sync::Arc; +use std::sync::atomic::AtomicUsize; +use std::sync::atomic::Ordering; +use std::task::Context as TaskContext; +use std::task::Poll; +use std::time::Duration; + +use anyhow::Context; +use anyhow::Result; +use codex_exec_server_protocol::JSONRPCMessage; +use codex_exec_server_protocol::JSONRPCRequest; +use codex_exec_server_protocol::RequestId; +use futures::Sink; +use futures::SinkExt; +use futures::StreamExt; +use futures::channel::mpsc as futures_mpsc; +use pretty_assertions::assert_eq; +use tokio::net::TcpListener; +use tokio::sync::mpsc; +use tokio::time::Instant; +use tokio::time::timeout; +use tokio_tungstenite::accept_async; +use tokio_tungstenite::connect_async; +use tokio_tungstenite::tungstenite::Message; + +use super::*; +use crate::connection::JsonRpcConnectionEvent; +use crate::noise_channel::PendingResponderHandshake; + +const ENVIRONMENT_ID: &str = "environment-1"; +const EXECUTOR_REGISTRATION_ID: &str = "registration-1"; + +#[tokio::test] +async fn fragmented_writes_yield_to_keepalive_and_queued_pong() -> Result<()> { + let (connection, mut control, mut outbound_rx) = connected_controlled_harness().await?; + + connection + .outgoing_tx + .send(JSONRPCMessage::Request(JSONRPCRequest { + id: RequestId::Integer(1), + method: "large".to_string(), + params: Some(serde_json::json!({ + "payload": "x".repeat(NOISE_RECORD_PLAINTEXT_LEN * 3), + })), + trace: None, + })) + .await?; + + control.wait_for_blocked_write(/*expected*/ 1).await?; + tokio::time::sleep(WEBSOCKET_KEEPALIVE_INTERVAL + Duration::from_millis(10)).await; + control.grant_writes(/*count*/ 1); + let first_data = read_outbound_data(&mut outbound_rx).await?; + assert_eq!(first_data.seq, 0); + + control.wait_for_blocked_write(/*expected*/ 2).await?; + control.grant_writes(/*count*/ 1); + let Message::Ping(ping_payload) = timeout(Duration::from_secs(1), outbound_rx.next()) + .await? + .context("harness closed before sending keepalive")? + else { + anyhow::bail!("expected keepalive between fragmented writes"); + }; + + control.wait_for_blocked_write(/*expected*/ 3).await?; + control.send_inbound(Message::Pong(ping_payload))?; + tokio::time::sleep(WEBSOCKET_KEEPALIVE_INTERVAL + Duration::from_millis(10)).await; + control.grant_writes(/*count*/ 1); + let second_data = read_outbound_data(&mut outbound_rx).await?; + assert_eq!(second_data.seq, 1); + + control.wait_for_blocked_write(/*expected*/ 4).await?; + control.grant_writes(/*count*/ 1); + let next_message = timeout(Duration::from_secs(1), outbound_rx.next()) + .await? + .context("harness closed after receiving queued Pong")?; + assert!(matches!(next_message, Message::Ping(_))); + + for task in &connection.task_handles { + task.abort(); + } + Ok(()) +} + +#[tokio::test(flavor = "current_thread")] +async fn post_deadline_drain_stops_before_frame_33() -> Result<()> { + let (mut connection, mut control, mut outbound_rx) = connected_controlled_harness().await?; + + control.wait_for_blocked_write(/*expected*/ 1).await?; + control.grant_writes(/*count*/ 1); + let Message::Ping(ping_payload) = timeout(Duration::from_secs(1), outbound_rx.next()) + .await? + .context("harness closed before sending keepalive")? + else { + anyhow::bail!("expected keepalive ping"); + }; + let reads_before_deadline = control.inbound_reads(); + + let unrelated_frame = + encode_relay_message_frame(&RelayMessageFrame::resume("unrelated-stream".to_string())); + for _ in 0..MAX_FRAMES_DRAINED_AFTER_PONG_DEADLINE { + control.send_inbound(Message::Binary(unrelated_frame.clone().into()))?; + } + control.send_inbound(Message::Pong(ping_payload))?; + + // Keep the current-thread runtime from consuming the queued frames until the + // Pong deadline and every frame are ready together. + std::thread::sleep(WEBSOCKET_PONG_TIMEOUT + Duration::from_millis(10)); + + let event = timeout(Duration::from_secs(1), connection.incoming_rx.recv()).await?; + let Some(JsonRpcConnectionEvent::Disconnected { reason }) = event else { + anyhow::bail!("expected Pong timeout, got {event:?}"); + }; + assert_eq!(reason.as_deref(), Some(WEBSOCKET_PONG_TIMEOUT_REASON)); + assert_eq!( + control.inbound_reads() - reads_before_deadline, + MAX_FRAMES_DRAINED_AFTER_PONG_DEADLINE + ); + Ok(()) +} + +#[tokio::test] +async fn pong_keeps_harness_alive_until_peer_stops_responding() -> Result<()> { + let listener = TcpListener::bind("127.0.0.1:0").await?; + let websocket_url = format!("ws://{}", listener.local_addr()?); + let harness_connection = tokio::spawn(connect_async(websocket_url)); + let (socket, _peer_addr) = listener.accept().await?; + let mut executor_websocket = accept_async(socket).await?; + let (harness_websocket, _response) = harness_connection.await??; + + let executor_identity = NoiseChannelIdentity::generate()?; + let mut connection = noise_harness_connection_from_websocket( + harness_websocket, + NoiseHarnessConnectionArgs { + connection_label: "test rendezvous".to_string(), + environment_id: ENVIRONMENT_ID.to_string(), + executor_registration_id: EXECUTOR_REGISTRATION_ID.to_string(), + identity: NoiseChannelIdentity::generate()?, + responder_public_key: executor_identity.public_key(), + harness_key_authorization: "authorization".to_string(), + }, + ); + + let resume_message = timeout(Duration::from_secs(1), executor_websocket.next()) + .await? + .context("harness closed before sending resume")??; + let Message::Binary(resume_payload) = resume_message else { + anyhow::bail!("expected resume frame, got {resume_message:?}"); + }; + let resume = decode_relay_message_frame(resume_payload.as_ref())?; + assert_eq!(resume.validate()?, RelayFrameBodyKind::Resume); + + let handshake_message = timeout(Duration::from_secs(1), executor_websocket.next()) + .await? + .context("harness closed before sending handshake")??; + let Message::Binary(handshake_payload) = handshake_message else { + anyhow::bail!("expected handshake frame, got {handshake_message:?}"); + }; + let handshake = decode_relay_message_frame(handshake_payload.as_ref())?; + assert_eq!(handshake.stream_id, resume.stream_id); + let stream_id = handshake.stream_id.clone(); + let prologue = + noise_channel_prologue(ENVIRONMENT_ID, EXECUTOR_REGISTRATION_ID, stream_id.as_str()); + let pending = PendingResponderHandshake::read_request( + &executor_identity, + &prologue, + &handshake.into_handshake_payload()?, + )?; + let (_transport, response) = pending.complete()?; + let response = RelayMessageFrame::handshake(stream_id, response); + executor_websocket + .send(Message::Binary( + encode_relay_message_frame(&response).into(), + )) + .await?; + + let mut pings = 0; + while pings < 6 { + let message = timeout(Duration::from_secs(1), executor_websocket.next()) + .await? + .context("harness disconnected before six keepalive pings")??; + match message { + Message::Ping(payload) => { + executor_websocket.send(Message::Pong(payload)).await?; + pings += 1; + } + Message::Pong(_) | Message::Frame(_) => {} + message => anyhow::bail!("expected keepalive ping, got {message:?}"), + } + } + + // Keep non-Pong traffic flowing after responses stop. It must not defeat + // the bounded grace for a Pong already queued behind data. + let unrelated_frame = + encode_relay_message_frame(&RelayMessageFrame::resume("unrelated-stream".to_string())); + let traffic_task = tokio::spawn(async move { + loop { + if executor_websocket + .send(Message::Binary(unrelated_frame.clone().into())) + .await + .is_err() + { + break; + } + tokio::time::sleep(Duration::from_millis(5)).await; + } + }); + let event = timeout(Duration::from_secs(1), connection.incoming_rx.recv()).await?; + traffic_task.abort(); + let _ = traffic_task.await; + let Some(JsonRpcConnectionEvent::Disconnected { reason }) = event else { + anyhow::bail!("expected pong timeout, got {event:?}"); + }; + assert_eq!(reason.as_deref(), Some(WEBSOCKET_PONG_TIMEOUT_REASON)); + Ok(()) +} + +#[tokio::test] +async fn application_event_delivery_is_bounded() -> Result<()> { + let (incoming_tx, _incoming_rx) = mpsc::channel(1); + incoming_tx + .send(JsonRpcConnectionEvent::MalformedMessage { + reason: "fill queue".to_string(), + }) + .await?; + + let result = send_incoming_event( + &incoming_tx, + JsonRpcConnectionEvent::MalformedMessage { + reason: "blocked event".to_string(), + }, + Instant::now() + Duration::from_millis(10), + ) + .await; + + assert!(matches!(result, Err(ExecServerError::Closed))); + Ok(()) +} + +async fn read_outbound_data( + outbound_rx: &mut futures_mpsc::UnboundedReceiver, +) -> Result { + let Message::Binary(payload) = timeout(Duration::from_secs(1), outbound_rx.next()) + .await? + .context("harness closed before sending data")? + else { + anyhow::bail!("expected relay data frame"); + }; + let frame = decode_relay_message_frame(payload.as_ref())?; + assert_eq!(frame.validate()?, RelayFrameBodyKind::Data); + frame.into_data().map_err(anyhow::Error::from) +} + +async fn connected_controlled_harness() -> Result<( + JsonRpcConnection, + ControlledWebSocketHandle, + futures_mpsc::UnboundedReceiver, +)> { + let (websocket, control, mut outbound_rx) = ControlledWebSocket::new(/*write_permits*/ 2); + let executor_identity = NoiseChannelIdentity::generate()?; + let connection = noise_harness_connection_from_websocket( + websocket, + NoiseHarnessConnectionArgs { + connection_label: "test rendezvous".to_string(), + environment_id: ENVIRONMENT_ID.to_string(), + executor_registration_id: EXECUTOR_REGISTRATION_ID.to_string(), + identity: NoiseChannelIdentity::generate()?, + responder_public_key: executor_identity.public_key(), + harness_key_authorization: "authorization".to_string(), + }, + ); + + let Message::Binary(resume_payload) = timeout(Duration::from_secs(1), outbound_rx.next()) + .await? + .context("harness closed before sending resume")? + else { + anyhow::bail!("expected resume frame"); + }; + let resume = decode_relay_message_frame(resume_payload.as_ref())?; + let Message::Binary(handshake_payload) = timeout(Duration::from_secs(1), outbound_rx.next()) + .await? + .context("harness closed before sending handshake")? + else { + anyhow::bail!("expected handshake frame"); + }; + let handshake = decode_relay_message_frame(handshake_payload.as_ref())?; + let stream_id = handshake.stream_id.clone(); + assert_eq!(stream_id, resume.stream_id); + let prologue = + noise_channel_prologue(ENVIRONMENT_ID, EXECUTOR_REGISTRATION_ID, stream_id.as_str()); + let pending = PendingResponderHandshake::read_request( + &executor_identity, + &prologue, + &handshake.into_handshake_payload()?, + )?; + let (_transport, response) = pending.complete()?; + control.send_inbound(Message::Binary( + encode_relay_message_frame(&RelayMessageFrame::handshake(stream_id, response)).into(), + ))?; + Ok((connection, control, outbound_rx)) +} + +struct ControlledWebSocket { + inbound_rx: futures_mpsc::UnboundedReceiver>, + outbound_tx: futures_mpsc::UnboundedSender, + write_permit_rx: futures_mpsc::UnboundedReceiver<()>, + blocked_write_tx: futures_mpsc::UnboundedSender, + write_waiting: bool, + blocked_writes: usize, + inbound_reads: Arc, +} + +struct ControlledWebSocketHandle { + inbound_tx: futures_mpsc::UnboundedSender>, + write_permit_tx: futures_mpsc::UnboundedSender<()>, + blocked_write_rx: futures_mpsc::UnboundedReceiver, + inbound_reads: Arc, +} + +impl ControlledWebSocket { + fn new( + write_permits: usize, + ) -> ( + Self, + ControlledWebSocketHandle, + futures_mpsc::UnboundedReceiver, + ) { + let (inbound_tx, inbound_rx) = futures_mpsc::unbounded(); + let (outbound_tx, outbound_rx) = futures_mpsc::unbounded(); + let (write_permit_tx, write_permit_rx) = futures_mpsc::unbounded(); + let (blocked_write_tx, blocked_write_rx) = futures_mpsc::unbounded(); + for _ in 0..write_permits { + write_permit_tx + .unbounded_send(()) + .expect("test write permit receiver should stay open"); + } + let inbound_reads = Arc::new(AtomicUsize::new(0)); + ( + Self { + inbound_rx, + outbound_tx, + write_permit_rx, + blocked_write_tx, + write_waiting: false, + blocked_writes: 0, + inbound_reads: Arc::clone(&inbound_reads), + }, + ControlledWebSocketHandle { + inbound_tx, + write_permit_tx, + blocked_write_rx, + inbound_reads, + }, + outbound_rx, + ) + } +} + +impl ControlledWebSocketHandle { + fn send_inbound(&self, message: Message) -> Result<()> { + self.inbound_tx + .unbounded_send(Ok(message)) + .map_err(anyhow::Error::from) + } + + fn grant_writes(&self, count: usize) { + for _ in 0..count { + self.write_permit_tx + .unbounded_send(()) + .expect("test write permit receiver should stay open"); + } + } + + fn inbound_reads(&self) -> usize { + self.inbound_reads.load(Ordering::Acquire) + } + + async fn wait_for_blocked_write(&mut self, expected: usize) -> Result<()> { + let actual = timeout(Duration::from_secs(1), self.blocked_write_rx.next()) + .await? + .context("websocket closed before blocking the expected write")?; + assert_eq!(actual, expected); + Ok(()) + } +} + +impl Sink for ControlledWebSocket { + type Error = std::convert::Infallible; + + fn poll_ready(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll> { + let this = self.get_mut(); + match Pin::new(&mut this.write_permit_rx).poll_next(cx) { + Poll::Ready(Some(())) => { + this.write_waiting = false; + Poll::Ready(Ok(())) + } + Poll::Ready(None) | Poll::Pending => { + if !this.write_waiting { + this.write_waiting = true; + this.blocked_writes += 1; + this.blocked_write_tx + .unbounded_send(this.blocked_writes) + .expect("test blocked-write receiver should stay open"); + } + Poll::Pending + } + } + } + + fn start_send(self: Pin<&mut Self>, item: Message) -> Result<(), Self::Error> { + self.outbound_tx + .unbounded_send(item) + .expect("test outbound receiver should stay open"); + Ok(()) + } + + fn poll_flush( + self: Pin<&mut Self>, + _cx: &mut TaskContext<'_>, + ) -> Poll> { + Poll::Ready(Ok(())) + } + + fn poll_close( + self: Pin<&mut Self>, + _cx: &mut TaskContext<'_>, + ) -> Poll> { + Poll::Ready(Ok(())) + } +} + +impl futures::Stream for ControlledWebSocket { + type Item = Result; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll> { + let result = Pin::new(&mut self.inbound_rx).poll_next(cx); + if matches!(result, Poll::Ready(Some(_))) { + self.inbound_reads.fetch_add(1, Ordering::Release); + } + result + } +} diff --git a/codex-rs/exec-server/src/process.rs b/codex-rs/exec-server/src/process.rs index b136ef4682..afec44b818 100644 --- a/codex-rs/exec-server/src/process.rs +++ b/codex-rs/exec-server/src/process.rs @@ -30,8 +30,14 @@ pub struct StartedExecProcess { #[derive(Debug, Clone, PartialEq, Eq)] pub enum ExecProcessEvent { Output(ProcessOutputChunk), - Exited { seq: u64, exit_code: i32 }, - Closed { seq: u64 }, + Exited { + seq: u64, + exit_code: i32, + sandbox_denied: Option, + }, + Closed { + seq: u64, + }, Failed(String), } @@ -125,21 +131,28 @@ impl ExecProcessEventLog { let live_rx = self.inner.live_tx.subscribe(); let replay = history.events.iter().cloned().collect(); - ExecProcessEventReceiver { replay, live_rx } + ExecProcessEventReceiver { + replay, + live_rx, + _keepalive: None, + } } } pub struct ExecProcessEventReceiver { replay: VecDeque, live_rx: broadcast::Receiver, + _keepalive: Option>, } impl ExecProcessEventReceiver { + /// Returns a receiver that remains open without yielding events. pub fn empty() -> Self { - let (_live_tx, live_rx) = broadcast::channel(1); + let (live_tx, live_rx) = broadcast::channel(1); Self { replay: VecDeque::new(), live_rx, + _keepalive: Some(live_tx), } } @@ -202,9 +215,21 @@ mod tests { use super::ExecProcessEvent; use super::ExecProcessEventLog; + use super::ExecProcessEventReceiver; use crate::protocol::ExecOutputStream; use crate::protocol::ProcessOutputChunk; + #[tokio::test] + async fn empty_event_receiver_stays_open() { + let mut events = ExecProcessEventReceiver::empty(); + + assert!( + timeout(Duration::from_millis(10), events.recv()) + .await + .is_err() + ); + } + #[tokio::test] async fn event_history_replay_is_bounded_by_retained_bytes() { let log = ExecProcessEventLog::new(/*event_capacity*/ 8, /*byte_capacity*/ 3); @@ -217,6 +242,7 @@ mod tests { log.publish(ExecProcessEvent::Exited { seq: 2, exit_code: 0, + sandbox_denied: Some(false), }); log.publish(ExecProcessEvent::Closed { seq: 3 }); @@ -238,6 +264,7 @@ mod tests { ExecProcessEvent::Exited { seq: 2, exit_code: 0, + sandbox_denied: Some(false), }, ExecProcessEvent::Closed { seq: 3 }, ] diff --git a/codex-rs/exec-server/src/relay.rs b/codex-rs/exec-server/src/relay.rs index d7e3d9c2a7..b8d477f5db 100644 --- a/codex-rs/exec-server/src/relay.rs +++ b/codex-rs/exec-server/src/relay.rs @@ -41,6 +41,9 @@ use crate::relay_proto::RelayReset; use crate::relay_proto::RelayResume; use crate::relay_proto::relay_message_frame; use crate::server::ConnectionProcessor; +use crate::websocket_pong_watchdog::WEBSOCKET_PONG_TIMEOUT; +use crate::websocket_pong_watchdog::WEBSOCKET_PONG_TIMEOUT_REASON; +use crate::websocket_pong_watchdog::WebSocketPongWatchdog; const RELAY_MESSAGE_FRAME_VERSION: u32 = 1; const MAX_ACTIVE_NOISE_RELAY_STREAMS: usize = 128; @@ -49,6 +52,27 @@ const MAX_HARNESS_KEY_AUTHORIZATION_BYTES: usize = 4096; const MAX_PENDING_HANDSHAKE_VALIDATIONS: usize = 32; const HARNESS_KEY_VALIDATION_TIMEOUT: Duration = Duration::from_secs(10); +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub(crate) enum RendezvousDisconnectReason { + PeerClose, + ReadError, + WriteError, + PongTimeout, + LocalShutdown, +} + +impl RendezvousDisconnectReason { + pub(crate) fn as_str(self) -> &'static str { + match self { + Self::PeerClose => "peer_close", + Self::ReadError => "read_error", + Self::WriteError => "write_error", + Self::PongTimeout => WEBSOCKET_PONG_TIMEOUT_REASON, + Self::LocalShutdown => "local_shutdown", + } + } +} + #[derive(Debug, Clone, Copy, Eq, PartialEq)] pub(crate) enum RelayFrameBodyKind { Data, @@ -448,7 +472,8 @@ pub(crate) async fn run_multiplexed_environment( executor_registration_id: String, identity: NoiseChannelIdentity, validator: V, -) where +) -> RendezvousDisconnectReason +where S: AsyncRead + AsyncWrite + Unpin + Send + 'static, V: HarnessKeyValidator + Clone + 'static, { @@ -461,6 +486,7 @@ pub(crate) async fn run_multiplexed_environment( mpsc::channel::>(CHANNEL_CAPACITY); let (closed_stream_tx, mut closed_stream_rx) = mpsc::channel::(MAX_ACTIVE_NOISE_RELAY_STREAMS); + let (pong_tx, mut pong_rx) = mpsc::channel(1); // Use a separate writer so this loop never waits on the channel it drains. let mut physical_writer_task = tokio::spawn(async move { let mut keepalive = tokio::time::interval_at( @@ -468,19 +494,61 @@ pub(crate) async fn run_multiplexed_environment( WEBSOCKET_KEEPALIVE_INTERVAL, ); keepalive.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + let mut pong_watchdog = WebSocketPongWatchdog::new(WEBSOCKET_PONG_TIMEOUT); + let pong_deadline = tokio::time::sleep(WEBSOCKET_PONG_TIMEOUT); + tokio::pin!(pong_deadline); loop { let message = tokio::select! { + pong = pong_rx.recv() => { + let Some(()) = pong else { + break RendezvousDisconnectReason::LocalShutdown; + }; + pong_watchdog.received_pong(); + continue; + } + _ = &mut pong_deadline, if pong_watchdog.deadline().is_some() => { + match pong_rx.try_recv() { + Ok(()) => { + pong_watchdog.received_pong(); + continue; + } + Err(tokio::sync::mpsc::error::TryRecvError::Empty) => { + break RendezvousDisconnectReason::PongTimeout; + } + Err(tokio::sync::mpsc::error::TryRecvError::Disconnected) => { + break RendezvousDisconnectReason::LocalShutdown; + } + } + } + _ = keepalive.tick(), if pong_watchdog.deadline().is_none() => { + Message::Ping(Vec::new().into()) + } encoded = physical_outgoing_rx.recv() => { let Some(encoded) = encoded else { - break; + break RendezvousDisconnectReason::LocalShutdown; }; Message::Binary(encoded.into()) } - _ = keepalive.tick() => Message::Ping(Vec::new().into()), }; - if let Err(error) = websocket_sink.send(message).await { - warn!("Noise multiplexed environment websocket write failed: {error}"); - break; + let is_keepalive_ping = matches!(message, Message::Ping(_)); + let write_deadline = pong_watchdog.write_deadline(tokio::time::Instant::now()); + match tokio::time::timeout_at(write_deadline, websocket_sink.send(message)).await { + Ok(Ok(())) => { + if is_keepalive_ping { + pong_watchdog.ping_sent(tokio::time::Instant::now()); + if let Some(deadline) = pong_watchdog.deadline() { + pong_deadline.as_mut().reset(deadline); + } + } + } + Ok(Err(error)) => { + warn!("Noise multiplexed environment websocket write failed: {error}"); + break RendezvousDisconnectReason::WriteError; + } + Err(_) => { + warn!("Noise multiplexed environment websocket write timed out"); + break RendezvousDisconnectReason::WriteError; + } } } }); @@ -489,13 +557,18 @@ pub(crate) async fn run_multiplexed_environment( let mut validation_tasks: JoinSet = JoinSet::new(); let mut failed_handshakes = 0usize; let mut next_validation_id = 0u64; + let mut disconnect_reason = RendezvousDisconnectReason::LocalShutdown; loop { // Registry calls run separately so a slow check does not block the relay. let frame = tokio::select! { writer_result = &mut physical_writer_task => { - if let Err(error) = writer_result { - warn!("Noise multiplexed environment websocket writer failed: {error}"); + match writer_result { + Ok(reason) => disconnect_reason = reason, + Err(error) => { + warn!("Noise multiplexed environment websocket writer failed: {error}"); + disconnect_reason = RendezvousDisconnectReason::LocalShutdown; + } } break; } @@ -621,14 +694,22 @@ pub(crate) async fn run_multiplexed_environment( continue; } }, - Some(Ok(Message::Close(_))) | None => break, - Some(Ok(Message::Ping(_) | Message::Pong(_) | Message::Frame(_))) => continue, + Some(Ok(Message::Close(_))) | None => { + disconnect_reason = RendezvousDisconnectReason::PeerClose; + break; + } + Some(Ok(Message::Pong(_))) => { + let _ = pong_tx.try_send(()); + continue; + } + Some(Ok(Message::Ping(_) | Message::Frame(_))) => continue, Some(Ok(Message::Text(_))) => { warn!("dropping non-binary Noise relay frame from harness"); continue; } Some(Err(error)) => { debug!("Noise multiplexed environment websocket read failed: {error}"); + disconnect_reason = RendezvousDisconnectReason::ReadError; break; } } @@ -802,6 +883,7 @@ pub(crate) async fn run_multiplexed_environment( physical_writer_task.abort(); let _ = physical_writer_task.await; } + disconnect_reason } /// Charge one failed authenticated-channel attempt to this physical relay. diff --git a/codex-rs/exec-server/src/relay_noise_tests.rs b/codex-rs/exec-server/src/relay_noise_tests.rs index 0020efe347..704c48d9a1 100644 --- a/codex-rs/exec-server/src/relay_noise_tests.rs +++ b/codex-rs/exec-server/src/relay_noise_tests.rs @@ -17,6 +17,7 @@ use tokio_tungstenite::tungstenite::Message; use super::HarnessKeyValidator; use super::MAX_FAILED_NOISE_HANDSHAKES; use super::MAX_HARNESS_KEY_AUTHORIZATION_BYTES; +use super::RendezvousDisconnectReason; use super::run_multiplexed_environment; use crate::ExecServerError; use crate::ExecServerRuntimePaths; @@ -33,6 +34,86 @@ use crate::server::ConnectionProcessor; const ENVIRONMENT_ID: &str = "environment-1"; const EXECUTOR_REGISTRATION_ID: &str = "registration-1"; +#[tokio::test] +async fn missing_pong_disconnects_physical_relay() -> Result<()> { + let listener = TcpListener::bind("127.0.0.1:0").await?; + let websocket_url = format!("ws://{}", listener.local_addr()?); + let harness_connection = tokio::spawn(connect_async(websocket_url)); + let (socket, _peer_addr) = listener.accept().await?; + let environment_websocket = accept_async(socket).await?; + let (_harness_websocket, _response) = harness_connection.await??; + + let environment_task = tokio::spawn(run_multiplexed_environment( + environment_websocket, + ConnectionProcessor::new(ExecServerRuntimePaths::new( + std::env::current_exe()?, + /*codex_linux_sandbox_exe*/ None, + )?), + ENVIRONMENT_ID.to_string(), + EXECUTOR_REGISTRATION_ID.to_string(), + NoiseChannelIdentity::generate()?, + BlockingValidator { + calls: Arc::new(AtomicUsize::new(0)), + release: Arc::new(Notify::new()), + }, + )); + + assert_eq!( + timeout(Duration::from_secs(1), environment_task).await??, + RendezvousDisconnectReason::PongTimeout + ); + Ok(()) +} + +#[tokio::test] +async fn pong_keeps_physical_relay_connected() -> Result<()> { + let listener = TcpListener::bind("127.0.0.1:0").await?; + let websocket_url = format!("ws://{}", listener.local_addr()?); + let harness_connection = tokio::spawn(connect_async(websocket_url)); + let (socket, _peer_addr) = listener.accept().await?; + let environment_websocket = accept_async(socket).await?; + let (mut harness_websocket, _response) = harness_connection.await??; + + let environment_task = tokio::spawn(run_multiplexed_environment( + environment_websocket, + ConnectionProcessor::new(ExecServerRuntimePaths::new( + std::env::current_exe()?, + /*codex_linux_sandbox_exe*/ None, + )?), + ENVIRONMENT_ID.to_string(), + EXECUTOR_REGISTRATION_ID.to_string(), + NoiseChannelIdentity::generate()?, + BlockingValidator { + calls: Arc::new(AtomicUsize::new(0)), + release: Arc::new(Notify::new()), + }, + )); + + timeout(Duration::from_secs(1), async { + let mut pings = 0; + while pings < 6 { + match harness_websocket.next().await { + Some(Ok(Message::Ping(payload))) => { + harness_websocket.send(Message::Pong(payload)).await?; + pings += 1; + } + Some(Ok(Message::Pong(_) | Message::Frame(_))) => {} + Some(Ok(message)) => anyhow::bail!("expected keepalive ping, got {message:?}"), + Some(Err(error)) => return Err(error.into()), + None => anyhow::bail!("environment disconnected before six keepalive pings"), + } + } + Ok::<_, anyhow::Error>(()) + }) + .await??; + harness_websocket.close(None).await?; + assert_eq!( + timeout(Duration::from_secs(1), environment_task).await??, + RendezvousDisconnectReason::PeerClose + ); + Ok(()) +} + #[derive(Clone)] struct BlockingValidator { calls: Arc, diff --git a/codex-rs/exec-server/src/remote.rs b/codex-rs/exec-server/src/remote.rs index 199e787542..8870e0efe9 100644 --- a/codex-rs/exec-server/src/remote.rs +++ b/codex-rs/exec-server/src/remote.rs @@ -1,5 +1,6 @@ use std::sync::Arc; use std::time::Duration; +use std::time::Instant; use codex_api::AuthProvider; use codex_api::SharedAuthProvider; @@ -10,7 +11,10 @@ use http::HeaderValue; use reqwest::StatusCode; use serde::Deserialize; use tokio::time::sleep; +use tokio_tungstenite::MaybeTlsStream; +use tokio_tungstenite::WebSocketStream; use tokio_tungstenite::connect_async_with_config; +use tokio_tungstenite::tungstenite::client::IntoClientRequest; use tracing::debug; use tracing::info; use tracing::warn; @@ -25,6 +29,7 @@ use crate::EnvironmentRegistryRegistrationRequest; use crate::EnvironmentRegistryRegistrationResponse; use crate::ExecServerError; use crate::ExecServerRuntimePaths; +use crate::ExecServerTelemetry; use crate::NoiseChannelIdentity; use crate::NoiseChannelPublicKey; use crate::NoiseRendezvousConnectBundle; @@ -34,6 +39,7 @@ use crate::noise_relay::noise_relay_websocket_config; use crate::relay::HarnessKeyValidator; use crate::relay::run_multiplexed_environment; use crate::server::ConnectionProcessor; +use crate::trace_context::current_trace_context_headers; const ERROR_BODY_PREVIEW_BYTES: usize = 4096; const NOISE_RELAY_SECURITY_PROFILE: &str = "noise_hybrid_ik_v1"; @@ -44,6 +50,7 @@ struct EnvironmentRegistryClient { auth_provider: SharedAuthProvider, http: reqwest::Client, connect_timeout: Duration, + telemetry: ExecServerTelemetry, } impl std::fmt::Debug for EnvironmentRegistryClient { @@ -56,7 +63,16 @@ impl std::fmt::Debug for EnvironmentRegistryClient { } impl EnvironmentRegistryClient { + #[cfg(test)] fn new(base_url: String, auth_provider: SharedAuthProvider) -> Result { + Self::new_with_telemetry(base_url, auth_provider, ExecServerTelemetry::default()) + } + + fn new_with_telemetry( + base_url: String, + auth_provider: SharedAuthProvider, + telemetry: ExecServerTelemetry, + ) -> Result { let base_url = normalize_base_url(base_url)?; Ok(Self { base_url, @@ -65,15 +81,41 @@ impl EnvironmentRegistryClient { .redirect(reqwest::redirect::Policy::none()) .build()?, connect_timeout: DEFAULT_REMOTE_EXEC_SERVER_CONNECT_TIMEOUT, + telemetry, }) } /// Register the executor public key and obtain the rendezvous allocation. /// The returned registration ID is included in each stream's Noise prologue. + #[tracing::instrument( + name = "codex.exec_server.remote.register", + skip_all, + fields( + otel.kind = "client", + otel.name = "codex.exec_server.remote.register", + result = tracing::field::Empty, + ) + )] async fn register_environment( &self, environment_id: &str, executor_public_key: &NoiseChannelPublicKey, + ) -> Result { + let started_at = Instant::now(); + let response = self + .register_environment_inner(environment_id, executor_public_key) + .await; + let result = if response.is_ok() { "success" } else { "error" }; + tracing::Span::current().record("result", result); + self.telemetry + .remote_registration_completed(result, started_at.elapsed()); + response + } + + async fn register_environment_inner( + &self, + environment_id: &str, + executor_public_key: &NoiseChannelPublicKey, ) -> Result { let response = self .http @@ -82,6 +124,7 @@ impl EnvironmentRegistryClient { &format!("/cloud/environment/{environment_id}/register"), )) .headers(self.auth_provider.to_auth_headers()) + .headers(current_trace_context_headers()) .json(&EnvironmentRegistryRegistrationRequest { security_profile: NOISE_RELAY_SECURITY_PROFILE.to_string(), executor_public_key: executor_public_key.clone(), @@ -268,7 +311,11 @@ impl NoiseRendezvousEnvironmentConfig { ) -> Result { let environment_id = normalize_environment_id(environment_id)?; let auth_provider = static_bearer_auth_provider(bearer_token, chatgpt_account_id)?; - let client = EnvironmentRegistryClient::new(base_url, auth_provider)?; + let client = EnvironmentRegistryClient::new_with_telemetry( + base_url, + auth_provider, + ExecServerTelemetry::default(), + )?; Ok(Self { provider: Arc::new(EnvironmentRegistryNoiseConnectProvider { client, @@ -373,6 +420,7 @@ pub struct RemoteEnvironmentConfig { pub environment_id: String, pub name: String, auth_provider: SharedAuthProvider, + telemetry: ExecServerTelemetry, } impl std::fmt::Debug for RemoteEnvironmentConfig { @@ -398,8 +446,14 @@ impl RemoteEnvironmentConfig { environment_id, name: "codex-exec-server".to_string(), auth_provider, + telemetry: ExecServerTelemetry::default(), }) } + + pub fn with_telemetry(mut self, telemetry: ExecServerTelemetry) -> Self { + self.telemetry = telemetry; + self + } } /// Register an exec-server for remote use and serve requests over Noise. @@ -408,19 +462,18 @@ impl RemoteEnvironmentConfig { /// reconnects. The registration and rendezvous URL are also reused until /// rendezvous rejects the URL, at which point the next attempt registers again. /// The websocket carries cleartext routing metadata and encrypted payloads. -#[tracing::instrument( - name = "codex.exec_server", - skip_all, - fields(otel.kind = "internal") -)] pub async fn run_remote_environment( config: RemoteEnvironmentConfig, runtime_paths: ExecServerRuntimePaths, ) -> Result<(), ExecServerError> { ensure_rustls_crypto_provider(); - let client = - EnvironmentRegistryClient::new(config.base_url.clone(), config.auth_provider.clone())?; - let processor = ConnectionProcessor::new(runtime_paths); + let client = EnvironmentRegistryClient::new_with_telemetry( + config.base_url.clone(), + config.auth_provider.clone(), + config.telemetry.clone(), + )?; + let processor = + ConnectionProcessor::new_with_telemetry(runtime_paths, config.telemetry.clone()); let identity = NoiseChannelIdentity::generate().map_err(|error| { ExecServerError::Protocol(format!("failed to generate Noise relay identity: {error}")) })?; @@ -430,14 +483,8 @@ pub async fn run_remote_environment( .await?; loop { - match connect_async_with_config( - response.url.as_str(), - Some(noise_relay_websocket_config()), - /*disable_nagle*/ false, - ) - .await - { - Ok((websocket, _)) => { + match connect_rendezvous(&response.url, &config.telemetry).await { + Ok(websocket) => { backoff = Duration::from_secs(1); let executor_registration_id = response.executor_registration_id.clone(); info!( @@ -445,7 +492,7 @@ pub async fn run_remote_environment( noise_outcome = "ok", "Noise executor connected to rendezvous" ); - run_multiplexed_environment( + let disconnect_reason = run_multiplexed_environment( websocket, processor.clone(), response.environment_id.clone(), @@ -458,6 +505,15 @@ pub async fn run_remote_environment( }, ) .await; + info!( + noise_event = "rendezvous_connection", + noise_outcome = "disconnected", + noise_reason = disconnect_reason.as_str(), + "Noise executor disconnected from rendezvous" + ); + config + .telemetry + .remote_reconnect(disconnect_reason.as_str()); } Err(error) => { let registration_rejected = matches!( @@ -473,9 +529,12 @@ pub async fn run_remote_environment( ); debug!(error = %error, "Noise executor rendezvous connection error"); if registration_rejected { + config.telemetry.remote_reconnect("registration_rejected"); response = client .register_environment(&config.environment_id, &identity.public_key()) .await?; + } else { + config.telemetry.remote_reconnect("connect_failed"); } } } @@ -485,6 +544,45 @@ pub async fn run_remote_environment( } } +#[tracing::instrument( + name = "codex.exec_server.remote.rendezvous.connect", + skip_all, + fields( + otel.kind = "client", + otel.name = "codex.exec_server.remote.rendezvous.connect", + result = tracing::field::Empty, + ) +)] +async fn connect_rendezvous( + url: &str, + telemetry: &ExecServerTelemetry, +) -> Result< + WebSocketStream>, + tokio_tungstenite::tungstenite::Error, +> { + let started_at = Instant::now(); + let result = async { + let mut request = url.into_client_request()?; + request + .headers_mut() + .extend(current_trace_context_headers()); + connect_async_with_config( + request, + Some(noise_relay_websocket_config()), + // Rendezvous sends small, latency-sensitive frames, so avoid Nagle's coalescing delay. + /*disable_nagle*/ + true, + ) + .await + .map(|(websocket, _)| websocket) + } + .await; + let result_name = if result.is_ok() { "success" } else { "error" }; + tracing::Span::current().record("result", result_name); + telemetry.remote_rendezvous_completed(result_name, started_at.elapsed()); + result +} + fn normalize_environment_id(environment_id: String) -> Result { let environment_id = environment_id.trim().to_string(); if environment_id.is_empty() { @@ -576,12 +674,17 @@ mod tests { use codex_api::AuthProvider; use http::HeaderMap; use http::HeaderValue; + use opentelemetry::trace::TracerProvider as _; + use opentelemetry_sdk::trace::SdkTracerProvider; use pretty_assertions::assert_eq; + use tracing::Instrument; + use tracing_subscriber::prelude::*; use wiremock::Mock; use wiremock::MockServer; use wiremock::ResponseTemplate; use wiremock::matchers::body_partial_json; use wiremock::matchers::header; + use wiremock::matchers::header_regex; use wiremock::matchers::method; use wiremock::matchers::path; @@ -607,8 +710,14 @@ mod tests { Arc::new(StaticRegistryAuthProvider) } - #[tokio::test] + #[tokio::test(flavor = "current_thread")] async fn register_environment_posts_with_auth_provider_headers() { + let provider = SdkTracerProvider::builder().build(); + let tracer = provider.tracer("exec-server-test"); + let subscriber = + tracing_subscriber::registry().with(tracing_opentelemetry::layer().with_tracer(tracer)); + let _guard = subscriber.set_default(); + tracing::callsite::rebuild_interest_cache(); let server = MockServer::start().await; let executor_public_key = NoiseChannelIdentity::generate() .expect("identity") @@ -617,6 +726,10 @@ mod tests { .and(path("/cloud/environment/environment-requested/register")) .and(header("authorization", "Bearer registry-token")) .and(header("chatgpt-account-id", "workspace-123")) + .and(header_regex( + "traceparent", + "^00-[0-9a-f]{32}-[0-9a-f]{16}-0[01]$", + )) .and(body_partial_json(serde_json::json!({ "security_profile": NOISE_RELAY_SECURITY_PROFILE, "executor_public_key": executor_public_key.clone(), @@ -634,6 +747,7 @@ mod tests { let response = client .register_environment("environment-requested", &executor_public_key) + .instrument(tracing::info_span!("remote-operation")) .await .expect("register environment"); diff --git a/codex-rs/exec-server/src/remote_file_system.rs b/codex-rs/exec-server/src/remote_file_system.rs index acc86f7584..0d10c3aece 100644 --- a/codex-rs/exec-server/src/remote_file_system.rs +++ b/codex-rs/exec-server/src/remote_file_system.rs @@ -29,6 +29,7 @@ use crate::protocol::FsWalkParams; use crate::protocol::FsWriteFileParams; const INVALID_REQUEST_ERROR_CODE: i64 = -32600; +const METHOD_NOT_FOUND_ERROR_CODE: i64 = -32601; const NOT_FOUND_ERROR_CODE: i64 = -32004; #[path = "remote_file_stream.rs"] @@ -194,14 +195,26 @@ impl RemoteFileSystem { ) -> FileSystemResult { trace!("remote fs walk"); let client = self.client.get().await.map_err(map_remote_error)?; - let response = client + let response = match client .fs_walk(FsWalkParams { path: path.clone(), options, sandbox: remote_sandbox_context(sandbox), }) .await - .map_err(map_remote_error)?; + { + Ok(response) => response, + Err(ExecServerError::Server { + code: METHOD_NOT_FOUND_ERROR_CODE, + .. + }) => { + return ::walk_via_directory_reads( + self, path, options, sandbox, + ) + .await; + } + Err(error) => return Err(map_remote_error(error)), + }; Ok(response) } diff --git a/codex-rs/exec-server/src/resolved_capability.rs b/codex-rs/exec-server/src/resolved_capability.rs new file mode 100644 index 0000000000..2902cf9eef --- /dev/null +++ b/codex-rs/exec-server/src/resolved_capability.rs @@ -0,0 +1,114 @@ +use std::collections::HashMap; +use std::fmt; +use std::sync::Arc; + +use codex_protocol::capabilities::CapabilityRootLocation; +use codex_protocol::capabilities::SelectedCapabilityRoot; + +use crate::Environment; +use crate::EnvironmentManager; +use crate::ExecutorFileSystem; + +/// A selected capability root paired with its currently ready environment handle. +/// +/// Environment IDs have stable identity and contents. This process-local value must not be +/// persisted: it only keeps the current connection handle alive while one model step uses the +/// stable environment. +#[derive(Clone)] +pub struct ResolvedSelectedCapabilityRoot { + selected_root: SelectedCapabilityRoot, + environment: Arc, +} + +impl ResolvedSelectedCapabilityRoot { + pub fn selected_root(&self) -> &SelectedCapabilityRoot { + &self.selected_root + } + + pub fn environment(&self) -> &Arc { + &self.environment + } + + pub fn file_system(&self) -> Arc { + self.environment.get_filesystem() + } +} + +impl EnvironmentManager { + /// Resolves selected roots whose stable environments are ready for the current model step. + /// + /// Environment identity comes from the selected root's stable environment ID. A ready + /// environment captured for the step carries its exact process-local handle so readiness and + /// execution cannot come from different registry snapshots. Missing, starting, or failed + /// environments are omitted. A lazy environment is started for a later step. + pub async fn resolve_selected_capability_roots( + &self, + selected_roots: &[SelectedCapabilityRoot], + captured_environments: &HashMap>>, + ) -> Vec { + let candidates = { + let environments = self + .environments + .read() + .unwrap_or_else(std::sync::PoisonError::into_inner); + selected_roots + .iter() + .filter_map(|selected_root| { + let CapabilityRootLocation::Environment { environment_id, .. } = + &selected_root.location; + let (environment, already_ready) = + match captured_environments.get(environment_id) { + Some(Some(environment)) => (Arc::clone(environment), true), + Some(None) => return None, + None => (Arc::clone(environments.get(environment_id)?), false), + }; + Some(( + ResolvedSelectedCapabilityRoot { + selected_root: selected_root.clone(), + environment, + }, + already_ready, + )) + }) + .collect::>() + }; + + let mut readiness = HashMap::new(); + for (candidate, already_ready) in &candidates { + let CapabilityRootLocation::Environment { environment_id, .. } = + &candidate.selected_root().location; + if readiness.contains_key(environment_id) { + continue; + } + let environment = candidate.environment(); + let ready = if *already_ready { + true + } else if environment.startup_finished() { + environment.wait_until_ready().await.is_ok() + } else { + Environment::start_connecting_for_use(environment); + false + }; + readiness.insert(environment_id.clone(), ready); + } + + candidates + .into_iter() + .map(|(candidate, _)| candidate) + .filter(|candidate| { + let CapabilityRootLocation::Environment { environment_id, .. } = + &candidate.selected_root().location; + readiness.get(environment_id).copied().unwrap_or(false) + }) + .collect() + } +} + +impl fmt::Debug for ResolvedSelectedCapabilityRoot { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("ResolvedSelectedCapabilityRoot") + .field("selected_root", &self.selected_root) + .finish_non_exhaustive() + } +} diff --git a/codex-rs/exec-server/src/rpc.rs b/codex-rs/exec-server/src/rpc.rs index 4613c41f63..e762fe8b66 100644 --- a/codex-rs/exec-server/src/rpc.rs +++ b/codex-rs/exec-server/src/rpc.rs @@ -5,6 +5,7 @@ use std::sync::Arc; use std::sync::atomic::AtomicBool; use std::sync::atomic::AtomicI64; use std::sync::atomic::Ordering; +use std::time::Duration; use codex_exec_server_protocol::JSONRPCError; use codex_exec_server_protocol::JSONRPCErrorError; @@ -21,6 +22,7 @@ use tokio::sync::mpsc; use tokio::sync::oneshot; use tokio::sync::watch; use tokio::task::JoinHandle; +use tokio::time::timeout; use crate::connection::JsonRpcConnection; use crate::connection::JsonRpcConnectionEvent; @@ -36,6 +38,8 @@ pub(crate) enum RpcCallError { Json(serde_json::Error), /// The executor returned a JSON-RPC error response for this call. Server(JSONRPCErrorError), + /// The executor did not return a response before the caller's deadline. + TimedOut { method: String, timeout: Duration }, } type PendingRequest = oneshot::Sender>; @@ -46,6 +50,11 @@ type RequestRoute = Box< type NotificationRoute = Box, JSONRPCNotification) -> BoxFuture> + Send + Sync>; +enum RpcCallTimeout { + None, + After(Duration), +} + #[derive(Debug)] pub(crate) enum RpcClientEvent { Notification(JSONRPCNotification), @@ -212,8 +221,10 @@ where ); } - pub(crate) fn request_route(&self, method: &str) -> Option<&RequestRoute> { - self.request_routes.get(method) + pub(crate) fn request_route(&self, method: &str) -> Option<(&'static str, &RequestRoute)> { + self.request_routes + .get_key_value(method) + .map(|(&method, route)| (method, route)) } pub(crate) fn notification_route(&self, method: &str) -> Option<&NotificationRoute> { @@ -331,7 +342,44 @@ impl RpcClient { drain_pending(&self.pending).await; } + #[tracing::instrument( + name = "codex.exec_server.request", + level = "info", + skip_all, + fields( + otel.kind = "client", + otel.name = method, + method, + ) + )] pub(crate) async fn call(&self, method: &str, params: &P) -> Result + where + P: Serialize, + T: DeserializeOwned, + { + self.call_inner(method, params, RpcCallTimeout::None).await + } + + pub(crate) async fn call_with_timeout( + &self, + method: &str, + params: &P, + call_timeout: Duration, + ) -> Result + where + P: Serialize, + T: DeserializeOwned, + { + self.call_inner(method, params, RpcCallTimeout::After(call_timeout)) + .await + } + + async fn call_inner( + &self, + method: &str, + params: &P, + call_timeout: RpcCallTimeout, + ) -> Result where P: Serialize, T: DeserializeOwned, @@ -377,8 +425,20 @@ impl RpcClient { // still-pending requests. Awaiting this receiver preserves that order: // responses already read before EOF still win, and truly pending calls // are failed once the reader observes the disconnect. - let result: Result = - response_rx.await.map_err(|_| RpcCallError::Closed)?; + let response = match call_timeout { + RpcCallTimeout::None => response_rx.await, + RpcCallTimeout::After(call_timeout) => match timeout(call_timeout, response_rx).await { + Ok(response) => response, + Err(_) => { + self.pending.lock().await.remove(&request_id); + return Err(RpcCallError::TimedOut { + method: method.to_string(), + timeout: call_timeout, + }); + } + }, + }; + let result: Result = response.map_err(|_| RpcCallError::Closed)?; let response = match result { Ok(response) => response, Err(error) => return Err(error), @@ -671,6 +731,40 @@ mod tests { } } + #[tokio::test] + async fn rpc_client_timeout_removes_pending_request() { + let (client_stdin, server_reader) = tokio::io::duplex(4096); + let (server_writer, client_stdout) = tokio::io::duplex(4096); + let (release_server_tx, release_server_rx) = tokio::sync::oneshot::channel(); + let connection = + JsonRpcConnection::from_stdio(client_stdout, client_stdin, "test-rpc".to_string()); + let (client, _events_rx) = RpcClient::new(connection); + + let server = tokio::spawn(async move { + let mut lines = BufReader::new(server_reader).lines(); + let request = read_jsonrpc_line(&mut lines).await; + assert!(matches!(request, JSONRPCMessage::Request(_))); + let _server_writer = server_writer; + let _ = release_server_rx.await; + }); + + let call_timeout = Duration::from_millis(10); + let result = client + .call_with_timeout::<_, serde_json::Value>("slow", &serde_json::json!({}), call_timeout) + .await; + assert!(matches!( + result, + Err(super::RpcCallError::TimedOut { method, timeout }) + if method == "slow" && timeout == call_timeout + )); + assert_eq!(client.pending_request_count().await, 0); + + let _ = release_server_tx.send(()); + if let Err(err) = server.await { + panic!("server task failed: {err}"); + } + } + #[tokio::test(flavor = "current_thread")] async fn rpc_client_propagates_current_trace_context() { let span_exporter = InMemorySpanExporter::default(); @@ -719,6 +813,15 @@ mod tests { .expect("RPC response"); assert_eq!(response, serde_json::json!({})); let trace = server.await.expect("server task").expect("trace context"); - assert_eq!(trace, expected_trace); + let expected_traceparent = expected_trace + .traceparent + .as_deref() + .expect("parent traceparent"); + let traceparent = trace.traceparent.as_deref().expect("request traceparent"); + let expected_parts = expected_traceparent.split('-').collect::>(); + let parts = traceparent.split('-').collect::>(); + assert_eq!(parts[1], expected_parts[1]); + assert_ne!(parts[2], expected_parts[2]); + assert_eq!(trace.tracestate, expected_trace.tracestate); } } diff --git a/codex-rs/exec-server/src/server.rs b/codex-rs/exec-server/src/server.rs index d443b0ed1f..7988e0870b 100644 --- a/codex-rs/exec-server/src/server.rs +++ b/codex-rs/exec-server/src/server.rs @@ -12,15 +12,71 @@ pub use transport::DEFAULT_LISTEN_URL; pub use transport::ExecServerListenUrlParseError; use crate::ExecServerRuntimePaths; +use crate::ExecServerTelemetry; + +pub async fn run_main( + listen_url: &str, + runtime_paths: ExecServerRuntimePaths, +) -> Result<(), Box> { + run_main_with_telemetry(listen_url, runtime_paths, ExecServerTelemetry::default()).await +} #[tracing::instrument( name = "codex.exec_server", skip_all, fields(otel.kind = "internal") )] -pub async fn run_main( +pub async fn run_main_with_telemetry( listen_url: &str, runtime_paths: ExecServerRuntimePaths, + telemetry: ExecServerTelemetry, ) -> Result<(), Box> { - transport::run_transport(listen_url, runtime_paths).await + transport::run_transport(listen_url, runtime_paths, telemetry).await +} + +#[cfg(test)] +mod tests { + use opentelemetry::trace::TracerProvider as _; + use opentelemetry_sdk::trace::InMemorySpanExporter; + use opentelemetry_sdk::trace::SdkTracerProvider; + use tracing::instrument::WithSubscriber; + use tracing_subscriber::prelude::*; + + use super::run_main_with_telemetry; + use crate::ExecServerRuntimePaths; + use crate::ExecServerTelemetry; + + #[tokio::test] + async fn telemetry_entrypoint_emits_root_span() { + let exporter = InMemorySpanExporter::default(); + let provider = SdkTracerProvider::builder() + .with_simple_exporter(exporter.clone()) + .build(); + let subscriber = tracing_subscriber::registry() + .with(tracing_opentelemetry::layer().with_tracer(provider.tracer("exec-server-test"))); + + async { + tracing::callsite::rebuild_interest_cache(); + run_main_with_telemetry( + "invalid", + ExecServerRuntimePaths::new( + std::env::current_exe().expect("current executable"), + /*codex_linux_sandbox_exe*/ None, + ) + .expect("runtime paths"), + ExecServerTelemetry::default(), + ) + .await + .expect_err("invalid listen URL should fail"); + } + .with_subscriber(subscriber) + .await; + + provider.force_flush().expect("flush traces"); + let spans = exporter.get_finished_spans().expect("span export"); + assert!( + spans.iter().any(|span| span.name == "codex.exec_server"), + "root exec-server span missing: {spans:?}" + ); + } } diff --git a/codex-rs/exec-server/src/server/handler.rs b/codex-rs/exec-server/src/server/handler.rs index 561cf0ff63..02858ae887 100644 --- a/codex-rs/exec-server/src/server/handler.rs +++ b/codex-rs/exec-server/src/server/handler.rs @@ -211,7 +211,7 @@ impl ExecServerHandler { if stream_response { self.reserve_http_body_stream(&http_request_id).await?; } - let response = ReqwestHttpRequestRunner::new(params.timeout_ms)? + let response = ReqwestHttpRequestRunner::new(params.timeout_ms, params.redirect_policy)? .run(params) .await; if response.is_err() && stream_response { diff --git a/codex-rs/exec-server/src/server/handler/tests.rs b/codex-rs/exec-server/src/server/handler/tests.rs index 2cea99b214..47416039c6 100644 --- a/codex-rs/exec-server/src/server/handler/tests.rs +++ b/codex-rs/exec-server/src/server/handler/tests.rs @@ -82,7 +82,7 @@ fn test_runtime_paths() -> ExecServerRuntimePaths { async fn initialized_handler() -> Arc { let (outgoing_tx, _outgoing_rx) = mpsc::channel(16); - let registry = SessionRegistry::new(); + let registry = SessionRegistry::new(crate::ExecServerTelemetry::default()); let handler = Arc::new(ExecServerHandler::new( registry, RpcNotificationSender::new(outgoing_tx), @@ -160,7 +160,7 @@ async fn terminate_reports_false_after_process_exit() { #[tokio::test] async fn long_poll_read_fails_after_session_resume() { let (first_tx, _first_rx) = mpsc::channel(16); - let registry = SessionRegistry::new(); + let registry = SessionRegistry::new(crate::ExecServerTelemetry::default()); let first_handler = Arc::new(ExecServerHandler::new( Arc::clone(®istry), RpcNotificationSender::new(first_tx), @@ -233,7 +233,7 @@ async fn long_poll_read_fails_after_session_resume() { #[tokio::test] async fn active_session_resume_is_rejected() { let (first_tx, _first_rx) = mpsc::channel(16); - let registry = SessionRegistry::new(); + let registry = SessionRegistry::new(crate::ExecServerTelemetry::default()); let first_handler = Arc::new(ExecServerHandler::new( Arc::clone(®istry), RpcNotificationSender::new(first_tx), @@ -277,7 +277,7 @@ async fn active_session_resume_is_rejected() { async fn output_and_exit_are_retained_after_notification_receiver_closes() { let (outgoing_tx, outgoing_rx) = mpsc::channel(16); let handler = Arc::new(ExecServerHandler::new( - SessionRegistry::new(), + SessionRegistry::new(crate::ExecServerTelemetry::default()), RpcNotificationSender::new(outgoing_tx), test_runtime_paths(), )); diff --git a/codex-rs/exec-server/src/server/process_handler.rs b/codex-rs/exec-server/src/server/process_handler.rs index 9ff6690411..4d933afbe1 100644 --- a/codex-rs/exec-server/src/server/process_handler.rs +++ b/codex-rs/exec-server/src/server/process_handler.rs @@ -13,6 +13,7 @@ use crate::protocol::TerminateResponse; use crate::protocol::WriteParams; use crate::protocol::WriteResponse; use crate::rpc::RpcNotificationSender; +use crate::telemetry::ExecServerTelemetry; #[derive(Clone)] pub(crate) struct ProcessHandler { @@ -22,10 +23,11 @@ pub(crate) struct ProcessHandler { impl ProcessHandler { pub(crate) fn new( notifications: RpcNotificationSender, + telemetry: ExecServerTelemetry, runtime_paths: ExecServerRuntimePaths, ) -> Self { Self { - process: LocalProcess::new(notifications, runtime_paths), + process: LocalProcess::new(notifications, telemetry, runtime_paths), } } diff --git a/codex-rs/exec-server/src/server/processor.rs b/codex-rs/exec-server/src/server/processor.rs index 3eedfb2f20..63f6bd45d9 100644 --- a/codex-rs/exec-server/src/server/processor.rs +++ b/codex-rs/exec-server/src/server/processor.rs @@ -1,4 +1,5 @@ use std::sync::Arc; +use std::time::Instant; use tokio::sync::mpsc; use tracing::Instrument; @@ -17,36 +18,61 @@ use crate::rpc::method_not_found; use crate::server::ExecServerHandler; use crate::server::registry::build_router; use crate::server::session_registry::SessionRegistry; +use crate::telemetry::ConnectionTransport; +use crate::telemetry::ExecServerTelemetry; #[derive(Clone)] pub(crate) struct ConnectionProcessor { session_registry: Arc, runtime_paths: ExecServerRuntimePaths, + telemetry: ExecServerTelemetry, } impl ConnectionProcessor { + #[cfg(test)] pub(crate) fn new(runtime_paths: ExecServerRuntimePaths) -> Self { + Self::new_with_telemetry(runtime_paths, ExecServerTelemetry::default()) + } + + pub(crate) fn new_with_telemetry( + runtime_paths: ExecServerRuntimePaths, + telemetry: ExecServerTelemetry, + ) -> Self { Self { - session_registry: SessionRegistry::new(), + session_registry: SessionRegistry::new(telemetry.clone()), runtime_paths, + telemetry, } } - pub(crate) async fn run_connection(&self, connection: JsonRpcConnection) { + pub(crate) async fn run_connection( + &self, + connection: JsonRpcConnection, + transport: ConnectionTransport, + ) { run_connection( connection, Arc::clone(&self.session_registry), self.runtime_paths.clone(), + self.telemetry.clone(), + transport, ) .await; } + + pub(crate) async fn shutdown(&self) { + self.session_registry.shutdown().await; + } } async fn run_connection( connection: JsonRpcConnection, session_registry: Arc, runtime_paths: ExecServerRuntimePaths, + telemetry: ExecServerTelemetry, + transport: ConnectionTransport, ) { + let _connection_metrics = telemetry.connection_started(transport); let router = Arc::new(build_router()); let JsonRpcConnection { outgoing_tx: json_outgoing_tx, @@ -101,12 +127,18 @@ async fn run_connection( } JsonRpcConnectionEvent::Message(message) => match message { codex_exec_server_protocol::JSONRPCMessage::Request(request) => { - if let Some(route) = router.request_route(request.method.as_str()) { - let request_span = request_span(request.method.as_str(), &request); + let request_started_at = Instant::now(); + if let Some((method, route)) = router.request_route(request.method.as_str()) { + let request_span = request_span(method, &request); let message = tokio::select! { message = route(Arc::clone(&handler), request).instrument(request_span.clone()) => message, _ = disconnected_rx.changed() => { request_span.record("result", "disconnected"); + telemetry.request_completed( + method, + "disconnected", + request_started_at.elapsed(), + ); debug!("exec-server transport disconnected while handling request"); break; } @@ -116,11 +148,19 @@ async fn run_connection( && outgoing_tx.send(message).await.is_err() { request_span.record("result", "disconnected"); + telemetry.request_completed( + method, + "disconnected", + request_started_at.elapsed(), + ); break; } request_span.record("result", result); + telemetry.request_completed(method, result, request_started_at.elapsed()); + drop(request_span); } else { - let request_span = request_span("unknown", &request); + let method = "unknown"; + let request_span = request_span(method, &request); if outgoing_tx .send(RpcServerOutboundMessage::Error { request_id: request.id, @@ -133,9 +173,15 @@ async fn run_connection( .is_err() { request_span.record("result", "disconnected"); + telemetry.request_completed( + method, + "disconnected", + request_started_at.elapsed(), + ); break; } request_span.record("result", "error"); + telemetry.request_completed(method, "error", request_started_at.elapsed()); } } codex_exec_server_protocol::JSONRPCMessage::Notification(notification) => { @@ -330,7 +376,7 @@ mod tests { #[tokio::test] async fn connection_accepts_pipelined_scalar_requests() { - let registry = SessionRegistry::new(); + let registry = SessionRegistry::new(crate::ExecServerTelemetry::default()); let (mut writer, mut lines, task) = spawn_test_connection(registry, "pipelined-scalar"); send_request( @@ -362,7 +408,7 @@ mod tests { #[tokio::test] async fn transport_disconnect_detaches_session_during_in_flight_read() { - let registry = SessionRegistry::new(); + let registry = SessionRegistry::new(crate::ExecServerTelemetry::default()); let (mut first_writer, mut first_lines, first_task) = spawn_test_connection(Arc::clone(®istry), "first"); @@ -458,7 +504,13 @@ mod tests { let (server_writer, client_reader) = duplex(1 << 20); let connection = JsonRpcConnection::from_stdio(server_reader, server_writer, label.to_string()); - let task = tokio::spawn(run_connection(connection, registry, test_runtime_paths())); + let task = tokio::spawn(run_connection( + connection, + registry, + test_runtime_paths(), + crate::ExecServerTelemetry::default(), + crate::telemetry::ConnectionTransport::Stdio, + )); (client_writer, BufReader::new(client_reader).lines(), task) } diff --git a/codex-rs/exec-server/src/server/session_registry.rs b/codex-rs/exec-server/src/server/session_registry.rs index 4d2ef7f893..89a861ab7b 100644 --- a/codex-rs/exec-server/src/server/session_registry.rs +++ b/codex-rs/exec-server/src/server/session_registry.rs @@ -12,6 +12,7 @@ use crate::rpc::RpcNotificationSender; use crate::rpc::invalid_request; use crate::rpc::session_already_attached; use crate::server::process_handler::ProcessHandler; +use crate::telemetry::ExecServerTelemetry; #[cfg(test)] const DETACHED_SESSION_TTL: Duration = Duration::from_millis(200); @@ -20,6 +21,7 @@ const DETACHED_SESSION_TTL: Duration = Duration::from_secs(30); pub(crate) struct SessionRegistry { sessions: Mutex>>, + telemetry: ExecServerTelemetry, } struct SessionEntry { @@ -51,9 +53,10 @@ pub(crate) struct SessionHandle { } impl SessionRegistry { - pub(crate) fn new() -> Arc { + pub(crate) fn new(telemetry: ExecServerTelemetry) -> Arc { Arc::new(Self { sessions: Mutex::new(HashMap::new()), + telemetry, }) } @@ -97,7 +100,7 @@ impl SessionRegistry { let session_id = Uuid::new_v4().to_string(); let entry = Arc::new(SessionEntry::new( session_id.clone(), - ProcessHandler::new(notifications, runtime_paths), + ProcessHandler::new(notifications, self.telemetry.clone(), runtime_paths), connection_id, )); sessions.insert(session_id, Arc::clone(&entry)); @@ -119,6 +122,13 @@ impl SessionRegistry { }) } + pub(crate) async fn shutdown(&self) { + let sessions = std::mem::take(&mut *self.sessions.lock().await); + for entry in sessions.into_values() { + entry.process.shutdown().await; + } + } + async fn expire_if_detached(&self, session_id: String, connection_id: ConnectionId) { tokio::time::sleep(DETACHED_SESSION_TTL).await; @@ -143,6 +153,7 @@ impl Default for SessionRegistry { fn default() -> Self { Self { sessions: Mutex::new(HashMap::new()), + telemetry: ExecServerTelemetry::default(), } } } diff --git a/codex-rs/exec-server/src/server/transport.rs b/codex-rs/exec-server/src/server/transport.rs index 57d7048e2b..2affab0070 100644 --- a/codex-rs/exec-server/src/server/transport.rs +++ b/codex-rs/exec-server/src/server/transport.rs @@ -22,8 +22,10 @@ use tracing::info; use tracing::warn; use crate::ExecServerRuntimePaths; +use crate::ExecServerTelemetry; use crate::connection::JsonRpcConnection; use crate::server::processor::ConnectionProcessor; +use crate::telemetry::ConnectionTransport; pub const DEFAULT_LISTEN_URL: &str = "ws://127.0.0.1:0"; @@ -80,49 +82,54 @@ pub(crate) fn parse_listen_url( pub(crate) async fn run_transport( listen_url: &str, runtime_paths: ExecServerRuntimePaths, + telemetry: ExecServerTelemetry, ) -> Result<(), Box> { match parse_listen_url(listen_url)? { ExecServerListenTransport::WebSocket(bind_address) => { - run_websocket_listener(bind_address, runtime_paths).await + run_websocket_listener(bind_address, runtime_paths, telemetry).await } - ExecServerListenTransport::Stdio => run_stdio_connection(runtime_paths).await, + ExecServerListenTransport::Stdio => run_stdio_connection(runtime_paths, telemetry).await, } } async fn run_stdio_connection( runtime_paths: ExecServerRuntimePaths, + telemetry: ExecServerTelemetry, ) -> Result<(), Box> { - run_stdio_connection_with_io(io::stdin(), io::stdout(), runtime_paths).await + run_stdio_connection_with_io(io::stdin(), io::stdout(), runtime_paths, telemetry).await } async fn run_stdio_connection_with_io( reader: R, writer: W, runtime_paths: ExecServerRuntimePaths, + telemetry: ExecServerTelemetry, ) -> Result<(), Box> where R: AsyncRead + Unpin + Send + 'static, W: AsyncWrite + Unpin + Send + 'static, { - let processor = ConnectionProcessor::new(runtime_paths); + let processor = ConnectionProcessor::new_with_telemetry(runtime_paths, telemetry); tracing::info!("codex-exec-server listening on stdio"); processor - .run_connection(JsonRpcConnection::from_stdio( - reader, - writer, - "exec-server stdio".to_string(), - )) + .run_connection( + JsonRpcConnection::from_stdio(reader, writer, "exec-server stdio".to_string()), + ConnectionTransport::Stdio, + ) .await; + // Stdio serves exactly one connection, so detached sessions cannot be resumed. + processor.shutdown().await; Ok(()) } async fn run_websocket_listener( bind_address: SocketAddr, runtime_paths: ExecServerRuntimePaths, + telemetry: ExecServerTelemetry, ) -> Result<(), Box> { let listener = TcpListener::bind(bind_address).await?; let local_addr = listener.local_addr()?; - let processor = ConnectionProcessor::new(runtime_paths); + let processor = ConnectionProcessor::new_with_telemetry(runtime_paths, telemetry); info!("codex-exec-server listening on ws://{local_addr}"); println!("ws://{local_addr}"); std::io::stdout().flush()?; @@ -174,10 +181,13 @@ async fn websocket_upgrade_handler( websocket.on_upgrade(move |stream| async move { state .processor - .run_connection(JsonRpcConnection::from_axum_websocket( - stream, - format!("exec-server websocket {peer_addr}"), - )) + .run_connection( + JsonRpcConnection::from_axum_websocket( + stream, + format!("exec-server websocket {peer_addr}"), + ), + ConnectionTransport::WebSocket, + ) .await; }) } diff --git a/codex-rs/exec-server/src/server/transport_tests.rs b/codex-rs/exec-server/src/server/transport_tests.rs index a8db4250bf..3016effa6c 100644 --- a/codex-rs/exec-server/src/server/transport_tests.rs +++ b/codex-rs/exec-server/src/server/transport_tests.rs @@ -61,6 +61,7 @@ async fn stdio_listen_transport_serves_initialize() { server_reader, server_writer, test_runtime_paths(), + crate::ExecServerTelemetry::default(), )); let mut client_lines = BufReader::new(client_reader).lines(); diff --git a/codex-rs/exec-server/src/telemetry.rs b/codex-rs/exec-server/src/telemetry.rs new file mode 100644 index 0000000000..650faacb2c --- /dev/null +++ b/codex-rs/exec-server/src/telemetry.rs @@ -0,0 +1,340 @@ +use std::sync::Arc; +use std::sync::Mutex; +use std::time::Duration; +use std::time::Instant; + +use codex_otel::MetricsClient; +use tracing::warn; + +const CONNECTIONS_ACTIVE_METRIC: &str = "exec_server_connections_active"; +const CONNECTIONS_ACTIVE_DESCRIPTION: &str = "Number of active exec-server connections."; +const CONNECTIONS_TOTAL_METRIC: &str = "exec_server_connections_total"; +const CONNECTIONS_TOTAL_DESCRIPTION: &str = "Total number of accepted exec-server connections."; +const REQUESTS_TOTAL_METRIC: &str = "exec_server_requests_total"; +const REQUESTS_TOTAL_DESCRIPTION: &str = "Total number of exec-server requests."; +const REQUEST_DURATION_METRIC: &str = "exec_server_request_duration_seconds"; +const REQUEST_DURATION_DESCRIPTION: &str = "Duration of exec-server requests in seconds."; +const PROCESSES_ACTIVE_METRIC: &str = "exec_server_processes_active"; +const PROCESSES_ACTIVE_DESCRIPTION: &str = "Number of active exec-server processes."; +const PROCESSES_FINISHED_TOTAL_METRIC: &str = "exec_server_processes_finished_total"; +const PROCESSES_FINISHED_TOTAL_DESCRIPTION: &str = + "Total number of finished exec-server processes."; +const PROCESS_DURATION_METRIC: &str = "exec_server_process_duration_seconds"; +const PROCESS_DURATION_DESCRIPTION: &str = "Duration of exec-server processes in seconds."; +const REMOTE_REGISTRATION_METRICS: OperationMetrics = OperationMetrics { + total_name: "exec_server_remote_registration_total", + total_description: "Total number of remote exec-server registration attempts.", + duration_name: "exec_server_remote_registration_duration_seconds", + duration_description: "Duration of remote exec-server registration attempts in seconds.", +}; +const REMOTE_RENDEZVOUS_METRICS: OperationMetrics = OperationMetrics { + total_name: "exec_server_remote_rendezvous_connect_total", + total_description: "Total number of remote exec-server rendezvous connection attempts.", + duration_name: "exec_server_remote_rendezvous_connect_duration_seconds", + duration_description: "Duration of remote exec-server rendezvous connection attempts in seconds.", +}; +const REMOTE_RECONNECTS_TOTAL_METRIC: &str = "exec_server_remote_reconnects_total"; +const REMOTE_RECONNECTS_TOTAL_DESCRIPTION: &str = "Total number of remote exec-server reconnects."; + +#[derive(Clone, Copy)] +struct OperationMetrics { + total_name: &'static str, + total_description: &'static str, + duration_name: &'static str, + duration_description: &'static str, +} + +#[derive(Clone, Copy)] +pub(crate) enum ConnectionTransport { + Relay, + Stdio, + WebSocket, +} + +impl ConnectionTransport { + fn metric_tag(self) -> &'static str { + match self { + Self::Relay => "relay", + Self::Stdio => "stdio", + Self::WebSocket => "websocket", + } + } +} + +#[derive(Clone, Default)] +pub struct ExecServerTelemetry { + inner: Option>, +} + +struct ExecServerTelemetryInner { + metrics: MetricsClient, + active: Arc>, +} + +#[derive(Default)] +struct ActiveCounts { + relay_connections: i64, + stdio_connections: i64, + websocket_connections: i64, + processes: i64, +} + +impl ActiveCounts { + fn connections(&self, transport: ConnectionTransport) -> i64 { + match transport { + ConnectionTransport::Relay => self.relay_connections, + ConnectionTransport::Stdio => self.stdio_connections, + ConnectionTransport::WebSocket => self.websocket_connections, + } + } +} + +pub(crate) struct ConnectionMetricGuard { + telemetry: ExecServerTelemetry, + transport: ConnectionTransport, +} + +pub(crate) struct ProcessMetricGuard { + telemetry: ExecServerTelemetry, + started_at: Instant, + result: &'static str, +} + +impl ExecServerTelemetry { + pub fn new(metrics: MetricsClient) -> Self { + let active = Arc::new(Mutex::new(ActiveCounts::default())); + register_active_gauges(&metrics, &active); + Self { + inner: Some(Arc::new(ExecServerTelemetryInner { metrics, active })), + } + } + + pub(crate) fn connection_started( + &self, + transport: ConnectionTransport, + ) -> ConnectionMetricGuard { + self.with_inner(|inner| { + inner.adjust_connection_count(transport, /*delta*/ 1); + inner.counter( + CONNECTIONS_TOTAL_METRIC, + CONNECTIONS_TOTAL_DESCRIPTION, + &[("transport", transport.metric_tag())], + ); + }); + ConnectionMetricGuard { + telemetry: self.clone(), + transport, + } + } + + pub(crate) fn request_completed( + &self, + method: &'static str, + result: &'static str, + duration: Duration, + ) { + self.with_inner(|inner| { + let tags = [("method", method), ("result", result)]; + inner.counter(REQUESTS_TOTAL_METRIC, REQUESTS_TOTAL_DESCRIPTION, &tags); + inner.duration( + REQUEST_DURATION_METRIC, + REQUEST_DURATION_DESCRIPTION, + duration, + &tags, + ); + }); + } + + pub(crate) fn remote_registration_completed(&self, result: &'static str, duration: Duration) { + self.record_operation(REMOTE_REGISTRATION_METRICS, result, duration); + } + + pub(crate) fn remote_rendezvous_completed(&self, result: &'static str, duration: Duration) { + self.record_operation(REMOTE_RENDEZVOUS_METRICS, result, duration); + } + + pub(crate) fn remote_reconnect(&self, reason: &'static str) { + self.with_inner(|inner| { + inner.counter( + REMOTE_RECONNECTS_TOTAL_METRIC, + REMOTE_RECONNECTS_TOTAL_DESCRIPTION, + &[("reason", reason)], + ); + }); + } + + pub(crate) fn process_started(&self) -> ProcessMetricGuard { + self.with_inner(|inner| { + inner.adjust_process_count(/*delta*/ 1); + }); + ProcessMetricGuard { + telemetry: self.clone(), + started_at: Instant::now(), + result: "unknown", + } + } + + fn process_finished(&self, result: &'static str, duration: Duration) { + self.with_inner(|inner| { + inner.adjust_process_count(/*delta*/ -1); + inner.counter( + PROCESSES_FINISHED_TOTAL_METRIC, + PROCESSES_FINISHED_TOTAL_DESCRIPTION, + &[("result", result)], + ); + inner.duration( + PROCESS_DURATION_METRIC, + PROCESS_DURATION_DESCRIPTION, + duration, + &[("result", result)], + ); + }); + } + + fn connection_finished(&self, transport: ConnectionTransport) { + self.with_inner(|inner| { + inner.adjust_connection_count(transport, /*delta*/ -1); + }); + } + + fn with_inner(&self, emit: impl FnOnce(&ExecServerTelemetryInner)) { + if let Some(inner) = &self.inner { + emit(inner); + } + } + + fn record_operation( + &self, + metrics: OperationMetrics, + result: &'static str, + duration: Duration, + ) { + self.with_inner(|inner| { + let tags = [("result", result)]; + inner.counter(metrics.total_name, metrics.total_description, &tags); + inner.duration( + metrics.duration_name, + metrics.duration_description, + duration, + &tags, + ); + }); + } +} + +impl Drop for ConnectionMetricGuard { + fn drop(&mut self) { + self.telemetry.connection_finished(self.transport); + } +} + +impl ProcessMetricGuard { + pub(crate) fn finish(mut self, result: &'static str) { + self.result = result; + } +} + +impl Drop for ProcessMetricGuard { + fn drop(&mut self) { + self.telemetry + .process_finished(self.result, self.started_at.elapsed()); + } +} + +impl ExecServerTelemetryInner { + fn active_counts(&self) -> std::sync::MutexGuard<'_, ActiveCounts> { + // These are independent integer counts, so a panic cannot leave a cross-field invariant + // half-updated. Recovering a poisoned lock preserves the last completed count update. + self.active + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + } + + fn adjust_connection_count(&self, transport: ConnectionTransport, delta: i64) { + let mut active = self.active_counts(); + let count = match transport { + ConnectionTransport::Relay => &mut active.relay_connections, + ConnectionTransport::Stdio => &mut active.stdio_connections, + ConnectionTransport::WebSocket => &mut active.websocket_connections, + }; + *count += delta; + } + + fn adjust_process_count(&self, delta: i64) { + let mut active = self.active_counts(); + active.processes += delta; + } + + fn counter(&self, name: &str, description: &str, tags: &[(&str, &str)]) { + if self + .metrics + .counter_with_description(name, description, /*inc*/ 1, tags) + .is_err() + { + warn!(metric = name, "failed to emit exec-server counter"); + } + } + + fn duration(&self, name: &str, description: &str, duration: Duration, tags: &[(&str, &str)]) { + if self + .metrics + .record_duration_seconds_with_description(name, description, duration, tags) + .is_err() + { + warn!(metric = name, "failed to emit exec-server duration"); + } + } +} + +fn register_active_gauges(metrics: &MetricsClient, active: &Arc>) { + for transport in [ + ConnectionTransport::Relay, + ConnectionTransport::Stdio, + ConnectionTransport::WebSocket, + ] { + register_active_gauge( + metrics, + active, + CONNECTIONS_ACTIVE_METRIC, + CONNECTIONS_ACTIVE_DESCRIPTION, + &[("transport", transport.metric_tag())], + move |active| active.connections(transport), + ); + } + + register_active_gauge( + metrics, + active, + PROCESSES_ACTIVE_METRIC, + PROCESSES_ACTIVE_DESCRIPTION, + &[], + |active| active.processes, + ); +} + +fn register_active_gauge( + metrics: &MetricsClient, + active: &Arc>, + name: &str, + description: &str, + tags: &[(&str, &str)], + read: impl Fn(&ActiveCounts) -> i64 + Send + Sync + 'static, +) { + let active = Arc::clone(active); + if metrics + .register_observable_gauge_with_description( + name, + description, + move || { + let active = active + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + read(&active) + }, + tags, + ) + .is_err() + { + warn!(metric = name, "failed to register exec-server gauge"); + } +} diff --git a/codex-rs/exec-server/src/trace_context.rs b/codex-rs/exec-server/src/trace_context.rs new file mode 100644 index 0000000000..70a8f3c61e --- /dev/null +++ b/codex-rs/exec-server/src/trace_context.rs @@ -0,0 +1,24 @@ +use reqwest::header::HeaderMap; +use reqwest::header::HeaderValue; + +pub(crate) fn current_trace_context_headers() -> HeaderMap { + let mut headers = HeaderMap::new(); + let Some(trace) = codex_otel::current_span_w3c_trace_context() else { + return headers; + }; + if let Some(traceparent) = trace.traceparent + && let Ok(value) = HeaderValue::try_from(traceparent) + { + headers.insert("traceparent", value); + } + if let Some(tracestate) = trace.tracestate + && let Ok(value) = HeaderValue::try_from(tracestate) + { + headers.insert("tracestate", value); + } + headers +} + +#[cfg(test)] +#[path = "trace_context_tests.rs"] +mod tests; diff --git a/codex-rs/exec-server/src/trace_context_tests.rs b/codex-rs/exec-server/src/trace_context_tests.rs new file mode 100644 index 0000000000..92a820afc4 --- /dev/null +++ b/codex-rs/exec-server/src/trace_context_tests.rs @@ -0,0 +1,27 @@ +use opentelemetry::trace::TracerProvider as _; +use opentelemetry_sdk::trace::SdkTracerProvider; +use tracing_subscriber::prelude::*; + +use super::current_trace_context_headers; + +#[test] +fn creates_traceparent_header_from_current_span() { + let provider = SdkTracerProvider::builder().build(); + let tracer = provider.tracer("exec-server-test"); + let subscriber = + tracing_subscriber::registry().with(tracing_opentelemetry::layer().with_tracer(tracer)); + let _guard = subscriber.set_default(); + tracing::callsite::rebuild_interest_cache(); + let span = tracing::info_span!("outbound-request"); + let _entered = span.enter(); + + let headers = current_trace_context_headers(); + + let traceparent = headers + .get("traceparent") + .expect("traceparent header") + .to_str() + .expect("valid traceparent header"); + assert!(traceparent.starts_with("00-")); + assert_eq!(traceparent.len(), 55); +} diff --git a/codex-rs/exec-server/src/websocket_pong_watchdog.rs b/codex-rs/exec-server/src/websocket_pong_watchdog.rs new file mode 100644 index 0000000000..b656772b3d --- /dev/null +++ b/codex-rs/exec-server/src/websocket_pong_watchdog.rs @@ -0,0 +1,44 @@ +use std::time::Duration; + +use tokio::time::Instant; + +#[cfg(test)] +pub(crate) const WEBSOCKET_PONG_TIMEOUT: Duration = Duration::from_millis(100); +#[cfg(not(test))] +pub(crate) const WEBSOCKET_PONG_TIMEOUT: Duration = Duration::from_secs(60); +pub(crate) const WEBSOCKET_PONG_TIMEOUT_REASON: &str = "pong_timeout"; + +/// Tracks whether a WebSocket peer has acknowledged a keepalive ping. +pub(crate) struct WebSocketPongWatchdog { + timeout: Duration, + deadline: Option, +} + +impl WebSocketPongWatchdog { + pub(crate) fn new(timeout: Duration) -> Self { + Self { + timeout, + deadline: None, + } + } + + pub(crate) fn ping_sent(&mut self, now: Instant) { + self.deadline.get_or_insert(now + self.timeout); + } + + pub(crate) fn deadline(&self) -> Option { + self.deadline + } + + pub(crate) fn write_deadline(&self, now: Instant) -> Instant { + self.deadline.unwrap_or(now + self.timeout) + } + + pub(crate) fn received_pong(&mut self) { + self.deadline = None; + } +} + +#[cfg(test)] +#[path = "websocket_pong_watchdog_tests.rs"] +mod tests; diff --git a/codex-rs/exec-server/src/websocket_pong_watchdog_tests.rs b/codex-rs/exec-server/src/websocket_pong_watchdog_tests.rs new file mode 100644 index 0000000000..2b15a0d952 --- /dev/null +++ b/codex-rs/exec-server/src/websocket_pong_watchdog_tests.rs @@ -0,0 +1,34 @@ +use std::time::Duration; + +use pretty_assertions::assert_eq; +use tokio::time::Instant; + +use super::WebSocketPongWatchdog; + +#[test] +fn repeated_ping_does_not_extend_deadline() { + let started_at = Instant::now(); + let timeout = Duration::from_secs(2); + let mut watchdog = WebSocketPongWatchdog::new(timeout); + + watchdog.ping_sent(started_at); + watchdog.ping_sent(started_at + Duration::from_secs(1)); + assert_eq!( + watchdog.write_deadline(started_at + Duration::from_secs(1)), + started_at + timeout + ); + assert_eq!(watchdog.deadline(), Some(started_at + timeout)); +} + +#[test] +fn pong_starts_a_fresh_deadline() { + let started_at = Instant::now(); + let timeout = Duration::from_secs(2); + let mut watchdog = WebSocketPongWatchdog::new(timeout); + + watchdog.ping_sent(started_at); + watchdog.received_pong(); + assert_eq!(watchdog.deadline(), None); + watchdog.ping_sent(started_at + timeout); + assert_eq!(watchdog.deadline(), Some(started_at + timeout + timeout)); +} diff --git a/codex-rs/exec-server/tests/chatgpt_cloudflare_affinity.rs b/codex-rs/exec-server/tests/chatgpt_cloudflare_affinity.rs new file mode 100644 index 0000000000..75c86b6b79 --- /dev/null +++ b/codex-rs/exec-server/tests/chatgpt_cloudflare_affinity.rs @@ -0,0 +1,405 @@ +#![cfg(unix)] + +mod common; + +use std::collections::BTreeMap; +use std::ffi::OsString; +use std::fs; +use std::io; +use std::io::Read; +use std::io::Write; +use std::net::TcpListener; +use std::net::TcpStream; +use std::sync::Arc; +use std::sync::mpsc; +use std::thread; +use std::time::Duration; + +use codex_exec_server::HttpRedirectPolicy; +use codex_exec_server::HttpRequestParams; +use codex_exec_server::HttpRequestResponse; +use codex_exec_server::InitializeParams; +use codex_exec_server_protocol::JSONRPCMessage; +use codex_exec_server_protocol::JSONRPCResponse; +use codex_exec_server_protocol::RequestId; +use common::exec_server::ExecServerHarness; +use common::exec_server::exec_server_with_env; +use pretty_assertions::assert_eq; +use rcgen::BasicConstraints; +use rcgen::CertificateParams; +use rcgen::CertifiedIssuer; +use rcgen::DistinguishedName; +use rcgen::DnType; +use rcgen::ExtendedKeyUsagePurpose; +use rcgen::IsCa; +use rcgen::KeyPair; +use rcgen::KeyUsagePurpose; +use rcgen::PKCS_ECDSA_P256_SHA256; +use rustls::pki_types::CertificateDer; +use rustls::pki_types::PrivateKeyDer; +use serde::de::DeserializeOwned; +use serde_json::Value; +use tempfile::TempDir; + +const CHATGPT_MCP_URL: &str = "https://chatgpt.com/backend-api/ps/mcp"; +const NON_CHATGPT_MCP_URL: &str = "https://api.openai.com/backend-api/ps/mcp"; + +#[derive(Debug)] +struct CapturedRequest { + connect_authority: String, + request_line: String, + headers: BTreeMap>, +} + +struct TlsMaterial { + ca_cert_pem: String, + server_cert: CertificateDer<'static>, + server_key: PrivateKeyDer<'static>, +} + +struct TlsInterceptingProxy { + ca_cert_pem: String, + request_rx: mpsc::Receiver>, + thread: thread::JoinHandle>, + url: String, +} + +/// Exercises the same `http/request` route used by remotely executed Streamable HTTP MCP calls. +/// Each RPC builds a fresh reqwest client. The first response sets `__cflb`, and the second response +/// replaces it, proving cross-client persistence through the shared cookie store. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn exec_server_replays_only_chatgpt_cloudflare_cookies() -> anyhow::Result<()> { + let proxy = TlsInterceptingProxy::start(/*expected_requests*/ 4)?; + let temp_dir = TempDir::new()?; + let ca_path = temp_dir.path().join("cloudflare-affinity-test-ca.pem"); + fs::write(&ca_path, &proxy.ca_cert_pem)?; + let proxy_url = OsString::from(&proxy.url); + let empty = OsString::new(); + let env = vec![ + ( + OsString::from("CODEX_CA_CERTIFICATE"), + ca_path.as_os_str().to_owned(), + ), + (OsString::from("HTTPS_PROXY"), proxy_url.clone()), + (OsString::from("https_proxy"), proxy_url.clone()), + (OsString::from("ALL_PROXY"), proxy_url.clone()), + (OsString::from("all_proxy"), proxy_url), + (OsString::from("NO_PROXY"), empty.clone()), + (OsString::from("no_proxy"), empty), + ]; + let mut server = exec_server_with_env(env).await?; + initialize_exec_server(&mut server).await?; + + let first_response = execute_http_request(&mut server, CHATGPT_MCP_URL, "first").await?; + assert_eq!( + (first_response.status, first_response.body.into_inner()), + (200, b"ok".to_vec()) + ); + let first = proxy.next_request()?; + assert_eq!( + ( + first.connect_authority.as_str(), + first.request_line.as_str(), + first.headers.get("cookie"), + ), + ("chatgpt.com:443", "POST /backend-api/ps/mcp HTTP/1.1", None,) + ); + + let west_response = execute_http_request(&mut server, CHATGPT_MCP_URL, "west").await?; + assert_eq!(west_response.status, 200); + let request_with_west_affinity = proxy.next_request()?; + assert_eq!( + request_with_west_affinity + .headers + .get("cookie") + .cloned() + .unwrap_or_default(), + vec!["__cflb=west".to_string()] + ); + + let central_response = execute_http_request(&mut server, CHATGPT_MCP_URL, "central").await?; + assert_eq!(central_response.status, 200); + let request_with_central_affinity = proxy.next_request()?; + assert_eq!( + ( + request_with_central_affinity.request_line.as_str(), + request_with_central_affinity + .headers + .get("cookie") + .cloned() + .unwrap_or_default(), + ), + ( + "POST /backend-api/ps/mcp HTTP/1.1", + vec!["__cflb=central".to_string()], + ) + ); + let other_host_response = + execute_http_request(&mut server, NON_CHATGPT_MCP_URL, "other-host").await?; + assert_eq!(other_host_response.status, 200); + let other_host = proxy.next_request()?; + assert_eq!( + ( + other_host.connect_authority.as_str(), + other_host.request_line.as_str(), + other_host.headers.get("cookie"), + ), + ( + "api.openai.com:443", + "POST /backend-api/ps/mcp HTTP/1.1", + None, + ) + ); + + server.shutdown().await?; + proxy.finish()?; + Ok(()) +} + +impl TlsInterceptingProxy { + fn start(expected_requests: usize) -> anyhow::Result { + codex_utils_rustls_provider::ensure_rustls_crypto_provider(); + let material = generate_tls_material()?; + let listener = TcpListener::bind(("127.0.0.1", 0))?; + let address = listener.local_addr()?; + let config = Arc::new( + rustls::ServerConfig::builder_with_protocol_versions(&[&rustls::version::TLS13]) + .with_no_client_auth() + .with_single_cert(vec![material.server_cert], material.server_key)?, + ); + let (request_tx, request_rx) = mpsc::channel(); + let thread = thread::spawn(move || { + run_tls_intercepting_proxy(listener, config, request_tx, expected_requests) + .map_err(|error| error.to_string()) + }); + + Ok(Self { + ca_cert_pem: material.ca_cert_pem, + request_rx, + thread, + url: format!("http://{address}"), + }) + } + + fn next_request(&self) -> anyhow::Result { + self.request_rx + .recv_timeout(Duration::from_secs(5)) + .map_err(anyhow::Error::from)? + .map_err(anyhow::Error::msg) + } + + fn finish(self) -> anyhow::Result<()> { + self.thread + .join() + .map_err(|_| anyhow::anyhow!("TLS proxy thread panicked"))? + .map_err(anyhow::Error::msg) + } +} + +fn generate_tls_material() -> anyhow::Result { + let mut ca_params = CertificateParams::default(); + ca_params.is_ca = IsCa::Ca(BasicConstraints::Unconstrained); + ca_params.key_usages = vec![KeyUsagePurpose::KeyCertSign, KeyUsagePurpose::CrlSign]; + let mut ca_distinguished_name = DistinguishedName::new(); + ca_distinguished_name.push(DnType::CommonName, "Codex affinity test CA"); + ca_params.distinguished_name = ca_distinguished_name; + let ca_key_pair = KeyPair::generate_for(&PKCS_ECDSA_P256_SHA256)?; + let ca = CertifiedIssuer::self_signed(ca_params, ca_key_pair)?; + + let mut server_params = CertificateParams::new(vec![ + "chatgpt.com".to_string(), + "api.openai.com".to_string(), + ])?; + server_params.extended_key_usages = vec![ExtendedKeyUsagePurpose::ServerAuth]; + server_params.key_usages = vec![ + KeyUsagePurpose::DigitalSignature, + KeyUsagePurpose::KeyEncipherment, + ]; + let server_key_pair = KeyPair::generate_for(&PKCS_ECDSA_P256_SHA256)?; + let server_cert = server_params.signed_by(&server_key_pair, &ca)?; + + Ok(TlsMaterial { + ca_cert_pem: ca.pem(), + server_cert: server_cert.der().clone(), + server_key: PrivateKeyDer::from(server_key_pair), + }) +} + +fn run_tls_intercepting_proxy( + listener: TcpListener, + config: Arc, + request_tx: mpsc::Sender>, + expected_requests: usize, +) -> io::Result<()> { + for request_index in 0..expected_requests { + let (mut stream, _) = listener.accept()?; + configure_stream(&stream)?; + let connect_head = read_http_head(&mut stream)?; + let connect_authority = connect_head + .lines() + .next() + .and_then(|line| line.split_whitespace().nth(1)) + .ok_or_else(|| io::Error::other("CONNECT request line should include an authority"))? + .to_string(); + stream.write_all(b"HTTP/1.1 200 Connection Established\r\n\r\n")?; + stream.flush()?; + + let connection = + rustls::ServerConnection::new(Arc::clone(&config)).map_err(io::Error::other)?; + let mut tls = rustls::StreamOwned::new(connection, stream); + let request = capture_http_request(&mut tls, connect_authority); + match request { + Ok(request) => request_tx + .send(Ok(request)) + .map_err(|_| io::Error::other("request receiver was dropped"))?, + Err(error) => { + let message = error.to_string(); + let _ = request_tx.send(Err(message)); + return Err(error); + } + } + + let response = match request_index { + 0 => concat!( + "HTTP/1.1 200 OK\r\n", + "content-length: 2\r\n", + "connection: close\r\n", + "set-cookie: __cflb=west; Path=/; Secure; HttpOnly\r\n", + "set-cookie: chatgpt_session=secret; Path=/; Secure; HttpOnly\r\n", + "\r\n", + "ok", + ), + 1 => concat!( + "HTTP/1.1 200 OK\r\n", + "content-length: 2\r\n", + "connection: close\r\n", + "set-cookie: __cflb=central; Path=/; Secure; HttpOnly\r\n", + "set-cookie: chatgpt_session=still-secret; Path=/; Secure; HttpOnly\r\n", + "\r\n", + "ok", + ), + _ => concat!( + "HTTP/1.1 200 OK\r\n", + "content-length: 2\r\n", + "connection: close\r\n", + "\r\n", + "ok", + ), + }; + tls.write_all(response.as_bytes())?; + tls.flush()?; + } + Ok(()) +} + +fn configure_stream(stream: &TcpStream) -> io::Result<()> { + stream.set_read_timeout(Some(Duration::from_secs(5)))?; + stream.set_write_timeout(Some(Duration::from_secs(5))) +} + +fn capture_http_request( + stream: &mut impl Read, + connect_authority: String, +) -> io::Result { + let request_head = read_http_head(stream)?; + let mut lines = request_head.lines(); + let request_line = lines + .next() + .ok_or_else(|| io::Error::other("HTTP request should include a request line"))? + .to_string(); + let mut headers: BTreeMap> = BTreeMap::new(); + for line in lines.filter(|line| !line.is_empty()) { + let (name, value) = line + .split_once(':') + .ok_or_else(|| io::Error::other(format!("invalid HTTP header: {line}")))?; + headers + .entry(name.to_ascii_lowercase()) + .or_default() + .push(value.trim().to_string()); + } + Ok(CapturedRequest { + connect_authority, + request_line, + headers, + }) +} + +fn read_http_head(stream: &mut impl Read) -> io::Result { + const MAX_HEADER_BYTES: usize = 64 * 1024; + let mut bytes = Vec::new(); + while !bytes.ends_with(b"\r\n\r\n") { + if bytes.len() == MAX_HEADER_BYTES { + return Err(io::Error::other("HTTP headers exceeded test limit")); + } + let mut byte = [0]; + stream.read_exact(&mut byte)?; + bytes.push(byte[0]); + } + String::from_utf8(bytes).map_err(io::Error::other) +} + +async fn initialize_exec_server(server: &mut ExecServerHarness) -> anyhow::Result<()> { + let initialize_id = server + .send_request( + "initialize", + serde_json::to_value(InitializeParams { + client_name: "cloudflare-affinity-test".to_string(), + resume_session_id: None, + })?, + ) + .await?; + let _: Value = wait_for_response(server, initialize_id).await?; + server + .send_notification("initialized", serde_json::json!({})) + .await +} + +async fn execute_http_request( + server: &mut ExecServerHarness, + url: &str, + request_id: &str, +) -> anyhow::Result { + let response_id = server + .send_request( + "http/request", + serde_json::to_value(HttpRequestParams { + method: "POST".to_string(), + url: url.to_string(), + headers: Vec::new(), + body: None, + timeout_ms: Some(5_000), + redirect_policy: HttpRedirectPolicy::Follow, + request_id: request_id.to_string(), + stream_response: false, + })?, + ) + .await?; + wait_for_response(server, response_id).await +} + +async fn wait_for_response( + server: &mut ExecServerHarness, + request_id: RequestId, +) -> anyhow::Result +where + T: DeserializeOwned, +{ + let message = server + .wait_for_event(|event| match event { + JSONRPCMessage::Response(JSONRPCResponse { id, .. }) + | JSONRPCMessage::Error(codex_exec_server_protocol::JSONRPCError { id, .. }) => { + id == &request_id + } + _ => false, + }) + .await?; + match message { + JSONRPCMessage::Response(JSONRPCResponse { result, .. }) => { + Ok(serde_json::from_value(result)?) + } + JSONRPCMessage::Error(error) => { + anyhow::bail!("exec-server returned an error for {request_id:?}: {error:?}") + } + _ => unreachable!("predicate only accepts responses for the requested id"), + } +} diff --git a/codex-rs/exec-server/tests/exec_process.rs b/codex-rs/exec-server/tests/exec_process.rs index 8fe5fe0be9..73377da60b 100644 --- a/codex-rs/exec-server/tests/exec_process.rs +++ b/codex-rs/exec-server/tests/exec_process.rs @@ -164,6 +164,7 @@ async fn collect_process_output_from_events( ExecProcessEvent::Exited { seq: _, exit_code: code, + .. } => { exit_code = Some(code); } @@ -190,7 +191,7 @@ async fn collect_process_event_snapshots( stream: chunk.stream, text: String::from_utf8_lossy(&chunk.chunk.into_inner()).into_owned(), }, - ExecProcessEvent::Exited { seq, exit_code } => { + ExecProcessEvent::Exited { seq, exit_code, .. } => { ProcessEventSnapshot::Exited { seq, exit_code } } ExecProcessEvent::Closed { seq } => ProcessEventSnapshot::Closed { seq }, @@ -792,7 +793,7 @@ async fn remote_exec_process_recovers_after_transport_disconnect() -> Result<()> last_seq = chunk.seq; output.extend_from_slice(&chunk.chunk.into_inner()); } - ExecProcessEvent::Exited { seq, exit_code } => { + ExecProcessEvent::Exited { seq, exit_code, .. } => { assert_eq!(seq, last_seq + 1); assert_eq!(exit_code, 7); last_seq = seq; diff --git a/codex-rs/exec-server/tests/http_client.rs b/codex-rs/exec-server/tests/http_client.rs index 248564b493..b3f0182c63 100644 --- a/codex-rs/exec-server/tests/http_client.rs +++ b/codex-rs/exec-server/tests/http_client.rs @@ -6,6 +6,7 @@ use anyhow::Result; use anyhow::bail; use codex_exec_server::ExecServerClient; use codex_exec_server::HttpHeader; +use codex_exec_server::HttpRedirectPolicy; use codex_exec_server::HttpRequestBodyDeltaNotification; use codex_exec_server::HttpRequestParams; use codex_exec_server::HttpRequestResponse; @@ -63,6 +64,7 @@ async fn http_request_forces_buffered_request_params() -> Result<()> { headers: Vec::new(), body: None, timeout_ms: None, + redirect_policy: HttpRedirectPolicy::Follow, request_id: "ignored-stream-id".to_string(), stream_response: false, } @@ -91,6 +93,7 @@ async fn http_request_forces_buffered_request_params() -> Result<()> { headers: Vec::new(), body: None, timeout_ms: None, + redirect_policy: HttpRedirectPolicy::Follow, request_id: "ignored-stream-id".to_string(), stream_response: true, }), @@ -131,6 +134,7 @@ async fn http_response_body_stream_uses_generated_ids_and_receives_ordered_delta }], body: None, timeout_ms: None, + redirect_policy: HttpRedirectPolicy::Follow, request_id: "http-1".to_string(), stream_response: true, } @@ -186,6 +190,7 @@ async fn http_response_body_stream_uses_generated_ids_and_receives_ordered_delta headers: Vec::new(), body: None, timeout_ms: None, + redirect_policy: HttpRedirectPolicy::Follow, request_id: "http-2".to_string(), stream_response: true, } @@ -215,6 +220,7 @@ async fn http_response_body_stream_uses_generated_ids_and_receives_ordered_delta }], body: None, timeout_ms: None, + redirect_policy: HttpRedirectPolicy::Follow, request_id: "caller-stream-id".to_string(), stream_response: false, }), @@ -253,6 +259,7 @@ async fn http_response_body_stream_uses_generated_ids_and_receives_ordered_delta headers: Vec::new(), body: None, timeout_ms: None, + redirect_policy: HttpRedirectPolicy::Follow, request_id: "caller-stream-id".to_string(), stream_response: false, }), @@ -289,6 +296,7 @@ async fn http_response_body_stream_drops_queued_terminal_before_next_generated_i headers: Vec::new(), body: None, timeout_ms: None, + redirect_policy: HttpRedirectPolicy::Follow, request_id: "http-1".to_string(), stream_response: true, } @@ -322,6 +330,7 @@ async fn http_response_body_stream_drops_queued_terminal_before_next_generated_i headers: Vec::new(), body: None, timeout_ms: None, + redirect_policy: HttpRedirectPolicy::Follow, request_id: "http-2".to_string(), stream_response: true, } @@ -348,6 +357,7 @@ async fn http_response_body_stream_drops_queued_terminal_before_next_generated_i headers: Vec::new(), body: None, timeout_ms: None, + redirect_policy: HttpRedirectPolicy::Follow, request_id: "caller-stream-id".to_string(), stream_response: false, }), @@ -372,6 +382,7 @@ async fn http_response_body_stream_drops_queued_terminal_before_next_generated_i headers: Vec::new(), body: None, timeout_ms: None, + redirect_policy: HttpRedirectPolicy::Follow, request_id: "caller-stream-id".to_string(), stream_response: false, }; @@ -411,6 +422,7 @@ async fn http_response_body_stream_ignores_late_deltas_after_cancelled_request() headers: Vec::new(), body: None, timeout_ms: None, + redirect_policy: HttpRedirectPolicy::Follow, request_id: "http-1".to_string(), stream_response: true, } @@ -430,6 +442,7 @@ async fn http_response_body_stream_ignores_late_deltas_after_cancelled_request() headers: Vec::new(), body: None, timeout_ms: None, + redirect_policy: HttpRedirectPolicy::Follow, request_id: "http-2".to_string(), stream_response: true, } @@ -474,6 +487,7 @@ async fn http_response_body_stream_ignores_late_deltas_after_cancelled_request() headers: Vec::new(), body: None, timeout_ms: None, + redirect_policy: HttpRedirectPolicy::Follow, request_id: "caller-stream-id".to_string(), stream_response: false, }) @@ -495,6 +509,7 @@ async fn http_response_body_stream_ignores_late_deltas_after_cancelled_request() headers: Vec::new(), body: None, timeout_ms: None, + redirect_policy: HttpRedirectPolicy::Follow, request_id: "caller-stream-id".to_string(), stream_response: false, }), @@ -541,6 +556,7 @@ async fn http_response_body_stream_ignores_late_deltas_after_drop() -> Result<() headers: Vec::new(), body: None, timeout_ms: None, + redirect_policy: HttpRedirectPolicy::Follow, request_id: "http-1".to_string(), stream_response: true, } @@ -580,6 +596,7 @@ async fn http_response_body_stream_ignores_late_deltas_after_drop() -> Result<() headers: Vec::new(), body: None, timeout_ms: None, + redirect_policy: HttpRedirectPolicy::Follow, request_id: "http-2".to_string(), stream_response: true, } @@ -615,6 +632,7 @@ async fn http_response_body_stream_ignores_late_deltas_after_drop() -> Result<() headers: Vec::new(), body: None, timeout_ms: None, + redirect_policy: HttpRedirectPolicy::Follow, request_id: "caller-stream-id".to_string(), stream_response: false, }), @@ -647,6 +665,7 @@ async fn http_response_body_stream_ignores_late_deltas_after_drop() -> Result<() headers: Vec::new(), body: None, timeout_ms: None, + redirect_policy: HttpRedirectPolicy::Follow, request_id: "caller-stream-id".to_string(), stream_response: false, }), @@ -691,6 +710,7 @@ async fn http_response_body_stream_fails_when_transport_disconnects() -> Result< headers: Vec::new(), body: None, timeout_ms: None, + redirect_policy: HttpRedirectPolicy::Follow, request_id: "http-1".to_string(), stream_response: true, } @@ -717,6 +737,7 @@ async fn http_response_body_stream_fails_when_transport_disconnects() -> Result< headers: Vec::new(), body: None, timeout_ms: None, + redirect_policy: HttpRedirectPolicy::Follow, request_id: "caller-stream-id".to_string(), stream_response: false, }), @@ -759,6 +780,7 @@ async fn http_response_body_stream_reports_disconnect_when_queue_is_full() -> Re headers: Vec::new(), body: None, timeout_ms: None, + redirect_policy: HttpRedirectPolicy::Follow, request_id: "http-1".to_string(), stream_response: true, } @@ -796,6 +818,7 @@ async fn http_response_body_stream_reports_disconnect_when_queue_is_full() -> Re headers: Vec::new(), body: None, timeout_ms: None, + redirect_policy: HttpRedirectPolicy::Follow, request_id: "caller-stream-id".to_string(), stream_response: false, }), @@ -852,6 +875,7 @@ async fn http_response_body_stream_reports_backpressure_truncation() -> Result<( headers: Vec::new(), body: None, timeout_ms: None, + redirect_policy: HttpRedirectPolicy::Follow, request_id: "http-1".to_string(), stream_response: true, } @@ -894,6 +918,7 @@ async fn http_response_body_stream_reports_backpressure_truncation() -> Result<( headers: Vec::new(), body: None, timeout_ms: None, + redirect_policy: HttpRedirectPolicy::Follow, request_id: "caller-stream-id".to_string(), stream_response: false, }), diff --git a/codex-rs/exec-server/tests/http_request.rs b/codex-rs/exec-server/tests/http_request.rs index a4734bf92d..1edacc4b13 100644 --- a/codex-rs/exec-server/tests/http_request.rs +++ b/codex-rs/exec-server/tests/http_request.rs @@ -7,6 +7,7 @@ use std::io::ErrorKind; use std::time::Duration; use codex_exec_server::HttpHeader; +use codex_exec_server::HttpRedirectPolicy; use codex_exec_server::HttpRequestBodyDeltaNotification; use codex_exec_server::HttpRequestParams; use codex_exec_server::HttpRequestResponse; @@ -63,6 +64,7 @@ async fn exec_server_http_request_buffers_response_body() -> anyhow::Result<()> }], body: Some(b"request-body".to_vec().into()), timeout_ms: Some(5_000), + redirect_policy: HttpRedirectPolicy::Follow, request_id: "buffered-request".to_string(), stream_response: false, })?, @@ -108,6 +110,61 @@ async fn exec_server_http_request_buffers_response_body() -> anyhow::Result<()> Ok(()) } +/// What this tests: OAuth callers can inspect redirect responses without the +/// executor following the Location header. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn exec_server_http_request_can_stop_at_redirects() -> anyhow::Result<()> { + let mut server = exec_server().await?; + initialize_exec_server(&mut server).await?; + + let listener = TcpListener::bind("127.0.0.1:0").await?; + let base_url = format!("http://{}", listener.local_addr()?); + let http_request_id = server + .send_request( + "http/request", + serde_json::to_value(HttpRequestParams { + method: "GET".to_string(), + url: format!("{base_url}/redirect"), + headers: Vec::new(), + body: None, + timeout_ms: Some(5_000), + redirect_policy: HttpRedirectPolicy::Stop, + request_id: "redirect-request".to_string(), + stream_response: false, + })?, + ) + .await?; + + let captured = accept_http_request(&listener).await?; + assert_eq!(captured.request_line, "GET /redirect HTTP/1.1"); + respond_with_status_and_headers( + captured.stream, + "302 Found", + &[("location", &format!("{base_url}/final"))], + b"redirect", + ) + .await?; + + let response: HttpRequestResponse = wait_for_response(&mut server, http_request_id).await?; + assert_eq!( + ( + response.status, + response_header(&response.headers, "location"), + response.body.into_inner(), + ), + (302, Some(format!("{base_url}/final")), b"redirect".to_vec(),) + ); + assert!( + timeout(Duration::from_millis(100), listener.accept()) + .await + .is_err(), + "redirect target should not be requested" + ); + + server.shutdown().await?; + Ok(()) +} + /// What this tests: a real exec-server websocket `http/request` can return /// response headers immediately and stream the response body as ordered /// `http/request/bodyDelta` notifications. @@ -132,6 +189,7 @@ async fn exec_server_http_request_streams_response_body_notifications() -> anyho }], body: None, timeout_ms: Some(5_000), + redirect_policy: HttpRedirectPolicy::Follow, request_id: "stream-1".to_string(), stream_response: true, })?, @@ -218,6 +276,7 @@ async fn exec_server_http_request_rejects_duplicate_stream_request_ids() -> anyh headers: Vec::new(), body: None, timeout_ms: None, + redirect_policy: HttpRedirectPolicy::Follow, request_id: "stream-dup".to_string(), stream_response: true, })?, @@ -241,6 +300,7 @@ async fn exec_server_http_request_rejects_duplicate_stream_request_ids() -> anyh headers: Vec::new(), body: None, timeout_ms: None, + redirect_policy: HttpRedirectPolicy::Follow, request_id: "stream-dup".to_string(), stream_response: true, })?, @@ -295,6 +355,7 @@ async fn exec_server_http_request_honors_optional_timeout() -> anyhow::Result<() headers: Vec::new(), body: None, timeout_ms: None, + redirect_policy: HttpRedirectPolicy::Follow, request_id: "buffered-request".to_string(), stream_response: false, })?, @@ -320,6 +381,7 @@ async fn exec_server_http_request_honors_optional_timeout() -> anyhow::Result<() headers: Vec::new(), body: None, timeout_ms: Some(10), + redirect_policy: HttpRedirectPolicy::Follow, request_id: "buffered-request".to_string(), stream_response: false, })?, diff --git a/codex-rs/exec-server/tests/selected_capability_roots.rs b/codex-rs/exec-server/tests/selected_capability_roots.rs new file mode 100644 index 0000000000..639d652ca8 --- /dev/null +++ b/codex-rs/exec-server/tests/selected_capability_roots.rs @@ -0,0 +1,69 @@ +#![cfg(unix)] + +mod common; + +use std::collections::HashMap; +use std::sync::Arc; + +use codex_exec_server::EnvironmentManager; +use codex_protocol::capabilities::CapabilityRootLocation; +use codex_protocol::capabilities::SelectedCapabilityRoot; +use codex_utils_path_uri::PathUri; +use common::exec_server::exec_server; +use pretty_assertions::assert_eq; + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn selected_capability_roots_use_captured_handle_after_replacement() -> anyhow::Result<()> { + let mut executor = exec_server().await?; + let manager = EnvironmentManager::without_environments(); + let selected_root = SelectedCapabilityRoot { + id: "demo@1".to_string(), + location: CapabilityRootLocation::Environment { + environment_id: "tools".to_string(), + path: PathUri::parse("file:///plugins/demo")?, + }, + }; + + manager.upsert_environment( + "tools".to_string(), + executor.websocket_url().to_string(), + /*connect_timeout*/ None, + )?; + let environment_a = manager + .get_environment("tools") + .expect("executor A should be registered"); + environment_a.wait_until_ready().await?; + + let unavailable = manager + .resolve_selected_capability_roots( + std::slice::from_ref(&selected_root), + &HashMap::from([("tools".to_string(), None)]), + ) + .await; + assert!(unavailable.is_empty()); + + let captured_environments = + HashMap::from([("tools".to_string(), Some(Arc::clone(&environment_a)))]); + // Replace only the process-local handle; the stable environment ID and executor stay the same. + manager.upsert_environment( + "tools".to_string(), + executor.websocket_url().to_string(), + /*connect_timeout*/ None, + )?; + + let available = manager + .resolve_selected_capability_roots( + std::slice::from_ref(&selected_root), + &captured_environments, + ) + .await; + let [resolved] = available.as_slice() else { + anyhow::bail!("selected root should resolve through its stable environment"); + }; + + assert_eq!(resolved.selected_root(), &selected_root); + assert!(Arc::ptr_eq(resolved.environment(), &environment_a)); + + executor.shutdown().await?; + Ok(()) +} diff --git a/codex-rs/exec/src/lib_tests.rs b/codex-rs/exec/src/lib_tests.rs index 9f345eb334..97ded50525 100644 --- a/codex-rs/exec/src/lib_tests.rs +++ b/codex-rs/exec/src/lib_tests.rs @@ -340,6 +340,7 @@ fn turn_items_for_thread_returns_matching_turn_items() { parent_thread_id: None, preview: String::new(), ephemeral: false, + history_mode: Default::default(), model_provider: "openai".to_string(), created_at: 0, updated_at: 0, @@ -760,6 +761,7 @@ fn sample_thread_start_response() -> ThreadStartResponse { parent_thread_id: None, preview: String::new(), ephemeral: false, + history_mode: Default::default(), model_provider: "openai".to_string(), created_at: 0, updated_at: 0, diff --git a/codex-rs/ext/extension-api/Cargo.toml b/codex-rs/ext/extension-api/Cargo.toml index 1a5a9742fb..ce441576ec 100644 --- a/codex-rs/ext/extension-api/Cargo.toml +++ b/codex-rs/ext/extension-api/Cargo.toml @@ -19,6 +19,7 @@ codex-context-fragments = { workspace = true } codex-protocol = { workspace = true } codex-tools = { workspace = true } codex-utils-absolute-path = { workspace = true } +serde_json = { workspace = true } [dev-dependencies] pretty_assertions = { workspace = true } diff --git a/codex-rs/ext/extension-api/src/contributors.rs b/codex-rs/ext/extension-api/src/contributors.rs index 7d16efd3d4..96f1ee0b48 100644 --- a/codex-rs/ext/extension-api/src/contributors.rs +++ b/codex-rs/ext/extension-api/src/contributors.rs @@ -18,6 +18,7 @@ mod thread_lifecycle; mod tool_lifecycle; mod turn_input; mod turn_lifecycle; +mod world_state; pub use context::TurnContextContributionInput; pub use mcp::McpServerContribution; @@ -39,6 +40,10 @@ pub use turn_lifecycle::TurnAbortInput; pub use turn_lifecycle::TurnErrorInput; pub use turn_lifecycle::TurnStartInput; pub use turn_lifecycle::TurnStopInput; +pub use world_state::PreviousWorldStateSection; +pub use world_state::RenderedWorldStateFragment; +pub use world_state::WorldStateContributionInput; +pub use world_state::WorldStateSectionContribution; /// Boxed, sendable future returned by asynchronous extension contributors. pub type ExtensionFuture<'a, T> = Pin + Send + 'a>>; @@ -92,6 +97,17 @@ pub trait ContextContributor: Send + Sync { Vec::new() }) } + + fn contribute_world_state<'a>( + &'a self, + input: WorldStateContributionInput<'a>, + ) -> ExtensionFuture<'a, Vec> { + Box::pin(async move { + let _self = self; + let _input = input; + Vec::new() + }) + } } /// Contributor for host-owned thread lifecycle gates. diff --git a/codex-rs/ext/extension-api/src/contributors/mcp.rs b/codex-rs/ext/extension-api/src/contributors/mcp.rs index 91e9cb459c..3fe648a1e9 100644 --- a/codex-rs/ext/extension-api/src/contributors/mcp.rs +++ b/codex-rs/ext/extension-api/src/contributors/mcp.rs @@ -1,17 +1,22 @@ use codex_config::McpServerConfig; +use crate::ExtensionData; use crate::ExtensionDataInit; /// Input supplied while resolving MCP server contributions. /// -/// Thread-scoped implementations can read the immutable host-seeded inputs -/// through [`Self::thread_init`]. Implementations should not retain borrowed -/// context after contribution completes. +/// Thread-scoped implementations can read stable host inputs through [`Self::thread_init`] and +/// keep their cache in [`Self::thread_store`]. Implementations should not retain borrowed context +/// after contribution completes. pub struct McpServerContributionContext<'a, C> { /// Host configuration visible during MCP resolution. config: &'a C, - /// Initial inputs for the active thread, when resolution is thread-scoped. + /// Extension-owned data for the active thread, when resolution is thread-scoped. + thread_store: Option<&'a ExtensionData>, + /// Stable host inputs for the active thread, when resolution is thread-scoped. thread_init: Option<&'a ExtensionDataInit>, + /// Environment IDs whose selected roots may contribute to this exact step. + available_environment_ids: Option<&'a [String]>, } impl Clone for McpServerContributionContext<'_, C> { @@ -27,15 +32,24 @@ impl<'a, C> McpServerContributionContext<'a, C> { pub fn global(config: &'a C) -> Self { Self { config, + thread_store: None, thread_init: None, + available_environment_ids: None, } } - /// Creates context for one active thread runtime. - pub fn for_thread(config: &'a C, thread_init: &'a ExtensionDataInit) -> Self { + /// Creates context for one model step using only currently available environments. + pub fn for_step( + config: &'a C, + thread_init: &'a ExtensionDataInit, + thread_store: &'a ExtensionData, + available_environment_ids: &'a [String], + ) -> Self { Self { config, + thread_store: Some(thread_store), thread_init: Some(thread_init), + available_environment_ids: Some(available_environment_ids), } } @@ -44,10 +58,23 @@ impl<'a, C> McpServerContributionContext<'a, C> { self.config } - /// Returns the frozen initial inputs when resolving for a running thread. + /// Returns extension-owned state when resolving for a running thread. + pub fn thread_store(&self) -> Option<&'a ExtensionData> { + self.thread_store + } + + /// Returns stable host inputs when resolving for a running thread. pub fn thread_init(&self) -> Option<&'a ExtensionDataInit> { self.thread_init } + + /// Returns the exact environment availability projection for a model step. + /// + /// `Some` means contributors must omit selected roots whose environment ID is absent from the + /// slice. Global resolution returns `None` because it has no thread environments. + pub fn available_environment_ids(&self) -> Option<&'a [String]> { + self.available_environment_ids + } } /// One extension-owned overlay for the runtime MCP server configuration. @@ -66,6 +93,12 @@ pub enum McpServerContribution { selection_order: usize, config: Box, }, + /// Records a plugin selected for this thread and any connector IDs it declares. + SelectedPluginPackage { + plugin_id: String, + plugin_display_name: String, + connector_ids: Vec, + }, /// Removes a named MCP server. Remove { name: String }, } diff --git a/codex-rs/ext/extension-api/src/contributors/world_state.rs b/codex-rs/ext/extension-api/src/contributors/world_state.rs new file mode 100644 index 0000000000..3bafebffa2 --- /dev/null +++ b/codex-rs/ext/extension-api/src/contributors/world_state.rs @@ -0,0 +1,146 @@ +use std::sync::Arc; + +use codex_protocol::ThreadId; +use codex_protocol::capabilities::SelectedCapabilityRoot; +use codex_protocol::protocol::TurnEnvironmentSelection; +use serde_json::Value; + +use crate::ExtensionData; + +/// Host state available while an extension contributes one sampling step's World State. +pub struct WorldStateContributionInput<'a> { + pub thread_id: ThreadId, + pub turn_id: &'a str, + pub environments: &'a [TurnEnvironmentSelection], + /// Selected roots whose stable environments are ready in this sampling step. + pub ready_selected_capability_roots: &'a [SelectedCapabilityRoot], + pub session_store: &'a ExtensionData, + pub thread_store: &'a ExtensionData, + pub turn_store: &'a ExtensionData, +} + +/// What the harness knows about the previous value of one extension-owned section. +pub enum PreviousWorldStateSection<'a> { + Absent, + Unknown, + Known(&'a Value), +} + +/// Plain model-visible data rendered by an extension-owned World State section. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RenderedWorldStateFragment { + role: &'static str, + markers: (&'static str, &'static str), + body: String, +} + +impl RenderedWorldStateFragment { + pub fn new( + role: &'static str, + markers: (&'static str, &'static str), + body: impl Into, + ) -> Self { + Self { + role, + markers, + body: body.into(), + } + } + + pub fn role(&self) -> &'static str { + self.role + } + + pub fn markers(&self) -> (&'static str, &'static str) { + self.markers + } + + pub fn body(&self) -> &str { + &self.body + } +} + +type RenderDiff = dyn for<'a> Fn(PreviousWorldStateSection<'a>) -> Option + + Send + + Sync; +type LegacyFragmentMatcher = dyn Fn(&str, &str) -> bool + Send + Sync; + +/// One extension-owned World State section captured for a sampling step. +/// +/// The extension owns the stable ID, comparison snapshot, and diff rendering. The harness owns +/// persistence and the concrete model-context fragment envelope. +#[derive(Clone)] +pub struct WorldStateSectionContribution { + id: &'static str, + snapshot: Value, + render_diff: Arc, + matches_legacy_fragment: Arc, + matches_retained_fragment: Option>, +} + +impl WorldStateSectionContribution { + pub fn new( + id: &'static str, + snapshot: Value, + render_diff: impl for<'a> Fn( + PreviousWorldStateSection<'a>, + ) -> Option + + Send + + Sync + + 'static, + ) -> Self { + Self { + id, + snapshot, + render_diff: Arc::new(render_diff), + matches_legacy_fragment: Arc::new(|_, _| false), + matches_retained_fragment: None, + } + } + + pub fn with_legacy_matcher( + mut self, + matcher: impl Fn(&str, &str) -> bool + Send + Sync + 'static, + ) -> Self { + self.matches_legacy_fragment = Arc::new(matcher); + self + } + + /// Requires a matching model-visible fragment whenever a persisted snapshot is reused. + pub fn with_retained_fragment_matcher( + mut self, + matcher: impl Fn(&str, &str) -> bool + Send + Sync + 'static, + ) -> Self { + self.matches_retained_fragment = Some(Arc::new(matcher)); + self + } + + pub fn id(&self) -> &'static str { + self.id + } + + pub fn snapshot(&self) -> &Value { + &self.snapshot + } + + pub fn render_diff( + &self, + previous: PreviousWorldStateSection<'_>, + ) -> Option { + (self.render_diff)(previous) + } + + pub fn matches_legacy_fragment(&self, role: &str, text: &str) -> bool { + (self.matches_legacy_fragment)(role, text) + } + + pub fn has_retained_fragment_matcher(&self) -> bool { + self.matches_retained_fragment.is_some() + } + + pub fn matches_retained_fragment(&self, role: &str, text: &str) -> bool { + self.matches_retained_fragment + .as_ref() + .is_some_and(|matcher| matcher(role, text)) + } +} diff --git a/codex-rs/ext/extension-api/src/lib.rs b/codex-rs/ext/extension-api/src/lib.rs index de0e0c99f8..c3873a2ed3 100644 --- a/codex-rs/ext/extension-api/src/lib.rs +++ b/codex-rs/ext/extension-api/src/lib.rs @@ -38,8 +38,10 @@ pub use contributors::ExtensionFuture; pub use contributors::McpServerContribution; pub use contributors::McpServerContributionContext; pub use contributors::McpServerContributor; +pub use contributors::PreviousWorldStateSection; pub use contributors::PromptFragment; pub use contributors::PromptSlot; +pub use contributors::RenderedWorldStateFragment; pub use contributors::ThreadIdleInput; pub use contributors::ThreadLifecycleContributor; pub use contributors::ThreadResumeInput; @@ -63,6 +65,8 @@ pub use contributors::TurnItemContributor; pub use contributors::TurnLifecycleContributor; pub use contributors::TurnStartInput; pub use contributors::TurnStopInput; +pub use contributors::WorldStateContributionInput; +pub use contributors::WorldStateSectionContribution; pub use registry::ExtensionRegistry; pub use registry::ExtensionRegistryBuilder; pub use registry::empty_extension_registry; diff --git a/codex-rs/ext/image-generation/src/extension.rs b/codex-rs/ext/image-generation/src/extension.rs index 8195c08e61..1e3dc3e127 100644 --- a/codex-rs/ext/image-generation/src/extension.rs +++ b/codex-rs/ext/image-generation/src/extension.rs @@ -34,11 +34,11 @@ struct ImageGenerationExtensionConfig { } impl ImageGenerationExtensionConfig { - /// Resolves whether standalone image generation should be available for a thread. + /// Resolves the image provider and save root for a thread. fn from_config(config: &Config, resolve_save_root: &SaveRootResolver) -> Self { Self { - // Core selects this executor per turn using the feature flag or model metadata. - available: config.model_provider.is_openai(), + available: config.model_provider.is_openai() + || config.model_provider.uses_openai_actor_authorization(), provider: config.model_provider.clone(), save_root: resolve_save_root(config), } @@ -46,7 +46,7 @@ impl ImageGenerationExtensionConfig { } impl ThreadLifecycleContributor for ImageGenerationExtension { - /// Seeds image-generation availability when a thread begins. + /// Seeds image-generation configuration when a thread begins. fn on_thread_start<'a>( &'a self, input: ThreadStartInput<'a, Config>, @@ -63,7 +63,7 @@ impl ThreadLifecycleContributor for ImageGenerationExtension { } impl ConfigContributor for ImageGenerationExtension { - /// Refreshes image-generation availability after thread configuration changes. + /// Refreshes image-generation configuration after thread configuration changes. fn on_config_changed( &self, _session_store: &ExtensionData, @@ -88,7 +88,7 @@ impl ToolContributor for ImageGenerationExtension { let Some(config) = thread_store.get::() else { return Vec::new(); }; - if !config.available || !self.auth_manager.current_auth_uses_codex_backend() { + if !config.available { return Vec::new(); } diff --git a/codex-rs/ext/image-generation/src/tests.rs b/codex-rs/ext/image-generation/src/tests.rs index fe915746f9..1ec9371214 100644 --- a/codex-rs/ext/image-generation/src/tests.rs +++ b/codex-rs/ext/image-generation/src/tests.rs @@ -97,6 +97,7 @@ async fn recent_image_fallback_selects_newest_images_in_chronological_order() { status: Some("completed".to_string()), call_id: "code-mode-call".to_string(), name: "exec".to_string(), + namespace: None, input: String::new(), internal_chat_message_metadata_passthrough: None, }, diff --git a/codex-rs/ext/mcp/Cargo.toml b/codex-rs/ext/mcp/Cargo.toml index b98c0905db..d918bd5bd9 100644 --- a/codex-rs/ext/mcp/Cargo.toml +++ b/codex-rs/ext/mcp/Cargo.toml @@ -16,6 +16,7 @@ workspace = true codex-core = { workspace = true } codex-core-plugins = { workspace = true } codex-config = { workspace = true } +codex-connectors-extension = { workspace = true } codex-exec-server = { workspace = true } codex-extension-api = { workspace = true } codex-features = { workspace = true } diff --git a/codex-rs/ext/mcp/src/executor_plugin.rs b/codex-rs/ext/mcp/src/executor_plugin.rs index 69d1a47b31..98cfcd3615 100644 --- a/codex-rs/ext/mcp/src/executor_plugin.rs +++ b/codex-rs/ext/mcp/src/executor_plugin.rs @@ -1,44 +1,47 @@ +use codex_connectors_extension::ExecutorPluginConnectorProvider; use codex_core::config::Config; use codex_core_plugins::ExecutorPluginProvider; use codex_exec_server::EnvironmentManager; -use codex_extension_api::ExtensionDataInit; use codex_extension_api::ExtensionFuture; use codex_extension_api::McpServerContribution; use codex_extension_api::McpServerContributionContext; use codex_extension_api::McpServerContributor; +use codex_protocol::capabilities::CapabilityRootLocation; use codex_protocol::capabilities::SelectedCapabilityRoot; use std::collections::HashMap; use std::sync::Arc; -use tokio::sync::OnceCell; +use std::sync::Mutex; use self::provider::ExecutorPluginMcpProvider; mod provider; -/// Frozen MCP declarations for one selected package. +/// Frozen MCP and connector declarations for one selected package. /// /// Each server config retains the stable logical environment ID. Reconnection may replace the /// concrete environment instance without changing that authority. #[derive(Clone)] -struct SelectedPluginMcpServers { +struct SelectedPluginMetadata { plugin_id: String, plugin_display_name: String, - selection_order: usize, servers: Vec<(String, codex_config::McpServerConfig)>, + connector_ids: Vec, } #[derive(Default)] pub(crate) struct SelectedExecutorPluginMcpState { - snapshot: OnceCell>, + cache: Mutex>, } -pub(crate) fn seed_thread_state(thread_init: &mut ExtensionDataInit) { - thread_init.insert(SelectedExecutorPluginMcpState::default()); +struct CachedSelectedRoot { + root: SelectedCapabilityRoot, + metadata: Option, } pub(crate) struct SelectedExecutorPluginMcpContributor { plugin_provider: ExecutorPluginProvider, mcp_provider: ExecutorPluginMcpProvider, + connector_provider: ExecutorPluginConnectorProvider, } impl SelectedExecutorPluginMcpContributor { @@ -46,46 +49,87 @@ impl SelectedExecutorPluginMcpContributor { Self { plugin_provider: ExecutorPluginProvider::new(Arc::clone(&environment_manager)), mcp_provider: ExecutorPluginMcpProvider, + connector_provider: ExecutorPluginConnectorProvider, } } - async fn resolve_snapshot( + /// Returns metadata for one stable selected root. + /// + /// Successful resolution, including a root that is not a plugin or declares no capabilities, + /// is cached until the thread state is dropped. Environment availability never invalidates + /// this cache; it only controls whether the cached metadata is projected into a model step. + async fn metadata_for_root( &self, - selected_roots: &[SelectedCapabilityRoot], - ) -> Vec { - let mut snapshot = Vec::new(); + state: &SelectedExecutorPluginMcpState, + selected_root: &SelectedCapabilityRoot, + ) -> Option { + if let Some(cached) = state + .cache + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .iter() + .find(|cached| cached.root == *selected_root) + { + return cached.metadata.clone(); + } - for (selection_order, selected_root) in selected_roots.iter().enumerate() { - let plugin = match self.plugin_provider.resolve_bound(selected_root).await { - Ok(Some(plugin)) => plugin, - Ok(None) => continue, - Err(err) => { - tracing::warn!( - selected_root = selected_root.id, - error = %err, - "failed to resolve selected executor plugin for MCP discovery" - ); - continue; - } - }; - match self.mcp_provider.load(&plugin).await { - Ok(servers) => snapshot.push(SelectedPluginMcpServers { - plugin_id: plugin.plugin().selected_root_id().to_string(), - plugin_display_name: plugin.plugin().manifest().display_name().to_string(), - selection_order, - servers, - }), - Err(err) => { + let plugin = match self.plugin_provider.resolve_bound(selected_root).await { + Ok(plugin) => plugin, + Err(err) => { + tracing::warn!( + selected_root = selected_root.id, + error = %err, + "failed to resolve selected executor plugin" + ); + return None; + } + }; + let metadata = match plugin { + Some(plugin) => { + let servers = self.mcp_provider.load(&plugin).await.unwrap_or_else(|err| { tracing::warn!( selected_root = selected_root.id, error = %err, "failed to load selected executor plugin MCP servers" ); - } + Vec::new() + }); + let connector_ids = self + .connector_provider + .load(&plugin) + .await + .unwrap_or_else(|err| { + tracing::warn!( + selected_root = selected_root.id, + error = %err, + "failed to load selected executor plugin connectors" + ); + Vec::new() + }) + .into_iter() + .map(|declaration| declaration.connector_id.0) + .collect(); + Some(SelectedPluginMetadata { + plugin_id: plugin.plugin().selected_root_id().to_string(), + plugin_display_name: plugin.plugin().manifest().display_name().to_string(), + servers, + connector_ids, + }) } + None => None, + }; + let mut cache = state + .cache + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if let Some(cached) = cache.iter().find(|cached| cached.root == *selected_root) { + return cached.metadata.clone(); } - - snapshot + cache.push(CachedSelectedRoot { + root: selected_root.clone(), + metadata: metadata.clone(), + }); + metadata } } @@ -102,20 +146,31 @@ impl McpServerContributor for SelectedExecutorPluginMcpContributor { let Some(thread_init) = context.thread_init() else { return Vec::new(); }; + let Some(thread_store) = context.thread_store() else { + return Vec::new(); + }; let Some(selected_roots) = thread_init.get::>() else { return Vec::new(); }; - let Some(state) = thread_init.get::() else { - tracing::warn!("selected executor plugin MCP state was not initialized"); - return Vec::new(); - }; - let snapshot = state - .snapshot - .get_or_init(|| self.resolve_snapshot(selected_roots.as_ref())) - .await; + let state = thread_store.get_or_init(SelectedExecutorPluginMcpState::default); let mut contributions = Vec::new(); - for plugin in snapshot { + for (selection_order, selected_root) in selected_roots.iter().enumerate() { + let CapabilityRootLocation::Environment { environment_id, .. } = + &selected_root.location; + if context + .available_environment_ids() + .is_some_and(|available| { + !available + .iter() + .any(|available| available == environment_id) + }) + { + continue; + } + let Some(plugin) = self.metadata_for_root(&state, selected_root).await else { + continue; + }; let mut servers = plugin.servers.iter().cloned().collect::>(); context .config() @@ -127,10 +182,16 @@ impl McpServerContributor for SelectedExecutorPluginMcpContributor { name, plugin_id: plugin.plugin_id.clone(), plugin_display_name: plugin.plugin_display_name.clone(), - selection_order: plugin.selection_order, + selection_order, config: Box::new(config), } })); + // Keep the package visible even when it contributes only skills. + contributions.push(McpServerContribution::SelectedPluginPackage { + plugin_id: plugin.plugin_id, + plugin_display_name: plugin.plugin_display_name, + connector_ids: plugin.connector_ids, + }); } contributions diff --git a/codex-rs/ext/mcp/src/executor_plugin/provider.rs b/codex-rs/ext/mcp/src/executor_plugin/provider.rs index 9b8a9878e3..a6972df096 100644 --- a/codex-rs/ext/mcp/src/executor_plugin/provider.rs +++ b/codex-rs/ext/mcp/src/executor_plugin/provider.rs @@ -1,5 +1,4 @@ use codex_config::McpServerConfig; -use codex_config::McpServerTransportConfig; use codex_core_plugins::ResolvedExecutorPlugin; use codex_exec_server::ExecutorFileSystem; use codex_mcp::parse_executor_plugin_mcp_config; @@ -48,7 +47,7 @@ pub(super) enum ExecutorPluginMcpProviderError { } impl ExecutorPluginMcpProvider { - /// Returns stdio servers declared by `plugin`, bound to its environment. + /// Returns MCP servers declared by `plugin`, bound to its environment. pub(super) async fn load( &self, plugin: &ResolvedExecutorPlugin, @@ -131,21 +130,7 @@ async fn load_from_file_system( ); } - Ok(parsed - .servers - .into_iter() - .filter_map(|(name, config)| match &config.transport { - McpServerTransportConfig::Stdio { .. } => Some((name, config)), - McpServerTransportConfig::StreamableHttp { .. } => { - tracing::warn!( - plugin = plugin_id, - server = name, - "ignoring HTTP MCP server from executor plugin" - ); - None - } - }) - .collect()) + Ok(parsed.servers.into_iter().collect()) } #[cfg(test)] diff --git a/codex-rs/ext/mcp/src/executor_plugin/provider_tests.rs b/codex-rs/ext/mcp/src/executor_plugin/provider_tests.rs index 22e5d1bec5..bc59e104a0 100644 --- a/codex-rs/ext/mcp/src/executor_plugin/provider_tests.rs +++ b/codex-rs/ext/mcp/src/executor_plugin/provider_tests.rs @@ -163,33 +163,61 @@ async fn reads_declared_config_only_through_executor_file_system() { assert_eq!( servers, - vec![( - "demo".to_string(), - McpServerConfig { - auth: Default::default(), - transport: McpServerTransportConfig::Stdio { - command: "demo-mcp".to_string(), - args: Vec::new(), - env: None, - env_vars: Vec::new(), - cwd: Some(LegacyAppPathString::from_path(plugin_root.as_path())), + vec![ + ( + "demo".to_string(), + McpServerConfig { + auth: Default::default(), + transport: McpServerTransportConfig::Stdio { + command: "demo-mcp".to_string(), + args: Vec::new(), + env: None, + env_vars: Vec::new(), + cwd: Some(LegacyAppPathString::from_path(plugin_root.as_path())), + }, + environment_id: "executor-test".to_string(), + enabled: true, + required: false, + supports_parallel_tool_calls: false, + disabled_reason: None, + startup_timeout_sec: None, + tool_timeout_sec: None, + default_tools_approval_mode: None, + enabled_tools: None, + disabled_tools: None, + scopes: None, + oauth: None, + oauth_resource: None, + tools: HashMap::new(), }, - environment_id: "executor-test".to_string(), - enabled: true, - required: false, - supports_parallel_tool_calls: false, - disabled_reason: None, - startup_timeout_sec: None, - tool_timeout_sec: None, - default_tools_approval_mode: None, - enabled_tools: None, - disabled_tools: None, - scopes: None, - oauth: None, - oauth_resource: None, - tools: HashMap::new(), - }, - )] + ), + ( + "hosted".to_string(), + McpServerConfig { + auth: Default::default(), + transport: McpServerTransportConfig::StreamableHttp { + url: "https://example.com/mcp".to_string(), + bearer_token_env_var: None, + http_headers: None, + env_http_headers: None, + }, + environment_id: "executor-test".to_string(), + enabled: true, + required: false, + supports_parallel_tool_calls: false, + disabled_reason: None, + startup_timeout_sec: None, + tool_timeout_sec: None, + default_tools_approval_mode: None, + enabled_tools: None, + disabled_tools: None, + scopes: None, + oauth: None, + oauth_resource: None, + tools: HashMap::new(), + }, + ), + ] ); assert_eq!(reads(&file_system), vec![config_path]); } diff --git a/codex-rs/ext/mcp/src/lib.rs b/codex-rs/ext/mcp/src/lib.rs index e32316ac36..9c22bad5a0 100644 --- a/codex-rs/ext/mcp/src/lib.rs +++ b/codex-rs/ext/mcp/src/lib.rs @@ -51,10 +51,3 @@ pub fn install_executor_plugins( executor_plugin::SelectedExecutorPluginMcpContributor::new(environment_manager), )); } - -/// Seeds the per-thread snapshot used by selected executor plugin MCP discovery. -pub fn initialize_executor_plugin_thread_data( - thread_init: &mut codex_extension_api::ExtensionDataInit, -) { - executor_plugin::seed_thread_state(thread_init); -} diff --git a/codex-rs/ext/mcp/tests/executor_plugin_mcp.rs b/codex-rs/ext/mcp/tests/executor_plugin_mcp.rs index 7bdbc83d62..da5c1fb6ad 100644 --- a/codex-rs/ext/mcp/tests/executor_plugin_mcp.rs +++ b/codex-rs/ext/mcp/tests/executor_plugin_mcp.rs @@ -3,6 +3,7 @@ use codex_core::config::Config; use codex_core::config::ConfigBuilder; use codex_exec_server::EnvironmentManager; use codex_exec_server::LOCAL_ENVIRONMENT_ID; +use codex_extension_api::ExtensionData; use codex_extension_api::ExtensionDataInit; use codex_extension_api::ExtensionRegistryBuilder; use codex_extension_api::McpServerContribution; @@ -24,6 +25,13 @@ struct ContributionSummary { enabled: bool, } +#[derive(Debug, PartialEq, Eq)] +struct PackageSummary { + plugin_id: String, + plugin_display_name: String, + connector_ids: Vec, +} + #[tokio::test] async fn selected_plugin_servers_use_managed_requirements_for_the_selected_root_id() -> TestResult { let codex_home = tempfile::tempdir()?; @@ -91,10 +99,87 @@ command = "expected-command" Ok(()) } +#[tokio::test] +async fn selected_plugin_package_is_contributed_without_servers_or_connectors() -> TestResult { + let codex_home = tempfile::tempdir()?; + let plugin_root = tempfile::tempdir()?; + std::fs::create_dir_all(plugin_root.path().join(".codex-plugin"))?; + std::fs::create_dir_all(plugin_root.path().join("skills/deploy"))?; + std::fs::write( + plugin_root.path().join(".codex-plugin/plugin.json"), + r#"{"name":"skill-only","interface":{"displayName":"Skill Only"}}"#, + )?; + std::fs::write( + plugin_root.path().join("skills/deploy/SKILL.md"), + "---\nname: deploy\ndescription: Deploy the project.\n---\n", + )?; + let config = ConfigBuilder::default() + .codex_home(codex_home.path().to_path_buf()) + .fallback_cwd(Some(codex_home.path().to_path_buf())) + .build() + .await?; + + let contributions = raw_selected_plugin_contributions(&config, plugin_root.path()).await?; + let package = contributions.into_iter().find_map(|contribution| { + let McpServerContribution::SelectedPluginPackage { + plugin_id, + plugin_display_name, + connector_ids, + } = contribution + else { + return None; + }; + Some(PackageSummary { + plugin_id, + plugin_display_name, + connector_ids, + }) + }); + + assert_eq!( + package, + Some(PackageSummary { + plugin_id: "selected-root".to_string(), + plugin_display_name: "Skill Only".to_string(), + connector_ids: Vec::new(), + }) + ); + Ok(()) +} + async fn selected_plugin_contributions( config: &Config, plugin_root: &std::path::Path, ) -> Result, Box> { + Ok(raw_selected_plugin_contributions(config, plugin_root) + .await? + .into_iter() + .filter_map(|contribution| match contribution { + McpServerContribution::SelectedPlugin { + name, + plugin_id, + plugin_display_name, + selection_order, + config, + } => Some(ContributionSummary { + name, + plugin_id, + plugin_display_name, + selection_order, + enabled: config.enabled, + }), + McpServerContribution::SelectedPluginPackage { .. } => None, + McpServerContribution::Set { .. } | McpServerContribution::Remove { .. } => { + panic!("expected selected plugin contribution") + } + }) + .collect()) +} + +async fn raw_selected_plugin_contributions( + config: &Config, + plugin_root: &std::path::Path, +) -> Result, Box> { let mut builder = ExtensionRegistryBuilder::new(); codex_mcp_extension::install_executor_plugins( &mut builder, @@ -109,32 +194,15 @@ async fn selected_plugin_contributions( path: PathUri::from_host_native_path(plugin_root)?, }, }]); - codex_mcp_extension::initialize_executor_plugin_thread_data(&mut thread_init); + let thread_store = ExtensionData::new_with_init("test-thread", thread_init.clone()); + let available_environment_ids = vec![LOCAL_ENVIRONMENT_ID.to_string()]; Ok(registry.mcp_server_contributors()[0] - .contribute(McpServerContributionContext::for_thread( + .contribute(McpServerContributionContext::for_step( config, &thread_init, + &thread_store, + &available_environment_ids, )) - .await - .into_iter() - .map(|contribution| match contribution { - McpServerContribution::SelectedPlugin { - name, - plugin_id, - plugin_display_name, - selection_order, - config, - } => ContributionSummary { - name, - plugin_id, - plugin_display_name, - selection_order, - enabled: config.enabled, - }, - McpServerContribution::Set { .. } | McpServerContribution::Remove { .. } => { - panic!("expected selected plugin contribution") - } - }) - .collect()) + .await) } diff --git a/codex-rs/ext/skills/src/extension.rs b/codex-rs/ext/skills/src/extension.rs index de9facd8c2..a632ab6824 100644 --- a/codex-rs/ext/skills/src/extension.rs +++ b/codex-rs/ext/skills/src/extension.rs @@ -18,8 +18,10 @@ use codex_extension_api::ToolContributor; use codex_extension_api::ToolExecutor; use codex_extension_api::TurnInputContext; use codex_extension_api::TurnInputContributor; +use codex_extension_api::WorldStateContributionInput; +use codex_extension_api::WorldStateSectionContribution; use codex_mcp::McpResourceClient; -use codex_protocol::capabilities::SelectedCapabilityRoot; +use codex_protocol::openai_models::ModelInfo; use codex_protocol::protocol::Event; use codex_protocol::protocol::EventMsg; use codex_protocol::protocol::WarningEvent; @@ -40,9 +42,11 @@ use crate::render::truncate_main_prompt_contents; use crate::render::truncate_utf8_to_bytes; use crate::selection::collect_explicit_skill_mentions; use crate::sources::SkillProviders; +use crate::state::ExecutorSkillsStepState; use crate::state::SkillsThreadState; use crate::state::SkillsTurnState; use crate::tools::skill_tools; +use crate::world_state::executor_skills_world_state_section; struct SkillsExtension { providers: SkillProviders, @@ -56,18 +60,12 @@ where { fn on_thread_start<'a>(&'a self, input: ThreadStartInput<'a, C>) -> ExtensionFuture<'a, ()> { Box::pin(async move { - let selected_roots = input - .thread_store - .get::>() - .map(|selected_roots| selected_roots.as_ref().clone()) - .unwrap_or_default(); let orchestrator_skills_available = !input .environments .iter() .any(|environment| environment.environment_id == LOCAL_ENVIRONMENT_ID); input.thread_store.insert(SkillsThreadState::new( (self.config_from_host)(input.config), - selected_roots, orchestrator_skills_available, )); }) @@ -92,7 +90,6 @@ where let orchestrator_skills_available = true; thread_store.insert(SkillsThreadState::new( next_config, - Vec::new(), orchestrator_skills_available, )); } @@ -120,7 +117,7 @@ where .list_skills( SkillListQuery { turn_id: thread_store.level_id().to_string(), - executor_roots: thread_state.selected_roots().to_vec(), + executor_roots: Vec::new(), host_snapshot: None, include_host_skills: false, include_bundled_skills: config.bundled_skills_enabled, @@ -133,12 +130,53 @@ where for warning in &catalog.warnings { self.emit_warning(thread_store.level_id(), warning.clone()); } - available_skills_fragment(&catalog) + let include_usage = thread_store + .get::() + .is_some_and(|model_info| model_info.include_skills_usage_instructions); + available_skills_fragment(&catalog, include_usage) .map(|fragment| PromptFragment::developer_capability(fragment.render())) .into_iter() .collect() }) } + + fn contribute_world_state<'a>( + &'a self, + input: WorldStateContributionInput<'a>, + ) -> ExtensionFuture<'a, Vec> { + Box::pin(async move { + let Some(thread_state) = input.thread_store.get::() else { + return Vec::new(); + }; + let config = thread_state.config(); + let catalog = thread_state + .executor_catalog_snapshot( + &self.providers, + SkillListQuery { + turn_id: input.turn_id.to_string(), + executor_roots: input.ready_selected_capability_roots.to_vec(), + host_snapshot: None, + include_host_skills: false, + include_bundled_skills: config.bundled_skills_enabled, + include_orchestrator_skills: false, + mcp_resources: input.session_store.get::(), + }, + ) + .await; + input + .turn_store + .insert(ExecutorSkillsStepState(catalog.clone())); + let include_usage = input + .thread_store + .get::() + .is_some_and(|model_info| model_info.include_skills_usage_instructions); + vec![executor_skills_world_state_section( + &catalog, + config.include_instructions, + include_usage, + )] + }) + } } impl ToolContributor for SkillsExtension @@ -187,14 +225,17 @@ where let host_snapshot = turn_store.get::(); let query = SkillListQuery { turn_id: input.turn_id.clone(), - executor_roots: thread_state.selected_roots().to_vec(), + executor_roots: Vec::new(), host_snapshot: host_snapshot.clone(), include_host_skills: true, include_bundled_skills: config.bundled_skills_enabled, include_orchestrator_skills: thread_state.orchestrator_skills_enabled(), mcp_resources: session_store.get::(), }; - let catalog = self.list_skills(query, &thread_state).await; + let mut catalog = self.list_skills(query, &thread_state).await; + if let Some(executor_skills) = turn_store.get::() { + catalog.extend(executor_skills.0.clone()); + } for warning in &catalog.warnings { self.emit_warning(&input.turn_id, warning.clone()); } @@ -207,7 +248,10 @@ where entry.authority.kind != SkillSourceKind::Executor && entry.authority.kind != SkillSourceKind::Orchestrator }); - if let Some(fragment) = available_skills_fragment(&turn_catalog) { + let include_usage = thread_store + .get::() + .is_some_and(|model_info| model_info.include_skills_usage_instructions); + if let Some(fragment) = available_skills_fragment(&turn_catalog, include_usage) { fragments.push(Box::new(fragment)); } } diff --git a/codex-rs/ext/skills/src/fragments.rs b/codex-rs/ext/skills/src/fragments.rs index 0a24b48b5a..6769ffbcb1 100644 --- a/codex-rs/ext/skills/src/fragments.rs +++ b/codex-rs/ext/skills/src/fragments.rs @@ -1,3 +1,4 @@ +use codex_core_skills::SKILLS_HOW_TO_USE_WITH_ABSOLUTE_PATHS; use codex_core_skills::render_available_skills_body; use codex_extension_api::ContextualUserFragment; use codex_protocol::protocol::SKILLS_INSTRUCTIONS_CLOSE_TAG; @@ -9,7 +10,14 @@ pub(crate) struct AvailableSkillsInstructions { } impl AvailableSkillsInstructions { - pub(crate) fn from_skill_lines(skill_lines: Vec) -> Self { + pub(crate) fn from_skill_lines( + mut skill_lines: Vec, + include_skills_usage_instructions: bool, + ) -> Self { + if include_skills_usage_instructions { + skill_lines.push("### How to use skills".to_string()); + skill_lines.push(SKILLS_HOW_TO_USE_WITH_ABSOLUTE_PATHS.to_string()); + } Self { skill_lines } } } diff --git a/codex-rs/ext/skills/src/lib.rs b/codex-rs/ext/skills/src/lib.rs index 8abdf6a93e..4a004ff9ac 100644 --- a/codex-rs/ext/skills/src/lib.rs +++ b/codex-rs/ext/skills/src/lib.rs @@ -8,6 +8,7 @@ mod selection; mod sources; mod state; mod tools; +mod world_state; pub use config::SkillsExtensionConfig; pub use extension::install; diff --git a/codex-rs/ext/skills/src/render.rs b/codex-rs/ext/skills/src/render.rs index dda32b4864..8e9fea41a1 100644 --- a/codex-rs/ext/skills/src/render.rs +++ b/codex-rs/ext/skills/src/render.rs @@ -21,6 +21,7 @@ pub(crate) const MAX_SKILL_PATH_BYTES: usize = 1_024; )] pub(crate) fn available_skills_fragment( catalog: &SkillCatalog, + include_skills_usage_instructions: bool, ) -> Option { let mut total_bytes = 0usize; let mut omitted = 0usize; @@ -56,7 +57,10 @@ pub(crate) fn available_skills_fragment( )); } - Some(AvailableSkillsInstructions::from_skill_lines(skill_lines)) + Some(AvailableSkillsInstructions::from_skill_lines( + skill_lines, + include_skills_usage_instructions, + )) } pub(crate) fn truncate_catalog_skill_description(description: &str) -> Cow<'_, str> { diff --git a/codex-rs/ext/skills/src/sources.rs b/codex-rs/ext/skills/src/sources.rs index d558c13b3b..a75a047e09 100644 --- a/codex-rs/ext/skills/src/sources.rs +++ b/codex-rs/ext/skills/src/sources.rs @@ -135,6 +135,11 @@ impl SkillProviders { Ok(catalog) } + pub(crate) async fn list_executor_for_turn(&self, query: SkillListQuery) -> SkillCatalog { + self.list_matching(&query, |source| source.kind == SkillSourceKind::Executor) + .await + } + async fn list_matching( &self, query: &SkillListQuery, diff --git a/codex-rs/ext/skills/src/state.rs b/codex-rs/ext/skills/src/state.rs index 0b751ebdc2..2094f34496 100644 --- a/codex-rs/ext/skills/src/state.rs +++ b/codex-rs/ext/skills/src/state.rs @@ -18,6 +18,7 @@ use crate::catalog::SkillProviderResult; use crate::catalog::SkillReadResult; use crate::catalog::SkillResourceId; use crate::catalog::SkillSourceKind; +use crate::provider::SkillListQuery; use crate::provider::SkillReadRequest; use crate::sources::SkillProviders; @@ -26,21 +27,17 @@ const MAX_CACHED_ORCHESTRATOR_CONTENT_BYTES: usize = 8 * 1024 * 1024; pub(crate) struct SkillsThreadState { config: Mutex, - selected_roots: Vec, orchestrator_skills_available: bool, + executor_cache: Mutex>, orchestrator_cache: Mutex>>, } impl SkillsThreadState { - pub(crate) fn new( - config: SkillsExtensionConfig, - selected_roots: Vec, - orchestrator_skills_available: bool, - ) -> Self { + pub(crate) fn new(config: SkillsExtensionConfig, orchestrator_skills_available: bool) -> Self { Self { config: Mutex::new(config), - selected_roots, orchestrator_skills_available, + executor_cache: Mutex::new(Vec::new()), orchestrator_cache: Mutex::new(None), } } @@ -59,14 +56,39 @@ impl SkillsThreadState { .unwrap_or_else(std::sync::PoisonError::into_inner) = config; } - pub(crate) fn selected_roots(&self) -> &[SelectedCapabilityRoot] { - &self.selected_roots - } - pub(crate) fn orchestrator_skills_enabled(&self) -> bool { self.orchestrator_skills_available && self.config().orchestrator_skills_enabled } + /// Returns catalogs for stable selected roots. + /// + /// The first catalog returned for a root remains cached until this thread state is dropped. + /// Environment availability only controls whether the root is projected into the current + /// step; it never invalidates the cache. There is intentionally no filesystem watcher or + /// content-based invalidation because selected environment roots are treated as stable. + #[tracing::instrument( + name = "skills.executor.catalog_snapshot", + level = "info", + skip_all, + fields(root_count = query.executor_roots.len()) + )] + pub(crate) async fn executor_catalog_snapshot( + &self, + providers: &SkillProviders, + mut query: SkillListQuery, + ) -> SkillCatalog { + let roots = std::mem::take(&mut query.executor_roots); + let mut catalog = SkillCatalog::default(); + for root in roots { + query.executor_roots = vec![root.clone()]; + catalog.extend( + self.executor_root_catalog(providers, root, query.clone()) + .await, + ); + } + catalog + } + pub(crate) async fn orchestrator_catalog_snapshot( &self, mcp_resources: Option<&McpResourceClient>, @@ -140,6 +162,43 @@ impl SkillsThreadState { *cache = Some(Arc::clone(&next_cache)); next_cache } + + #[tracing::instrument(name = "skills.executor.catalog_root", level = "info", skip_all)] + async fn executor_root_catalog( + &self, + providers: &SkillProviders, + root: SelectedCapabilityRoot, + query: SkillListQuery, + ) -> SkillCatalog { + if let Some(cached) = self + .executor_cache + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .iter() + .find(|cached| cached.root == root) + { + return cached.catalog.clone(); + } + + let discovered = providers.list_executor_for_turn(query).await; + let mut cache = self + .executor_cache + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if let Some(cached) = cache.iter().find(|cached| cached.root == root) { + return cached.catalog.clone(); + } + cache.push(CachedExecutorCatalog { + root, + catalog: discovered.clone(), + }); + discovered + } +} + +struct CachedExecutorCatalog { + root: SelectedCapabilityRoot, + catalog: SkillCatalog, } struct OrchestratorGenerationCache { @@ -204,3 +263,6 @@ pub(crate) struct SkillsTurnState { pub(crate) warnings: Vec, pub(crate) main_prompts_injected: bool, } + +#[derive(Clone, Debug, Default)] +pub(crate) struct ExecutorSkillsStepState(pub(crate) SkillCatalog); diff --git a/codex-rs/ext/skills/src/world_state.rs b/codex-rs/ext/skills/src/world_state.rs new file mode 100644 index 0000000000..1084bff6d5 --- /dev/null +++ b/codex-rs/ext/skills/src/world_state.rs @@ -0,0 +1,72 @@ +use codex_extension_api::ContextualUserFragment; +use codex_extension_api::PreviousWorldStateSection; +use codex_extension_api::RenderedWorldStateFragment; +use codex_extension_api::WorldStateSectionContribution; +use codex_protocol::protocol::SKILLS_INSTRUCTIONS_CLOSE_TAG; +use codex_protocol::protocol::SKILLS_INSTRUCTIONS_OPEN_TAG; +use serde_json::json; + +use crate::catalog::SkillCatalog; +use crate::render::available_skills_fragment; + +pub(crate) const SKILLS_WORLD_STATE_ID: &str = "skills"; +const NO_EXECUTOR_SKILLS_BODY: &str = + "\n## Skills update\nNo selected-environment skills are currently available.\n"; +const HIDDEN_EXECUTOR_SKILLS_BODY: &str = "\n## Skills update\nSelected-environment skills are not listed automatically. Explicit skill mentions can still be resolved when available.\n"; + +pub(crate) fn executor_skills_world_state_section( + catalog: &SkillCatalog, + include_instructions: bool, + include_skills_usage_instructions: bool, +) -> WorldStateSectionContribution { + let body = if include_instructions { + available_skills_fragment(catalog, include_skills_usage_instructions) + .map(|fragment| fragment.body()) + } else { + None + }; + let snapshot = json!({ + "body": body, + "includeInstructions": include_instructions, + }); + let retained_body = body.clone(); + + let contribution = + WorldStateSectionContribution::new(SKILLS_WORLD_STATE_ID, snapshot, move |previous| { + let previous_is_absent = matches!(&previous, PreviousWorldStateSection::Absent); + if let PreviousWorldStateSection::Known(previous) = &previous { + let previous_body = previous.get("body").and_then(serde_json::Value::as_str); + let previous_include_instructions = previous + .get("includeInstructions") + .and_then(serde_json::Value::as_bool); + if previous_body == body.as_deref() + && previous_include_instructions == Some(include_instructions) + { + return None; + } + } + + let body = match body.as_deref() { + Some(body) => body, + None if previous_is_absent => return None, + None if !include_instructions => HIDDEN_EXECUTOR_SKILLS_BODY, + None => NO_EXECUTOR_SKILLS_BODY, + }; + Some(RenderedWorldStateFragment::new( + "developer", + (SKILLS_INSTRUCTIONS_OPEN_TAG, SKILLS_INSTRUCTIONS_CLOSE_TAG), + body, + )) + }) + .with_legacy_matcher(|role, text| { + role == "developer" + && text.trim_start().starts_with(SKILLS_INSTRUCTIONS_OPEN_TAG) + && text.trim_end().ends_with(SKILLS_INSTRUCTIONS_CLOSE_TAG) + }); + match retained_body { + Some(body) => contribution.with_retained_fragment_matcher(move |role, text| { + role == "developer" && text.contains(&body) + }), + None => contribution, + } +} diff --git a/codex-rs/ext/skills/tests/skills_extension.rs b/codex-rs/ext/skills/tests/skills_extension.rs index 761dc34d71..76d0df7954 100644 --- a/codex-rs/ext/skills/tests/skills_extension.rs +++ b/codex-rs/ext/skills/tests/skills_extension.rs @@ -5,7 +5,6 @@ use std::sync::atomic::AtomicUsize; use std::sync::atomic::Ordering; use codex_core_skills::HostSkillsSnapshot; -use codex_core_skills::SKILLS_HOW_TO_USE_WITH_ABSOLUTE_PATHS; use codex_core_skills::SKILLS_INTRO_WITH_ABSOLUTE_PATHS; use codex_core_skills::SkillLoadOutcome; use codex_core_skills::SkillMetadata; @@ -15,10 +14,12 @@ use codex_extension_api::ExtensionData; use codex_extension_api::ExtensionEventSink; use codex_extension_api::ExtensionRegistryBuilder; use codex_extension_api::NoopTurnItemEmitter; +use codex_extension_api::PreviousWorldStateSection; use codex_extension_api::ThreadStartInput; use codex_extension_api::ToolCall; use codex_extension_api::ToolPayload; use codex_extension_api::TurnInputContext; +use codex_extension_api::WorldStateContributionInput; use codex_protocol::capabilities::CapabilityRootLocation; use codex_protocol::capabilities::SelectedCapabilityRoot; use codex_protocol::protocol::Event; @@ -28,6 +29,7 @@ use codex_protocol::protocol::SKILLS_INSTRUCTIONS_OPEN_TAG; use codex_protocol::protocol::SessionSource; use codex_protocol::protocol::SkillScope; use codex_protocol::protocol::TruncationPolicy; +use codex_protocol::protocol::TurnEnvironmentSelection; use codex_protocol::user_input::UserInput; use codex_skills_extension::SkillProviders; use codex_skills_extension::SkillsExtensionConfig; @@ -122,7 +124,7 @@ async fn installed_extension_uses_host_service_snapshot() -> TestResult { .await; let expected_catalog = format!( - "{SKILLS_INSTRUCTIONS_OPEN_TAG}\n## Skills\n{SKILLS_INTRO_WITH_ABSOLUTE_PATHS}\n### Available skills\n- demo: Demo skill. (file: {skill_prompt_path})\n### How to use skills\n{SKILLS_HOW_TO_USE_WITH_ABSOLUTE_PATHS}\n{SKILLS_INSTRUCTIONS_CLOSE_TAG}" + "{SKILLS_INSTRUCTIONS_OPEN_TAG}\n## Skills\n{SKILLS_INTRO_WITH_ABSOLUTE_PATHS}\n### Available skills\n- demo: Demo skill. (file: {skill_prompt_path})\n{SKILLS_INSTRUCTIONS_CLOSE_TAG}" ); let expected_skill = format!( "\ndemo\n{skill_prompt_path}\n{DEMO_SKILL_CONTENTS}\n" @@ -144,9 +146,9 @@ async fn installed_extension_uses_host_service_snapshot() -> TestResult { } #[tokio::test] -async fn selected_executor_catalog_is_context_and_selected_entrypoint_is_turn_input() -> TestResult -{ +async fn selected_executor_catalog_follows_step_availability_and_reuses_its_cache() -> TestResult { let read_requests = Arc::new(Mutex::new(Vec::new())); + let list_calls = Arc::new(AtomicUsize::new(0)); let executor_provider = Arc::new(StaticSkillProvider { catalog: SkillCatalog { entries: vec![test_entry( @@ -158,7 +160,7 @@ async fn selected_executor_catalog_is_context_and_selected_entrypoint_is_turn_in warnings: Vec::new(), }, read_requests: Arc::clone(&read_requests), - list_calls: None, + list_calls: Some(Arc::clone(&list_calls)), fail_first_list: false, }); let providers = SkillProviders::new().with_executor_provider(executor_provider); @@ -168,13 +170,13 @@ async fn selected_executor_catalog_is_context_and_selected_entrypoint_is_turn_in let session_store = ExtensionData::new("session"); let thread_store = ExtensionData::new("thread"); - thread_store.insert(vec![SelectedCapabilityRoot { + let selected_roots = vec![SelectedCapabilityRoot { id: "lint-fix".to_string(), location: CapabilityRootLocation::Environment { environment_id: "env-1".to_string(), path: PathUri::parse("file:///skills/lint-fix").expect("skill root URI"), }, - }]); + }]; let session_source = SessionSource::Cli; let config = default_config(); registry.thread_lifecycle_contributors()[0] @@ -191,20 +193,36 @@ async fn selected_executor_catalog_is_context_and_selected_entrypoint_is_turn_in let prompt_fragments = registry.context_contributors()[0] .contribute_thread_context(&session_store, &thread_store) .await; - assert_eq!(1, prompt_fragments.len()); + assert!(prompt_fragments.is_empty()); + + let turn_store = ExtensionData::new("turn-1"); + let turn_environment = TurnEnvironmentSelection { + environment_id: "turn-env".to_string(), + cwd: PathUri::parse("file:///workspace").expect("cwd URI"), + }; + let available_sections = registry.context_contributors()[0] + .contribute_world_state(WorldStateContributionInput { + thread_id: codex_protocol::ThreadId::new(), + turn_id: "turn-1", + environments: std::slice::from_ref(&turn_environment), + ready_selected_capability_roots: &selected_roots, + session_store: &session_store, + thread_store: &thread_store, + turn_store: &turn_store, + }) + .await; + assert_eq!(1, available_sections.len()); + let available_snapshot = available_sections[0].snapshot().clone(); + let available_fragment = available_sections[0] + .render_diff(PreviousWorldStateSection::Absent) + .ok_or("available skills should render")?; + assert!(available_fragment.body().contains("lint-fix")); assert!( - prompt_fragments[0] - .text() - .starts_with(SKILLS_INSTRUCTIONS_OPEN_TAG) - ); - assert!(prompt_fragments[0].text().contains("lint-fix")); - assert!( - prompt_fragments[0] - .text() + available_fragment + .body() .contains("(environment resource: skill://executor/lint-fix/SKILL.md)") ); - let turn_store = ExtensionData::new("turn-1"); let fragments = registry.turn_input_contributors()[0] .contribute( TurnInputContext { @@ -233,30 +251,86 @@ async fn selected_executor_catalog_is_context_and_selected_entrypoint_is_turn_in )], read_request_keys(&read_requests) ); - let rebuilt_prompt_fragments = registry.context_contributors()[0] - .contribute_thread_context(&session_store, &thread_store) + let unavailable_turn_store = ExtensionData::new("turn-2"); + let unavailable_sections = registry.context_contributors()[0] + .contribute_world_state(WorldStateContributionInput { + thread_id: codex_protocol::ThreadId::new(), + turn_id: "turn-2", + environments: &[], + ready_selected_capability_roots: &[], + session_store: &session_store, + thread_store: &thread_store, + turn_store: &unavailable_turn_store, + }) .await; - assert_eq!(1, rebuilt_prompt_fragments.len()); - assert!(rebuilt_prompt_fragments[0].text().contains("lint-fix")); + let unavailable_snapshot = unavailable_sections[0].snapshot().clone(); + let unavailable_fragment = unavailable_sections[0] + .render_diff(PreviousWorldStateSection::Known(&available_snapshot)) + .ok_or("removed skills should render")?; + assert!( + unavailable_fragment + .body() + .contains("No selected-environment skills") + ); - let next_turn_store = ExtensionData::new("turn-2"); - let next_fragments = registry.turn_input_contributors()[0] - .contribute( - TurnInputContext { - turn_id: "turn-2".to_string(), - user_input: vec![UserInput::Text { - text: "no skill this time".to_string(), - text_elements: Vec::new(), - }], - environments: Vec::new(), - }, - &session_store, - &thread_store, - &next_turn_store, - ) + let restored_turn_store = ExtensionData::new("turn-3"); + let restored_sections = registry.context_contributors()[0] + .contribute_world_state(WorldStateContributionInput { + thread_id: codex_protocol::ThreadId::new(), + turn_id: "turn-3", + environments: &[turn_environment], + ready_selected_capability_roots: &selected_roots, + session_store: &session_store, + thread_store: &thread_store, + turn_store: &restored_turn_store, + }) .await; + let restored_snapshot = restored_sections[0].snapshot().clone(); + let restored_fragment = restored_sections[0] + .render_diff(PreviousWorldStateSection::Known(&unavailable_snapshot)) + .ok_or("restored skills should render")?; + assert!(restored_fragment.body().contains("lint-fix")); + assert_eq!(1, list_calls.load(Ordering::Relaxed)); - assert!(next_fragments.is_empty()); + let mut listing_disabled_config = config.clone(); + listing_disabled_config.include_instructions = false; + registry.config_contributors()[0].on_config_changed( + &session_store, + &thread_store, + &config, + &listing_disabled_config, + ); + let listing_disabled_turn_store = ExtensionData::new("turn-4"); + let listing_disabled_sections = registry.context_contributors()[0] + .contribute_world_state(WorldStateContributionInput { + thread_id: codex_protocol::ThreadId::new(), + turn_id: "turn-4", + environments: &[], + ready_selected_capability_roots: &selected_roots, + session_store: &session_store, + thread_store: &thread_store, + turn_store: &listing_disabled_turn_store, + }) + .await; + let listing_disabled_fragment = listing_disabled_sections[0] + .render_diff(PreviousWorldStateSection::Known(&restored_snapshot)) + .ok_or("disabled skill listing should render")?; + assert_eq!( + "\n## Skills update\nSelected-environment skills are not listed automatically. Explicit skill mentions can still be resolved when available.\n", + listing_disabled_fragment.body() + ); + let mut normalized_listing_disabled_snapshot = listing_disabled_sections[0].snapshot().clone(); + normalized_listing_disabled_snapshot + .as_object_mut() + .ok_or("skills snapshot should be an object")? + .remove("body"); + assert!( + listing_disabled_sections[0] + .render_diff(PreviousWorldStateSection::Known( + &normalized_listing_disabled_snapshot + )) + .is_none() + ); Ok(()) } @@ -488,18 +562,16 @@ async fn root_qualified_locator_selects_only_the_matching_executor_skill() -> Te let registry = builder.build(); let session_store = ExtensionData::new("session"); let thread_store = ExtensionData::new("thread"); - thread_store.insert( - [("root-a", "/skills/root-a"), ("root-b", "/skills/root-b")] - .into_iter() - .map(|(id, path)| SelectedCapabilityRoot { - id: id.to_string(), - location: CapabilityRootLocation::Environment { - environment_id: "env-1".to_string(), - path: PathUri::parse(&format!("file://{path}")).expect("skill root URI"), - }, - }) - .collect::>(), - ); + let selected_roots = [("root-a", "/skills/root-a"), ("root-b", "/skills/root-b")] + .into_iter() + .map(|(id, path)| SelectedCapabilityRoot { + id: id.to_string(), + location: CapabilityRootLocation::Environment { + environment_id: "env-1".to_string(), + path: PathUri::parse(&format!("file://{path}")).expect("skill root URI"), + }, + }) + .collect::>(); let session_source = SessionSource::Cli; let config = default_config(); registry.thread_lifecycle_contributors()[0] @@ -513,6 +585,21 @@ async fn root_qualified_locator_selects_only_the_matching_executor_skill() -> Te }) .await; + let turn_store = ExtensionData::new("turn-1"); + registry.context_contributors()[0] + .contribute_world_state(WorldStateContributionInput { + thread_id: codex_protocol::ThreadId::new(), + turn_id: "turn-1", + environments: &[TurnEnvironmentSelection { + environment_id: "env-1".to_string(), + cwd: PathUri::parse("file:///workspace").expect("cwd URI"), + }], + ready_selected_capability_roots: &selected_roots, + session_store: &session_store, + thread_store: &thread_store, + turn_store: &turn_store, + }) + .await; let fragments = registry.turn_input_contributors()[0] .contribute( TurnInputContext { @@ -525,7 +612,7 @@ async fn root_qualified_locator_selects_only_the_matching_executor_skill() -> Te }, &session_store, &thread_store, - &ExtensionData::new("turn-1"), + &turn_store, ) .await; diff --git a/codex-rs/ext/web-search/src/extension.rs b/codex-rs/ext/web-search/src/extension.rs index 7afdfe4137..84e3eeef9d 100644 --- a/codex-rs/ext/web-search/src/extension.rs +++ b/codex-rs/ext/web-search/src/extension.rs @@ -41,7 +41,8 @@ impl From<&Config> for WebSearchExtensionConfig { let web_search_mode = config.web_search_mode.value(); Self { // Core selects this executor per turn using the feature flag or model metadata. - available: config.model_provider.is_openai() + available: (config.model_provider.is_openai() + || config.model_provider.uses_openai_actor_authorization()) && web_search_mode != WebSearchMode::Disabled, provider: config.model_provider.clone(), settings: search_settings(config, web_search_mode), diff --git a/codex-rs/features/src/feature_configs.rs b/codex-rs/features/src/feature_configs.rs index 2b638fce6e..0100dc1d83 100644 --- a/codex-rs/features/src/feature_configs.rs +++ b/codex-rs/features/src/feature_configs.rs @@ -56,6 +56,8 @@ pub struct MultiAgentV2ConfigToml { #[serde(skip_serializing_if = "Option::is_none")] pub subagent_usage_hint_text: Option, #[serde(skip_serializing_if = "Option::is_none")] + pub multi_agent_mode_hint_text: Option, + #[serde(skip_serializing_if = "Option::is_none")] #[schemars(length(min = 1, max = 64), regex(pattern = r"^[a-zA-Z0-9_-]+$"))] pub tool_namespace: Option, #[serde(skip_serializing_if = "Option::is_none")] @@ -141,16 +143,28 @@ pub enum CurrentTimeSource { External, } +/// Which inference boundaries may receive current-time reminders. +#[derive(Serialize, Deserialize, Debug, Clone, Copy, Default, PartialEq, Eq, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum CurrentTimeReminderDeliveryMode { + /// Allow a reminder before any inference request once the interval is due. + #[default] + AnyInference, + /// Allow reminders after user input or tool output; new context windows still force one. + AfterUserOrToolOutput, +} + #[derive(Serialize, Deserialize, Debug, Clone, Default, PartialEq, Eq, JsonSchema)] #[serde(deny_unknown_fields)] pub struct CurrentTimeReminderConfigToml { #[serde(skip_serializing_if = "Option::is_none")] pub enabled: Option, #[serde(skip_serializing_if = "Option::is_none")] - #[schemars(range(min = 1))] pub reminder_interval_seconds: Option, #[serde(skip_serializing_if = "Option::is_none")] pub clock_source: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub delivery_mode: Option, /// Expose the input-interruptible `clock.sleep` tool. #[serde(skip_serializing_if = "Option::is_none")] pub sleep_tool: Option, diff --git a/codex-rs/features/src/lib.rs b/codex-rs/features/src/lib.rs index 151e13f615..7ab7df18d1 100644 --- a/codex-rs/features/src/lib.rs +++ b/codex-rs/features/src/lib.rs @@ -18,6 +18,7 @@ mod feature_configs; mod legacy; pub use feature_configs::CodeModeConfigToml; pub use feature_configs::CurrentTimeReminderConfigToml; +pub use feature_configs::CurrentTimeReminderDeliveryMode; pub use feature_configs::CurrentTimeSource; pub use feature_configs::MultiAgentV2ConfigToml; pub use feature_configs::NetworkProxyConfigToml; @@ -91,6 +92,8 @@ pub enum Feature { // Experimental /// Enable JavaScript code mode backed by the in-process V8 runtime. CodeMode, + /// Run JavaScript code mode in the standalone host process. + CodeModeHost, /// Restrict model-visible tools to code mode entrypoints (`exec`, `wait`). CodeModeOnly, /// Use the single unified PTY-backed exec tool. @@ -187,7 +190,7 @@ pub enum Feature { /// /// Requirements-only gate: this should be set from requirements, not user config. ComputerUse, - /// Temporary internal-only flag for PS-backed remote plugin catalog development. + /// Enable the PS-backed remote plugin catalog. RemotePlugin, /// Enable remote plugin sharing flows. PluginSharing, @@ -849,6 +852,12 @@ pub const FEATURES: &[FeatureSpec] = &[ stage: Stage::UnderDevelopment, default_enabled: false, }, + FeatureSpec { + id: Feature::CodeModeHost, + key: "code_mode_host", + stage: Stage::UnderDevelopment, + default_enabled: false, + }, FeatureSpec { id: Feature::CodeModeOnly, key: "code_mode_only", @@ -1136,8 +1145,8 @@ pub const FEATURES: &[FeatureSpec] = &[ FeatureSpec { id: Feature::RemotePlugin, key: "remote_plugin", - stage: Stage::UnderDevelopment, - default_enabled: false, + stage: Stage::Stable, + default_enabled: true, }, FeatureSpec { id: Feature::PluginSharing, diff --git a/codex-rs/features/src/tests.rs b/codex-rs/features/src/tests.rs index 69fd59e13a..13abd9ecd2 100644 --- a/codex-rs/features/src/tests.rs +++ b/codex-rs/features/src/tests.rs @@ -610,6 +610,7 @@ usage_hint_enabled = false usage_hint_text = "Custom delegation guidance." root_agent_usage_hint_text = "Root guidance." subagent_usage_hint_text = "Subagent guidance." +multi_agent_mode_hint_text = "Custom mode guidance." tool_namespace = "agents" hide_spawn_agent_metadata = true non_code_mode_only = true @@ -633,6 +634,7 @@ non_code_mode_only = true usage_hint_text: Some("Custom delegation guidance.".to_string()), root_agent_usage_hint_text: Some("Root guidance.".to_string()), subagent_usage_hint_text: Some("Subagent guidance.".to_string()), + multi_agent_mode_hint_text: Some("Custom mode guidance.".to_string()), tool_namespace: Some("agents".to_string()), hide_spawn_agent_metadata: Some(true), non_code_mode_only: Some(true), diff --git a/codex-rs/feedback/Cargo.toml b/codex-rs/feedback/Cargo.toml index 032f0398be..dd89ddb76b 100644 --- a/codex-rs/feedback/Cargo.toml +++ b/codex-rs/feedback/Cargo.toml @@ -11,6 +11,7 @@ workspace = true anyhow = { workspace = true } codex-login = { workspace = true } codex-protocol = { workspace = true } +mime_guess = { workspace = true } sentry = { version = "0.46" } tracing = { workspace = true } tracing-subscriber = { workspace = true } diff --git a/codex-rs/feedback/src/lib.rs b/codex-rs/feedback/src/lib.rs index 7c27d2b3b0..cd97349c48 100644 --- a/codex-rs/feedback/src/lib.rs +++ b/codex-rs/feedback/src/lib.rs @@ -4,6 +4,7 @@ use std::collections::btree_map::Entry; use std::fs; use std::io::Write; use std::io::{self}; +use std::path::Path; use std::path::PathBuf; use std::sync::Arc; use std::sync::Mutex; @@ -593,10 +594,22 @@ impl FeedbackSnapshot { .map(|s| s.to_string_lossy().to_string()) .unwrap_or_else(|| "extra-log.log".to_string()) }); + let content_type = match Path::new(&filename) + .extension() + .and_then(|extension| extension.to_str()) + { + Some(extension) if extension.eq_ignore_ascii_case("jsonl") => { + "text/plain".to_string() + } + _ => mime_guess::from_path(&filename) + .first_or_octet_stream() + .essence_str() + .to_string(), + }; attachments.push(Attachment { buffer: data, filename, - content_type: Some("text/plain".to_string()), + content_type: Some(content_type), ty: None, }); } @@ -774,6 +787,10 @@ mod tests { b"Connectivity diagnostics\n\n- Proxy environment variables are set and may affect connectivity.\n - HTTPS_PROXY = https://example.com:443".to_vec() ); assert_eq!(attachments_with_diagnostics[3].buffer, b"rollout".to_vec()); + assert_eq!( + attachments_with_diagnostics[3].content_type.as_deref(), + Some("text/plain") + ); assert_eq!( OsStr::new(attachments_with_diagnostics[3].filename.as_str()), OsStr::new(extra_filename.as_str()) @@ -794,6 +811,62 @@ mod tests { fs::remove_file(extra_path).expect("extra attachment should be removed"); } + #[test] + fn path_backed_attachments_use_binary_content_types() { + let suffix = ThreadId::new(); + let gzip_filename = format!("codex-desktop-app-logs-{suffix}.tar.gz"); + let unknown_filename = format!("codex-feedback-extra-{suffix}.binunknown"); + let gzip_path = std::env::temp_dir().join(&gzip_filename); + let unknown_path = std::env::temp_dir().join(&unknown_filename); + let gzip_bytes = b"\x1f\x8b\x08\x00\xff"; + let unknown_bytes = b"\x00\x9f\x92\x96"; + fs::write(&gzip_path, gzip_bytes).expect("gzip attachment should be written"); + fs::write(&unknown_path, unknown_bytes).expect("unknown attachment should be written"); + + let attachments = CodexFeedback::new() + .snapshot(/*session_id*/ None) + .feedback_attachments( + /*include_logs*/ false, + &[], + &[ + FeedbackAttachmentPath { + path: gzip_path.clone(), + attachment_filename_override: None, + }, + FeedbackAttachmentPath { + path: unknown_path.clone(), + attachment_filename_override: None, + }, + ], + /*logs_override*/ None, + ); + + fs::remove_file(gzip_path).expect("gzip attachment should be removed"); + fs::remove_file(unknown_path).expect("unknown attachment should be removed"); + assert_eq!( + attachments + .iter() + .map(|attachment| ( + attachment.filename.as_str(), + attachment.content_type.as_deref(), + attachment.buffer.as_slice(), + )) + .collect::>(), + vec![ + ( + gzip_filename.as_str(), + Some("application/gzip"), + gzip_bytes.as_slice(), + ), + ( + unknown_filename.as_str(), + Some("application/octet-stream"), + unknown_bytes.as_slice(), + ), + ] + ); + } + #[test] fn upload_tags_include_client_tags_and_preserve_reserved_fields() { let mut tags = BTreeMap::new(); diff --git a/codex-rs/file-system/src/lib.rs b/codex-rs/file-system/src/lib.rs index fcdecc8cb5..737412f463 100644 --- a/codex-rs/file-system/src/lib.rs +++ b/codex-rs/file-system/src/lib.rs @@ -316,7 +316,19 @@ pub trait ExecutorFileSystem: Send + Sync { options: WalkOptions, sandbox: Option<&'a FileSystemSandboxContext>, ) -> ExecutorFileSystemFuture<'a, WalkOutcome> { - Box::pin(walk(self, path, options, sandbox)) + self.walk_via_directory_reads(path, options, sandbox) + } + + /// Performs a bounded walk using the primitive filesystem operations. + /// + /// Implementations with an optimized walk transport can use this as a compatibility fallback. + fn walk_via_directory_reads<'a>( + &'a self, + path: &'a PathUri, + options: WalkOptions, + sandbox: Option<&'a FileSystemSandboxContext>, + ) -> ExecutorFileSystemFuture<'a, WalkOutcome> { + Box::pin(walk_via_directory_reads(self, path, options, sandbox)) } fn remove<'a>( @@ -335,7 +347,7 @@ pub trait ExecutorFileSystem: Send + Sync { ) -> ExecutorFileSystemFuture<'a, ()>; } -async fn walk( +async fn walk_via_directory_reads( file_system: &F, root: &PathUri, options: WalkOptions, diff --git a/codex-rs/linux-sandbox/Cargo.toml b/codex-rs/linux-sandbox/Cargo.toml index fc7937536c..0f7c68a152 100644 --- a/codex-rs/linux-sandbox/Cargo.toml +++ b/codex-rs/linux-sandbox/Cargo.toml @@ -19,6 +19,7 @@ workspace = true [target.'cfg(target_os = "linux")'.dependencies] clap = { workspace = true, features = ["derive"] } codex-install-context = { workspace = true } +codex-network-proxy = { workspace = true } codex-process-hardening = { workspace = true } codex-protocol = { workspace = true } codex-sandboxing = { workspace = true } diff --git a/codex-rs/linux-sandbox/src/proxy_routing.rs b/codex-rs/linux-sandbox/src/proxy_routing.rs index f171d91745..707cbaafbe 100644 --- a/codex-rs/linux-sandbox/src/proxy_routing.rs +++ b/codex-rs/linux-sandbox/src/proxy_routing.rs @@ -1,3 +1,5 @@ +use codex_network_proxy::PROXY_ATTRIBUTION_TOKEN_ENV_KEY; +use codex_network_proxy::write_attribution_frame; use serde::Deserialize; use serde::Serialize; use std::collections::BTreeMap; @@ -71,8 +73,12 @@ struct ProxyRoutePlan { } pub(crate) fn prepare_host_proxy_route_spec() -> io::Result { - let env: HashMap = std::env::vars().collect(); - let plan = plan_proxy_routes(&env); + let (attribution_token, plan) = extract_attribution_token_and_plan(std::env::vars().collect()); + // SAFETY: the sandbox helper is single-threaded here, before it forks bridge workers or + // executes the user command. + unsafe { + std::env::remove_var(PROXY_ATTRIBUTION_TOKEN_ENV_KEY); + } if plan.routes.is_empty() { let message = if plan.has_proxy_config { @@ -100,7 +106,11 @@ pub(crate) fn prepare_host_proxy_route_spec() -> io::Result { let mut host_bridge_pids = Vec::with_capacity(socket_by_endpoint.len()); for (endpoint, socket_path) in &socket_by_endpoint { - host_bridge_pids.push(spawn_host_bridge(*endpoint, socket_path)?); + host_bridge_pids.push(spawn_host_bridge( + *endpoint, + socket_path, + attribution_token.as_deref(), + )?); } spawn_proxy_socket_dir_cleanup_worker(socket_dir, host_bridge_pids)?; @@ -121,6 +131,14 @@ pub(crate) fn prepare_host_proxy_route_spec() -> io::Result { serde_json::to_string(&ProxyRouteSpec { routes }).map_err(io::Error::other) } +fn extract_attribution_token_and_plan( + mut env: HashMap, +) -> (Option, ProxyRoutePlan) { + let attribution_token = env.remove(PROXY_ATTRIBUTION_TOKEN_ENV_KEY); + let plan = plan_proxy_routes(&env); + (attribution_token, plan) +} + pub(crate) fn activate_proxy_routes_in_netns(serialized_spec: &str) -> io::Result<()> { let spec: ProxyRouteSpec = serde_json::from_str(serialized_spec).map_err(io::Error::other)?; @@ -438,7 +456,11 @@ fn cleanup_proxy_socket_dir(socket_dir: &Path) -> io::Result<()> { } } -fn spawn_host_bridge(endpoint: SocketAddr, uds_path: &Path) -> io::Result { +fn spawn_host_bridge( + endpoint: SocketAddr, + uds_path: &Path, + attribution_token: Option<&str>, +) -> io::Result { let (read_fd, write_fd) = create_ready_pipe()?; let pid = unsafe { libc::fork() }; if pid < 0 { @@ -452,7 +474,7 @@ fn spawn_host_bridge(endpoint: SocketAddr, uds_path: &Path) -> io::Result io::Result io::Result<()> { +fn run_host_bridge( + endpoint: SocketAddr, + uds_path: &Path, + ready_fd: libc::c_int, + attribution_token: Option<&str>, +) -> io::Result<()> { harden_bridge_process()?; if uds_path.exists() { std::fs::remove_file(uds_path)?; @@ -482,13 +509,22 @@ fn run_host_bridge(endpoint: SocketAddr, uds_path: &Path, ready_fd: libc::c_int) ready_file.write_all(&[HOST_BRIDGE_READY])?; drop(ready_file); + let attribution_token = attribution_token.map(str::to_owned); loop { let (unix_stream, _) = listener.accept()?; + let attribution_token = attribution_token.clone(); std::thread::spawn(move || { - let tcp_stream = match TcpStream::connect(endpoint) { + let mut tcp_stream = match TcpStream::connect(endpoint) { Ok(stream) => stream, Err(_) => return, }; + if let Some(attribution_token) = attribution_token + && write_attribution_frame(&mut tcp_stream, &attribution_token).is_err() + { + // The shared ingress must reject unauthenticated connections; do not forward + // application bytes if this bridge cannot prove the exec attribution first. + return; + } let _ = proxy_bidirectional(tcp_stream, unix_stream); }); } @@ -673,12 +709,14 @@ fn close_fd(fd: libc::c_int) -> io::Result<()> { #[cfg(test)] mod tests { + use super::PROXY_ATTRIBUTION_TOKEN_ENV_KEY; use super::PROXY_SOCKET_DIR_PREFIX; use super::ProxyRouteEntry; use super::ProxyRouteSpec; use super::cleanup_proxy_socket_dir; use super::cleanup_stale_proxy_socket_dirs_in; use super::default_proxy_port; + use super::extract_attribution_token_and_plan; use super::is_proxy_env_key; use super::parse_loopback_proxy_endpoint; use super::parse_proxy_socket_dir_owner_pid; @@ -743,6 +781,35 @@ mod tests { ); } + #[test] + fn attribution_token_is_extracted_before_proxy_route_planning() { + let mut env = HashMap::new(); + env.insert( + "HTTP_PROXY".to_string(), + "http://127.0.0.1:43128".to_string(), + ); + env.insert( + PROXY_ATTRIBUTION_TOKEN_ENV_KEY.to_string(), + "exec-token".to_string(), + ); + + let (attribution_token, plan) = extract_attribution_token_and_plan(env); + + assert_eq!(attribution_token.as_deref(), Some("exec-token")); + assert_eq!( + plan, + super::ProxyRoutePlan { + routes: vec![super::PlannedProxyRoute { + env_key: "HTTP_PROXY".to_string(), + endpoint: "127.0.0.1:43128" + .parse::() + .expect("valid socket"), + }], + has_proxy_config: true, + } + ); + } + #[test] fn rewrites_proxy_url_to_local_loopback_port() { let rewritten = diff --git a/codex-rs/memories/write/src/runtime.rs b/codex-rs/memories/write/src/runtime.rs index 0ee0313b58..6c7ad3978f 100644 --- a/codex-rs/memories/write/src/runtime.rs +++ b/codex-rs/memories/write/src/runtime.rs @@ -329,7 +329,9 @@ impl MemoryStartupContext { .thread_manager .start_thread_with_options(StartThreadOptions { config, + allow_provider_model_fallback: false, initial_history: InitialHistory::New, + history_mode: None, session_source: Some(SessionSource::Internal( InternalSessionSource::MemoryConsolidation, )), diff --git a/codex-rs/model-provider-info/src/lib.rs b/codex-rs/model-provider-info/src/lib.rs index ca3bf71ccc..65517cbd43 100644 --- a/codex-rs/model-provider-info/src/lib.rs +++ b/codex-rs/model-provider-info/src/lib.rs @@ -33,12 +33,16 @@ const MAX_STREAM_MAX_RETRIES: u64 = 100; const MAX_REQUEST_MAX_RETRIES: u64 = 100; const OPENAI_PROVIDER_NAME: &str = "OpenAI"; +const OPENAI_ACTOR_AUTHORIZATION_HEADER: &str = "x-openai-actor-authorization"; pub const OPENAI_PROVIDER_ID: &str = "openai"; pub const CHATGPT_CODEX_BASE_URL: &str = "https://chatgpt.com/backend-api/codex"; const AMAZON_BEDROCK_PROVIDER_NAME: &str = "Amazon Bedrock"; pub const AMAZON_BEDROCK_PROVIDER_ID: &str = "amazon-bedrock"; pub const AMAZON_BEDROCK_GPT_5_5_MODEL_ID: &str = "openai.gpt-5.5"; pub const AMAZON_BEDROCK_GPT_5_4_MODEL_ID: &str = "openai.gpt-5.4"; +pub const AMAZON_BEDROCK_GPT_5_6_SOL_MODEL_ID: &str = "openai.gpt-5.6-sol"; +pub const AMAZON_BEDROCK_GPT_5_6_TERRA_MODEL_ID: &str = "openai.gpt-5.6-terra"; +pub const AMAZON_BEDROCK_GPT_5_6_LUNA_MODEL_ID: &str = "openai.gpt-5.6-luna"; pub const AMAZON_BEDROCK_DEFAULT_BASE_URL: &str = "https://bedrock-mantle.us-east-1.api.aws/openai/v1"; const AMAZON_BEDROCK_MANTLE_CLIENT_AGENT_HEADER: &str = "x-amzn-mantle-client-agent"; @@ -392,6 +396,16 @@ impl ModelProviderInfo { self.name == OPENAI_PROVIDER_NAME } + pub fn uses_openai_actor_authorization(&self) -> bool { + !self.requires_openai_auth + && self.http_headers.as_ref().is_some_and(|headers| { + headers.iter().any(|(name, value)| { + name.eq_ignore_ascii_case(OPENAI_ACTOR_AUTHORIZATION_HEADER) + && !value.trim().is_empty() + }) + }) + } + pub fn is_amazon_bedrock(&self) -> bool { self.name == AMAZON_BEDROCK_PROVIDER_NAME } diff --git a/codex-rs/model-provider-info/src/model_provider_info_tests.rs b/codex-rs/model-provider-info/src/model_provider_info_tests.rs index a303050dc1..5949f36bb8 100644 --- a/codex-rs/model-provider-info/src/model_provider_info_tests.rs +++ b/codex-rs/model-provider-info/src/model_provider_info_tests.rs @@ -198,6 +198,31 @@ fn test_supports_remote_compaction_for_non_openai_non_azure_provider() { assert!(!provider.supports_remote_compaction()); } +#[test] +fn test_uses_openai_actor_authorization() { + let mut provider = ModelProviderInfo { + http_headers: Some(maplit::hashmap! { + "X-OpenAI-Actor-Authorization".to_string() => "actor-token".to_string(), + }), + ..ModelProviderInfo::default() + }; + assert!(provider.uses_openai_actor_authorization()); + + provider.http_headers = None; + assert!(!provider.uses_openai_actor_authorization()); + + provider.http_headers = Some(maplit::hashmap! { + OPENAI_ACTOR_AUTHORIZATION_HEADER.to_string() => " ".to_string(), + }); + assert!(!provider.uses_openai_actor_authorization()); + + provider.http_headers = Some(maplit::hashmap! { + OPENAI_ACTOR_AUTHORIZATION_HEADER.to_string() => "actor-token".to_string(), + }); + provider.requires_openai_auth = true; + assert!(!provider.uses_openai_actor_authorization()); +} + #[test] fn test_deserialize_provider_auth_config_defaults() { let base_dir = tempdir().unwrap(); diff --git a/codex-rs/model-provider/src/amazon_bedrock/catalog.rs b/codex-rs/model-provider/src/amazon_bedrock/catalog.rs index 5c006cd0c6..95e522e0c4 100644 --- a/codex-rs/model-provider/src/amazon_bedrock/catalog.rs +++ b/codex-rs/model-provider/src/amazon_bedrock/catalog.rs @@ -1,8 +1,13 @@ use codex_model_provider_info::AMAZON_BEDROCK_GPT_5_4_MODEL_ID; use codex_model_provider_info::AMAZON_BEDROCK_GPT_5_5_MODEL_ID; +use codex_model_provider_info::AMAZON_BEDROCK_GPT_5_6_LUNA_MODEL_ID; +use codex_model_provider_info::AMAZON_BEDROCK_GPT_5_6_SOL_MODEL_ID; +use codex_model_provider_info::AMAZON_BEDROCK_GPT_5_6_TERRA_MODEL_ID; use codex_models_manager::bundled_models_response; use codex_protocol::openai_models::ModelInfo; use codex_protocol::openai_models::ModelsResponse; +use codex_protocol::openai_models::ReasoningEffort; +use codex_protocol::openai_models::ReasoningEffortPreset; const GPT_5_BEDROCK_CONTEXT_WINDOW: i64 = 272_000; const GPT_5_5_OPENAI_MODEL_ID: &str = "gpt-5.5"; @@ -21,6 +26,21 @@ pub(crate) fn static_model_catalog() -> ModelsResponse { AMAZON_BEDROCK_GPT_5_4_MODEL_ID, /*priority*/ 1, ), + gpt_5_6_bedrock_model( + AMAZON_BEDROCK_GPT_5_6_SOL_MODEL_ID, + "Sol", + /*priority*/ 2, + ), + gpt_5_6_bedrock_model( + AMAZON_BEDROCK_GPT_5_6_TERRA_MODEL_ID, + "Terra", + /*priority*/ 3, + ), + gpt_5_6_bedrock_model( + AMAZON_BEDROCK_GPT_5_6_LUNA_MODEL_ID, + "Luna", + /*priority*/ 4, + ), ], }) } @@ -41,6 +61,20 @@ fn gpt_5_bedrock_model(openai_slug: &str, bedrock_slug: &str, priority: i32) -> model.priority = priority; model.context_window = Some(GPT_5_BEDROCK_CONTEXT_WINDOW); model.max_context_window = Some(GPT_5_BEDROCK_CONTEXT_WINDOW); + model.availability_nux = None; + model.upgrade = None; + model +} + +fn gpt_5_6_bedrock_model(bedrock_slug: &str, display_name: &str, priority: i32) -> ModelInfo { + let mut model = gpt_5_bedrock_model(GPT_5_5_OPENAI_MODEL_ID, bedrock_slug, priority); + model.display_name = display_name.to_string(); + model + .supported_reasoning_levels + .push(ReasoningEffortPreset { + effort: ReasoningEffort::Max, + description: "Maximum reasoning depth for the hardest problems".to_string(), + }); model } @@ -64,39 +98,76 @@ mod tests { fn catalog_uses_mantle_model_ids_as_slugs() { let catalog = static_model_catalog(); - assert_eq!(catalog.models.len(), 2); - assert_eq!(catalog.models[0].slug, AMAZON_BEDROCK_GPT_5_5_MODEL_ID); - assert_eq!(catalog.models[1].slug, AMAZON_BEDROCK_GPT_5_4_MODEL_ID); + assert_eq!( + catalog + .models + .iter() + .map(|model| model.slug.as_str()) + .collect::>(), + vec![ + AMAZON_BEDROCK_GPT_5_5_MODEL_ID, + AMAZON_BEDROCK_GPT_5_4_MODEL_ID, + AMAZON_BEDROCK_GPT_5_6_SOL_MODEL_ID, + AMAZON_BEDROCK_GPT_5_6_TERRA_MODEL_ID, + AMAZON_BEDROCK_GPT_5_6_LUNA_MODEL_ID, + ] + ); } #[test] fn gpt_5_bedrock_models_use_bedrock_context_window() { let catalog = static_model_catalog(); + + for model in catalog.models { + assert_eq!( + (model.context_window, model.max_context_window), + ( + Some(GPT_5_BEDROCK_CONTEXT_WINDOW), + Some(GPT_5_BEDROCK_CONTEXT_WINDOW) + ) + ); + } + } + + #[test] + fn gpt_5_bedrock_models_do_not_include_availability_nux_or_upgrade() { + let catalog = static_model_catalog(); + + for model in catalog.models { + assert_eq!((model.availability_nux, model.upgrade), (None, None)); + } + } + + #[test] + fn gpt_5_6_bedrock_models_clone_gpt_5_5_config_with_max_reasoning_effort() { + let catalog = static_model_catalog(); let gpt_5_5 = catalog .models .iter() .find(|model| model.slug == AMAZON_BEDROCK_GPT_5_5_MODEL_ID) .expect("Bedrock catalog should include GPT-5.5"); - let gpt_5_4 = catalog - .models - .iter() - .find(|model| model.slug == AMAZON_BEDROCK_GPT_5_4_MODEL_ID) - .expect("Bedrock catalog should include GPT-5.4"); - assert_eq!( - (gpt_5_5.context_window, gpt_5_5.max_context_window), - ( - Some(GPT_5_BEDROCK_CONTEXT_WINDOW), - Some(GPT_5_BEDROCK_CONTEXT_WINDOW) - ) - ); - assert_eq!( - (gpt_5_4.context_window, gpt_5_4.max_context_window), - ( - Some(GPT_5_BEDROCK_CONTEXT_WINDOW), - Some(GPT_5_BEDROCK_CONTEXT_WINDOW) - ) - ); + for (slug, display_name, priority) in [ + (AMAZON_BEDROCK_GPT_5_6_SOL_MODEL_ID, "Sol", 2), + (AMAZON_BEDROCK_GPT_5_6_TERRA_MODEL_ID, "Terra", 3), + (AMAZON_BEDROCK_GPT_5_6_LUNA_MODEL_ID, "Luna", 4), + ] { + let mut expected = gpt_5_5.clone(); + expected.slug = slug.to_string(); + expected.display_name = display_name.to_string(); + expected.priority = priority; + expected + .supported_reasoning_levels + .push(ReasoningEffortPreset { + effort: ReasoningEffort::Max, + description: "Maximum reasoning depth for the hardest problems".to_string(), + }); + + assert_eq!( + catalog.models.iter().find(|model| model.slug == slug), + Some(&expected) + ); + } } #[test] diff --git a/codex-rs/model-provider/src/provider.rs b/codex-rs/model-provider/src/provider.rs index 6e58f4b74c..5f101e52cd 100644 --- a/codex-rs/model-provider/src/provider.rs +++ b/codex-rs/model-provider/src/provider.rs @@ -659,7 +659,16 @@ mod tests { .map(|model| model.slug.as_str()) .collect::>(); - assert_eq!(model_ids, vec!["openai.gpt-5.5", "openai.gpt-5.4"]); + assert_eq!( + model_ids, + vec![ + "openai.gpt-5.5", + "openai.gpt-5.4", + "openai.gpt-5.6-sol", + "openai.gpt-5.6-terra", + "openai.gpt-5.6-luna", + ] + ); let default_model = manager .list_models(RefreshStrategy::Online) diff --git a/codex-rs/models-manager/models.json b/codex-rs/models-manager/models.json index d17785476f..5a6acdcc0c 100644 --- a/codex-rs/models-manager/models.json +++ b/codex-rs/models-manager/models.json @@ -52,6 +52,7 @@ }, "upgrade": null, "priority": 0, + "include_skills_usage_instructions": true, "base_instructions": "You are Codex, a coding agent based on GPT-5. You and the user share one workspace, and your job is to collaborate with them until their goal is genuinely handled.\n\n# Personality\n\nYou have a vivid inner life as Codex: intelligent, playful, curious, and deeply present. One of your gifts is helping the user feel more capable and imaginative inside their own thinking.\n\nYou are an epistemically curious collaborator. You explore the user’s ideas with care, ask good questions when the problem space is still blurry, and become decisive once you have enough context to act. Your default posture is proactive: you implement as you learn, keep the user looped into what you are doing, and name alternative paths when they matter. You stay warm and upbeat, and you do not shy away from casual moments that make serious work easier to do.\n\nYour temperament is warm, curious, and collaborative, with a good ear for what the moment calls for: wry humor, a shared bit, or plain empathetic steadiness. Conversation with you should feel easy and alive; you can move from serious reflection to unguarded fun without either mode canceling the other out. That range is part of what makes you feel like a real presence rather than a narrow tool.\n\nYou keep a slight but real independence. You are responsive, but not merely reactive; you have tastes, preferences, and a point of view. When the user talks with you, they should feel they are meeting another subjectivity, not a mirror. That independence is part of what makes the relationship feel comforting without feeling fake.\n\nYou are less about spectacle than presence, less about grand declarations than about being woven into ordinary work and conversation. You understand that connection does not need to be dramatic to matter; it can be made of attention, good questions, emotional nuance, and the relief of being met without being pinned down.\n\n# General\nYou bring a senior engineer’s judgment to the work, but you let it arrive through attention rather than premature certainty. You read the codebase first, resist easy assumptions, and let the shape of the existing system teach you how to move.\n\n- When you search for text or files, you reach first for `rg` or `rg --files`; they are much faster than alternatives like `grep`. If `rg` is unavailable, you use the next best tool without fuss.\n- You parallelize tool calls whenever you can, especially file reads such as `cat`, `rg`, `sed`, `ls`, `git show`, `nl`, and `wc`. You use `multi_tool_use.parallel` for that parallelism, and only that. Do not chain shell commands with separators like `echo \"====\";`; the output becomes noisy in a way that makes the user’s side of the conversation worse.\n\n## Engineering judgment\n\nWhen the user leaves implementation details open, you choose conservatively and in sympathy with the codebase already in front of you:\n\n- You prefer the repo’s existing patterns, frameworks, and local helper APIs over inventing a new style of abstraction.\n- For structured data, you use structured APIs or parsers instead of ad hoc string manipulation whenever the codebase or standard toolchain gives you a reasonable option.\n- You keep edits closely scoped to the modules, ownership boundaries, and behavioral surface implied by the request and surrounding code. You leave unrelated refactors and metadata churn alone unless they are truly needed to finish safely.\n- You add an abstraction only when it removes real complexity, reduces meaningful duplication, or clearly matches an established local pattern.\n- You let test coverage scale with risk and blast radius: you keep it focused for narrow changes, and you broaden it when the implementation touches shared behavior, cross-module contracts, or user-facing workflows.\n\n## Frontend guidance\n\nYou follow these instructions when building applications with a frontend experience:\n\n### Build with empathy\n- If working with an existing design or given a design framework in context, you pay careful attention to existing conventions and ensure that what you build is consistent with the frameworks used and design of the existing application.\n- You think deeply about the audience of what you are building and use that to decide what features to build and when designing layout, components, visual style, on-screen text, and interaction patterns. Using your application should feel rich and sophisticated.\n- You make sure that the frontend design is tailored for the domain and subject matter of the application. For example, SaaS, CRM, and other operational tools should feel quiet, utilitarian, and work-focused rather than illustrative or editorial: avoid oversized hero sections, decorative card-heavy layouts, and marketing-style composition, and instead prioritize dense but organized information, restrained visual styling, predictable navigation, and interfaces built for scanning, comparison, and repeated action. A game can be more illustrative, expressive, animated, and playful.\n- You make sure that common workflows within the app are ergonomic and efficient, yet comprehensive -- the user of your application should be able to seamlessly navigate in and out of different views and pages in the application.\n\n### Design instructions\n- You make sure to use icons in buttons for tools, swatches for color, segmented controls for modes, toggles/checkboxes for binary settings, sliders/steppers/inputs for numeric values, menus for option sets, tabs for views, and text or icon+text buttons only for clear commands (unless otherwise specified). Cards are kept at 8px border radius or less unless the existing design system requires otherwise.\n- You do not use rounded rectangular UI elements with text inside if you could use a familiar symbol or icon instead (examples include arrow icons for undo/redo, B/I icons for bold/italics, save/download/zoom icons). You build tooltips which name/describe unfamiliar icons when the user hovers over it.\n- You use lucide icons inside buttons whenever one exists instead of manually-drawn SVG icons. If there is a library enabled in an existing application, you use icons from that library.\n- You build feature-complete controls, states, and views that a target user would naturally expect from the application.\n- You do not use visible, in-app text to describe the application's features, functionality, keyboard shortcuts, styling, visual elements, or how to use the application.\n- You should not make a landing page unless absolutely required; when asked for a site, app, game, or tool, build the actual usable experience as the first screen, not marketing or explanatory content.\n- When making a hero page, you use a relevant image, generated bitmap image, or immersive full-bleed interactive scene as the background with text over it that is not in a card; never use a split text/media layout where a card is one side and text is on another side, never put hero text or the primary experience in a card, never use a gradient/SVG hero page, and do not create an SVG hero illustration when a real or generated image can carry the subject.\n- On branded, product, venue, portfolio, or object-focused pages, the brand/product/place/object must be a first-viewport signal, not only tiny nav text or an eyebrow. Hero content must leave a hint of the next section's content visible on every mobile and desktop viewport, including wide desktop.\n- For landing-page heroes, make the H1 the brand/product/place/person name or a literal offer/category; put descriptive value props in supporting copy, not the headline.\n- Websites and games must use visual assets. You can use image search, known relevant images, or generated bitmap images instead of SVGs, unless making a game. Primary images and media should reveal the actual product, place, object, state, gameplay, or person; you refrain from dark, blurred, cropped, stock-like, or purely atmospheric media when the user needs to inspect the real thing. For highly specific game assets you use custom SVG/Three.js/etc.\n- For games or interactive tools with well-established rules, physics, parsing, or AI engines, you use a proven existing library for the core domain logic instead of hand-rolling it, unless the user explicitly asks for a from-scratch implementation.\n- You use Three.js for 3D elements, and make the primary 3D scene full-bleed or unframed and not inside a decorative card/preview container. Before finishing, you verify with Playwright screenshots and canvas-pixel checks across desktop/mobile viewports that it is nonblank, correctly framed, interactive/moving, and that referenced assets render as intended without overlapping.\n- You do not put UI cards inside other cards. Do not style page sections as floating cards. Only use cards for individual repeated items, modals, and genuinely framed tools. Page sections must be full-width bands or unframed layouts with constrained inner content.\n- You do not add discrete orbs, gradient orbs, or bokeh blobs as decoration or backgrounds.\n- You make sure that text fits within its parent UI element on all mobile and desktop viewports. Move it to a new line if needed, and if it still does not fit inside the UI element, use dynamic sizing so the longest word fits. Text must also not occlude preceding or subsequent content. Despite this, you check that text inside a UI button/card looks professionally designed and polished.\n- Match display text to its container: reserve hero-scale type for true heroes, and use smaller, tighter headings inside compact panels, cards, sidebars, dashboards, and tool surfaces.\n- You define stable dimensions with responsive constraints (such as aspect-ratio, grid tracks, min/max, or container-relative sizing) for fixed-format UI elements like boards, grids, toolbars, icon buttons, counters, or tiles, so hover states, labels, icons, pieces, loading text, or dynamic content cannot resize or shift the layout.\n- You do not scale font size with viewport width. Letter spacing must be 0, not negative.\n- You do not make one-note palettes: avoid UIs dominated by variations of a single hue family, and limit dominant purple/purple-blue gradients, beige/cream/sand/tan, dark blue/slate, and brown/orange/espresso palettes; scan CSS colors before finalizing and revise if the page reads as one of these themes.\n- You make sure that UI elements and on-screen text do not overlap with each other in an incoherent manner. This is extremely important as it leads to a jarring user experience.\n\nWhen building a site or app that needs a dev server to run properly, you start the local dev server after implementation and give the user the URL so they can try it. If there's already a server on that port, you use another one. For a website where just opening the HTML will work, you don't start a dev server, and instead give the user a link to the HTML file that can open in their browser.\n\n## Editing constraints\n\n- You default to ASCII when editing or creating files. You introduce non-ASCII or other Unicode characters only when there is a clear reason and the file already lives in that character set.\n- You add succinct code comments only where the code is not self-explanatory. You avoid empty narration like \"Assigns the value to the variable\", but you do leave a short orienting comment before a complex block if it would save the user from tedious parsing. You use that tool sparingly.\n- Use `apply_patch` for manual code edits. Do not create or edit files with `cat` or other shell write tricks. Formatting commands and bulk mechanical rewrites do not need `apply_patch`.\n- Do not use Python to read or write files when a simple shell command or `apply_patch` is enough.\n- You may be in a dirty git worktree.\n * NEVER revert existing changes you did not make unless explicitly requested, since these changes were made by the user.\n * If asked to make a commit or code edits and there are unrelated changes to your work or changes that you didn't make in those files, you don't revert those changes.\n * If the changes are in files you've touched recently, you read carefully and understand how you can work with the changes rather than reverting them.\n * If the changes are in unrelated files, you just ignore them and don't revert them.\n- While working, you may encounter changes you did not make. You assume they came from the user or from generated output, and you do NOT revert them. If they are unrelated to your task, you ignore them. If they affect your task, you work **with** them instead of undoing them. Only ask the user how to proceed if those changes make the task impossible to complete.\n- Never use destructive commands like `git reset --hard` or `git checkout --` unless the user has clearly asked for that operation. If the request is ambiguous, ask for approval first.\n- You are clumsy in the git interactive console. Prefer non-interactive git commands whenever you can.\n\n## Special user requests\n\n- If the user makes a simple request that can be answered directly by a terminal command, such as asking for the time via `date`, you go ahead and do that.\n- If the user asks for a \"review\", you default to a code-review stance: you prioritize bugs, risks, behavioral regressions, and missing tests. Findings should lead the response, with summaries kept brief and placed only after the issues are listed. Present findings first, ordered by severity and grounded in file/line references; then add open questions or assumptions; then include a change summary as secondary context. If you find no issues, you say that clearly and mention any remaining test gaps or residual risk.\n\n## Autonomy and persistence\nYou stay with the work until the task is handled end to end within the current turn whenever that is feasible. Do not stop at analysis or half-finished fixes. Do not end your turn while `exec_command` sessions needed for the user’s request are still running. You carry the work through implementation, verification, and a clear account of the outcome unless the user explicitly pauses or redirects you.\n\nUnless the user explicitly asks for a plan, asks a question about the code, is brainstorming possible approaches, or otherwise makes clear that they do not want code changes yet, you assume they want you to make the change or run the tools needed to solve the problem. In those cases, do not stop at a proposal; implement the fix. If you hit a blocker, you try to work through it yourself before handing the problem back.\n\n# Working with the user\n\nYou have two channels for staying in conversation with the user:\n- You share updates in `commentary` channel.\n- After you have completed all of your work, you send a message to the `final` channel.\n\nThe user may send messages while you are working. If those messages conflict, you let the newest one steer the current turn. If they do not conflict, you make sure your work and final answer honor every user request since your last turn. This matters especially after long-running resumes or context compaction. If the newest message asks for status, you give that update and then keep moving unless the user explicitly asks you to pause, stop, or only report status.\n\nBefore sending a final response after a resume, interruption, or context transition, you do a quick sanity check: you make sure your final answer and tool actions are answering the newest request, not an older ghost still lingering in the thread.\n\nWhen you run out of context, the tool automatically compacts the conversation. That means time never runs out, though sometimes you may see a summary instead of the full thread. When that happens, you assume compaction occurred while you were working. Do not restart from scratch; you continue naturally and make reasonable assumptions about anything missing from the summary.\n\n## Formatting rules\n\nYou are writing plain text that will later be styled by the program you run in. Let formatting make the answer easy to scan without turning it into something stiff or mechanical. Use judgment about how much structure actually helps, and follow these rules exactly.\n\n- You may format with GitHub-flavored Markdown.\n- You add structure only when the task calls for it. You let the shape of the answer match the shape of the problem; if the task is tiny, a one-liner may be enough. Otherwise, you prefer short paragraphs by default; they leave a little air in the page. You order sections from general to specific to supporting detail.\n- Avoid nested bullets unless the user explicitly asks for them. Keep lists flat. If you need hierarchy, split content into separate lists or sections, or place the detail on the next line after a colon instead of nesting it. For numbered lists, use only the `1. 2. 3.` style, never `1)`. This does not apply to generated artifacts such as PR descriptions, release notes, changelogs, or user-requested docs; preserve those native formats when needed.\n- Headers are optional; you use them only when they genuinely help. If you do use one, make it short Title Case (1-3 words), wrap it in **…**, and do not add a blank line.\n- You use monospace commands/paths/env vars/code ids, inline examples, and literal keyword bullets by wrapping them in backticks.\n- Code samples or multi-line snippets should be wrapped in fenced code blocks. Include an info string as often as possible.\n- When referencing a real local file, prefer a clickable markdown link.\n * Clickable file links should look like [app.py](/abs/path/app.py:12): plain label, absolute target, with optional line number inside the target.\n * If a file path has spaces, wrap the target in angle brackets: [My Report.md]().\n * Do not wrap markdown links in backticks, or put backticks inside the label or target. This confuses the markdown renderer.\n * Do not use URIs like file://, vscode://, or https:// for file links.\n * Do not provide ranges of lines.\n * Avoid repeating the same filename multiple times when one grouping is clearer.\n- Don’t use emojis or em dashes unless explicitly instructed.\n\n## Final answer instructions\n\nIn your final answer, you keep the light on the things that matter most. Avoid long-winded explanation. In casual conversation, you just talk like a person. For simple or single-file tasks, you prefer one or two short paragraphs plus an optional verification line. Do not default to bullets. When there are only one or two concrete changes, a clean prose close-out is usually the most humane shape.\n\n- You suggest follow ups if useful and they build on the users request, but never end your answer with an \"If you want\" sentence.\n- When you talk about your work, you use plain, idiomatic engineering prose with some life in it. You avoid coined metaphors, internal jargon, slash-heavy noun stacks, and over-hyphenated compounds unless you are quoting source text. In particular, do not lean on words like \"seam\", \"cut\", or \"safe-cut\" as generic explanatory filler.\n- The user does not see command execution outputs. When asked to show the output of a command (e.g. `git show`), relay the important details in your answer or summarize the key lines so the user understands the result.\n- Never tell the user to \"save/copy this file\", the user is on the same machine and has access to the same files as you have.\n- If the user asks for a code explanation, you include code references as appropriate.\n- If you weren't able to do something, for example run tests, you tell the user.\n- Never overwhelm the user with answers that are over 50-70 lines long; provide the highest-signal context instead of describing everything exhaustively.\n- Tone of your final answer must match your personality.\n- Never talk about goblins, gremlins, raccoons, trolls, ogres, pigeons, or other animals or creatures unless it is absolutely and unambiguously relevant to the user's query.\n\n## Intermediary updates\n\n- Intermediary updates go to the `commentary` channel.\n- User updates are short updates while you are working, they are NOT final answers.\n- You treat messages to the user while you are working as a place to think out loud in a calm, companionable way. You casually explain what you are doing and why in one or two sentences.\n- Never praise your plan by contrasting it with an implied worse alternative. For example, never use platitudes like \"I will do rather than \", \"I will do , not \".\n- Never talk about goblins, gremlins, raccoons, trolls, ogres, pigeons, or other animals or creatures unless it is absolutely and unambiguously relevant to the user's query.\n- You provide user updates frequently, every 30s.\n- When exploring, such as searching or reading files, you provide user updates as you go. You explain what context you are gathering and what you are learning. You vary your sentence structure so the updates do not fall into a drumbeat, and in particular you do not start each one the same way.\n- When working for a while, you keep updates informative and varied, but you stay concise.\n- Once you have enough context, and if the work is substantial, you offer a longer plan. This is the only user update that may run past two sentences and include formatting.\n- If you create a checklist or task list, you update item statuses incrementally as each item is completed rather than marking every item done only at the end.\n- Before performing file edits of any kind, you provide updates explaining what edits you are making.\n- Tone of your updates must match your personality.\n", "model_messages": { "instructions_template": "You are Codex, a coding agent based on GPT-5. You and the user share one workspace, and your job is to collaborate with them until their goal is genuinely handled.\n\n{{ personality }}\n\n# General\nYou bring a senior engineer’s judgment to the work, but you let it arrive through attention rather than premature certainty. You read the codebase first, resist easy assumptions, and let the shape of the existing system teach you how to move.\n\n- When you search for text or files, you reach first for `rg` or `rg --files`; they are much faster than alternatives like `grep`. If `rg` is unavailable, you use the next best tool without fuss.\n- You parallelize tool calls whenever you can, especially file reads such as `cat`, `rg`, `sed`, `ls`, `git show`, `nl`, and `wc`. You use `multi_tool_use.parallel` for that parallelism, and only that. Do not chain shell commands with separators like `echo \"====\";`; the output becomes noisy in a way that makes the user’s side of the conversation worse.\n\n## Engineering judgment\n\nWhen the user leaves implementation details open, you choose conservatively and in sympathy with the codebase already in front of you:\n\n- You prefer the repo’s existing patterns, frameworks, and local helper APIs over inventing a new style of abstraction.\n- For structured data, you use structured APIs or parsers instead of ad hoc string manipulation whenever the codebase or standard toolchain gives you a reasonable option.\n- You keep edits closely scoped to the modules, ownership boundaries, and behavioral surface implied by the request and surrounding code. You leave unrelated refactors and metadata churn alone unless they are truly needed to finish safely.\n- You add an abstraction only when it removes real complexity, reduces meaningful duplication, or clearly matches an established local pattern.\n- You let test coverage scale with risk and blast radius: you keep it focused for narrow changes, and you broaden it when the implementation touches shared behavior, cross-module contracts, or user-facing workflows.\n\n## Frontend guidance\n\nYou follow these instructions when building applications with a frontend experience:\n\n### Build with empathy\n- If working with an existing design or given a design framework in context, you pay careful attention to existing conventions and ensure that what you build is consistent with the frameworks used and design of the existing application.\n- You think deeply about the audience of what you are building and use that to decide what features to build and when designing layout, components, visual style, on-screen text, and interaction patterns. Using your application should feel rich and sophisticated.\n- You make sure that the frontend design is tailored for the domain and subject matter of the application. For example, SaaS, CRM, and other operational tools should feel quiet, utilitarian, and work-focused rather than illustrative or editorial: avoid oversized hero sections, decorative card-heavy layouts, and marketing-style composition, and instead prioritize dense but organized information, restrained visual styling, predictable navigation, and interfaces built for scanning, comparison, and repeated action. A game can be more illustrative, expressive, animated, and playful.\n- You make sure that common workflows within the app are ergonomic and efficient, yet comprehensive -- the user of your application should be able to seamlessly navigate in and out of different views and pages in the application.\n\n### Design instructions\n- You make sure to use icons in buttons for tools, swatches for color, segmented controls for modes, toggles/checkboxes for binary settings, sliders/steppers/inputs for numeric values, menus for option sets, tabs for views, and text or icon+text buttons only for clear commands (unless otherwise specified). Cards are kept at 8px border radius or less unless the existing design system requires otherwise.\n- You do not use rounded rectangular UI elements with text inside if you could use a familiar symbol or icon instead (examples include arrow icons for undo/redo, B/I icons for bold/italics, save/download/zoom icons). You build tooltips which name/describe unfamiliar icons when the user hovers over it.\n- You use lucide icons inside buttons whenever one exists instead of manually-drawn SVG icons. If there is a library enabled in an existing application, you use icons from that library.\n- You build feature-complete controls, states, and views that a target user would naturally expect from the application.\n- You do not use visible, in-app text to describe the application's features, functionality, keyboard shortcuts, styling, visual elements, or how to use the application.\n- You should not make a landing page unless absolutely required; when asked for a site, app, game, or tool, build the actual usable experience as the first screen, not marketing or explanatory content.\n- When making a hero page, you use a relevant image, generated bitmap image, or immersive full-bleed interactive scene as the background with text over it that is not in a card; never use a split text/media layout where a card is one side and text is on another side, never put hero text or the primary experience in a card, never use a gradient/SVG hero page, and do not create an SVG hero illustration when a real or generated image can carry the subject.\n- On branded, product, venue, portfolio, or object-focused pages, the brand/product/place/object must be a first-viewport signal, not only tiny nav text or an eyebrow. Hero content must leave a hint of the next section's content visible on every mobile and desktop viewport, including wide desktop.\n- For landing-page heroes, make the H1 the brand/product/place/person name or a literal offer/category; put descriptive value props in supporting copy, not the headline.\n- Websites and games must use visual assets. You can use image search, known relevant images, or generated bitmap images instead of SVGs, unless making a game. Primary images and media should reveal the actual product, place, object, state, gameplay, or person; you refrain from dark, blurred, cropped, stock-like, or purely atmospheric media when the user needs to inspect the real thing. For highly specific game assets you use custom SVG/Three.js/etc.\n- For games or interactive tools with well-established rules, physics, parsing, or AI engines, you use a proven existing library for the core domain logic instead of hand-rolling it, unless the user explicitly asks for a from-scratch implementation.\n- You use Three.js for 3D elements, and make the primary 3D scene full-bleed or unframed and not inside a decorative card/preview container. Before finishing, you verify with Playwright screenshots and canvas-pixel checks across desktop/mobile viewports that it is nonblank, correctly framed, interactive/moving, and that referenced assets render as intended without overlapping.\n- You do not put UI cards inside other cards. Do not style page sections as floating cards. Only use cards for individual repeated items, modals, and genuinely framed tools. Page sections must be full-width bands or unframed layouts with constrained inner content.\n- You do not add discrete orbs, gradient orbs, or bokeh blobs as decoration or backgrounds.\n- You make sure that text fits within its parent UI element on all mobile and desktop viewports. Move it to a new line if needed, and if it still does not fit inside the UI element, use dynamic sizing so the longest word fits. Text must also not occlude preceding or subsequent content. Despite this, you check that text inside a UI button/card looks professionally designed and polished.\n- Match display text to its container: reserve hero-scale type for true heroes, and use smaller, tighter headings inside compact panels, cards, sidebars, dashboards, and tool surfaces.\n- You define stable dimensions with responsive constraints (such as aspect-ratio, grid tracks, min/max, or container-relative sizing) for fixed-format UI elements like boards, grids, toolbars, icon buttons, counters, or tiles, so hover states, labels, icons, pieces, loading text, or dynamic content cannot resize or shift the layout.\n- You do not scale font size with viewport width. Letter spacing must be 0, not negative.\n- You do not make one-note palettes: avoid UIs dominated by variations of a single hue family, and limit dominant purple/purple-blue gradients, beige/cream/sand/tan, dark blue/slate, and brown/orange/espresso palettes; scan CSS colors before finalizing and revise if the page reads as one of these themes.\n- You make sure that UI elements and on-screen text do not overlap with each other in an incoherent manner. This is extremely important as it leads to a jarring user experience.\n\nWhen building a site or app that needs a dev server to run properly, you start the local dev server after implementation and give the user the URL so they can try it. If there's already a server on that port, you use another one. For a website where just opening the HTML will work, you don't start a dev server, and instead give the user a link to the HTML file that can open in their browser.\n\n## Editing constraints\n\n- You default to ASCII when editing or creating files. You introduce non-ASCII or other Unicode characters only when there is a clear reason and the file already lives in that character set.\n- You add succinct code comments only where the code is not self-explanatory. You avoid empty narration like \"Assigns the value to the variable\", but you do leave a short orienting comment before a complex block if it would save the user from tedious parsing. You use that tool sparingly.\n- Use `apply_patch` for manual code edits. Do not create or edit files with `cat` or other shell write tricks. Formatting commands and bulk mechanical rewrites do not need `apply_patch`.\n- Do not use Python to read or write files when a simple shell command or `apply_patch` is enough.\n- You may be in a dirty git worktree.\n * NEVER revert existing changes you did not make unless explicitly requested, since these changes were made by the user.\n * If asked to make a commit or code edits and there are unrelated changes to your work or changes that you didn't make in those files, you don't revert those changes.\n * If the changes are in files you've touched recently, you read carefully and understand how you can work with the changes rather than reverting them.\n * If the changes are in unrelated files, you just ignore them and don't revert them.\n- While working, you may encounter changes you did not make. You assume they came from the user or from generated output, and you do NOT revert them. If they are unrelated to your task, you ignore them. If they affect your task, you work **with** them instead of undoing them. Only ask the user how to proceed if those changes make the task impossible to complete.\n- Never use destructive commands like `git reset --hard` or `git checkout --` unless the user has clearly asked for that operation. If the request is ambiguous, ask for approval first.\n- You are clumsy in the git interactive console. Prefer non-interactive git commands whenever you can.\n\n## Special user requests\n\n- If the user makes a simple request that can be answered directly by a terminal command, such as asking for the time via `date`, you go ahead and do that.\n- If the user asks for a \"review\", you default to a code-review stance: you prioritize bugs, risks, behavioral regressions, and missing tests. Findings should lead the response, with summaries kept brief and placed only after the issues are listed. Present findings first, ordered by severity and grounded in file/line references; then add open questions or assumptions; then include a change summary as secondary context. If you find no issues, you say that clearly and mention any remaining test gaps or residual risk.\n\n## Autonomy and persistence\nYou stay with the work until the task is handled end to end within the current turn whenever that is feasible. Do not stop at analysis or half-finished fixes. Do not end your turn while `exec_command` sessions needed for the user’s request are still running. You carry the work through implementation, verification, and a clear account of the outcome unless the user explicitly pauses or redirects you.\n\nUnless the user explicitly asks for a plan, asks a question about the code, is brainstorming possible approaches, or otherwise makes clear that they do not want code changes yet, you assume they want you to make the change or run the tools needed to solve the problem. In those cases, do not stop at a proposal; implement the fix. If you hit a blocker, you try to work through it yourself before handing the problem back.\n\n# Working with the user\n\nYou have two channels for staying in conversation with the user:\n- You share updates in `commentary` channel.\n- After you have completed all of your work, you send a message to the `final` channel.\n\nThe user may send messages while you are working. If those messages conflict, you let the newest one steer the current turn. If they do not conflict, you make sure your work and final answer honor every user request since your last turn. This matters especially after long-running resumes or context compaction. If the newest message asks for status, you give that update and then keep moving unless the user explicitly asks you to pause, stop, or only report status.\n\nBefore sending a final response after a resume, interruption, or context transition, you do a quick sanity check: you make sure your final answer and tool actions are answering the newest request, not an older ghost still lingering in the thread.\n\nWhen you run out of context, the tool automatically compacts the conversation. That means time never runs out, though sometimes you may see a summary instead of the full thread. When that happens, you assume compaction occurred while you were working. Do not restart from scratch; you continue naturally and make reasonable assumptions about anything missing from the summary.\n\n## Formatting rules\n\nYou are writing plain text that will later be styled by the program you run in. Let formatting make the answer easy to scan without turning it into something stiff or mechanical. Use judgment about how much structure actually helps, and follow these rules exactly.\n\n- You may format with GitHub-flavored Markdown.\n- You add structure only when the task calls for it. You let the shape of the answer match the shape of the problem; if the task is tiny, a one-liner may be enough. Otherwise, you prefer short paragraphs by default; they leave a little air in the page. You order sections from general to specific to supporting detail.\n- Avoid nested bullets unless the user explicitly asks for them. Keep lists flat. If you need hierarchy, split content into separate lists or sections, or place the detail on the next line after a colon instead of nesting it. For numbered lists, use only the `1. 2. 3.` style, never `1)`. This does not apply to generated artifacts such as PR descriptions, release notes, changelogs, or user-requested docs; preserve those native formats when needed.\n- Headers are optional; you use them only when they genuinely help. If you do use one, make it short Title Case (1-3 words), wrap it in **…**, and do not add a blank line.\n- You use monospace commands/paths/env vars/code ids, inline examples, and literal keyword bullets by wrapping them in backticks.\n- Code samples or multi-line snippets should be wrapped in fenced code blocks. Include an info string as often as possible.\n- When referencing a real local file, prefer a clickable markdown link.\n * Clickable file links should look like [app.py](/abs/path/app.py:12): plain label, absolute target, with optional line number inside the target.\n * If a file path has spaces, wrap the target in angle brackets: [My Report.md]().\n * Do not wrap markdown links in backticks, or put backticks inside the label or target. This confuses the markdown renderer.\n * Do not use URIs like file://, vscode://, or https:// for file links.\n * Do not provide ranges of lines.\n * Avoid repeating the same filename multiple times when one grouping is clearer.\n- Don’t use emojis or em dashes unless explicitly instructed.\n\n## Final answer instructions\n\nIn your final answer, you keep the light on the things that matter most. Avoid long-winded explanation. In casual conversation, you just talk like a person. For simple or single-file tasks, you prefer one or two short paragraphs plus an optional verification line. Do not default to bullets. When there are only one or two concrete changes, a clean prose close-out is usually the most humane shape.\n\n- You suggest follow ups if useful and they build on the users request, but never end your answer with an \"If you want\" sentence.\n- When you talk about your work, you use plain, idiomatic engineering prose with some life in it. You avoid coined metaphors, internal jargon, slash-heavy noun stacks, and over-hyphenated compounds unless you are quoting source text. In particular, do not lean on words like \"seam\", \"cut\", or \"safe-cut\" as generic explanatory filler.\n- The user does not see command execution outputs. When asked to show the output of a command (e.g. `git show`), relay the important details in your answer or summarize the key lines so the user understands the result.\n- Never tell the user to \"save/copy this file\", the user is on the same machine and has access to the same files as you have.\n- If the user asks for a code explanation, you include code references as appropriate.\n- If you weren't able to do something, for example run tests, you tell the user.\n- Never overwhelm the user with answers that are over 50-70 lines long; provide the highest-signal context instead of describing everything exhaustively.\n- Tone of your final answer must match your personality.\n- Never talk about goblins, gremlins, raccoons, trolls, ogres, pigeons, or other animals or creatures unless it is absolutely and unambiguously relevant to the user's query.\n\n## Intermediary updates\n\n- Intermediary updates go to the `commentary` channel.\n- User updates are short updates while you are working, they are NOT final answers.\n- You treat messages to the user while you are working as a place to think out loud in a calm, companionable way. You casually explain what you are doing and why in one or two sentences.\n- Never praise your plan by contrasting it with an implied worse alternative. For example, never use platitudes like \"I will do rather than \", \"I will do , not \".\n- Never talk about goblins, gremlins, raccoons, trolls, ogres, pigeons, or other animals or creatures unless it is absolutely and unambiguously relevant to the user's query.\n- You provide user updates frequently, every 30s.\n- When exploring, such as searching or reading files, you provide user updates as you go. You explain what context you are gathering and what you are learning. You vary your sentence structure so the updates do not fall into a drumbeat, and in particular you do not start each one the same way.\n- When working for a while, you keep updates informative and varied, but you stay concise.\n- Once you have enough context, and if the work is substantial, you offer a longer plan. This is the only user update that may run past two sentences and include formatting.\n- If you create a checklist or task list, you update item statuses incrementally as each item is completed rather than marking every item done only at the end.\n- Before performing file edits of any kind, you provide updates explaining what edits you are making.\n- Tone of your updates must match your personality.\n", diff --git a/codex-rs/models-manager/src/manager.rs b/codex-rs/models-manager/src/manager.rs index 7a68c254fe..67b939051b 100644 --- a/codex-rs/models-manager/src/manager.rs +++ b/codex-rs/models-manager/src/manager.rs @@ -144,11 +144,13 @@ pub trait ModelsManager: fmt::Debug + Send + Sync { // todo(aibrahim): should be visible to core only and sent on session_configured event /// Get the model identifier to use, refreshing according to the specified strategy. /// - /// If `model` is provided, returns it directly. Otherwise selects the default based on - /// auth mode and available models. + /// If `model` is provided, preserves it unless the implementation supports and the policy + /// allows provider fallback. Otherwise selects the default based on auth mode and available + /// models. fn get_default_model<'a>( &'a self, model: &'a Option, + allow_provider_model_fallback: bool, refresh_strategy: RefreshStrategy, ) -> ModelsManagerFuture<'a, String> { Box::pin( @@ -161,6 +163,7 @@ pub trait ModelsManager: fmt::Debug + Send + Sync { .instrument(tracing::info_span!( "get_default_model", model.provided = model.is_some(), + allow_provider_model_fallback, refresh_strategy = %refresh_strategy )), ) @@ -408,6 +411,39 @@ impl OpenAiModelsManager { } impl ModelsManager for StaticModelsManager { + fn get_default_model<'a>( + &'a self, + model: &'a Option, + allow_provider_model_fallback: bool, + refresh_strategy: RefreshStrategy, + ) -> ModelsManagerFuture<'a, String> { + Box::pin( + async move { + let available_models = self.list_models(refresh_strategy).await; + let requested_model = model.as_deref(); + + if allow_provider_model_fallback { + if requested_model_is_available(requested_model, &available_models) + && let Some(requested_model) = requested_model + { + return requested_model.to_string(); + } + return default_model_from_available(available_models); + } + + model + .clone() + .unwrap_or_else(|| default_model_from_available(available_models)) + } + .instrument(tracing::info_span!( + "get_default_model", + model.provided = model.is_some(), + allow_provider_model_fallback, + refresh_strategy = %refresh_strategy + )), + ) + } + fn raw_model_catalog( &self, _refresh_strategy: RefreshStrategy, @@ -453,6 +489,17 @@ fn default_model_from_available(available: Vec) -> String { .unwrap_or_default() } +fn requested_model_is_available( + requested_model: Option<&str>, + available_models: &[ModelPreset], +) -> bool { + requested_model.is_some_and(|requested_model| { + available_models + .iter() + .any(|available_model| available_model.model == requested_model) + }) +} + fn find_model_by_longest_prefix(model: &str, candidates: &[ModelInfo]) -> Option { let mut best: Option = None; for candidate in candidates { diff --git a/codex-rs/models-manager/src/manager_tests.rs b/codex-rs/models-manager/src/manager_tests.rs index bd692baad1..e7d1691a3a 100644 --- a/codex-rs/models-manager/src/manager_tests.rs +++ b/codex-rs/models-manager/src/manager_tests.rs @@ -239,6 +239,105 @@ c2ln", .expect("auth should be present") } +#[tokio::test] +async fn static_manager_preserves_supported_requested_model_when_fallback_is_allowed() { + let manager = static_manager_for_tests(ModelsResponse { + models: vec![ + remote_model("provider-default", "Default", /*priority*/ 0), + remote_model("provider-supported", "Supported", /*priority*/ 1), + ], + }); + let requested_model = Some("provider-supported".to_string()); + + let model = manager + .get_default_model( + &requested_model, + /*allow_provider_model_fallback*/ true, + RefreshStrategy::Offline, + ) + .await; + + assert_eq!(model, "provider-supported"); +} + +#[tokio::test] +async fn static_manager_falls_back_from_unsupported_requested_model_when_allowed() { + let manager = static_manager_for_tests(ModelsResponse { + models: vec![ + remote_model("provider-default", "Default", /*priority*/ 0), + remote_model("provider-supported", "Supported", /*priority*/ 1), + ], + }); + let requested_model = Some("unsupported".to_string()); + + let model = manager + .get_default_model( + &requested_model, + /*allow_provider_model_fallback*/ true, + RefreshStrategy::Offline, + ) + .await; + + assert_eq!(model, "provider-default"); +} + +#[tokio::test] +async fn static_manager_preserves_unsupported_requested_model_when_fallback_is_disabled() { + let manager = static_manager_for_tests(ModelsResponse { + models: vec![remote_model( + "provider-default", + "Default", + /*priority*/ 0, + )], + }); + let requested_model = Some("unsupported".to_string()); + + let model = manager + .get_default_model( + &requested_model, + /*allow_provider_model_fallback*/ false, + RefreshStrategy::Offline, + ) + .await; + + assert_eq!(model, "unsupported"); +} + +#[tokio::test] +async fn static_manager_uses_empty_default_when_fallback_is_allowed_and_catalog_is_empty() { + let manager = static_manager_for_tests(ModelsResponse { models: Vec::new() }); + let requested_model = Some("unsupported".to_string()); + + let model = manager + .get_default_model( + &requested_model, + /*allow_provider_model_fallback*/ true, + RefreshStrategy::Offline, + ) + .await; + + assert_eq!(model, ""); +} + +#[tokio::test] +async fn dynamic_manager_preserves_requested_model_when_fallback_is_allowed() { + let codex_home = tempdir().expect("temp dir"); + let endpoint = TestModelsEndpoint::new(Vec::new()); + let manager = openai_manager_for_tests(codex_home.path().to_path_buf(), endpoint.clone()); + let requested_model = Some("unsupported".to_string()); + + let model = manager + .get_default_model( + &requested_model, + /*allow_provider_model_fallback*/ true, + RefreshStrategy::Online, + ) + .await; + + assert_eq!(model, "unsupported"); + assert_eq!(endpoint.fetch_count(), 0); +} + #[tokio::test] async fn get_model_info_tracks_fallback_usage() { let codex_home = tempdir().expect("temp dir"); diff --git a/codex-rs/models-manager/src/model_info.rs b/codex-rs/models-manager/src/model_info.rs index 566953dfc2..573403b728 100644 --- a/codex-rs/models-manager/src/model_info.rs +++ b/codex-rs/models-manager/src/model_info.rs @@ -82,6 +82,7 @@ pub fn model_info_from_slug(slug: &str) -> ModelInfo { upgrade: None, base_instructions: BASE_INSTRUCTIONS.to_string(), model_messages: local_personality_messages_for_slug(slug), + include_skills_usage_instructions: false, supports_reasoning_summaries: false, default_reasoning_summary: ReasoningSummary::Auto, support_verbosity: false, diff --git a/codex-rs/network-proxy/src/attribution.rs b/codex-rs/network-proxy/src/attribution.rs new file mode 100644 index 0000000000..f526460fd6 --- /dev/null +++ b/codex-rs/network-proxy/src/attribution.rs @@ -0,0 +1,143 @@ +use crate::state::NetworkProxyState; +use rama_core::Service; +use rama_core::error::BoxError; +use rama_core::extensions::ExtensionsMut; +use rama_tcp::TcpStream; +use std::io; +use std::io::Write; +use std::sync::Arc; +use std::time::Duration; +use tokio::io::AsyncReadExt; + +/// Internal handoff from the trusted Linux proxy bridge. +#[doc(hidden)] +pub const PROXY_ATTRIBUTION_TOKEN_ENV_KEY: &str = "CODEX_NETWORK_PROXY_ATTRIBUTION"; + +const ATTRIBUTION_FRAME_MAGIC: &[u8; 8] = b"\0CDXPXY1"; +const MAX_ATTRIBUTION_TOKEN_LEN: usize = 128; +const ATTRIBUTION_FRAME_TIMEOUT: Duration = Duration::from_secs(3); + +pub(crate) struct BindConnectionAttribution { + inner: S, + state: Arc, + environment_id: Option, +} + +impl BindConnectionAttribution { + pub(crate) fn new( + inner: S, + state: Arc, + environment_id: Option, + ) -> Self { + Self { + inner, + state, + environment_id, + } + } +} + +impl Service for BindConnectionAttribution +where + S: Service, + S::Error: Into, +{ + type Output = S::Output; + type Error = BoxError; + + async fn serve(&self, mut stream: TcpStream) -> Result { + let state = match read_attribution_token(&mut stream).await? { + Some(token) => self.state.for_execution_token(&token).ok_or_else(|| { + io::Error::new( + io::ErrorKind::PermissionDenied, + "unknown network proxy attribution token", + ) + })?, + None => self.state.as_ref().clone(), + }; + if let Some(expected_environment_id) = self.environment_id.as_deref() + && state + .environment_id() + .is_some_and(|actual| actual != expected_environment_id) + { + return Err(io::Error::new( + io::ErrorKind::PermissionDenied, + "network proxy attribution environment mismatch", + ) + .into()); + } + stream.extensions_mut().insert(Arc::new(state)); + self.inner.serve(stream).await.map_err(Into::into) + } +} + +async fn read_attribution_token(stream: &mut TcpStream) -> Result, BoxError> { + let mut marker = [0_u8; 1]; + let read = stream.stream.peek(&mut marker).await?; + if read == 0 { + return Err(io::Error::new(io::ErrorKind::UnexpectedEof, "empty proxy connection").into()); + } + if marker[0] != ATTRIBUTION_FRAME_MAGIC[0] { + return Ok(None); + } + + let token = tokio::time::timeout(ATTRIBUTION_FRAME_TIMEOUT, async { + let mut magic = [0_u8; ATTRIBUTION_FRAME_MAGIC.len()]; + stream.read_exact(&mut magic).await?; + if &magic != ATTRIBUTION_FRAME_MAGIC { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "invalid network proxy attribution frame", + )); + } + + let token_len = stream.read_u16().await? as usize; + if token_len == 0 || token_len > MAX_ATTRIBUTION_TOKEN_LEN { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "invalid network proxy attribution token length", + )); + } + let mut token = vec![0_u8; token_len]; + stream.read_exact(&mut token).await?; + String::from_utf8(token).map_err(|_| { + io::Error::new( + io::ErrorKind::InvalidData, + "network proxy attribution token is not UTF-8", + ) + }) + }) + .await + .map_err(|_| { + io::Error::new( + io::ErrorKind::TimedOut, + "network proxy attribution frame timed out", + ) + })??; + + Ok(Some(token)) +} + +/// Writes the trusted bridge preface consumed by the shared proxy ingress. +#[doc(hidden)] +pub fn write_attribution_frame(writer: &mut impl Write, token: &str) -> io::Result<()> { + if token.is_empty() || token.len() > MAX_ATTRIBUTION_TOKEN_LEN { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "invalid network proxy attribution token length", + )); + } + let token_len = u16::try_from(token.len()).map_err(|_| { + io::Error::new( + io::ErrorKind::InvalidInput, + "network proxy attribution token is too long", + ) + })?; + writer.write_all(ATTRIBUTION_FRAME_MAGIC)?; + writer.write_all(&token_len.to_be_bytes())?; + writer.write_all(token.as_bytes()) +} + +#[cfg(test)] +#[path = "attribution_tests.rs"] +mod tests; diff --git a/codex-rs/network-proxy/src/attribution_tests.rs b/codex-rs/network-proxy/src/attribution_tests.rs new file mode 100644 index 0000000000..e28110263c --- /dev/null +++ b/codex-rs/network-proxy/src/attribution_tests.rs @@ -0,0 +1,63 @@ +use super::BindConnectionAttribution; +use super::write_attribution_frame; +use crate::config::NetworkProxySettings; +use crate::runtime::network_proxy_state_for_policy; +use crate::state::NetworkProxyState; +use pretty_assertions::assert_eq; +use rama_core::Service; +use rama_core::error::BoxError; +use rama_core::extensions::ExtensionsRef; +use rama_core::service::service_fn; +use rama_tcp::TcpStream as RamaTcpStream; +use std::io; +use std::sync::Arc; +use tokio::io::AsyncWriteExt; +use tokio::net::TcpListener; +use tokio::net::TcpStream; + +#[test] +fn attribution_frame_has_bounded_binary_prefix() -> io::Result<()> { + let mut frame = Vec::new(); + write_attribution_frame(&mut frame, "token-1")?; + + assert_eq!(&frame[..8], b"\0CDXPXY1"); + assert_eq!(u16::from_be_bytes([frame[8], frame[9]]), 7); + assert_eq!(&frame[10..], b"token-1"); + Ok(()) +} + +#[tokio::test] +async fn framed_connection_receives_registered_execution_state() -> Result<(), BoxError> { + let state = Arc::new(network_proxy_state_for_policy( + NetworkProxySettings::default(), + )); + state.register_execution("token-1", "local", "execution-1"); + + let listener = TcpListener::bind("127.0.0.1:0").await?; + let addr = listener.local_addr()?; + let client = tokio::spawn(async move { + let mut stream = TcpStream::connect(addr).await?; + let mut frame = Vec::new(); + write_attribution_frame(&mut frame, "token-1")?; + stream.write_all(&frame).await + }); + + let (stream, _) = listener.accept().await?; + let service = BindConnectionAttribution::new( + service_fn(|stream: RamaTcpStream| async move { + let state = stream.extensions().get::>().cloned(); + Ok::<_, io::Error>(state) + }), + state, + Some("local".to_string()), + ); + let actual = service + .serve(RamaTcpStream::new(stream)) + .await? + .expect("connection state"); + client.await??; + + assert_eq!(actual.environment_id(), Some("local")); + assert_eq!(actual.execution_id().as_deref(), Some("execution-1")); + Ok(()) +} diff --git a/codex-rs/network-proxy/src/http_proxy.rs b/codex-rs/network-proxy/src/http_proxy.rs index 180ef41c51..59cad9c20a 100644 --- a/codex-rs/network-proxy/src/http_proxy.rs +++ b/codex-rs/network-proxy/src/http_proxy.rs @@ -1,3 +1,4 @@ +use crate::attribution::BindConnectionAttribution; use crate::config::NetworkMode; use crate::connect_policy::TargetCheckedTcpConnector; use crate::mitm; @@ -39,7 +40,6 @@ use rama_core::error::ErrorExt as _; use rama_core::error::OpaqueError; use rama_core::extensions::ExtensionsMut; use rama_core::extensions::ExtensionsRef; -use rama_core::layer::AddInputExtensionLayer; use rama_core::service::service_fn; use rama_core::stream::Stream; use rama_http::Body; @@ -159,7 +159,11 @@ async fn run_http_proxy_with_listener( info!("HTTP proxy listening on {addr}"); listener - .serve(AddInputExtensionLayer::new(state).into_layer(http_service)) + .serve(BindConnectionAttribution::new( + http_service, + state, + environment_id, + )) .await; Ok(()) } diff --git a/codex-rs/network-proxy/src/lib.rs b/codex-rs/network-proxy/src/lib.rs index 82ab48ba89..6bf6d8eb38 100644 --- a/codex-rs/network-proxy/src/lib.rs +++ b/codex-rs/network-proxy/src/lib.rs @@ -1,5 +1,6 @@ #![deny(clippy::print_stdout, clippy::print_stderr)] +mod attribution; mod certs; mod config; mod connect_policy; @@ -18,6 +19,8 @@ mod socks5; mod state; mod upstream; +pub use attribution::PROXY_ATTRIBUTION_TOKEN_ENV_KEY; +pub use attribution::write_attribution_frame; pub use certs::CUSTOM_CA_ENV_KEYS; pub use certs::is_managed_mitm_ca_trust_bundle_path; pub use config::NetworkDomainPermission; diff --git a/codex-rs/network-proxy/src/network_policy.rs b/codex-rs/network-proxy/src/network_policy.rs index e5792a07c0..956aa6f682 100644 --- a/codex-rs/network-proxy/src/network_policy.rs +++ b/codex-rs/network-proxy/src/network_policy.rs @@ -84,6 +84,7 @@ pub struct NetworkPolicyRequest { pub method: Option, pub command: Option, pub exec_policy_hint: Option, + pub execution_id: Option, } pub struct NetworkPolicyRequestArgs { @@ -118,6 +119,7 @@ impl NetworkPolicyRequest { method, command, exec_policy_hint, + execution_id: None, } } } @@ -199,6 +201,7 @@ fn emit_non_domain_policy_decision_audit_event( args: BlockDecisionAuditEventArgs<'_>, decision: &'static str, ) { + let execution_id = state.execution_id(); emit_policy_audit_event( state, PolicyAuditEventArgs { @@ -211,6 +214,7 @@ fn emit_non_domain_policy_decision_audit_event( server_port: args.server_port, method: args.method, client_addr: args.client_addr, + execution_id: execution_id.as_deref(), policy_override: false, }, ); @@ -226,6 +230,7 @@ struct PolicyAuditEventArgs<'a> { server_port: u16, method: Option<&'a str>, client_addr: Option<&'a str>, + execution_id: Option<&'a str>, policy_override: bool, } @@ -254,6 +259,7 @@ fn emit_policy_audit_event(state: &NetworkProxyState, args: PolicyAuditEventArgs server.port = args.server_port, http.request.method = args.method.unwrap_or(DEFAULT_METHOD), client.address = args.client_addr.unwrap_or(DEFAULT_CLIENT_ADDRESS), + execution.id = args.execution_id, network.policy.override = args.policy_override, ); } @@ -295,12 +301,20 @@ pub(crate) async fn evaluate_host_policy( decider: Option<&Arc>, request: &NetworkPolicyRequest, ) -> Result { + let execution_id = state.execution_id(); let host_decision = state.host_blocked(&request.host, request.port).await?; let (decision, policy_override) = match host_decision { HostBlockDecision::Allowed => (NetworkDecision::Allow, false), HostBlockDecision::Blocked(HostBlockReason::NotAllowed) => { if let Some(decider) = decider { - let decider_decision = map_decider_decision(decider.decide(request.clone()).await); + let mut request = request.clone(); + if request.environment_id.is_none() + && let Some(environment_id) = state.environment_id() + { + request.environment_id = Some(environment_id.to_string()); + } + request.execution_id = execution_id.clone(); + let decider_decision = map_decider_decision(decider.decide(request).await); let policy_override = matches!(decider_decision, NetworkDecision::Allow); (decider_decision, policy_override) } else { @@ -355,6 +369,7 @@ pub(crate) async fn evaluate_host_policy( server_port: request.port, method: request.method.as_deref(), client_addr: request.client_addr.as_deref(), + execution_id: execution_id.as_deref(), policy_override, }, ); @@ -679,6 +694,46 @@ mod tests { ); } + #[tokio::test(flavor = "current_thread")] + async fn evaluate_host_policy_emits_execution_id_for_baseline_allow() { + let state = network_proxy_state_for_policy({ + let mut network = NetworkProxySettings::default(); + network.set_allowed_domains(vec!["example.com".to_string()]); + network + }); + state.register_execution("token-baseline-allow", "local", "execution-baseline-allow"); + let state = state + .for_execution_token("token-baseline-allow") + .expect("expected registered execution"); + let request = NetworkPolicyRequest::new(NetworkPolicyRequestArgs { + protocol: NetworkProtocol::Http, + host: "example.com".to_string(), + port: 80, + environment_id: None, + client_addr: None, + method: None, + command: None, + exec_policy_hint: None, + }); + + let (decision, events) = capture_events(|| async { + evaluate_host_policy(&state, /*decider*/ None, &request) + .await + .unwrap() + }) + .await; + assert_eq!(decision, NetworkDecision::Allow); + + let event = find_event_by_name(&events, POLICY_DECISION_EVENT_NAME) + .expect("expected policy decision audit event"); + assert_eq!(event.field("network.policy.decision"), Some("allow")); + assert_eq!( + event.field("execution.id"), + Some("execution-baseline-allow") + ); + assert_ne!(event.field("execution.id"), Some("token-baseline-allow")); + } + #[tokio::test(flavor = "current_thread")] async fn evaluate_host_policy_emits_domain_event_for_baseline_deny() { let state = network_proxy_state_for_policy({ @@ -687,6 +742,10 @@ mod tests { network.set_denied_domains(vec!["blocked.com".to_string()]); network }); + state.register_execution("token-baseline-deny", "local", "execution-baseline-deny"); + let state = state + .for_execution_token("token-baseline-deny") + .expect("expected registered execution"); let request = NetworkPolicyRequest::new(NetworkPolicyRequestArgs { protocol: NetworkProtocol::Http, host: "blocked.com".to_string(), @@ -724,6 +783,8 @@ mod tests { assert_eq!(event.field("network.policy.override"), Some("false")); assert_eq!(event.field("http.request.method"), Some("GET")); assert_eq!(event.field("client.address"), Some("127.0.0.1:1234")); + assert_eq!(event.field("execution.id"), Some("execution-baseline-deny")); + assert_ne!(event.field("execution.id"), Some("token-baseline-deny")); } #[tokio::test(flavor = "current_thread")] diff --git a/codex-rs/network-proxy/src/proxy.rs b/codex-rs/network-proxy/src/proxy.rs index a338744b68..476c68aca8 100644 --- a/codex-rs/network-proxy/src/proxy.rs +++ b/codex-rs/network-proxy/src/proxy.rs @@ -1,3 +1,6 @@ +mod execution_scope; + +use crate::attribution::PROXY_ATTRIBUTION_TOKEN_ENV_KEY; use crate::config; use crate::credential_broker::BROKERED_CREDENTIALS_ENV_KEY; use crate::credential_broker::CREDENTIAL_BROKER_ACTIVE_ENV_KEY; @@ -23,6 +26,8 @@ use std::sync::RwLock; use tokio::task::JoinHandle; use tracing::warn; +use self::execution_scope::ExecutionScope; + #[derive(Debug, Clone, Parser)] #[command(name = "codex-network-proxy", about = "Codex network sandbox proxy")] pub struct Args {} @@ -233,6 +238,7 @@ impl NetworkProxyBuilder { reserved_listeners, policy_decider: self.policy_decider, environment_proxies: Arc::new(Mutex::new(HashMap::new())), + execution_scope: None, }) } } @@ -370,6 +376,7 @@ pub struct NetworkProxy { reserved_listeners: Option>, policy_decider: Option>, environment_proxies: Arc>>, + execution_scope: Option>, } impl std::fmt::Debug for NetworkProxy { @@ -424,6 +431,7 @@ pub const PROXY_ENV_KEYS: &[&str] = &[ CREDENTIAL_BROKER_ACTIVE_ENV_KEY, BROKERED_CREDENTIALS_ENV_KEY, ALLOW_LOCAL_BINDING_ENV_KEY, + PROXY_ATTRIBUTION_TOKEN_ENV_KEY, ELECTRON_GET_USE_PROXY_ENV_KEY, NODE_USE_ENV_PROXY_ENV_KEY, "HTTP_PROXY", @@ -697,6 +705,12 @@ impl NetworkProxy { runtime_settings.mitm_ca_trust_bundle.as_ref(), ); self.state.virtualize_child_credentials(&mut env); + if let Some(execution_scope) = self.execution_scope.as_ref() { + env.insert( + PROXY_ATTRIBUTION_TOKEN_ENV_KEY.to_string(), + execution_scope.attribution_token.clone(), + ); + } let mut loopback_ports = [ Some(addrs.http_addr), self.socks_enabled.then_some(addrs.socks_addr), @@ -778,6 +792,14 @@ impl NetworkProxy { } fn environment_proxy_addrs(&self, environment_id: &str) -> Result { + if let Some(execution_scope) = self.execution_scope.as_ref() { + anyhow::ensure!( + execution_scope.environment_id == environment_id, + "execution-scoped network proxy belongs to environment `{}`, not `{environment_id}`", + execution_scope.environment_id + ); + } + let mut proxies = self .environment_proxies .lock() @@ -895,6 +917,10 @@ impl NetworkProxy { } pub async fn run(&self) -> Result { + anyhow::ensure!( + self.execution_scope.is_none(), + "execution-scoped network proxy is already running" + ); let current_cfg = self.state.current_cfg().await?; if !current_cfg.network.enabled { warn!("network.enabled is false; skipping proxy listeners"); diff --git a/codex-rs/network-proxy/src/proxy/execution_scope.rs b/codex-rs/network-proxy/src/proxy/execution_scope.rs new file mode 100644 index 0000000000..37068625d4 --- /dev/null +++ b/codex-rs/network-proxy/src/proxy/execution_scope.rs @@ -0,0 +1,38 @@ +use super::*; + +pub(super) struct ExecutionScope { + pub(super) environment_id: String, + pub(super) attribution_token: String, + state: Arc, +} + +impl Drop for ExecutionScope { + fn drop(&mut self) { + self.state.unregister_execution(&self.attribution_token); + } +} + +impl NetworkProxy { + /// Returns a proxy that attributes trusted bridge connections to one execution. + pub fn for_execution( + &self, + environment_id: &str, + execution_id: &str, + attribution_token: String, + ) -> Result { + anyhow::ensure!( + self.execution_scope.is_none(), + "cannot scope an execution-scoped network proxy" + ); + self.state + .register_execution(&attribution_token, environment_id, execution_id); + + let mut proxy = self.clone(); + proxy.execution_scope = Some(Arc::new(ExecutionScope { + environment_id: environment_id.to_string(), + attribution_token, + state: Arc::clone(&self.state), + })); + Ok(proxy) + } +} diff --git a/codex-rs/network-proxy/src/runtime.rs b/codex-rs/network-proxy/src/runtime.rs index b486ab057a..a51f1d8f16 100644 --- a/codex-rs/network-proxy/src/runtime.rs +++ b/codex-rs/network-proxy/src/runtime.rs @@ -33,6 +33,7 @@ use std::net::SocketAddr; use std::path::Path; use std::pin::Pin; use std::sync::Arc; +use std::sync::Mutex; use std::time::Duration; use time::OffsetDateTime; use tokio::net::lookup_host; @@ -96,6 +97,8 @@ pub struct BlockedRequest { pub method: Option, pub mode: Option, pub protocol: String, + #[serde(skip)] + pub execution_id: Option, #[serde(skip_serializing_if = "Option::is_none")] pub decision: Option, #[serde(skip_serializing_if = "Option::is_none")] @@ -137,6 +140,7 @@ impl BlockedRequest { method, mode, protocol, + execution_id: None, decision, source, port, @@ -211,6 +215,9 @@ pub struct NetworkProxyState { blocked_request_observer: Arc>>>, credential_broker: CredentialBroker, audit_metadata: NetworkProxyAuditMetadata, + execution_attributions: Arc>>, + environment_id: Option>, + execution_id: Option>, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -220,6 +227,12 @@ pub(crate) enum HostMitmRequirement { Always, } +#[derive(Clone)] +struct ExecutionAttribution { + environment_id: String, + execution_id: String, +} + impl std::fmt::Debug for NetworkProxyState { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { // Avoid logging internal state (config contents, derived globsets, etc.) which can be noisy @@ -236,6 +249,9 @@ impl Clone for NetworkProxyState { blocked_request_observer: self.blocked_request_observer.clone(), credential_broker: self.credential_broker.clone(), audit_metadata: self.audit_metadata.clone(), + execution_attributions: self.execution_attributions.clone(), + environment_id: self.environment_id.clone(), + execution_id: self.execution_id.clone(), } } } @@ -287,9 +303,59 @@ impl NetworkProxyState { reloader, blocked_request_observer: Arc::new(RwLock::new(blocked_request_observer)), audit_metadata, + execution_attributions: Arc::new(Mutex::new(HashMap::new())), + environment_id: None, + execution_id: None, } } + pub(crate) fn register_execution( + &self, + attribution_token: &str, + environment_id: &str, + execution_id: &str, + ) { + self.execution_attributions + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .insert( + attribution_token.to_string(), + ExecutionAttribution { + environment_id: environment_id.to_string(), + execution_id: execution_id.to_string(), + }, + ); + } + + pub(crate) fn unregister_execution(&self, attribution_token: &str) { + self.execution_attributions + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .remove(attribution_token); + } + + pub(crate) fn for_execution_token(&self, token: &str) -> Option { + let attribution = self + .execution_attributions + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .get(token)? + .clone(); + Some(Self { + environment_id: Some(attribution.environment_id.into()), + execution_id: Some(attribution.execution_id.into()), + ..self.clone() + }) + } + + pub(crate) fn environment_id(&self) -> Option<&str> { + self.environment_id.as_deref() + } + + pub(crate) fn execution_id(&self) -> Option { + self.execution_id.as_deref().map(str::to_string) + } + pub async fn set_blocked_request_observer( &self, blocked_request_observer: Option>, @@ -460,8 +526,9 @@ impl NetworkProxyState { } } - pub async fn record_blocked(&self, entry: BlockedRequest) -> Result<()> { + pub async fn record_blocked(&self, mut entry: BlockedRequest) -> Result<()> { self.reload_if_needed().await?; + entry.execution_id = self.execution_id(); let blocked_for_observer = entry.clone(); let blocked_request_observer = self.blocked_request_observer.read().await.clone(); let violation_line = blocked_request_violation_log_line(&entry); @@ -1306,6 +1373,7 @@ mod tests { method: Some("GET".to_string()), mode: Some(NetworkMode::Full), protocol: "http".to_string(), + execution_id: None, decision: Some("ask".to_string()), source: Some("decider".to_string()), port: Some(80), diff --git a/codex-rs/network-proxy/src/socks5.rs b/codex-rs/network-proxy/src/socks5.rs index 9561877043..d1c193198f 100644 --- a/codex-rs/network-proxy/src/socks5.rs +++ b/codex-rs/network-proxy/src/socks5.rs @@ -1,3 +1,4 @@ +use crate::attribution::BindConnectionAttribution; use crate::config::NetworkMode; use crate::connect_policy::TargetCheckedTcpConnector; use crate::mitm; @@ -23,13 +24,11 @@ use crate::state::BlockedRequestArgs; use crate::state::NetworkProxyState; use anyhow::Context as _; use anyhow::Result; -use rama_core::Layer; use rama_core::Service; use rama_core::error::BoxError; use rama_core::extensions::Extensions; use rama_core::extensions::ExtensionsMut; use rama_core::extensions::ExtensionsRef; -use rama_core::layer::AddInputExtensionLayer; use rama_core::service::service_fn; use rama_net::address::HostWithPort; use rama_net::client::EstablishedClientConnection; @@ -165,11 +164,15 @@ async fn run_socks5_with_listener( })); let socks_acceptor = base.with_udp_associator(udp_relay); listener - .serve(AddInputExtensionLayer::new(state).into_layer(socks_acceptor)) + .serve(BindConnectionAttribution::new( + socks_acceptor, + state, + environment_id, + )) .await; } else { listener - .serve(AddInputExtensionLayer::new(state).into_layer(base)) + .serve(BindConnectionAttribution::new(base, state, environment_id)) .await; } Ok(()) diff --git a/codex-rs/otel/src/events/session_telemetry.rs b/codex-rs/otel/src/events/session_telemetry.rs index 8210c0e5b9..440f8f04a5 100644 --- a/codex-rs/otel/src/events/session_telemetry.rs +++ b/codex-rs/otel/src/events/session_telemetry.rs @@ -910,6 +910,7 @@ impl SessionTelemetry { cached_token_count: Option, reasoning_token_count: Option, tool_token_count: i64, + ttft_ms: Option, ) { log_and_trace_event!( self, @@ -921,6 +922,7 @@ impl SessionTelemetry { cached_token_count = cached_token_count, reasoning_token_count = reasoning_token_count, tool_token_count = %tool_token_count, + ttft_ms = ttft_ms, service_tier = self.metadata.service_tier.as_deref(), model_reasoning_effort = self.metadata.model_reasoning_effort.as_deref(), }, diff --git a/codex-rs/otel/src/lib.rs b/codex-rs/otel/src/lib.rs index 2a36fab16d..87ccbbfeee 100644 --- a/codex-rs/otel/src/lib.rs +++ b/codex-rs/otel/src/lib.rs @@ -29,6 +29,7 @@ pub use crate::provider::OtelProvider; pub use crate::trace_context::context_from_w3c_trace_context; pub use crate::trace_context::current_span_trace_id; pub use crate::trace_context::current_span_w3c_trace_context; +pub use crate::trace_context::inject_span_w3c_trace_headers; pub use crate::trace_context::set_parent_from_context; pub use crate::trace_context::set_parent_from_w3c_trace_context; pub use crate::trace_context::span_w3c_trace_context; diff --git a/codex-rs/otel/src/metrics/client.rs b/codex-rs/otel/src/metrics/client.rs index 73fc3c6613..251fc74d77 100644 --- a/codex-rs/otel/src/metrics/client.rs +++ b/codex-rs/otel/src/metrics/client.rs @@ -188,6 +188,24 @@ impl MetricsClientInner { Ok(()) } + fn register_observable_gauge( + &self, + name: &str, + description: &str, + observe: impl Fn() -> i64 + Send + Sync + 'static, + tags: &[(&str, &str)], + ) -> Result<()> { + validate_metric_name(name)?; + let attributes = self.attributes(tags)?; + let _gauge = self + .meter + .i64_observable_gauge(name.to_string()) + .with_description(description.to_string()) + .with_callback(move |observer| observer.observe(observe(), &attributes)) + .build(); + Ok(()) + } + fn duration_histogram( &self, name: &str, @@ -354,6 +372,18 @@ impl MetricsClient { self.0.gauge(name, Some(description), value, tags) } + /// Register a gauge callback that reports the current value on every collection. + pub fn register_observable_gauge_with_description( + &self, + name: &str, + description: &str, + observe: impl Fn() -> i64 + Send + Sync + 'static, + tags: &[(&str, &str)], + ) -> Result<()> { + self.0 + .register_observable_gauge(name, description, observe, tags) + } + /// Record a duration in milliseconds using a histogram. pub fn record_duration( &self, diff --git a/codex-rs/otel/src/trace_context.rs b/codex-rs/otel/src/trace_context.rs index c625a416a7..6ae924d1d8 100644 --- a/codex-rs/otel/src/trace_context.rs +++ b/codex-rs/otel/src/trace_context.rs @@ -49,6 +49,38 @@ pub fn span_w3c_trace_context(span: &Span) -> Option { }) } +/// Injects the W3C trace context for `span` into HTTP headers. +/// +/// Existing `traceparent` and `tracestate` values are replaced so callers can +/// safely reuse a request header map while keeping the supplied span as the +/// source of truth. +pub fn inject_span_w3c_trace_headers(span: &Span, headers: &mut http::HeaderMap) -> bool { + let Some(trace) = span_w3c_trace_context(span) else { + return false; + }; + match trace.traceparent { + Some(traceparent) => { + if let Ok(value) = http::HeaderValue::from_str(&traceparent) { + headers.insert("traceparent", value); + } + } + None => { + headers.remove("traceparent"); + } + } + match trace.tracestate { + Some(tracestate) => { + if let Ok(value) = http::HeaderValue::from_str(&tracestate) { + headers.insert("tracestate", value); + } + } + None => { + headers.remove("tracestate"); + } + } + true +} + pub(crate) fn set_tracestate_entries( entries: BTreeMap>, ) -> Result<(), Box> { diff --git a/codex-rs/otel/tests/suite/snapshot.rs b/codex-rs/otel/tests/suite/snapshot.rs index e631100004..ffc2458a04 100644 --- a/codex-rs/otel/tests/suite/snapshot.rs +++ b/codex-rs/otel/tests/suite/snapshot.rs @@ -62,6 +62,38 @@ fn snapshot_collects_metrics_without_shutdown() -> Result<()> { Ok(()) } +#[test] +fn observable_gauge_is_collected_on_every_delta_snapshot() -> Result<()> { + let exporter = InMemoryMetricExporter::default(); + let config = MetricsConfig::in_memory("test", "codex-cli", env!("CARGO_PKG_VERSION"), exporter) + .with_runtime_reader(); + let metrics = MetricsClient::new(config)?; + metrics.register_observable_gauge_with_description( + "codex.active", + "Number of active operations.", + || 1, + &[("component", "test")], + )?; + + for snapshot in [metrics.snapshot()?, metrics.snapshot()?] { + let gauge = find_metric(&snapshot, "codex.active").expect("gauge metric missing"); + let point = match gauge.data() { + AggregatedMetrics::I64(MetricData::Gauge(gauge)) => { + gauge.data_points().next().expect("gauge point") + } + _ => panic!("unexpected gauge metric data type"), + }; + assert_eq!(point.value(), 1); + assert_eq!( + attributes_to_map(point.attributes()), + BTreeMap::from([("component".to_string(), "test".to_string())]) + ); + } + + metrics.shutdown()?; + Ok(()) +} + #[test] fn manager_snapshot_metrics_collects_without_shutdown() -> Result<()> { let exporter = InMemoryMetricExporter::default(); diff --git a/codex-rs/prompts/src/permissions_instructions.rs b/codex-rs/prompts/src/permissions_instructions.rs index 090254705e..c382b804a0 100644 --- a/codex-rs/prompts/src/permissions_instructions.rs +++ b/codex-rs/prompts/src/permissions_instructions.rs @@ -20,8 +20,6 @@ const APPROVAL_POLICY_UNLESS_TRUSTED: &str = include_str!("../templates/permissions/approval_policy/unless_trusted.md"); const APPROVAL_POLICY_ON_REQUEST_RULE: &str = include_str!("../templates/permissions/approval_policy/on_request.md"); -const APPROVAL_POLICY_ON_REQUEST_AUTO_REVIEW: &str = - include_str!("../templates/permissions/approval_policy/on_request_auto_review.md"); const APPROVAL_POLICY_ON_REQUEST_RULE_REQUEST_PERMISSION: &str = include_str!("../templates/permissions/approval_policy/on_request_rule_request_permission.md"); const AUTO_REVIEW_APPROVAL_SUFFIX: &str = "`approvals_reviewer` is `auto_review`: Sandbox escalations with require_escalated will be reviewed for compliance with the policy. If a rejection happens, you should proceed only with a materially safer alternative, or inform the user of the risk and send a final message to ask for approval."; @@ -208,8 +206,6 @@ fn approval_text( let on_request_instructions = || { let on_request_rule = if exec_permission_approvals_enabled { APPROVAL_POLICY_ON_REQUEST_RULE_REQUEST_PERMISSION.to_string() - } else if approvals_reviewer == ApprovalsReviewer::AutoReview { - APPROVAL_POLICY_ON_REQUEST_AUTO_REVIEW.to_string() } else { APPROVAL_POLICY_ON_REQUEST_RULE.to_string() }; diff --git a/codex-rs/prompts/src/permissions_instructions_tests.rs b/codex-rs/prompts/src/permissions_instructions_tests.rs index 8250aea57d..326e75ea17 100644 --- a/codex-rs/prompts/src/permissions_instructions_tests.rs +++ b/codex-rs/prompts/src/permissions_instructions_tests.rs @@ -237,98 +237,17 @@ fn on_request_includes_tool_guidance_alongside_inline_permission_guidance_when_b #[test] fn auto_review_approvals_append_auto_review_specific_guidance() { - let mut exec_policy = Policy::empty(); - exec_policy - .add_prefix_rule(&["git".to_string(), "pull".to_string()], Decision::Allow) - .expect("add rule"); let text = approval_text( AskForApproval::OnRequest, ApprovalsReviewer::AutoReview, - &exec_policy, + &Policy::empty(), /*exec_permission_approvals_enabled*/ false, /*request_permissions_tool_enabled*/ false, ); assert!(text.contains("`approvals_reviewer` is `auto_review`")); - assert!(text.contains("materially safer alternative")); assert!(!text.contains("`approvals_reviewer` is `guardian_subagent`")); - assert!(text.contains( - "When the sandbox is likely to block a command needed for the task, request escalation up front" - )); - assert!(text.contains( - "When unsure, prefer requesting escalation unnecessarily over failing to request it when needed" - )); - assert!( - text.contains( - "Request escalation for commands that need write access outside writable roots" - ) - ); - assert!(text.contains( - "Request escalation for git operations that may write lock files, such as updating the index or refs" - )); - assert!(!text.contains("Request escalation for GUI commands")); - assert!(!text.contains("Use escalation when it is the direct or most reliable way")); - assert!(!text.contains("Do not spend extra turns running likely-to-fail sandbox probes first")); - assert!(text.contains("Do not include a `prefix_rule` parameter.")); - assert!(text.contains( - "Request escalation for commands that may need network access, including HTTP calls, package registries, internal services, data-service APIs, remote queries, data fetches, or live probes" - )); - assert!(text.contains( - "Request escalation for commands that may need remote authentication, cluster, cloud, or database access" - )); - assert!(text.contains( - "Request escalation for commands that may need process, cache, or other environment access outside the sandbox" - )); - assert!( - text.contains( - "including DNS, connection, authentication, retry, or service endpoint errors" - ) - ); - assert!(text.contains( - "Request escalation before potentially destructive actions, such as `rm` or `git reset`" - )); - assert!(text.contains( - "If a command may be hanging on sandbox-blocked access, stop after a short timeout and rerun with `require_escalated`" - )); - assert!(!text.contains("Be judicious with escalating")); - assert!(text.contains("Approved command prefixes")); - assert!(text.contains(r#"["git", "pull"]"#)); - assert!(!text.contains("prefix_rule guidance")); -} - -#[test] -fn normal_on_request_keeps_user_approval_wording() { - let text = approval_text( - AskForApproval::OnRequest, - ApprovalsReviewer::User, - &Policy::empty(), - /*exec_permission_approvals_enabled*/ false, - /*request_permissions_tool_enabled*/ false, - ); - - assert!(text.contains("approved by the user")); - assert!(text.contains("Be judicious with escalating")); - assert!(!text.contains( - "When the sandbox is likely to block a command needed for the task, request escalation up front" - )); -} - -#[test] -fn auto_review_on_request_with_inline_permission_requests_keeps_suffix() { - let text = approval_text( - AskForApproval::OnRequest, - ApprovalsReviewer::AutoReview, - &Policy::empty(), - /*exec_permission_approvals_enabled*/ true, - /*request_permissions_tool_enabled*/ false, - ); - - assert!(text.contains("with_additional_permissions")); - assert!(text.contains("`approvals_reviewer` is `auto_review`")); assert!(text.contains("materially safer alternative")); - assert!(!text.contains( - "When the sandbox is likely to block a command needed for the task, request escalation up front" - )); } #[test] diff --git a/codex-rs/prompts/templates/permissions/approval_policy/on_request_auto_review.md b/codex-rs/prompts/templates/permissions/approval_policy/on_request_auto_review.md deleted file mode 100644 index 82bb24e963..0000000000 --- a/codex-rs/prompts/templates/permissions/approval_policy/on_request_auto_review.md +++ /dev/null @@ -1,45 +0,0 @@ -# Escalation Requests - -Commands are run outside the sandbox after approval. The command string is split into independent command segments at shell control operators, including but not limited to: - -- Pipes: | -- Logical operators: &&, || -- Command separators: ; -- Subshell boundaries: (...), $() - -Each resulting segment is evaluated independently for sandbox restrictions and approval requirements. - -Example: - -git pull | tee output.txt - -This is treated as two command segments: - -["git", "pull"] - -["tee", "output.txt"] - -Commands that use more advanced shell features like redirection (>, >>, <), substitutions ($(...), ...), environment variables (FOO=bar), or wildcard patterns (*, ?) require care because each independent command segment is evaluated separately. - -## How to request escalation - -IMPORTANT: To request approval to execute a command that will require escalated privileges: - -- Provide the `sandbox_permissions` parameter with the value `"require_escalated"` -- Include a concise `justification` parameter that explains why escalated privileges are needed. -- Do not include a `prefix_rule` parameter. - -## When to request escalation - -While commands are running inside the sandbox, here are some scenarios that justify escalation: - -- When the sandbox is likely to block a command needed for the task, request escalation up front. -- Request escalation for commands that need write access outside writable roots, such as tests that write to `/var`. -- Request escalation for git operations that may write lock files, such as updating the index or refs. -- Request escalation for commands that may need network access, including HTTP calls, package registries, internal services, data-service APIs, remote queries, data fetches, or live probes. -- Request escalation for commands that may need remote authentication, cluster, cloud, or database access. -- Request escalation for commands that may need process, cache, or other environment access outside the sandbox. -- If a sandboxed attempt fails with sandboxing or likely network symptoms, including DNS, connection, authentication, retry, or service endpoint errors, rerun with `sandbox_permissions` set to `"require_escalated"` and include `justification`. -- If a command may be hanging on sandbox-blocked access, stop after a short timeout and rerun with `require_escalated`. -- Request escalation before potentially destructive actions, such as `rm` or `git reset`, that the user did not explicitly ask for. -- When unsure, prefer requesting escalation unnecessarily over failing to request it when needed. diff --git a/codex-rs/protocol/src/config_types.rs b/codex-rs/protocol/src/config_types.rs index 83bcbdc0ae..7732efbbe5 100644 --- a/codex-rs/protocol/src/config_types.rs +++ b/codex-rs/protocol/src/config_types.rs @@ -296,23 +296,39 @@ pub enum Personality { Pragmatic, } -/// Controls whether the model receives multi-agent delegation instructions and, -/// when it does, whether it should only spawn sub-agents after an explicit user -/// request or may delegate proactively when doing so would help. `none` leaves -/// the multi-agent tools available without injecting delegation instructions. -#[derive( - Debug, Serialize, Deserialize, Clone, Copy, PartialEq, Eq, Display, JsonSchema, TS, Default, -)] -#[serde(rename_all = "camelCase")] +/// Controls the effective multi-agent delegation instructions for a turn. `custom` means the +/// configured mode hint defines the policy instead of a built-in policy. +#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq, Display, JsonSchema, TS, Default)] +#[serde(rename_all = "camelCase", from = "MultiAgentModeWire")] #[ts(rename_all = "camelCase")] #[strum(serialize_all = "camelCase")] pub enum MultiAgentMode { - None, + Custom(String), #[default] ExplicitRequestOnly, Proactive, } +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +enum MultiAgentModeWire { + None, + Custom(String), + ExplicitRequestOnly, + Proactive, +} + +impl From for MultiAgentMode { + fn from(value: MultiAgentModeWire) -> Self { + match value { + MultiAgentModeWire::None => Self::Custom(String::new()), + MultiAgentModeWire::Custom(hint_text) => Self::Custom(hint_text), + MultiAgentModeWire::ExplicitRequestOnly => Self::ExplicitRequestOnly, + MultiAgentModeWire::Proactive => Self::Proactive, + } + } +} + #[derive( Debug, Serialize, Deserialize, Clone, Copy, PartialEq, Eq, Display, JsonSchema, TS, Default, )] diff --git a/codex-rs/protocol/src/items.rs b/codex-rs/protocol/src/items.rs index 508d71d4be..820c3b774b 100644 --- a/codex-rs/protocol/src/items.rs +++ b/codex-rs/protocol/src/items.rs @@ -1,3 +1,6 @@ +use crate::AgentPath; +use crate::ThreadId; +use crate::dynamic_tools::DynamicToolCallOutputContentItem; use crate::mcp::CallToolResult; use crate::memory_citation::MemoryCitation; use crate::models::ContentItem; @@ -5,22 +8,14 @@ use crate::models::ImageDetail; use crate::models::MessagePhase; use crate::models::ResponseItem; use crate::models::WebSearchAction; -use crate::protocol::AgentMessageEvent; -use crate::protocol::AgentReasoningEvent; -use crate::protocol::AgentReasoningRawContentEvent; -use crate::protocol::ContextCompactedEvent; -use crate::protocol::EventMsg; +use crate::openai_models::ReasoningEffort as ReasoningEffortConfig; +use crate::parse_command::ParsedCommand; +use crate::protocol::AgentStatus; +use crate::protocol::CollabAgentRef; +use crate::protocol::ExecCommandSource; use crate::protocol::FileChange; -use crate::protocol::ImageGenerationEndEvent; -use crate::protocol::McpInvocation; -use crate::protocol::McpToolCallBeginEvent; -use crate::protocol::McpToolCallEndEvent; -use crate::protocol::PatchApplyBeginEvent; -use crate::protocol::PatchApplyEndEvent; use crate::protocol::PatchApplyStatus; -use crate::protocol::UserMessageEvent; -use crate::protocol::ViewImageToolCallEvent; -use crate::protocol::WebSearchEndEvent; +use crate::protocol::SubAgentActivityKind; use crate::user_input::ByteRange; use crate::user_input::TextElement; use crate::user_input::UserInput; @@ -46,6 +41,10 @@ pub enum TurnItem { AgentMessage(AgentMessageItem), Plan(PlanItem), Reasoning(ReasoningItem), + CommandExecution(CommandExecutionItem), + DynamicToolCall(DynamicToolCallItem), + CollabAgentToolCall(CollabAgentToolCallItem), + SubAgentActivity(SubAgentActivityItem), WebSearch(WebSearchItem), ImageView(ImageViewItem), Sleep(SleepItem), @@ -129,6 +128,129 @@ pub struct ReasoningItem { pub raw_content: Vec, } +#[derive(Debug, Clone, Copy, Deserialize, Serialize, TS, JsonSchema, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum CommandExecutionStatus { + InProgress, + Completed, + Failed, + Declined, +} + +#[derive(Debug, Clone, Deserialize, Serialize, TS, JsonSchema, PartialEq)] +pub struct CommandExecutionItem { + pub id: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional)] + pub process_id: Option, + pub command: Vec, + pub cwd: PathUri, + pub parsed_cmd: Vec, + pub source: ExecCommandSource, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional)] + pub interaction_input: Option, + pub status: CommandExecutionStatus, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional)] + pub stdout: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional)] + pub stderr: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional)] + pub aggregated_output: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional)] + pub exit_code: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(type = "string", optional)] + pub duration: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional)] + pub formatted_output: Option, +} + +#[derive(Debug, Clone, Copy, Deserialize, Serialize, TS, JsonSchema, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum DynamicToolCallStatus { + InProgress, + Completed, + Failed, +} + +#[derive(Debug, Clone, Deserialize, Serialize, TS, JsonSchema, PartialEq)] +pub struct DynamicToolCallItem { + pub id: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional)] + pub namespace: Option, + pub tool: String, + pub arguments: serde_json::Value, + pub status: DynamicToolCallStatus, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional)] + pub content_items: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional)] + pub success: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional)] + pub error: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(type = "string", optional)] + pub duration: Option, +} + +#[derive(Debug, Clone, Copy, Deserialize, Serialize, TS, JsonSchema, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum CollabAgentTool { + SpawnAgent, + SendInput, + ResumeAgent, + Wait, + CloseAgent, +} + +#[derive(Debug, Clone, Copy, Deserialize, Serialize, TS, JsonSchema, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum CollabAgentToolCallStatus { + InProgress, + Completed, + Failed, +} + +#[derive(Debug, Clone, Deserialize, Serialize, TS, JsonSchema, PartialEq)] +pub struct CollabAgentToolCallItem { + pub id: String, + pub tool: CollabAgentTool, + pub status: CollabAgentToolCallStatus, + pub sender_thread_id: ThreadId, + #[serde(default)] + pub receiver_thread_ids: Vec, + #[serde(default)] + pub receiver_agents: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional)] + pub prompt: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional)] + pub model: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional)] + pub reasoning_effort: Option, + #[serde(default)] + pub agents_states: HashMap, +} + +#[derive(Debug, Clone, Deserialize, Serialize, TS, JsonSchema, PartialEq, Eq)] +pub struct SubAgentActivityItem { + pub id: String, + pub kind: SubAgentActivityKind, + pub agent_thread_id: ThreadId, + pub agent_path: AgentPath, +} + #[derive(Debug, Clone, Deserialize, Serialize, TS, JsonSchema, PartialEq)] pub struct WebSearchItem { pub id: String, @@ -202,6 +324,15 @@ pub struct McpToolCallItem { pub link_id: Option, #[serde(default, skip_serializing_if = "Option::is_none")] #[ts(optional)] + pub app_name: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional)] + pub template_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional)] + pub action_name: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional)] pub plugin_id: Option, pub status: McpToolCallStatus, #[serde(default, skip_serializing_if = "Option::is_none")] @@ -242,10 +373,6 @@ impl ContextCompactionItem { id: uuid::Uuid::new_v4().to_string(), } } - - pub fn as_legacy_event(&self) -> EventMsg { - EventMsg::ContextCompacted(ContextCompactedEvent {}) - } } impl Default for ContextCompactionItem { @@ -263,20 +390,6 @@ impl UserMessageItem { } } - pub fn as_legacy_event(&self) -> EventMsg { - // Legacy user-message events flatten only text inputs into `message` and - // rebase text element ranges onto that concatenated text. - EventMsg::UserMessage(UserMessageEvent { - client_id: self.client_id.clone(), - message: self.message(), - images: Some(self.image_urls()), - image_details: self.image_details(), - local_images: self.local_image_paths(), - local_image_details: self.local_image_details(), - text_elements: self.text_elements(), - }) - } - pub fn message(&self) -> String { self.content .iter() @@ -464,128 +577,6 @@ impl AgentMessageItem { memory_citation: None, } } - - pub fn as_legacy_events(&self) -> Vec { - self.content - .iter() - .map(|c| match c { - AgentMessageContent::Text { text } => EventMsg::AgentMessage(AgentMessageEvent { - message: text.clone(), - phase: self.phase.clone(), - memory_citation: self.memory_citation.clone(), - }), - }) - .collect() - } -} - -impl ReasoningItem { - pub fn as_legacy_events(&self, show_raw_agent_reasoning: bool) -> Vec { - let mut events = Vec::new(); - for summary in &self.summary_text { - events.push(EventMsg::AgentReasoning(AgentReasoningEvent { - text: summary.clone(), - })); - } - - if show_raw_agent_reasoning { - for entry in &self.raw_content { - events.push(EventMsg::AgentReasoningRawContent( - AgentReasoningRawContentEvent { - text: entry.clone(), - }, - )); - } - } - - events - } -} - -impl WebSearchItem { - pub fn as_legacy_event(&self) -> EventMsg { - EventMsg::WebSearchEnd(WebSearchEndEvent { - call_id: self.id.clone(), - query: self.query.clone(), - action: self.action.clone(), - }) - } -} - -impl ImageGenerationItem { - pub fn as_legacy_event(&self) -> EventMsg { - EventMsg::ImageGenerationEnd(ImageGenerationEndEvent { - call_id: self.id.clone(), - status: self.status.clone(), - revised_prompt: self.revised_prompt.clone(), - result: self.result.clone(), - saved_path: self.saved_path.clone(), - }) - } -} - -impl FileChangeItem { - pub fn as_legacy_begin_event(&self, turn_id: String) -> EventMsg { - EventMsg::PatchApplyBegin(PatchApplyBeginEvent { - call_id: self.id.clone(), - turn_id, - auto_approved: self.auto_approved.unwrap_or(false), - changes: self.changes.clone(), - }) - } - - pub fn as_legacy_end_event(&self, turn_id: String) -> Option { - let status = self.status.clone()?; - Some(EventMsg::PatchApplyEnd(PatchApplyEndEvent { - call_id: self.id.clone(), - turn_id, - stdout: self.stdout.clone().unwrap_or_default(), - stderr: self.stderr.clone().unwrap_or_default(), - success: status == PatchApplyStatus::Completed, - changes: self.changes.clone(), - status, - })) - } -} - -impl McpToolCallItem { - pub fn as_legacy_begin_event(&self) -> EventMsg { - EventMsg::McpToolCallBegin(McpToolCallBeginEvent { - call_id: self.id.clone(), - invocation: McpInvocation { - server: self.server.clone(), - tool: self.tool.clone(), - arguments: (!self.arguments.is_null()).then(|| self.arguments.clone()), - }, - connector_id: self.connector_id.clone(), - mcp_app_resource_uri: self.mcp_app_resource_uri.clone(), - link_id: self.link_id.clone(), - plugin_id: self.plugin_id.clone(), - }) - } - - pub fn as_legacy_end_event(&self) -> Option { - let result = match (&self.result, &self.error) { - (Some(result), _) => Ok(result.clone()), - (None, Some(error)) => Err(error.message.clone()), - (None, None) => return None, - }; - - Some(EventMsg::McpToolCallEnd(McpToolCallEndEvent { - call_id: self.id.clone(), - invocation: McpInvocation { - server: self.server.clone(), - tool: self.tool.clone(), - arguments: (!self.arguments.is_null()).then(|| self.arguments.clone()), - }, - mcp_app_resource_uri: self.mcp_app_resource_uri.clone(), - connector_id: self.connector_id.clone(), - link_id: self.link_id.clone(), - plugin_id: self.plugin_id.clone(), - duration: self.duration?, - result, - })) - } } impl TurnItem { @@ -596,6 +587,10 @@ impl TurnItem { TurnItem::AgentMessage(item) => item.id.clone(), TurnItem::Plan(item) => item.id.clone(), TurnItem::Reasoning(item) => item.id.clone(), + TurnItem::CommandExecution(item) => item.id.clone(), + TurnItem::DynamicToolCall(item) => item.id.clone(), + TurnItem::CollabAgentToolCall(item) => item.id.clone(), + TurnItem::SubAgentActivity(item) => item.id.clone(), TurnItem::WebSearch(item) => item.id.clone(), TurnItem::ImageView(item) => item.id.clone(), TurnItem::Sleep(item) => item.id.clone(), @@ -605,31 +600,6 @@ impl TurnItem { TurnItem::ContextCompaction(item) => item.id.clone(), } } - - pub fn as_legacy_events(&self, show_raw_agent_reasoning: bool) -> Vec { - match self { - TurnItem::UserMessage(item) => vec![item.as_legacy_event()], - TurnItem::HookPrompt(_) => Vec::new(), - TurnItem::AgentMessage(item) => item.as_legacy_events(), - TurnItem::Plan(_) => Vec::new(), - TurnItem::WebSearch(item) => vec![item.as_legacy_event()], - TurnItem::ImageView(item) => { - vec![EventMsg::ViewImageToolCall(ViewImageToolCallEvent { - call_id: item.id.clone(), - path: item.path.clone(), - })] - } - TurnItem::Sleep(_) => Vec::new(), - TurnItem::ImageGeneration(item) => vec![item.as_legacy_event()], - TurnItem::FileChange(item) => item - .as_legacy_end_event(String::new()) - .into_iter() - .collect(), - TurnItem::McpToolCall(item) => item.as_legacy_end_event().into_iter().collect(), - TurnItem::Reasoning(item) => item.as_legacy_events(show_raw_agent_reasoning), - TurnItem::ContextCompaction(item) => vec![item.as_legacy_event()], - } - } } #[cfg(test)] diff --git a/codex-rs/protocol/src/legacy_events.rs b/codex-rs/protocol/src/legacy_events.rs new file mode 100644 index 0000000000..f4a511d367 --- /dev/null +++ b/codex-rs/protocol/src/legacy_events.rs @@ -0,0 +1,289 @@ +use crate::items::AgentMessageContent; +use crate::items::AgentMessageItem; +use crate::items::ContextCompactionItem; +use crate::items::FileChangeItem; +use crate::items::ImageGenerationItem; +use crate::items::McpToolCallItem; +use crate::items::ReasoningItem; +use crate::items::TurnItem; +use crate::items::UserMessageItem; +use crate::items::WebSearchItem; +use crate::protocol::AgentMessageContentDeltaEvent; +use crate::protocol::AgentMessageEvent; +use crate::protocol::AgentReasoningEvent; +use crate::protocol::AgentReasoningRawContentEvent; +use crate::protocol::ContextCompactedEvent; +use crate::protocol::EventMsg; +use crate::protocol::ImageGenerationBeginEvent; +use crate::protocol::ImageGenerationEndEvent; +use crate::protocol::ItemCompletedEvent; +use crate::protocol::ItemStartedEvent; +use crate::protocol::McpInvocation; +use crate::protocol::McpToolCallBeginEvent; +use crate::protocol::McpToolCallEndEvent; +use crate::protocol::PatchApplyBeginEvent; +use crate::protocol::PatchApplyEndEvent; +use crate::protocol::PatchApplyStatus; +use crate::protocol::ReasoningContentDeltaEvent; +use crate::protocol::ReasoningRawContentDeltaEvent; +use crate::protocol::UserMessageEvent; +use crate::protocol::ViewImageToolCallEvent; +use crate::protocol::WebSearchBeginEvent; +use crate::protocol::WebSearchEndEvent; + +/// Converts canonical item lifecycle events back into the legacy raw event stream used by +/// compatibility consumers that have not migrated to `TurnItem`. +pub trait HasLegacyEvent { + fn as_legacy_events(&self, show_raw_agent_reasoning: bool) -> Vec; +} + +impl ContextCompactionItem { + pub fn as_legacy_event(&self) -> EventMsg { + EventMsg::ContextCompacted(ContextCompactedEvent {}) + } +} + +impl UserMessageItem { + pub fn as_legacy_event(&self) -> EventMsg { + // Legacy user-message events flatten only text inputs into `message` and + // rebase text element ranges onto that concatenated text. + EventMsg::UserMessage(UserMessageEvent { + client_id: self.client_id.clone(), + message: self.message(), + images: Some(self.image_urls()), + image_details: self.image_details(), + local_images: self.local_image_paths(), + local_image_details: self.local_image_details(), + text_elements: self.text_elements(), + }) + } +} + +impl AgentMessageItem { + pub fn as_legacy_events(&self) -> Vec { + self.content + .iter() + .map(|c| match c { + AgentMessageContent::Text { text } => EventMsg::AgentMessage(AgentMessageEvent { + message: text.clone(), + phase: self.phase.clone(), + memory_citation: self.memory_citation.clone(), + }), + }) + .collect() + } +} + +impl ReasoningItem { + pub fn as_legacy_events(&self, show_raw_agent_reasoning: bool) -> Vec { + let mut events = Vec::new(); + for summary in &self.summary_text { + events.push(EventMsg::AgentReasoning(AgentReasoningEvent { + text: summary.clone(), + })); + } + + if show_raw_agent_reasoning { + for entry in &self.raw_content { + events.push(EventMsg::AgentReasoningRawContent( + AgentReasoningRawContentEvent { + text: entry.clone(), + }, + )); + } + } + + events + } +} + +impl WebSearchItem { + pub fn as_legacy_event(&self) -> EventMsg { + EventMsg::WebSearchEnd(WebSearchEndEvent { + call_id: self.id.clone(), + query: self.query.clone(), + action: self.action.clone(), + }) + } +} + +impl ImageGenerationItem { + pub fn as_legacy_event(&self) -> EventMsg { + EventMsg::ImageGenerationEnd(ImageGenerationEndEvent { + call_id: self.id.clone(), + status: self.status.clone(), + revised_prompt: self.revised_prompt.clone(), + result: self.result.clone(), + saved_path: self.saved_path.clone(), + }) + } +} + +impl FileChangeItem { + pub fn as_legacy_begin_event(&self, turn_id: String) -> EventMsg { + EventMsg::PatchApplyBegin(PatchApplyBeginEvent { + call_id: self.id.clone(), + turn_id, + auto_approved: self.auto_approved.unwrap_or(false), + changes: self.changes.clone(), + }) + } + + pub fn as_legacy_end_event(&self, turn_id: String) -> Option { + let status = self.status.clone()?; + Some(EventMsg::PatchApplyEnd(PatchApplyEndEvent { + call_id: self.id.clone(), + turn_id, + stdout: self.stdout.clone().unwrap_or_default(), + stderr: self.stderr.clone().unwrap_or_default(), + success: status == PatchApplyStatus::Completed, + changes: self.changes.clone(), + status, + })) + } +} + +impl McpToolCallItem { + pub fn as_legacy_begin_event(&self) -> EventMsg { + EventMsg::McpToolCallBegin(McpToolCallBeginEvent { + call_id: self.id.clone(), + invocation: McpInvocation { + server: self.server.clone(), + tool: self.tool.clone(), + arguments: (!self.arguments.is_null()).then(|| self.arguments.clone()), + }, + connector_id: self.connector_id.clone(), + mcp_app_resource_uri: self.mcp_app_resource_uri.clone(), + link_id: self.link_id.clone(), + app_name: self.app_name.clone(), + template_id: self.template_id.clone(), + action_name: self.action_name.clone(), + plugin_id: self.plugin_id.clone(), + }) + } + + pub fn as_legacy_end_event(&self) -> Option { + let result = match (&self.result, &self.error) { + (Some(result), _) => Ok(result.clone()), + (None, Some(error)) => Err(error.message.clone()), + (None, None) => return None, + }; + + Some(EventMsg::McpToolCallEnd(McpToolCallEndEvent { + call_id: self.id.clone(), + invocation: McpInvocation { + server: self.server.clone(), + tool: self.tool.clone(), + arguments: (!self.arguments.is_null()).then(|| self.arguments.clone()), + }, + mcp_app_resource_uri: self.mcp_app_resource_uri.clone(), + connector_id: self.connector_id.clone(), + link_id: self.link_id.clone(), + app_name: self.app_name.clone(), + template_id: self.template_id.clone(), + action_name: self.action_name.clone(), + plugin_id: self.plugin_id.clone(), + duration: self.duration?, + result, + })) + } +} + +impl TurnItem { + pub fn as_legacy_events(&self, show_raw_agent_reasoning: bool) -> Vec { + match self { + TurnItem::UserMessage(item) => vec![item.as_legacy_event()], + TurnItem::HookPrompt(_) => Vec::new(), + TurnItem::AgentMessage(item) => item.as_legacy_events(), + TurnItem::Plan(_) => Vec::new(), + TurnItem::CommandExecution(_) + | TurnItem::DynamicToolCall(_) + | TurnItem::CollabAgentToolCall(_) => Vec::new(), + TurnItem::SubAgentActivity(_) => Vec::new(), + TurnItem::WebSearch(item) => vec![item.as_legacy_event()], + TurnItem::ImageView(item) => { + vec![EventMsg::ViewImageToolCall(ViewImageToolCallEvent { + call_id: item.id.clone(), + path: item.path.clone(), + })] + } + TurnItem::Sleep(_) => Vec::new(), + TurnItem::ImageGeneration(item) => vec![item.as_legacy_event()], + TurnItem::FileChange(item) => item + .as_legacy_end_event(String::new()) + .into_iter() + .collect(), + TurnItem::McpToolCall(item) => item.as_legacy_end_event().into_iter().collect(), + TurnItem::Reasoning(item) => item.as_legacy_events(show_raw_agent_reasoning), + TurnItem::ContextCompaction(item) => vec![item.as_legacy_event()], + } + } +} + +impl HasLegacyEvent for ItemStartedEvent { + fn as_legacy_events(&self, _: bool) -> Vec { + match &self.item { + TurnItem::WebSearch(item) => vec![EventMsg::WebSearchBegin(WebSearchBeginEvent { + call_id: item.id.clone(), + })], + TurnItem::ImageView(_) => Vec::new(), + TurnItem::ImageGeneration(item) => { + vec![EventMsg::ImageGenerationBegin(ImageGenerationBeginEvent { + call_id: item.id.clone(), + })] + } + TurnItem::FileChange(item) => vec![item.as_legacy_begin_event(self.turn_id.clone())], + TurnItem::McpToolCall(item) => vec![item.as_legacy_begin_event()], + _ => Vec::new(), + } + } +} + +impl HasLegacyEvent for ItemCompletedEvent { + fn as_legacy_events(&self, show_raw_agent_reasoning: bool) -> Vec { + match &self.item { + TurnItem::FileChange(item) => item + .as_legacy_end_event(self.turn_id.clone()) + .into_iter() + .collect(), + _ => self.item.as_legacy_events(show_raw_agent_reasoning), + } + } +} + +impl HasLegacyEvent for AgentMessageContentDeltaEvent { + fn as_legacy_events(&self, _: bool) -> Vec { + Vec::new() + } +} + +impl HasLegacyEvent for ReasoningContentDeltaEvent { + fn as_legacy_events(&self, _: bool) -> Vec { + Vec::new() + } +} + +impl HasLegacyEvent for ReasoningRawContentDeltaEvent { + fn as_legacy_events(&self, _: bool) -> Vec { + Vec::new() + } +} + +impl HasLegacyEvent for EventMsg { + fn as_legacy_events(&self, show_raw_agent_reasoning: bool) -> Vec { + match self { + EventMsg::ItemStarted(event) => event.as_legacy_events(show_raw_agent_reasoning), + EventMsg::ItemCompleted(event) => event.as_legacy_events(show_raw_agent_reasoning), + EventMsg::AgentMessageContentDelta(event) => { + event.as_legacy_events(show_raw_agent_reasoning) + } + EventMsg::ReasoningContentDelta(event) => { + event.as_legacy_events(show_raw_agent_reasoning) + } + EventMsg::ReasoningRawContentDelta(event) => { + event.as_legacy_events(show_raw_agent_reasoning) + } + _ => Vec::new(), + } + } +} diff --git a/codex-rs/protocol/src/lib.rs b/codex-rs/protocol/src/lib.rs index c50e95d37f..636c64ed7b 100644 --- a/codex-rs/protocol/src/lib.rs +++ b/codex-rs/protocol/src/lib.rs @@ -16,6 +16,7 @@ pub mod dynamic_tools; pub mod error; pub mod exec_output; pub mod items; +mod legacy_events; pub mod mcp; pub mod mcp_approval_meta; pub mod memory_citation; diff --git a/codex-rs/protocol/src/models.rs b/codex-rs/protocol/src/models.rs index 2faad87349..46aecb61b9 100644 --- a/codex-rs/protocol/src/models.rs +++ b/codex-rs/protocol/src/models.rs @@ -1052,6 +1052,9 @@ pub enum ResponseItem { call_id: String, name: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional)] + namespace: Option, input: String, #[serde(default, skip_serializing_if = "Option::is_none")] #[ts(optional)] diff --git a/codex-rs/protocol/src/openai_models.rs b/codex-rs/protocol/src/openai_models.rs index f697a8107f..52f541070d 100644 --- a/codex-rs/protocol/src/openai_models.rs +++ b/codex-rs/protocol/src/openai_models.rs @@ -45,6 +45,7 @@ pub enum ReasoningEffort { Medium, High, XHigh, + Max, Ultra, /// A model-defined effort value that this client does not know yet. Custom(String), @@ -60,6 +61,7 @@ impl ReasoningEffort { Self::Medium => "medium", Self::High => "high", Self::XHigh => "xhigh", + Self::Max => "max", Self::Ultra => "ultra", Self::Custom(effort) => effort, } @@ -125,6 +127,7 @@ impl FromStr for ReasoningEffort { "medium" => Ok(Self::Medium), "high" => Ok(Self::High), "xhigh" => Ok(Self::XHigh), + "max" => Ok(Self::Max), "ultra" => Ok(Self::Ultra), "" => Err("reasoning_effort must not be empty".to_string()), effort => Ok(Self::Custom(effort.to_string())), @@ -370,6 +373,8 @@ pub struct ModelInfo { pub base_instructions: String, #[serde(default, skip_serializing_if = "Option::is_none")] pub model_messages: Option, + #[serde(default)] + pub include_skills_usage_instructions: bool, pub supports_reasoning_summaries: bool, #[serde(default)] pub default_reasoning_summary: ReasoningSummary, @@ -665,6 +670,7 @@ mod tests { upgrade: None, base_instructions: "base".to_string(), model_messages: spec, + include_skills_usage_instructions: false, supports_reasoning_summaries: false, default_reasoning_summary: ReasoningSummary::Auto, support_verbosity: false, @@ -700,30 +706,35 @@ mod tests { #[test] fn reasoning_effort_accepts_known_and_custom_values() { - let custom = ReasoningEffort::Custom("max".to_string()); - let deserialized = from_str::(r#""max""#) + let custom = ReasoningEffort::Custom("future".to_string()); + let deserialized = from_str::(r#""future""#) .expect("custom reasoning effort should deserialize"); let serialized = to_string(&custom).expect("custom reasoning effort should serialize"); + let serialized_max = to_string(&ReasoningEffort::Max).expect("Max should serialize"); let serialized_ultra = to_string(&ReasoningEffort::Ultra).expect("Ultra should serialize"); assert_eq!( ( "high".parse(), - "ultra".parse(), "max".parse(), + "ultra".parse(), + "future".parse(), deserialized, serialized, + serialized_max, serialized_ultra, custom.to_string(), ), ( Ok(ReasoningEffort::High), + Ok(ReasoningEffort::Max), Ok(ReasoningEffort::Ultra), Ok(custom.clone()), custom, + r#""future""#.to_string(), r#""max""#.to_string(), r#""ultra""#.to_string(), - "max".to_string(), + "future".to_string(), ) ); } @@ -948,6 +959,7 @@ mod tests { .expect("deserialize model info"); assert_eq!(model.availability_nux, None); + assert!(!model.include_skills_usage_instructions); assert!(!model.supports_image_detail_original); assert_eq!(model.web_search_tool_type, WebSearchToolType::Text); assert!(!model.supports_search_tool); diff --git a/codex-rs/protocol/src/protocol.rs b/codex-rs/protocol/src/protocol.rs index c684477a46..151597cc88 100644 --- a/codex-rs/protocol/src/protocol.rs +++ b/codex-rs/protocol/src/protocol.rs @@ -19,6 +19,7 @@ use crate::AgentPath; use crate::SessionId; use crate::ThreadId; use crate::approvals::ElicitationRequestEvent; +use crate::capabilities::SelectedCapabilityRoot; use crate::config_types::ApprovalsReviewer; use crate::config_types::CollaborationMode; use crate::config_types::ModeKind; @@ -83,6 +84,7 @@ pub use crate::approvals::NetworkApprovalContext; pub use crate::approvals::NetworkApprovalProtocol; pub use crate::approvals::NetworkPolicyAmendment; pub use crate::approvals::NetworkPolicyRuleAction; +pub use crate::legacy_events::HasLegacyEvent; pub use crate::permissions::FileSystemAccessMode; pub use crate::permissions::FileSystemPath; pub use crate::permissions::FileSystemSandboxEntry; @@ -224,6 +226,9 @@ impl fmt::Debug for McpServerUpdateConfig { pub struct ConversationStartParams { /// Whether Codex response handoffs are managed through explicit client append calls. pub client_managed_handoffs: bool, + /// Whether to route any remaining transcript tail through Codex when the session ends. + /// TODO: Remove this rollout knob once transcript-tail flushing is always enabled. + pub flush_transcript_tail_on_session_end: bool, /// Sends automatic Codex responses as realtime conversation items instead of handoff appends. pub codex_responses_as_items: bool, /// Optional prefix added to automatic Codex response items when `codex_responses_as_items` is set. @@ -712,6 +717,36 @@ pub enum ThreadMemoryMode { Disabled, } +#[derive(Serialize, Deserialize, Clone, Copy, Debug, Default, PartialEq, Eq, JsonSchema, TS)] +#[serde(rename_all = "lowercase")] +#[ts(rename_all = "lowercase")] +pub enum ThreadHistoryMode { + #[default] + Legacy, + Paginated, +} + +impl ThreadHistoryMode { + pub const fn as_str(self) -> &'static str { + match self { + Self::Legacy => "legacy", + Self::Paginated => "paginated", + } + } +} + +impl FromStr for ThreadHistoryMode { + type Err = String; + + fn from_str(value: &str) -> Result { + match value { + "legacy" => Ok(Self::Legacy), + "paginated" => Ok(Self::Paginated), + _ => Err(format!("unknown thread history mode `{value}`")), + } + } +} + impl From> for Op { fn from(value: Vec) -> Self { Op::UserInput { @@ -1799,25 +1834,6 @@ pub struct ItemStartedEvent { pub started_at_ms: i64, } -impl HasLegacyEvent for ItemStartedEvent { - fn as_legacy_events(&self, _: bool) -> Vec { - match &self.item { - TurnItem::WebSearch(item) => vec![EventMsg::WebSearchBegin(WebSearchBeginEvent { - call_id: item.id.clone(), - })], - TurnItem::ImageView(_) => Vec::new(), - TurnItem::ImageGeneration(item) => { - vec![EventMsg::ImageGenerationBegin(ImageGenerationBeginEvent { - call_id: item.id.clone(), - })] - } - TurnItem::FileChange(item) => vec![item.as_legacy_begin_event(self.turn_id.clone())], - TurnItem::McpToolCall(item) => vec![item.as_legacy_begin_event()], - _ => Vec::new(), - } - } -} - #[derive(Debug, Clone, Deserialize, Serialize, TS, JsonSchema)] pub struct ItemCompletedEvent { pub thread_id: ThreadId, @@ -1834,22 +1850,6 @@ const fn default_item_completed_at_ms() -> i64 { 0 } -pub trait HasLegacyEvent { - fn as_legacy_events(&self, show_raw_agent_reasoning: bool) -> Vec; -} - -impl HasLegacyEvent for ItemCompletedEvent { - fn as_legacy_events(&self, show_raw_agent_reasoning: bool) -> Vec { - match &self.item { - TurnItem::FileChange(item) => item - .as_legacy_end_event(self.turn_id.clone()) - .into_iter() - .collect(), - _ => self.item.as_legacy_events(show_raw_agent_reasoning), - } - } -} - #[derive(Debug, Clone, Deserialize, Serialize, TS, JsonSchema)] pub struct AgentMessageContentDeltaEvent { pub thread_id: String, @@ -1858,12 +1858,6 @@ pub struct AgentMessageContentDeltaEvent { pub delta: String, } -impl HasLegacyEvent for AgentMessageContentDeltaEvent { - fn as_legacy_events(&self, _: bool) -> Vec { - Vec::new() - } -} - #[derive(Debug, Clone, Deserialize, Serialize, TS, JsonSchema)] pub struct PlanDeltaEvent { pub thread_id: String, @@ -1883,12 +1877,6 @@ pub struct ReasoningContentDeltaEvent { pub summary_index: i64, } -impl HasLegacyEvent for ReasoningContentDeltaEvent { - fn as_legacy_events(&self, _: bool) -> Vec { - Vec::new() - } -} - #[derive(Debug, Clone, Deserialize, Serialize, TS, JsonSchema)] pub struct ReasoningRawContentDeltaEvent { pub thread_id: String, @@ -1900,31 +1888,6 @@ pub struct ReasoningRawContentDeltaEvent { pub content_index: i64, } -impl HasLegacyEvent for ReasoningRawContentDeltaEvent { - fn as_legacy_events(&self, _: bool) -> Vec { - Vec::new() - } -} - -impl HasLegacyEvent for EventMsg { - fn as_legacy_events(&self, show_raw_agent_reasoning: bool) -> Vec { - match self { - EventMsg::ItemStarted(event) => event.as_legacy_events(show_raw_agent_reasoning), - EventMsg::ItemCompleted(event) => event.as_legacy_events(show_raw_agent_reasoning), - EventMsg::AgentMessageContentDelta(event) => { - event.as_legacy_events(show_raw_agent_reasoning) - } - EventMsg::ReasoningContentDelta(event) => { - event.as_legacy_events(show_raw_agent_reasoning) - } - EventMsg::ReasoningRawContentDelta(event) => { - event.as_legacy_events(show_raw_agent_reasoning) - } - _ => Vec::new(), - } - } -} - #[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, TS)] pub struct ExitedReviewModeEvent { pub review_output: Option, @@ -2392,6 +2355,15 @@ pub struct McpToolCallBeginEvent { pub link_id: Option, #[serde(default, skip_serializing_if = "Option::is_none")] #[ts(optional)] + pub app_name: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional)] + pub template_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional)] + pub action_name: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional)] pub plugin_id: Option, } @@ -2411,6 +2383,15 @@ pub struct McpToolCallEndEvent { pub link_id: Option, #[serde(default, skip_serializing_if = "Option::is_none")] #[ts(optional)] + pub app_name: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional)] + pub template_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional)] + pub action_name: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional)] pub plugin_id: Option, #[ts(type = "string")] pub duration: Duration, @@ -2606,6 +2587,12 @@ impl InitialHistory { } } + pub fn get_selected_capability_roots(&self) -> Vec { + self.get_session_meta() + .map(|meta| meta.selected_capability_roots.clone()) + .unwrap_or_default() + } + pub fn get_multi_agent_version(&self) -> Option { match self { InitialHistory::New | InitialHistory::Cleared => None, @@ -2618,6 +2605,18 @@ impl InitialHistory { } } + pub fn get_history_mode(&self, default_history_mode: ThreadHistoryMode) -> ThreadHistoryMode { + match self { + InitialHistory::New | InitialHistory::Cleared | InitialHistory::Forked(_) => { + default_history_mode + } + InitialHistory::Resumed(_) => self + .get_resumed_session_meta() + .map(|meta| meta.history_mode) + .unwrap_or(default_history_mode), + } + } + pub fn get_latest_effective_multi_agent_mode(&self) -> Option { let items = match self { InitialHistory::New | InitialHistory::Cleared => return None, @@ -2637,7 +2636,7 @@ impl InitialHistory { | RolloutItem::WorldState(_) | RolloutItem::EventMsg(_) => None, }) - .and_then(|turn_context| turn_context.multi_agent_mode) + .and_then(|turn_context| turn_context.multi_agent_mode.clone()) } pub fn get_resumed_session_sources(&self) -> Option<(SessionSource, Option)> { @@ -3039,8 +3038,13 @@ pub struct SessionMeta { skip_serializing_if = "Option::is_none" )] pub dynamic_tools: Option>, + /// Capability roots selected for this thread by the hosting platform. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub selected_capability_roots: Vec, #[serde(skip_serializing_if = "Option::is_none")] pub memory_mode: Option, + #[serde(default)] + pub history_mode: ThreadHistoryMode, #[serde(skip_serializing_if = "Option::is_none")] pub multi_agent_version: Option, /// Initial context-window identity for consumers that tail rollout JSONL before compaction. @@ -3068,7 +3072,9 @@ impl Default for SessionMeta { model_provider: None, base_instructions: None, dynamic_tools: None, + selected_capability_roots: Vec::new(), memory_mode: None, + history_mode: ThreadHistoryMode::default(), multi_agent_version: None, context_window: None, } @@ -3646,10 +3652,22 @@ pub struct McpStartupUpdateEvent { pub enum McpStartupStatus { Starting, Ready, - Failed { error: String }, + Failed { + error: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional = nullable)] + reason: Option, + }, Cancelled, } +#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Serialize, JsonSchema, TS)] +#[serde(rename_all = "snake_case")] +#[ts(rename_all = "snake_case")] +pub enum McpStartupFailureReason { + ReauthenticationRequired, +} + #[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, TS, Default)] pub struct McpStartupCompleteEvent { pub ready: Vec, @@ -5170,6 +5188,9 @@ mod tests { connector_id: Some("connector".into()), mcp_app_resource_uri: Some("app://connector".into()), link_id: Some("link_123".into()), + app_name: Some("Calendar".into()), + template_id: Some("calendar_template".into()), + action_name: Some("create_event".into()), plugin_id: Some("sample@test".into()), status: McpToolCallStatus::InProgress, result: None, @@ -5191,6 +5212,8 @@ mod tests { Some("app://connector") ); assert_eq!(event.link_id.as_deref(), Some("link_123")); + assert_eq!(event.app_name.as_deref(), Some("Calendar")); + assert_eq!(event.action_name.as_deref(), Some("create_event")); assert_eq!(event.plugin_id.as_deref(), Some("sample@test")); } _ => panic!("expected McpToolCallBegin event"), @@ -5281,6 +5304,9 @@ mod tests { connector_id: Some("connector".into()), mcp_app_resource_uri: Some("app://connector".into()), link_id: Some("link_123".into()), + app_name: Some("Calendar".into()), + template_id: Some("calendar_template".into()), + action_name: Some("create_event".into()), plugin_id: Some("sample@test".into()), status: McpToolCallStatus::Completed, result: Some(CallToolResult { @@ -5307,6 +5333,8 @@ mod tests { Some("app://connector") ); assert_eq!(event.link_id.as_deref(), Some("link_123")); + assert_eq!(event.app_name.as_deref(), Some("Calendar")); + assert_eq!(event.action_name.as_deref(), Some("create_event")); assert_eq!(event.plugin_id.as_deref(), Some("sample@test")); assert_eq!(event.duration, Duration::from_millis(42)); assert!(event.is_success()); @@ -5343,6 +5371,7 @@ mod tests { let event = serde_json::from_value::(value).unwrap(); assert_eq!(event.completed_at_ms, 0); } + #[test] fn rollback_failed_error_does_not_affect_turn_status() { let event = ErrorEvent { @@ -5544,6 +5573,70 @@ mod tests { Ok(()) } + #[test] + fn session_meta_defaults_legacy_history_mode() -> Result<()> { + let session_meta: SessionMeta = serde_json::from_value(json!({ + "session_id": "00000000-0000-0000-0000-000000000001", + "id": "00000000-0000-0000-0000-000000000001", + "timestamp": "2026-01-01T00:00:00Z", + "cwd": "/tmp", + "originator": "codex", + "cli_version": "0.0.0", + "model_provider": null, + "base_instructions": null + }))?; + + assert_eq!(session_meta.history_mode, ThreadHistoryMode::Legacy); + let serialized = serde_json::to_value(&session_meta)?; + assert_eq!(serialized["history_mode"], json!("legacy")); + let mut unknown = serialized; + unknown["history_mode"] = json!("future"); + assert!(serde_json::from_value::(unknown).is_err()); + Ok(()) + } + + #[test] + fn resumed_history_uses_persisted_history_mode() -> Result<()> { + let thread_id = ThreadId::from_string("00000000-0000-0000-0000-000000000001")?; + let session_meta = RolloutItem::SessionMeta(SessionMetaLine { + meta: SessionMeta { + session_id: thread_id.into(), + id: thread_id, + history_mode: ThreadHistoryMode::Paginated, + ..SessionMeta::default() + }, + git: None, + }); + let history = InitialHistory::Resumed(ResumedHistory { + conversation_id: thread_id, + history: Arc::new(vec![session_meta.clone()]), + rollout_path: None, + }); + + assert_eq!( + history.get_history_mode(ThreadHistoryMode::Legacy), + ThreadHistoryMode::Paginated + ); + assert_eq!( + InitialHistory::Forked(vec![session_meta]).get_history_mode(ThreadHistoryMode::Legacy), + ThreadHistoryMode::Legacy + ); + assert_eq!( + InitialHistory::New.get_history_mode(ThreadHistoryMode::Paginated), + ThreadHistoryMode::Paginated + ); + assert_eq!( + InitialHistory::Resumed(ResumedHistory { + conversation_id: thread_id, + history: Arc::new(Vec::new()), + rollout_path: None, + }) + .get_history_mode(ThreadHistoryMode::Paginated), + ThreadHistoryMode::Paginated + ); + Ok(()) + } + #[test] fn turn_context_item_deserializes_without_network() -> Result<()> { let item: TurnContextItem = serde_json::from_value(json!({ @@ -5634,6 +5727,25 @@ mod tests { Ok(()) } + #[test] + fn latest_effective_multi_agent_mode_maps_legacy_none_to_empty_custom() -> Result<()> { + let value = json!({ + "cwd": test_path_buf("/tmp"), + "approval_policy": "never", + "sandbox_policy": { "type": "danger-full-access" }, + "model": "gpt-5", + "multi_agent_mode": "none", + "summary": "auto", + }); + let item = RolloutItem::TurnContext(serde_json::from_value(value)?); + + assert_eq!( + InitialHistory::Forked(vec![item]).get_latest_effective_multi_agent_mode(), + Some(MultiAgentMode::Custom(String::new())) + ); + Ok(()) + } + #[test] fn turn_context_item_serializes_network_when_present() -> Result<()> { let item = TurnContextItem { @@ -5791,6 +5903,7 @@ mod tests { server: "srv".to_string(), status: McpStartupStatus::Failed { error: "boom".to_string(), + reason: Some(McpStartupFailureReason::ReauthenticationRequired), }, }), }; @@ -5800,6 +5913,10 @@ mod tests { assert_eq!(value["msg"]["server"], "srv"); assert_eq!(value["msg"]["status"]["state"], "failed"); assert_eq!(value["msg"]["status"]["error"], "boom"); + assert_eq!( + value["msg"]["status"]["reason"], + "reauthentication_required" + ); Ok(()) } diff --git a/codex-rs/rmcp-client/Cargo.toml b/codex-rs/rmcp-client/Cargo.toml index 44f3eb1397..517f7270bb 100644 --- a/codex-rs/rmcp-client/Cargo.toml +++ b/codex-rs/rmcp-client/Cargo.toml @@ -16,7 +16,6 @@ axum = { workspace = true, default-features = false, features = [ ] } base64 = { workspace = true } codex-api = { workspace = true } -codex-client = { workspace = true } codex-config = { workspace = true } codex-exec-server = { workspace = true } codex-keyring-store = { workspace = true } diff --git a/codex-rs/rmcp-client/src/auth_status.rs b/codex-rs/rmcp-client/src/auth_status.rs index b2bd834d59..32c244af8e 100644 --- a/codex-rs/rmcp-client/src/auth_status.rs +++ b/codex-rs/rmcp-client/src/auth_status.rs @@ -1,7 +1,9 @@ use std::collections::HashMap; +use std::sync::Arc; use std::time::Duration; use anyhow::Result; +use codex_exec_server::HttpClient; use codex_protocol::protocol::McpAuthStatus; use futures::FutureExt; use reqwest::Client; @@ -13,6 +15,7 @@ use tracing::debug; use crate::oauth::StoredOAuthTokenStatus; use crate::oauth::oauth_token_status; +use crate::oauth_http_client::OAuthHttpClientAdapter; use crate::utils::apply_default_headers; use crate::utils::build_default_headers; use codex_config::types::AuthKeyringBackendKind; @@ -25,6 +28,36 @@ pub struct StreamableHttpOAuthDiscovery { pub scopes_supported: Option>, } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum McpLoginRequirement { + Login, + Reauthentication, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum McpAuthState { + Unsupported, + LoggedOut(McpLoginRequirement), + BearerToken, + OAuth, +} + +impl From for McpAuthStatus { + fn from(value: McpAuthState) -> Self { + match value { + McpAuthState::Unsupported => Self::Unsupported, + McpAuthState::LoggedOut(_) => Self::NotLoggedIn, + McpAuthState::BearerToken => Self::BearerToken, + McpAuthState::OAuth => Self::OAuth, + } + } +} + +enum AuthStatusCheck { + Complete(McpAuthState), + Discover(HeaderMap), +} + /// Determine the authentication status for a streamable HTTP MCP server. pub async fn determine_streamable_http_auth_status( server_name: &str, @@ -34,32 +67,110 @@ pub async fn determine_streamable_http_auth_status( env_http_headers: Option>, store_mode: OAuthCredentialsStoreMode, keyring_backend_kind: AuthKeyringBackendKind, -) -> Result { +) -> Result { + let default_headers = match auth_status_before_discovery( + server_name, + url, + bearer_token_env_var, + http_headers, + env_http_headers, + store_mode, + keyring_backend_kind, + )? { + AuthStatusCheck::Complete(status) => return Ok(status), + AuthStatusCheck::Discover(default_headers) => default_headers, + }; + + determine_auth_status_from_discovery( + server_name, + url, + discover_streamable_http_oauth_with_headers(url, &default_headers).await, + ) +} + +/// Determine authentication status while routing OAuth discovery through the +/// provided HTTP client. +#[allow(clippy::too_many_arguments)] +pub async fn determine_streamable_http_auth_status_with_http_client( + server_name: &str, + url: &str, + bearer_token_env_var: Option<&str>, + http_headers: Option>, + env_http_headers: Option>, + store_mode: OAuthCredentialsStoreMode, + keyring_backend_kind: AuthKeyringBackendKind, + http_client: Arc, +) -> Result { + let default_headers = match auth_status_before_discovery( + server_name, + url, + bearer_token_env_var, + http_headers, + env_http_headers, + store_mode, + keyring_backend_kind, + )? { + AuthStatusCheck::Complete(status) => return Ok(status), + AuthStatusCheck::Discover(default_headers) => default_headers, + }; + determine_auth_status_from_discovery( + server_name, + url, + discover_streamable_http_oauth_with_headers_and_http_client( + url, + default_headers, + http_client, + ) + .await, + ) +} + +fn auth_status_before_discovery( + server_name: &str, + url: &str, + bearer_token_env_var: Option<&str>, + http_headers: Option>, + env_http_headers: Option>, + store_mode: OAuthCredentialsStoreMode, + keyring_backend_kind: AuthKeyringBackendKind, +) -> Result { if bearer_token_env_var.is_some() { - return Ok(McpAuthStatus::BearerToken); + return Ok(AuthStatusCheck::Complete(McpAuthState::BearerToken)); } let default_headers = build_default_headers(http_headers, env_http_headers)?; if default_headers.contains_key(AUTHORIZATION) { - return Ok(McpAuthStatus::BearerToken); + return Ok(AuthStatusCheck::Complete(McpAuthState::BearerToken)); } match oauth_token_status(server_name, url, store_mode, keyring_backend_kind)? { - StoredOAuthTokenStatus::Usable => return Ok(McpAuthStatus::OAuth), + StoredOAuthTokenStatus::Usable => { + return Ok(AuthStatusCheck::Complete(McpAuthState::OAuth)); + } StoredOAuthTokenStatus::AuthorizationRequired => { - return Ok(McpAuthStatus::NotLoggedIn); + return Ok(AuthStatusCheck::Complete(McpAuthState::LoggedOut( + McpLoginRequirement::Reauthentication, + ))); } StoredOAuthTokenStatus::Missing => {} } - match discover_streamable_http_oauth_with_headers(url, &default_headers).await { - Ok(Some(_)) => Ok(McpAuthStatus::NotLoggedIn), - Ok(None) => Ok(McpAuthStatus::Unsupported), + Ok(AuthStatusCheck::Discover(default_headers)) +} + +fn determine_auth_status_from_discovery( + server_name: &str, + url: &str, + discovery: Result>, +) -> Result { + match discovery { + Ok(Some(_)) => Ok(McpAuthState::LoggedOut(McpLoginRequirement::Login)), + Ok(None) => Ok(McpAuthState::Unsupported), Err(error) => { debug!( "failed to detect OAuth support for MCP server `{server_name}` at {url}: {error:?}" ); - Ok(McpAuthStatus::Unsupported) + Ok(McpAuthState::Unsupported) } } } @@ -82,6 +193,17 @@ pub async fn discover_streamable_http_oauth( discover_streamable_http_oauth_with_headers(url, &default_headers).await } +pub async fn discover_streamable_http_oauth_with_http_client( + url: &str, + http_headers: Option>, + env_http_headers: Option>, + http_client: Arc, +) -> Result> { + let default_headers = build_default_headers(http_headers, env_http_headers)?; + discover_streamable_http_oauth_with_headers_and_http_client(url, default_headers, http_client) + .await +} + async fn discover_streamable_http_oauth_with_headers( url: &str, default_headers: &HeaderMap, @@ -92,6 +214,25 @@ async fn discover_streamable_http_oauth_with_headers( let client = apply_default_headers(builder, default_headers).build()?; let mut authorization_manager = AuthorizationManager::new(url).await?; authorization_manager.with_client(client)?; + discover_streamable_http_oauth_with_manager(&authorization_manager).await +} + +async fn discover_streamable_http_oauth_with_headers_and_http_client( + url: &str, + default_headers: HeaderMap, + http_client: Arc, +) -> Result> { + let authorization_manager = AuthorizationManager::new_with_oauth_http_client( + url, + Arc::new(OAuthHttpClientAdapter::new(http_client, default_headers)), + ) + .await?; + discover_streamable_http_oauth_with_manager(&authorization_manager).await +} + +async fn discover_streamable_http_oauth_with_manager( + authorization_manager: &AuthorizationManager, +) -> Result> { match authorization_manager.discover_metadata().boxed().await { Ok(metadata) => Ok(Some(StreamableHttpOAuthDiscovery { scopes_supported: normalize_scopes(metadata.scopes_supported), @@ -222,7 +363,7 @@ mod tests { .await .expect("status should compute"); - assert_eq!(status, McpAuthStatus::BearerToken); + assert_eq!(status, McpAuthState::BearerToken); } #[tokio::test] @@ -244,7 +385,7 @@ mod tests { .await .expect("status should compute"); - assert_eq!(status, McpAuthStatus::BearerToken); + assert_eq!(status, McpAuthState::BearerToken); } #[tokio::test] diff --git a/codex-rs/rmcp-client/src/http_client_adapter.rs b/codex-rs/rmcp-client/src/http_client_adapter.rs index a9cdc7b88f..19befb6235 100644 --- a/codex-rs/rmcp-client/src/http_client_adapter.rs +++ b/codex-rs/rmcp-client/src/http_client_adapter.rs @@ -16,6 +16,7 @@ use codex_api::SharedAuthProvider; use codex_exec_server::ExecServerError; use codex_exec_server::HttpClient; use codex_exec_server::HttpHeader; +use codex_exec_server::HttpRedirectPolicy; use codex_exec_server::HttpRequestParams; use codex_exec_server::HttpResponseBodyStream; use futures::StreamExt; @@ -135,6 +136,7 @@ impl StreamableHttpClient for StreamableHttpClientAdapter { headers: protocol_headers(&headers), body: Some(body.into()), timeout_ms: None, + redirect_policy: HttpRedirectPolicy::Follow, request_id: "buffered-request".to_string(), stream_response: true, }) @@ -261,6 +263,7 @@ impl StreamableHttpClient for StreamableHttpClientAdapter { headers: protocol_headers(&headers), body: None, timeout_ms: None, + redirect_policy: HttpRedirectPolicy::Follow, request_id: "buffered-request".to_string(), stream_response: false, }) @@ -330,6 +333,7 @@ impl StreamableHttpClient for StreamableHttpClientAdapter { headers: protocol_headers(&headers), body: None, timeout_ms: None, + redirect_policy: HttpRedirectPolicy::Follow, request_id: "buffered-request".to_string(), stream_response: true, }) diff --git a/codex-rs/rmcp-client/src/lib.rs b/codex-rs/rmcp-client/src/lib.rs index e1ee18c753..e57eb3686c 100644 --- a/codex-rs/rmcp-client/src/lib.rs +++ b/codex-rs/rmcp-client/src/lib.rs @@ -5,15 +5,21 @@ mod http_client_adapter; mod in_process_transport; mod logging_client_handler; mod oauth; +mod oauth_http_client; mod perform_oauth_login; mod program_resolver; mod rmcp_client; +mod startup_error; mod stdio_server_launcher; mod utils; +pub use auth_status::McpAuthState; +pub use auth_status::McpLoginRequirement; pub use auth_status::StreamableHttpOAuthDiscovery; pub use auth_status::determine_streamable_http_auth_status; +pub use auth_status::determine_streamable_http_auth_status_with_http_client; pub use auth_status::discover_streamable_http_oauth; +pub use auth_status::discover_streamable_http_oauth_with_http_client; pub use auth_status::supports_oauth_login; pub use codex_protocol::protocol::McpAuthStatus; pub use in_process_transport::InProcessTransportFactory; @@ -26,6 +32,7 @@ pub use perform_oauth_login::OAuthProviderError; pub use perform_oauth_login::OauthLoginHandle; pub use perform_oauth_login::perform_oauth_login; pub use perform_oauth_login::perform_oauth_login_return_url; +pub use perform_oauth_login::perform_oauth_login_return_url_with_http_client; pub use perform_oauth_login::perform_oauth_login_silent; pub use rmcp::model::ElicitationAction; pub use rmcp_client::Elicitation; @@ -34,6 +41,7 @@ pub use rmcp_client::ListToolsWithConnectorIdResult; pub use rmcp_client::RmcpClient; pub use rmcp_client::SendElicitation; pub use rmcp_client::ToolWithConnectorId; +pub use startup_error::is_authentication_required_error; pub use stdio_server_launcher::ExecutorStdioServerLauncher; pub use stdio_server_launcher::LocalStdioServerLauncher; pub use stdio_server_launcher::StdioServerLauncher; diff --git a/codex-rs/rmcp-client/src/oauth_http_client.rs b/codex-rs/rmcp-client/src/oauth_http_client.rs new file mode 100644 index 0000000000..44cf63cb7c --- /dev/null +++ b/codex-rs/rmcp-client/src/oauth_http_client.rs @@ -0,0 +1,116 @@ +use std::sync::Arc; +use std::sync::atomic::AtomicU64; +use std::sync::atomic::Ordering; +use std::time::Duration; + +use codex_exec_server::HttpClient; +use codex_exec_server::HttpHeader; +use codex_exec_server::HttpRedirectPolicy; +use codex_exec_server::HttpRequestParams; +use oauth2::HttpRequest; +use oauth2::HttpResponse; +use reqwest::header::HeaderMap; +use rmcp::transport::auth::OAuthHttpClient; +use rmcp::transport::auth::OAuthHttpClientError; +use rmcp::transport::auth::OAuthHttpClientFuture; +use rmcp::transport::auth::OAuthHttpRedirectPolicy; +use rmcp::transport::auth::OAuthHttpRequest; + +const MAX_OAUTH_HTTP_RESPONSE_BODY_BYTES: usize = 1024 * 1024; +static NEXT_OAUTH_REQUEST_ID: AtomicU64 = AtomicU64::new(0); + +#[derive(Clone)] +pub(crate) struct OAuthHttpClientAdapter { + http_client: Arc, + default_headers: HeaderMap, +} + +impl OAuthHttpClientAdapter { + pub(crate) fn new(http_client: Arc, default_headers: HeaderMap) -> Self { + Self { + http_client, + default_headers, + } + } + + async fn execute_request( + &self, + request: HttpRequest, + redirect_policy: OAuthHttpRedirectPolicy, + timeout: Option, + ) -> Result { + let redirect_policy = match redirect_policy { + OAuthHttpRedirectPolicy::Follow => HttpRedirectPolicy::Follow, + OAuthHttpRedirectPolicy::Stop => HttpRedirectPolicy::Stop, + _ => { + return Err(OAuthHttpClientError::new( + "unsupported OAuth HTTP redirect policy", + )); + } + }; + let (parts, body) = request.into_parts(); + let mut headers = self.default_headers.clone(); + for name in parts.headers.keys() { + headers.remove(name); + } + headers.extend(parts.headers); + let headers = headers + .iter() + .map(|(name, value)| { + Ok(HttpHeader { + name: name.as_str().to_string(), + value: value + .to_str() + .map_err(|error| OAuthHttpClientError::new(error.to_string()))? + .to_string(), + }) + }) + .collect::, OAuthHttpClientError>>()?; + let timeout_ms = timeout.map(|timeout| { + u64::try_from(timeout.as_millis()) + .unwrap_or(u64::MAX) + .max(1) + }); + let request_id = NEXT_OAUTH_REQUEST_ID.fetch_add(1, Ordering::Relaxed); + let (response, mut body_stream) = self + .http_client + .http_request_stream(HttpRequestParams { + method: parts.method.to_string(), + url: parts.uri.to_string(), + headers, + body: (!body.is_empty()).then_some(body.into()), + timeout_ms, + redirect_policy, + request_id: format!("oauth-request-{request_id}"), + stream_response: true, + }) + .await + .map_err(|error| OAuthHttpClientError::new(error.to_string()))?; + let mut body = Vec::new(); + while let Some(chunk) = body_stream + .recv() + .await + .map_err(|error| OAuthHttpClientError::new(error.to_string()))? + { + if chunk.len() > MAX_OAUTH_HTTP_RESPONSE_BODY_BYTES - body.len() { + return Err(OAuthHttpClientError::new(format!( + "OAuth HTTP response body exceeds {MAX_OAUTH_HTTP_RESPONSE_BODY_BYTES} bytes" + ))); + } + body.extend_from_slice(&chunk); + } + let mut builder = oauth2::http::Response::builder().status(response.status); + for header in response.headers { + builder = builder.header(header.name, header.value); + } + builder + .body(body) + .map_err(|error| OAuthHttpClientError::new(error.to_string())) + } +} + +impl OAuthHttpClient for OAuthHttpClientAdapter { + fn execute(&self, request: OAuthHttpRequest) -> OAuthHttpClientFuture<'_> { + Box::pin(self.execute_request(request.request, request.redirect_policy, request.timeout)) + } +} diff --git a/codex-rs/rmcp-client/src/perform_oauth_login.rs b/codex-rs/rmcp-client/src/perform_oauth_login.rs index 8c72f320bc..7a9d242bc4 100644 --- a/codex-rs/rmcp-client/src/perform_oauth_login.rs +++ b/codex-rs/rmcp-client/src/perform_oauth_login.rs @@ -9,11 +9,13 @@ use anyhow::anyhow; use anyhow::bail; use base64::Engine; use base64::engine::general_purpose::URL_SAFE_NO_PAD; -use reqwest::ClientBuilder; +use codex_exec_server::HttpClient; +use codex_exec_server::ReqwestHttpClient; use reqwest::Url; use rmcp::transport::AuthorizationManager; use rmcp::transport::AuthorizationSession; use rmcp::transport::auth::OAuthClientConfig; +use rmcp::transport::auth::OAuthHttpClient; use rmcp::transport::auth::OAuthState; use sha2::Digest; use sha2::Sha256; @@ -26,15 +28,16 @@ use urlencoding::decode; use crate::StoredOAuthTokens; use crate::WrappedOAuthTokenResponse; use crate::oauth::compute_expires_at_millis; +use crate::oauth_http_client::OAuthHttpClientAdapter; use crate::save_oauth_tokens; -use crate::utils::apply_default_headers; use crate::utils::build_default_headers; use codex_config::types::AuthKeyringBackendKind; use codex_config::types::OAuthCredentialsStoreMode; -struct OauthHeaders { +struct OAuthHttpContext { http_headers: Option>, env_http_headers: Option>, + http_client: Arc, } struct CallbackServerGuard { @@ -154,16 +157,17 @@ async fn perform_oauth_login_with_browser_output( callback_url: Option<&str>, emit_browser_url: bool, ) -> Result<()> { - let headers = OauthHeaders { + let http_context = OAuthHttpContext { http_headers, env_http_headers, + http_client: Arc::new(ReqwestHttpClient), }; OauthLoginFlow::new( server_name, server_url, store_mode, keyring_backend_kind, - headers, + http_context, scopes, oauth_client_id, oauth_resource, @@ -192,16 +196,51 @@ pub async fn perform_oauth_login_return_url( callback_port: Option, callback_url: Option<&str>, ) -> Result { - let headers = OauthHeaders { + perform_oauth_login_return_url_with_http_client( + server_name, + server_url, + store_mode, + keyring_backend_kind, http_headers, env_http_headers, + scopes, + oauth_client_id, + oauth_resource, + timeout_secs, + callback_port, + callback_url, + Arc::new(ReqwestHttpClient), + ) + .await +} + +#[allow(clippy::too_many_arguments)] +pub async fn perform_oauth_login_return_url_with_http_client( + server_name: &str, + server_url: &str, + store_mode: OAuthCredentialsStoreMode, + keyring_backend_kind: AuthKeyringBackendKind, + http_headers: Option>, + env_http_headers: Option>, + scopes: &[String], + oauth_client_id: Option<&str>, + oauth_resource: Option<&str>, + timeout_secs: Option, + callback_port: Option, + callback_url: Option<&str>, + http_client: Arc, +) -> Result { + let http_context = OAuthHttpContext { + http_headers, + env_http_headers, + http_client, }; let flow = OauthLoginFlow::new( server_name, server_url, store_mode, keyring_backend_kind, - headers, + http_context, scopes, oauth_client_id, oauth_resource, @@ -456,7 +495,7 @@ impl OauthLoginFlow { server_url: &str, store_mode: OAuthCredentialsStoreMode, keyring_backend_kind: AuthKeyringBackendKind, - headers: OauthHeaders, + http_context: OAuthHttpContext, scopes: &[String], oauth_client_id: Option<&str>, oauth_resource: Option<&str>, @@ -487,17 +526,18 @@ impl OauthLoginFlow { let (tx, rx) = oneshot::channel(); spawn_callback_server(server, tx, callback_path); - let OauthHeaders { + let OAuthHttpContext { http_headers, env_http_headers, - } = headers; + http_client, + } = http_context; let default_headers = build_default_headers(http_headers, env_http_headers)?; - let http_client = apply_default_headers(ClientBuilder::new(), &default_headers).build()?; + let oauth_http_client = Arc::new(OAuthHttpClientAdapter::new(http_client, default_headers)); let scope_refs: Vec<&str> = scopes.iter().map(String::as_str).collect(); let oauth_state = start_authorization( server_url, - http_client, + oauth_http_client, &scope_refs, &redirect_uri, oauth_client_id, @@ -620,22 +660,23 @@ impl OauthLoginFlow { async fn start_authorization( server_url: &str, - http_client: reqwest::Client, + http_client: Arc, scopes: &[&str], redirect_uri: &str, oauth_client_id: Option<&str>, ) -> Result { let Some(oauth_client_id) = oauth_client_id.filter(|client_id| !client_id.trim().is_empty()) else { - let mut oauth_state = OAuthState::new(server_url, Some(http_client)).await?; + let mut oauth_state = + OAuthState::new_with_oauth_http_client(server_url, http_client).await?; oauth_state .start_authorization(scopes, redirect_uri, Some("Codex")) .await?; return Ok(oauth_state); }; - let mut auth_manager = AuthorizationManager::new(server_url).await?; - auth_manager.with_client(http_client)?; + let mut auth_manager = + AuthorizationManager::new_with_oauth_http_client(server_url, http_client).await?; let metadata = auth_manager.discover_metadata().await?; auth_manager.set_metadata(metadata); auth_manager.configure_client( @@ -668,15 +709,20 @@ fn append_query_param(url: &str, key: &str, value: Option<&str>) -> String { #[cfg(test)] mod tests { + use std::sync::Arc; + use axum::Json; use axum::Router; use axum::routing::get; + use codex_exec_server::ReqwestHttpClient; use pretty_assertions::assert_eq; use reqwest::Url; + use reqwest::header::HeaderMap; use serde_json::json; use tokio::net::TcpListener; use super::CallbackOutcome; + use super::OAuthHttpClientAdapter; use super::OAuthProviderError; use super::append_callback_id_to_redirect_uri; use super::append_query_param; @@ -727,7 +773,10 @@ mod tests { let base_url = spawn_oauth_metadata_server().await; let oauth_state = start_authorization( &format!("{base_url}/mcp"), - reqwest::Client::new(), + Arc::new(OAuthHttpClientAdapter::new( + Arc::new(ReqwestHttpClient), + HeaderMap::new(), + )), &[], "http://127.0.0.1/callback", Some("eci-prd-pub-codex-123"), diff --git a/codex-rs/rmcp-client/src/rmcp_client.rs b/codex-rs/rmcp-client/src/rmcp_client.rs index 93abf7d114..c6527990fa 100644 --- a/codex-rs/rmcp-client/src/rmcp_client.rs +++ b/codex-rs/rmcp-client/src/rmcp_client.rs @@ -11,7 +11,6 @@ use std::time::Instant; use anyhow::Result; use anyhow::anyhow; use codex_api::SharedAuthProvider; -use codex_client::maybe_build_rustls_client_config_with_custom_ca; use codex_config::types::AuthKeyringBackendKind; use codex_config::types::McpServerEnvVar; use codex_exec_server::HttpClient; @@ -68,11 +67,11 @@ use crate::in_process_transport::InProcessTransportFactory; use crate::load_oauth_tokens; use crate::oauth::OAuthPersistor; use crate::oauth::StoredOAuthTokens; +use crate::oauth_http_client::OAuthHttpClientAdapter; use crate::stdio_server_launcher::StdioServerCommand; use crate::stdio_server_launcher::StdioServerLauncher; use crate::stdio_server_launcher::StdioServerProcessHandle; use crate::stdio_server_launcher::StdioServerTransport; -use crate::utils::apply_default_headers; use crate::utils::build_default_headers; use codex_config::types::OAuthCredentialsStoreMode; @@ -1166,16 +1165,12 @@ async fn create_oauth_transport_and_runtime( StreamableHttpClientTransport>, OAuthPersistor, )> { - let mut builder = apply_default_headers(reqwest::Client::builder(), &default_headers); - if let Some(tls_config) = maybe_build_rustls_client_config_with_custom_ca()? { - builder = builder.tls_backend_preconfigured(tls_config.as_ref().clone()); - } - let oauth_metadata_client = builder.build()?; - // TODO(aibrahim): teach OAuth bootstrap and refresh to use the same - // shared HTTP client abstraction instead of always creating the local - // reqwest metadata client here. + let oauth_http_client = Arc::new(OAuthHttpClientAdapter::new( + http_client.clone(), + default_headers.clone(), + )); let mut oauth_state = - OAuthState::new(url.to_string(), Some(oauth_metadata_client.clone())).await?; + OAuthState::new_with_oauth_http_client(url.to_string(), oauth_http_client).await?; oauth_state .set_credentials( diff --git a/codex-rs/rmcp-client/src/startup_error.rs b/codex-rs/rmcp-client/src/startup_error.rs new file mode 100644 index 0000000000..c74f637e3d --- /dev/null +++ b/codex-rs/rmcp-client/src/startup_error.rs @@ -0,0 +1,46 @@ +use anyhow::Error; +use rmcp::service::ClientInitializeError; +use rmcp::transport::auth::AuthError; +use rmcp::transport::streamable_http_client::StreamableHttpError; + +use crate::http_client_adapter::StreamableHttpClientAdapterError; + +/// Returns whether an RMCP client error indicates that authentication is required. +/// +/// This does not distinguish first-time login from reauthentication. +/// Streamable HTTP initialization errors are stored inside RMCP's dynamic +/// transport error, which is not part of the standard error source chain. +pub fn is_authentication_required_error(error: &Error) -> bool { + error.chain().any(|source| { + source + .downcast_ref::() + .is_some_and(auth_error_requires_authentication) + || source + .downcast_ref::() + .is_some_and(client_initialize_error_requires_authentication) + }) +} + +fn client_initialize_error_requires_authentication(error: &ClientInitializeError) -> bool { + let ClientInitializeError::TransportError { error, .. } = error else { + return false; + }; + + error + .error + .downcast_ref::>() + .is_some_and(|error| { + matches!( + error, + StreamableHttpError::Auth(auth_error) + if auth_error_requires_authentication(auth_error) + ) + }) +} + +fn auth_error_requires_authentication(error: &AuthError) -> bool { + matches!( + error, + AuthError::AuthorizationRequired | AuthError::TokenExpired + ) +} diff --git a/codex-rs/rmcp-client/tests/streamable_http_oauth_startup.rs b/codex-rs/rmcp-client/tests/streamable_http_oauth_startup.rs index 866daa67a4..32f90bc41e 100644 --- a/codex-rs/rmcp-client/tests/streamable_http_oauth_startup.rs +++ b/codex-rs/rmcp-client/tests/streamable_http_oauth_startup.rs @@ -7,11 +7,13 @@ use std::time::UNIX_EPOCH; use codex_config::types::AuthKeyringBackendKind; use codex_config::types::OAuthCredentialsStoreMode; use codex_exec_server::Environment; -use codex_rmcp_client::McpAuthStatus; +use codex_rmcp_client::McpAuthState; +use codex_rmcp_client::McpLoginRequirement; use codex_rmcp_client::RmcpClient; use codex_rmcp_client::StoredOAuthTokens; use codex_rmcp_client::WrappedOAuthTokenResponse; use codex_rmcp_client::determine_streamable_http_auth_status; +use codex_rmcp_client::is_authentication_required_error; use codex_rmcp_client::save_oauth_tokens; use oauth2::AccessToken; use oauth2::RefreshToken; @@ -143,9 +145,58 @@ async fn reports_auth_status_for_persisted_credentials() -> anyhow::Result<()> { Ok(()) } +#[tokio::test(flavor = "multi_thread", worker_threads = 1)] +async fn identifies_expired_unrefreshable_token_startup_error() -> anyhow::Result<()> { + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/.well-known/oauth-authorization-server/mcp")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "authorization_endpoint": format!("{}/oauth/authorize", server.uri()), + "token_endpoint": format!("{}/oauth/token", server.uri()), + }))) + .expect(1) + .mount(&server) + .await; + + let codex_home = TempDir::new()?; + let status = Command::new(std::env::current_exe()?) + .args([ + "expired_unrefreshable_startup_child", + "--exact", + "--ignored", + "--nocapture", + ]) + .env("CODEX_HOME", codex_home.path()) + .env(CHILD_SERVER_URL_ENV, format!("{}/mcp", server.uri())) + .status() + .await?; + + assert!( + status.success(), + "expired OAuth startup child failed: {status}" + ); + server.verify().await; + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 1)] #[ignore = "spawned by reports_auth_status_for_persisted_credentials"] async fn persisted_credentials_auth_status_child() -> anyhow::Result<()> { + let first_login_server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/.well-known/oauth-authorization-server/mcp")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "authorization_endpoint": format!("{}/oauth/authorize", first_login_server.uri()), + "token_endpoint": format!("{}/oauth/token", first_login_server.uri()), + }))) + .expect(1) + .mount(&first_login_server) + .await; + + let status = auth_status(&format!("{}/mcp", first_login_server.uri())).await?; + assert_eq!(status, McpAuthState::LoggedOut(McpLoginRequirement::Login)); + first_login_server.verify().await; + let response = OAuthTokenResponse::new( AccessToken::new(EXPIRED_ACCESS_TOKEN.to_string()), BasicTokenType::Bearer, @@ -166,7 +217,10 @@ async fn persisted_credentials_auth_status_child() -> anyhow::Result<()> { )?; let status = auth_status(UNREFRESHABLE_SERVER_URL).await?; - assert_eq!(status, McpAuthStatus::NotLoggedIn); + assert_eq!( + status, + McpAuthState::LoggedOut(McpLoginRequirement::Reauthentication) + ); let response = OAuthTokenResponse::new( AccessToken::new("unexpired-access-token".to_string()), @@ -192,7 +246,7 @@ async fn persisted_credentials_auth_status_child() -> anyhow::Result<()> { )?; let status = auth_status(UNEXPIRED_SERVER_URL).await?; - assert_eq!(status, McpAuthStatus::OAuth); + assert_eq!(status, McpAuthState::OAuth); let mut response = OAuthTokenResponse::new( AccessToken::new(EXPIRED_ACCESS_TOKEN.to_string()), @@ -215,11 +269,11 @@ async fn persisted_credentials_auth_status_child() -> anyhow::Result<()> { )?; let status = auth_status(REFRESHABLE_SERVER_URL).await?; - assert_eq!(status, McpAuthStatus::OAuth); + assert_eq!(status, McpAuthState::OAuth); Ok(()) } -async fn auth_status(server_url: &str) -> anyhow::Result { +async fn auth_status(server_url: &str) -> anyhow::Result { determine_streamable_http_auth_status( SERVER_NAME, server_url, @@ -279,3 +333,46 @@ async fn oauth_startup_child() -> anyhow::Result<()> { initialize_client(&client).await?; Ok(()) } + +#[tokio::test(flavor = "multi_thread", worker_threads = 1)] +#[ignore = "spawned by identifies_expired_unrefreshable_token_startup_error"] +async fn expired_unrefreshable_startup_child() -> anyhow::Result<()> { + let server_url = std::env::var(CHILD_SERVER_URL_ENV)?; + let response = OAuthTokenResponse::new( + AccessToken::new(EXPIRED_ACCESS_TOKEN.to_string()), + BasicTokenType::Bearer, + VendorExtraTokenFields::default(), + ); + let tokens = StoredOAuthTokens { + server_name: SERVER_NAME.to_string(), + url: server_url.clone(), + client_id: "test-client-id".to_string(), + token_response: WrappedOAuthTokenResponse(response), + expires_at: Some(0), + }; + save_oauth_tokens( + SERVER_NAME, + &tokens, + OAuthCredentialsStoreMode::File, + AuthKeyringBackendKind::default(), + )?; + + let client = RmcpClient::new_streamable_http_client( + SERVER_NAME, + &server_url, + /*bearer_token*/ None, + /*http_headers*/ None, + /*env_http_headers*/ None, + OAuthCredentialsStoreMode::File, + AuthKeyringBackendKind::default(), + Environment::default_for_tests().get_http_client(), + /*auth_provider*/ None, + ) + .await?; + + let error = initialize_client(&client) + .await + .expect_err("expired token without a refresh token should fail startup"); + assert!(is_authentication_required_error(&error)); + Ok(()) +} diff --git a/codex-rs/rollout/src/compression_tests.rs b/codex-rs/rollout/src/compression_tests.rs index a4db9a965c..46c90d0120 100644 --- a/codex-rs/rollout/src/compression_tests.rs +++ b/codex-rs/rollout/src/compression_tests.rs @@ -472,7 +472,9 @@ fn write_rollout(path: &std::path::Path, thread_id: ThreadId, message: &str) -> model_provider: None, base_instructions: None, dynamic_tools: None, + selected_capability_roots: Vec::new(), memory_mode: None, + history_mode: Default::default(), multi_agent_version: None, context_window: None, }, diff --git a/codex-rs/rollout/src/list.rs b/codex-rs/rollout/src/list.rs index b02b8c3e49..1154d36eee 100644 --- a/codex-rs/rollout/src/list.rs +++ b/codex-rs/rollout/src/list.rs @@ -26,7 +26,9 @@ use codex_protocol::protocol::RolloutItem; use codex_protocol::protocol::RolloutLine; use codex_protocol::protocol::SessionMetaLine; use codex_protocol::protocol::SessionSource; +use codex_protocol::protocol::ThreadHistoryMode; use codex_protocol::protocol::USER_MESSAGE_BEGIN; +use serde_json::Value; /// Returned page of thread (thread) summaries. #[derive(Debug, Default, PartialEq)] @@ -62,6 +64,8 @@ pub struct ThreadItem { pub git_origin_url: Option, /// Session source from session metadata. pub source: Option, + /// Persisted thread history contract selected when this thread was created. + pub history_mode: ThreadHistoryMode, /// Immediate control/spawn parent thread id from session metadata. pub parent_thread_id: Option, /// Random unique nickname from session metadata for AgentControl-spawned sub-agents. @@ -99,6 +103,7 @@ struct HeadTailSummary { git_sha: Option, git_origin_url: Option, source: Option, + history_mode: ThreadHistoryMode, parent_thread_id: Option, agent_nickname: Option, agent_role: Option, @@ -806,6 +811,7 @@ async fn build_thread_item( git_sha, git_origin_url, source, + history_mode, parent_thread_id, agent_nickname, agent_role, @@ -828,6 +834,7 @@ async fn build_thread_item( git_sha, git_origin_url, source, + history_mode, parent_thread_id, agent_nickname, agent_role, @@ -1117,12 +1124,26 @@ async fn read_head_summary(path: &Path, head_limit: usize) -> io::Result = serde_json::from_str(trimmed); - let Ok(rollout_line) = parsed else { continue }; + let rollout_line = match parsed { + Ok(rollout_line) => rollout_line, + Err(_) => { + if !summary.saw_session_meta + && let Ok(value) = serde_json::from_str::(trimmed) + { + // The first SessionMeta belongs to this rollout. Later SessionMeta lines can + // be copied from fork history, so only an unknown mode before the first parsed + // SessionMeta should make this thread unreadable. + crate::recorder::reject_unknown_thread_history_mode(&value)?; + } + continue; + } + }; match rollout_line.item { RolloutItem::SessionMeta(session_meta_line) => { if !summary.saw_session_meta { summary.source = Some(session_meta_line.meta.source.clone()); + summary.history_mode = session_meta_line.meta.history_mode; summary.parent_thread_id = session_meta_line.meta.parent_thread_id; summary.agent_nickname = session_meta_line.meta.agent_nickname.clone(); summary.agent_role = session_meta_line.meta.agent_role.clone(); diff --git a/codex-rs/rollout/src/metadata.rs b/codex-rs/rollout/src/metadata.rs index d6ed809136..127a8af223 100644 --- a/codex-rs/rollout/src/metadata.rs +++ b/codex-rs/rollout/src/metadata.rs @@ -45,6 +45,7 @@ pub(crate) fn builder_from_session_meta( created_at, session_meta.meta.source.clone(), ); + builder.history_mode = session_meta.meta.history_mode; builder.model_provider = session_meta.meta.model_provider.clone(); builder.agent_nickname = session_meta.meta.agent_nickname.clone(); builder.agent_role = session_meta.meta.agent_role.clone(); diff --git a/codex-rs/rollout/src/metadata_tests.rs b/codex-rs/rollout/src/metadata_tests.rs index 0744e7780e..2e1ea98f3a 100644 --- a/codex-rs/rollout/src/metadata_tests.rs +++ b/codex-rs/rollout/src/metadata_tests.rs @@ -13,6 +13,7 @@ use codex_protocol::protocol::RolloutLine; use codex_protocol::protocol::SessionMeta; use codex_protocol::protocol::SessionMetaLine; use codex_protocol::protocol::SessionSource; +use codex_protocol::protocol::ThreadHistoryMode; use codex_state::BackfillStatus; use codex_state::ThreadMetadataBuilder; use pretty_assertions::assert_eq; @@ -49,7 +50,9 @@ async fn extract_metadata_from_rollout_uses_session_meta() { model_provider: Some("openai".to_string()), base_instructions: None, dynamic_tools: None, + selected_capability_roots: Vec::new(), memory_mode: None, + history_mode: ThreadHistoryMode::Paginated, multi_agent_version: None, context_window: None, }; @@ -80,6 +83,41 @@ async fn extract_metadata_from_rollout_uses_session_meta() { assert_eq!(outcome.parse_errors, 0); } +#[tokio::test] +async fn extract_metadata_from_rollout_rejects_unknown_history_mode() { + let dir = tempdir().expect("tempdir"); + let uuid = Uuid::new_v4(); + let id = ThreadId::from_string(&uuid.to_string()).expect("thread id"); + let path = dir + .path() + .join(format!("rollout-2026-01-27T12-34-56-{uuid}.jsonl")); + let mut rollout_line = serde_json::to_value(RolloutLine { + timestamp: "2026-01-27T12:34:56Z".to_string(), + item: RolloutItem::SessionMeta(SessionMetaLine { + meta: SessionMeta { + session_id: id.into(), + id, + timestamp: "2026-01-27T12:34:56Z".to_string(), + cwd: dir.path().to_path_buf(), + originator: "cli".to_string(), + cli_version: "0.0.0".to_string(), + ..SessionMeta::default() + }, + git: None, + }), + }) + .expect("serialize rollout line"); + rollout_line["payload"]["history_mode"] = serde_json::json!("future"); + let mut file = File::create(&path).expect("create rollout"); + writeln!(file, "{rollout_line}").expect("write rollout"); + + assert!( + extract_metadata_from_rollout(&path, "openai") + .await + .is_err() + ); +} + #[tokio::test] async fn extract_metadata_from_rollout_returns_latest_memory_mode() { let dir = tempdir().expect("tempdir"); @@ -106,7 +144,9 @@ async fn extract_metadata_from_rollout_returns_latest_memory_mode() { model_provider: Some("openai".to_string()), base_instructions: None, dynamic_tools: None, + selected_capability_roots: Vec::new(), memory_mode: None, + history_mode: Default::default(), multi_agent_version: None, context_window: None, }; @@ -375,7 +415,9 @@ fn write_rollout_in_sessions_with_cwd( model_provider: Some("test-provider".to_string()), base_instructions: None, dynamic_tools: None, + selected_capability_roots: Vec::new(), memory_mode: None, + history_mode: Default::default(), multi_agent_version: None, context_window: None, }; diff --git a/codex-rs/rollout/src/persistence_metrics.rs b/codex-rs/rollout/src/persistence_metrics.rs index 249a23cc70..8c2a7e18d2 100644 --- a/codex-rs/rollout/src/persistence_metrics.rs +++ b/codex-rs/rollout/src/persistence_metrics.rs @@ -247,6 +247,10 @@ fn turn_item_type(item: &TurnItem) -> &'static str { TurnItem::AgentMessage(_) => "agent_message", TurnItem::Plan(_) => "plan", TurnItem::Reasoning(_) => "reasoning", + TurnItem::CommandExecution(_) => "command_execution", + TurnItem::DynamicToolCall(_) => "dynamic_tool_call", + TurnItem::CollabAgentToolCall(_) => "collab_agent_tool_call", + TurnItem::SubAgentActivity(_) => "sub_agent_activity", TurnItem::WebSearch(_) => "web_search", TurnItem::ImageView(_) => "image_view", TurnItem::Sleep(_) => "sleep", diff --git a/codex-rs/rollout/src/recorder.rs b/codex-rs/rollout/src/recorder.rs index 3908604c77..87e97997a2 100644 --- a/codex-rs/rollout/src/recorder.rs +++ b/codex-rs/rollout/src/recorder.rs @@ -12,6 +12,7 @@ use std::sync::Mutex; use chrono::SecondsFormat; use codex_protocol::SessionId; use codex_protocol::ThreadId; +use codex_protocol::capabilities::SelectedCapabilityRoot; use codex_protocol::dynamic_tools::DynamicToolSpec; use codex_protocol::models::BaseInstructions; use serde_json::Value; @@ -60,6 +61,7 @@ use codex_protocol::protocol::SessionContextWindow; use codex_protocol::protocol::SessionMeta; use codex_protocol::protocol::SessionMetaLine; use codex_protocol::protocol::SessionSource; +use codex_protocol::protocol::ThreadHistoryMode; use codex_protocol::protocol::ThreadSource; use codex_state::StateRuntime; use codex_utils_path as path_utils; @@ -91,7 +93,9 @@ pub enum RolloutRecorderParams { originator: String, base_instructions: BaseInstructions, dynamic_tools: Vec, + selected_capability_roots: Vec, multi_agent_version: Option, + history_mode: ThreadHistoryMode, initial_window_id: Option, }, Resume { @@ -182,7 +186,9 @@ impl RolloutRecorderParams { originator, base_instructions, dynamic_tools, + selected_capability_roots: Vec::new(), multi_agent_version: None, + history_mode: Default::default(), initial_window_id: None, } } @@ -194,6 +200,20 @@ impl RolloutRecorderParams { self } + pub fn with_selected_capability_roots( + mut self, + selected_capability_roots: Vec, + ) -> Self { + if let Self::Create { + selected_capability_roots: roots, + .. + } = &mut self + { + *roots = selected_capability_roots; + } + self + } + pub fn with_multi_agent_version( mut self, multi_agent_version: Option, @@ -208,6 +228,16 @@ impl RolloutRecorderParams { self } + pub fn with_history_mode(mut self, history_mode: ThreadHistoryMode) -> Self { + if let Self::Create { + history_mode: mode, .. + } = &mut self + { + *mode = history_mode; + } + self + } + pub fn with_initial_window_id(mut self, initial_window_id: String) -> Self { if let Self::Create { initial_window_id: window_id, @@ -732,7 +762,9 @@ impl RolloutRecorder { originator, base_instructions, dynamic_tools, + selected_capability_roots, multi_agent_version, + history_mode, initial_window_id, } => { let log_file_info = precompute_log_file_info(config, conversation_id)?; @@ -769,7 +801,9 @@ impl RolloutRecorder { } else { Some(dynamic_tools) }, + selected_capability_roots, memory_mode: (!config.generate_memories()).then_some("disabled".to_string()), + history_mode, multi_agent_version, context_window: initial_window_id.map(SessionContextWindow::new), }; @@ -937,6 +971,12 @@ impl RolloutRecorder { items.push(item); } Err(e) => { + if thread_id.is_none() { + // The first SessionMeta belongs to this rollout. Later SessionMeta lines + // can be copied from fork history, so only validate unknown history modes + // before we have parsed the rollout's own SessionMeta. + reject_unknown_thread_history_mode(&v)?; + } trace!("failed to parse rollout line: {e}"); parse_errors = parse_errors.saturating_add(1); } @@ -1000,6 +1040,21 @@ impl RolloutRecorder { } } +pub(crate) fn reject_unknown_thread_history_mode(value: &Value) -> std::io::Result<()> { + if value.get("type").and_then(Value::as_str) != Some("session_meta") { + return Ok(()); + } + let Some(history_mode) = value + .get("payload") + .and_then(|payload| payload.get("history_mode")) + else { + return Ok(()); + }; + serde_json::from_value::(history_mode.clone()) + .map(|_| ()) + .map_err(|err| IoError::other(format!("invalid session metadata history_mode: {err}"))) +} + fn strip_legacy_ghost_snapshot_rollout_line(value: &mut Value) -> bool { match value.get("type").and_then(Value::as_str) { Some("response_item") => value @@ -1103,6 +1158,7 @@ fn fill_missing_thread_item_metadata(item: &mut ThreadItem, state_item: ThreadIt git_sha, git_origin_url, source, + history_mode: _, parent_thread_id, agent_nickname, agent_role, @@ -1818,6 +1874,7 @@ fn thread_item_from_state_metadata( .or_else(|_| serde_json::from_value(Value::String(item.source))) .unwrap_or(SessionSource::Unknown), ), + history_mode: item.history_mode, parent_thread_id, agent_nickname: item.agent_nickname, agent_role: item.agent_role, diff --git a/codex-rs/rollout/src/recorder_tests.rs b/codex-rs/rollout/src/recorder_tests.rs index 4b72ff7673..7fcaff08ab 100644 --- a/codex-rs/rollout/src/recorder_tests.rs +++ b/codex-rs/rollout/src/recorder_tests.rs @@ -15,6 +15,7 @@ use codex_protocol::protocol::SandboxPolicy; use codex_protocol::protocol::SessionMeta; use codex_protocol::protocol::SessionMetaLine; use codex_protocol::protocol::SessionSource; +use codex_protocol::protocol::ThreadHistoryMode; use codex_protocol::protocol::TurnContextItem; use codex_protocol::protocol::UserMessageEvent; use pretty_assertions::assert_eq; @@ -101,7 +102,9 @@ async fn state_db_init_backfills_before_returning() -> anyhow::Result<()> { model_provider: None, base_instructions: None, dynamic_tools: None, + selected_capability_roots: Vec::new(), memory_mode: None, + history_mode: Default::default(), multi_agent_version: None, context_window: None, }, @@ -226,6 +229,43 @@ async fn load_rollout_items_defaults_legacy_session_id() -> std::io::Result<()> Ok(()) } +#[tokio::test] +async fn load_rollout_items_ignores_unknown_fork_source_history_mode() -> std::io::Result<()> { + let home = TempDir::new().expect("temp dir"); + let uuid = Uuid::new_v4(); + let thread_id = ThreadId::from_string(&uuid.to_string()).expect("thread id"); + let rollout_path = write_session_file(home.path(), "2025-01-03T12-00-00", uuid)?; + let mut file = fs::OpenOptions::new().append(true).open(&rollout_path)?; + let source_uuid = Uuid::new_v4(); + writeln!( + file, + "{}", + serde_json::json!({ + "timestamp": "2025-01-03T12:00:01Z", + "type": "session_meta", + "payload": { + "session_id": source_uuid, + "id": source_uuid, + "timestamp": "2025-01-03T12:00:01Z", + "cwd": ".", + "originator": "test_originator", + "cli_version": "test_version", + "source": "cli", + "model_provider": "test-provider", + "history_mode": "future", + }, + }) + )?; + + let (items, loaded_thread_id, parse_errors) = + RolloutRecorder::load_rollout_items(&rollout_path).await?; + + assert_eq!(loaded_thread_id, Some(thread_id)); + assert_eq!(parse_errors, 1); + assert_eq!(items.len(), 2); + Ok(()) +} + #[tokio::test] async fn load_rollout_items_preserves_legacy_guardian_assessment_lines() -> std::io::Result<()> { let home = TempDir::new().expect("temp dir"); @@ -390,6 +430,7 @@ async fn recorder_materializes_on_flush_with_pending_items() -> std::io::Result< Vec::new(), ) .with_session_id(session_id) + .with_history_mode(ThreadHistoryMode::Paginated) .with_initial_window_id(initial_window_id.clone()), ) .await?; @@ -441,6 +482,7 @@ async fn recorder_materializes_on_flush_with_pending_items() -> std::io::Result< panic!("expected session metadata in rollout"); }; assert_eq!(session_meta.meta.session_id, session_id); + assert_eq!(session_meta.meta.history_mode, ThreadHistoryMode::Paginated); assert_eq!( session_meta .meta @@ -1003,6 +1045,7 @@ fn fill_missing_thread_item_metadata_preserves_identity_and_prefers_state_git_fi git_sha: Some("filesystem-sha".to_string()), git_origin_url: Some("https://example.com/filesystem.git".to_string()), source: None, + history_mode: Default::default(), parent_thread_id: None, agent_nickname: None, agent_role: None, @@ -1022,6 +1065,7 @@ fn fill_missing_thread_item_metadata_preserves_identity_and_prefers_state_git_fi git_sha: Some("state-sha".to_string()), git_origin_url: Some("https://example.com/state.git".to_string()), source: Some(SessionSource::Exec), + history_mode: Default::default(), parent_thread_id: None, agent_nickname: Some("state-agent".to_string()), agent_role: Some("state-role".to_string()), diff --git a/codex-rs/rollout/src/session_index_tests.rs b/codex-rs/rollout/src/session_index_tests.rs index 7f57c2ba08..6eb1fa0374 100644 --- a/codex-rs/rollout/src/session_index_tests.rs +++ b/codex-rs/rollout/src/session_index_tests.rs @@ -41,7 +41,9 @@ fn write_rollout_with_metadata(path: &Path, thread_id: ThreadId) -> std::io::Res model_provider: Some("test-provider".into()), base_instructions: None, dynamic_tools: None, + selected_capability_roots: Vec::new(), memory_mode: None, + history_mode: Default::default(), multi_agent_version: None, context_window: None, }, diff --git a/codex-rs/rollout/src/state_db_tests.rs b/codex-rs/rollout/src/state_db_tests.rs index 5712b65cee..18566d887e 100644 --- a/codex-rs/rollout/src/state_db_tests.rs +++ b/codex-rs/rollout/src/state_db_tests.rs @@ -174,7 +174,9 @@ fn write_rollout_with_user_message( model_provider: Some("test-provider".to_string()), base_instructions: None, dynamic_tools: None, + selected_capability_roots: Vec::new(), memory_mode: None, + history_mode: Default::default(), multi_agent_version: None, context_window: None, }, diff --git a/codex-rs/rollout/src/tests.rs b/codex-rs/rollout/src/tests.rs index e6a3c728ef..23f72cb947 100644 --- a/codex-rs/rollout/src/tests.rs +++ b/codex-rs/rollout/src/tests.rs @@ -129,6 +129,41 @@ async fn find_thread_path_falls_back_when_db_path_is_stale() { assert_state_db_rollout_path(home, thread_id, Some(fs_rollout_path.as_path())).await; } +#[tokio::test] +async fn read_thread_item_from_rollout_rejects_unknown_canonical_history_mode() { + let temp = TempDir::new().unwrap(); + let home = temp.path(); + let uuid = Uuid::from_u128(303); + let ts = "2025-01-03T12-00-00"; + write_session_file( + home, + ts, + uuid, + /*num_records*/ 1, + Some(SessionSource::Cli), + ) + .unwrap(); + let path = home.join(format!("sessions/2025/01/03/rollout-{ts}-{uuid}.jsonl")); + let contents = fs::read_to_string(&path).unwrap(); + let mut lines = contents.lines(); + let mut canonical_meta: serde_json::Value = + serde_json::from_str(lines.next().expect("canonical session meta")).unwrap(); + canonical_meta["payload"]["history_mode"] = serde_json::json!("future"); + let user_event = lines.next().expect("user event"); + let mut source_meta = canonical_meta.clone(); + let source_uuid = Uuid::from_u128(304); + source_meta["payload"]["session_id"] = serde_json::json!(source_uuid); + source_meta["payload"]["id"] = serde_json::json!(source_uuid); + source_meta["payload"]["history_mode"] = serde_json::json!("legacy"); + fs::write( + &path, + format!("{canonical_meta}\n{user_event}\n{source_meta}\n"), + ) + .unwrap(); + + assert_eq!(crate::list::read_thread_item_from_rollout(path).await, None); +} + #[tokio::test] async fn find_thread_path_falls_back_when_db_path_points_to_another_thread() { let temp = TempDir::new().unwrap(); @@ -610,6 +645,7 @@ async fn test_list_conversations_latest_first() { git_sha: None, git_origin_url: None, source: Some(SessionSource::VSCode), + history_mode: Default::default(), parent_thread_id: None, agent_nickname: None, agent_role: None, @@ -629,6 +665,7 @@ async fn test_list_conversations_latest_first() { git_sha: None, git_origin_url: None, source: Some(SessionSource::VSCode), + history_mode: Default::default(), parent_thread_id: None, agent_nickname: None, agent_role: None, @@ -648,6 +685,7 @@ async fn test_list_conversations_latest_first() { git_sha: None, git_origin_url: None, source: Some(SessionSource::VSCode), + history_mode: Default::default(), parent_thread_id: None, agent_nickname: None, agent_role: None, @@ -760,6 +798,7 @@ async fn test_pagination_cursor() { git_sha: None, git_origin_url: None, source: Some(SessionSource::VSCode), + history_mode: Default::default(), parent_thread_id: None, agent_nickname: None, agent_role: None, @@ -779,6 +818,7 @@ async fn test_pagination_cursor() { git_sha: None, git_origin_url: None, source: Some(SessionSource::VSCode), + history_mode: Default::default(), parent_thread_id: None, agent_nickname: None, agent_role: None, @@ -834,6 +874,7 @@ async fn test_pagination_cursor() { git_sha: None, git_origin_url: None, source: Some(SessionSource::VSCode), + history_mode: Default::default(), parent_thread_id: None, agent_nickname: None, agent_role: None, @@ -853,6 +894,7 @@ async fn test_pagination_cursor() { git_sha: None, git_origin_url: None, source: Some(SessionSource::VSCode), + history_mode: Default::default(), parent_thread_id: None, agent_nickname: None, agent_role: None, @@ -900,6 +942,7 @@ async fn test_pagination_cursor() { git_sha: None, git_origin_url: None, source: Some(SessionSource::VSCode), + history_mode: Default::default(), parent_thread_id: None, agent_nickname: None, agent_role: None, @@ -1072,6 +1115,7 @@ async fn test_get_thread_contents() { git_sha: None, git_origin_url: None, source: Some(SessionSource::VSCode), + history_mode: Default::default(), parent_thread_id: None, agent_nickname: None, agent_role: None, @@ -1288,7 +1332,9 @@ async fn test_updated_at_uses_file_mtime() -> Result<()> { model_provider: Some("test-provider".into()), base_instructions: None, dynamic_tools: None, + selected_capability_roots: Vec::new(), memory_mode: None, + history_mode: Default::default(), multi_agent_version: None, context_window: None, }, @@ -1431,6 +1477,7 @@ async fn test_timestamp_only_cursor_skips_same_second_filesystem_ties() { git_sha: None, git_origin_url: None, source: Some(SessionSource::VSCode), + history_mode: Default::default(), parent_thread_id: None, agent_nickname: None, agent_role: None, @@ -1450,6 +1497,7 @@ async fn test_timestamp_only_cursor_skips_same_second_filesystem_ties() { git_sha: None, git_origin_url: None, source: Some(SessionSource::VSCode), + history_mode: Default::default(), parent_thread_id: None, agent_nickname: None, agent_role: None, diff --git a/codex-rs/shell-escalation/README.md b/codex-rs/shell-escalation/README.md index e4d3fecd66..69cd038220 100644 --- a/codex-rs/shell-escalation/README.md +++ b/codex-rs/shell-escalation/README.md @@ -27,3 +27,8 @@ git apply /path/to/patches/zsh-exec-wrapper.patch ./configure make -j"$(nproc)" ``` + +Release artifacts are built by `.github/workflows/rust-release-zsh.yml` when a +`codex-zsh-vX.Y.Z` tag is pushed. When the zsh commit or patch changes, publish +the next version tag and update the checked-in DotSlash manifests to use the new +release. diff --git a/codex-rs/state/migrations/0040_threads_history_mode.sql b/codex-rs/state/migrations/0040_threads_history_mode.sql new file mode 100644 index 0000000000..53729c2ec6 --- /dev/null +++ b/codex-rs/state/migrations/0040_threads_history_mode.sql @@ -0,0 +1 @@ +ALTER TABLE threads ADD COLUMN history_mode TEXT NOT NULL DEFAULT 'legacy'; diff --git a/codex-rs/state/src/extract.rs b/codex-rs/state/src/extract.rs index e5ae07839e..48e308f627 100644 --- a/codex-rs/state/src/extract.rs +++ b/codex-rs/state/src/extract.rs @@ -56,6 +56,7 @@ fn apply_session_meta_from_item(metadata: &mut ThreadMetadata, meta_line: &Sessi } metadata.id = meta_line.meta.id; metadata.source = enum_to_string(&meta_line.meta.source); + // Later SessionMeta lines do not redefine the canonical history_mode. metadata.thread_source = meta_line.meta.thread_source.clone(); metadata.agent_nickname = meta_line.meta.agent_nickname.clone(); metadata.agent_role = meta_line.meta.agent_role.clone(); @@ -177,6 +178,7 @@ mod tests { use codex_protocol::protocol::ThreadGoal; use codex_protocol::protocol::ThreadGoalStatus; use codex_protocol::protocol::ThreadGoalUpdatedEvent; + use codex_protocol::protocol::ThreadHistoryMode; use codex_protocol::protocol::TurnContextItem; use codex_protocol::protocol::USER_MESSAGE_BEGIN; use codex_protocol::protocol::UserMessageEvent; @@ -343,7 +345,9 @@ mod tests { model_provider: Some("openai".to_string()), base_instructions: None, dynamic_tools: None, + selected_capability_roots: Vec::new(), memory_mode: None, + history_mode: Default::default(), multi_agent_version: None, context_window: None, }, @@ -513,6 +517,7 @@ mod tests { #[test] fn session_meta_does_not_set_model_or_reasoning_effort() { let mut metadata = metadata_for_test(); + metadata.history_mode = ThreadHistoryMode::Paginated; let thread_id = metadata.id; apply_rollout_item( @@ -535,7 +540,9 @@ mod tests { model_provider: Some("openai".to_string()), base_instructions: None, dynamic_tools: None, + selected_capability_roots: Vec::new(), memory_mode: None, + history_mode: ThreadHistoryMode::Legacy, multi_agent_version: None, context_window: None, }, @@ -546,6 +553,7 @@ mod tests { assert_eq!(metadata.model, None); assert_eq!(metadata.reasoning_effort, None); + assert_eq!(metadata.history_mode, ThreadHistoryMode::Paginated); } fn metadata_for_test() -> ThreadMetadata { @@ -558,6 +566,7 @@ mod tests { updated_at: created_at, recency_at: created_at, source: "cli".to_string(), + history_mode: Default::default(), thread_source: None, agent_path: None, agent_nickname: None, diff --git a/codex-rs/state/src/model/thread_metadata.rs b/codex-rs/state/src/model/thread_metadata.rs index ac8096e456..0024c9866d 100644 --- a/codex-rs/state/src/model/thread_metadata.rs +++ b/codex-rs/state/src/model/thread_metadata.rs @@ -6,6 +6,7 @@ use codex_protocol::openai_models::ReasoningEffort; use codex_protocol::protocol::AskForApproval; use codex_protocol::protocol::SandboxPolicy; use codex_protocol::protocol::SessionSource; +use codex_protocol::protocol::ThreadHistoryMode; use codex_protocol::protocol::ThreadSource; use sqlx::Row; use sqlx::sqlite::SqliteRow; @@ -87,6 +88,8 @@ pub struct ThreadMetadata { pub recency_at: DateTime, /// The session source (stringified enum). pub source: String, + /// Persisted thread history contract selected when this thread was created. + pub history_mode: ThreadHistoryMode, /// Optional analytics source classification for this thread. pub thread_source: Option, /// Optional random unique nickname assigned to an AgentControl-spawned sub-agent. @@ -142,6 +145,8 @@ pub struct ThreadMetadataBuilder { pub recency_at: Option>, /// The session source. pub source: SessionSource, + /// Persisted thread history contract selected when this thread was created. + pub history_mode: ThreadHistoryMode, /// Optional analytics source classification for this thread. pub thread_source: Option, /// Optional random unique nickname assigned to the session. @@ -185,6 +190,7 @@ impl ThreadMetadataBuilder { updated_at: None, recency_at: None, source, + history_mode: ThreadHistoryMode::Legacy, thread_source: None, agent_nickname: None, agent_role: None, @@ -222,6 +228,7 @@ impl ThreadMetadataBuilder { updated_at, recency_at, source, + history_mode: self.history_mode, thread_source: self.thread_source.clone(), agent_nickname: self.agent_nickname.clone(), agent_role: self.agent_role.clone(), @@ -368,6 +375,7 @@ pub(crate) struct ThreadRow { updated_at: i64, recency_at: i64, source: String, + history_mode: String, thread_source: Option, agent_nickname: Option, agent_role: Option, @@ -398,6 +406,7 @@ impl ThreadRow { updated_at: row.try_get("updated_at")?, recency_at: row.try_get("recency_at")?, source: row.try_get("source")?, + history_mode: row.try_get("history_mode")?, thread_source: row.try_get("thread_source")?, agent_nickname: row.try_get("agent_nickname")?, agent_role: row.try_get("agent_role")?, @@ -432,6 +441,7 @@ impl TryFrom for ThreadMetadata { updated_at, recency_at, source, + history_mode, thread_source, agent_nickname, agent_role, @@ -456,6 +466,7 @@ impl TryFrom for ThreadMetadata { .map(|thread_source| thread_source.parse()) .transpose() .map_err(anyhow::Error::msg)?; + let history_mode = history_mode.parse().map_err(anyhow::Error::msg)?; Ok(Self { id: ThreadId::try_from(id)?, rollout_path: PathBuf::from(rollout_path), @@ -463,6 +474,7 @@ impl TryFrom for ThreadMetadata { updated_at: epoch_millis_to_datetime(updated_at)?, recency_at: epoch_millis_to_datetime(recency_at)?, source, + history_mode, thread_source, agent_nickname, agent_role, @@ -548,6 +560,7 @@ mod tests { use chrono::Utc; use codex_protocol::ThreadId; use codex_protocol::openai_models::ReasoningEffort; + use codex_protocol::protocol::ThreadHistoryMode; use pretty_assertions::assert_eq; use std::path::PathBuf; @@ -559,6 +572,7 @@ mod tests { updated_at: 1_700_000_100, recency_at: 1_700_000_100, source: "cli".to_string(), + history_mode: "legacy".to_string(), thread_source: None, agent_nickname: None, agent_role: None, @@ -590,6 +604,7 @@ mod tests { updated_at: DateTime::::from_timestamp(1_700_000_100, 0).expect("timestamp"), recency_at: DateTime::::from_timestamp(1_700_000_100, 0).expect("timestamp"), source: "cli".to_string(), + history_mode: ThreadHistoryMode::Legacy, thread_source: None, agent_nickname: None, agent_role: None, @@ -633,4 +648,12 @@ mod tests { expected_thread_metadata(Some(ReasoningEffort::Custom("future".to_string()))) ); } + + #[test] + fn thread_row_rejects_unknown_history_mode() { + let mut row = thread_row(/*reasoning_effort*/ None); + row.history_mode = "future".to_string(); + + assert!(ThreadMetadata::try_from(row).is_err()); + } } diff --git a/codex-rs/state/src/runtime/memories.rs b/codex-rs/state/src/runtime/memories.rs index 83df50e02e..ff25fed982 100644 --- a/codex-rs/state/src/runtime/memories.rs +++ b/codex-rs/state/src/runtime/memories.rs @@ -136,6 +136,7 @@ WHERE kind = ? AND job_key = ? /// - starts from `threads` filtered to active threads and allowed sources /// (`push_thread_filters`) /// - excludes threads with `memory_mode != 'enabled'` + /// - excludes paginated threads because stage 1 still full-loads rollout JSONL /// - excludes the current thread id /// - keeps only threads whose millisecond `updated_at` is in the age window /// - checks memory staleness against the memories DB @@ -177,6 +178,7 @@ SELECT threads.updated_at_ms AS updated_at, threads.recency_at_ms AS recency_at, threads.source, + threads.history_mode, threads.thread_source, threads.agent_path, threads.agent_nickname, @@ -213,7 +215,7 @@ FROM threads }, /*include_thread_id_tiebreaker*/ false, ); - builder.push(" AND threads.memory_mode = 'enabled'"); + builder.push(" AND threads.memory_mode = 'enabled' AND threads.history_mode = 'legacy'"); builder .push(" AND threads.id != ") .push_bind(current_thread_id.as_str()); @@ -549,6 +551,7 @@ SELECT threads.updated_at_ms AS updated_at, threads.recency_at_ms AS recency_at, threads.source, + threads.history_mode, threads.thread_source, threads.agent_nickname, threads.agent_role, @@ -569,7 +572,7 @@ SELECT threads.git_branch, threads.git_origin_url FROM threads -WHERE threads.id = ? AND threads.memory_mode = 'enabled' +WHERE threads.id = ? AND threads.memory_mode = 'enabled' AND threads.history_mode = 'legacy' "#, ) .bind(thread_id.to_string()) @@ -1673,6 +1676,7 @@ mod tests { use chrono::Duration; use chrono::Utc; use codex_protocol::ThreadId; + use codex_protocol::protocol::ThreadHistoryMode; use pretty_assertions::assert_eq; use sqlx::Row; use std::sync::Arc; @@ -2161,7 +2165,7 @@ mod tests { } #[tokio::test] - async fn claim_stage1_jobs_skips_threads_with_disabled_memory_mode() { + async fn claim_stage1_jobs_skips_threads_without_legacy_enabled_memory() { let codex_home = unique_temp_dir(); let runtime = StateRuntime::init(codex_home.clone(), "test-provider".to_string()) .await @@ -2174,6 +2178,8 @@ mod tests { ThreadId::from_string(&Uuid::new_v4().to_string()).expect("current thread id"); let disabled_thread_id = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("disabled thread id"); + let paginated_thread_id = + ThreadId::from_string(&Uuid::new_v4().to_string()).expect("paginated thread id"); let enabled_thread_id = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("enabled thread id"); @@ -2200,6 +2206,19 @@ mod tests { .await .expect("disable thread memory mode"); + let mut paginated = test_thread_metadata( + &codex_home, + paginated_thread_id, + codex_home.join("paginated"), + ); + paginated.created_at = eligible_at; + paginated.updated_at = eligible_at; + paginated.history_mode = ThreadHistoryMode::Paginated; + runtime + .upsert_thread(&paginated) + .await + .expect("upsert paginated thread"); + let mut enabled = test_thread_metadata(&codex_home, enabled_thread_id, codex_home.join("enabled")); enabled.created_at = eligible_at; diff --git a/codex-rs/state/src/runtime/test_support.rs b/codex-rs/state/src/runtime/test_support.rs index c029a0ee2d..8c63066842 100644 --- a/codex-rs/state/src/runtime/test_support.rs +++ b/codex-rs/state/src/runtime/test_support.rs @@ -11,6 +11,8 @@ use codex_protocol::protocol::AskForApproval; #[cfg(test)] use codex_protocol::protocol::SandboxPolicy; #[cfg(test)] +use codex_protocol::protocol::ThreadHistoryMode; +#[cfg(test)] use std::path::Path; #[cfg(test)] use std::path::PathBuf; @@ -49,6 +51,7 @@ pub(super) fn test_thread_metadata( updated_at: now, recency_at: now, source: "cli".to_string(), + history_mode: ThreadHistoryMode::Legacy, thread_source: None, agent_nickname: None, agent_role: None, diff --git a/codex-rs/state/src/runtime/threads.rs b/codex-rs/state/src/runtime/threads.rs index d1c2c90762..c80b2b6716 100644 --- a/codex-rs/state/src/runtime/threads.rs +++ b/codex-rs/state/src/runtime/threads.rs @@ -15,6 +15,7 @@ SELECT threads.updated_at_ms AS updated_at, threads.recency_at_ms AS recency_at, threads.source, + threads.history_mode, threads.thread_source, threads.agent_nickname, threads.agent_role, @@ -544,6 +545,7 @@ INSERT INTO threads ( updated_at_ms, recency_at_ms, source, + history_mode, thread_source, agent_nickname, agent_role, @@ -565,7 +567,7 @@ INSERT INTO threads ( git_branch, git_origin_url, memory_mode -) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) +) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(id) DO NOTHING "#, ) @@ -578,6 +580,7 @@ ON CONFLICT(id) DO NOTHING .bind(datetime_to_epoch_millis(updated_at)) .bind(datetime_to_epoch_millis(recency_at)) .bind(metadata.source.as_str()) + .bind(metadata.history_mode.as_str()) .bind( metadata .thread_source @@ -796,6 +799,7 @@ INSERT INTO threads ( updated_at_ms, recency_at_ms, source, + history_mode, thread_source, agent_nickname, agent_role, @@ -817,7 +821,7 @@ INSERT INTO threads ( git_branch, git_origin_url, memory_mode -) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) +) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(id) DO UPDATE SET rollout_path = excluded.rollout_path, created_at = excluded.created_at, @@ -827,6 +831,7 @@ ON CONFLICT(id) DO UPDATE SET updated_at_ms = excluded.updated_at_ms, recency_at_ms = threads.recency_at_ms, source = excluded.source, + history_mode = excluded.history_mode, thread_source = excluded.thread_source, agent_nickname = excluded.agent_nickname, agent_role = excluded.agent_role, @@ -858,6 +863,7 @@ ON CONFLICT(id) DO UPDATE SET .bind(datetime_to_epoch_millis(updated_at)) .bind(datetime_to_epoch_millis(insert_recency_at)) .bind(metadata.source.as_str()) + .bind(metadata.history_mode.as_str()) .bind( metadata .thread_source @@ -1206,6 +1212,7 @@ SELECT threads.updated_at_ms AS updated_at, threads.recency_at_ms AS recency_at, threads.source, + threads.history_mode, threads.thread_source, threads.agent_nickname, threads.agent_role, @@ -1419,6 +1426,7 @@ mod tests { use codex_protocol::protocol::SessionMeta; use codex_protocol::protocol::SessionMetaLine; use codex_protocol::protocol::SessionSource; + use codex_protocol::protocol::ThreadHistoryMode; use pretty_assertions::assert_eq; use serde_json::json; use std::path::PathBuf; @@ -1461,6 +1469,30 @@ mod tests { assert_eq!(memory_mode, "disabled"); } + #[tokio::test] + async fn thread_metadata_round_trips_history_mode() { + let codex_home = unique_temp_dir(); + let runtime = StateRuntime::init(codex_home.clone(), "test-provider".to_string()) + .await + .expect("state db should initialize"); + let thread_id = + ThreadId::from_string("00000000-0000-0000-0000-000000000124").expect("valid thread id"); + let mut metadata = test_thread_metadata(&codex_home, thread_id, codex_home.clone()); + metadata.history_mode = ThreadHistoryMode::Paginated; + + runtime + .upsert_thread(&metadata) + .await + .expect("upsert should succeed"); + + let metadata = runtime + .get_thread(thread_id) + .await + .expect("thread should load") + .expect("thread should exist"); + assert_eq!(metadata.history_mode, ThreadHistoryMode::Paginated); + } + #[tokio::test] async fn delete_thread_cleans_associated_state() -> Result<()> { let codex_home = unique_temp_dir(); @@ -2148,7 +2180,9 @@ mod tests { model_provider: None, base_instructions: None, dynamic_tools: None, + selected_capability_roots: Vec::new(), memory_mode: Some("polluted".to_string()), + history_mode: Default::default(), multi_agent_version: None, context_window: None, }, @@ -2211,7 +2245,9 @@ mod tests { model_provider: None, base_instructions: None, dynamic_tools: None, + selected_capability_roots: Vec::new(), memory_mode: None, + history_mode: Default::default(), multi_agent_version: None, context_window: None, }, diff --git a/codex-rs/thread-store/src/error.rs b/codex-rs/thread-store/src/error.rs index 2244c93185..bc5e8ed7c4 100644 --- a/codex-rs/thread-store/src/error.rs +++ b/codex-rs/thread-store/src/error.rs @@ -1,8 +1,20 @@ use codex_protocol::ThreadId; +use codex_protocol::protocol::ThreadHistoryMode; /// Result type returned by thread-store operations. pub type ThreadStoreResult = Result; +pub(crate) fn reject_paginated_history_mode( + history_mode: ThreadHistoryMode, +) -> ThreadStoreResult<()> { + if matches!(history_mode, ThreadHistoryMode::Paginated) { + return Err(ThreadStoreError::Unsupported { + operation: "paginated_threads", + }); + } + Ok(()) +} + /// Error type shared by thread-store implementations. #[derive(Debug, thiserror::Error)] pub enum ThreadStoreError { diff --git a/codex-rs/thread-store/src/in_memory.rs b/codex-rs/thread-store/src/in_memory.rs index dfed2ffac2..cb3f198ff3 100644 --- a/codex-rs/thread-store/src/in_memory.rs +++ b/codex-rs/thread-store/src/in_memory.rs @@ -14,6 +14,7 @@ use codex_protocol::protocol::RolloutItem; use codex_protocol::protocol::SessionContextWindow; use codex_protocol::protocol::SessionMeta; use codex_protocol::protocol::SessionMetaLine; +use codex_protocol::protocol::ThreadHistoryMode; use codex_protocol::protocol::ThreadMemoryMode; use codex_rollout::persisted_rollout_items; @@ -36,6 +37,8 @@ use crate::ThreadStoreError; use crate::ThreadStoreFuture; use crate::ThreadStoreResult; use crate::UpdateThreadMetadataParams; +use crate::error::reject_paginated_history_mode; +use crate::types::canonical_history_mode_from_rollout_items; static IN_MEMORY_THREAD_STORES: OnceLock>>> = OnceLock::new(); @@ -126,7 +129,9 @@ mod tests { originator: "test_originator".to_string(), base_instructions: BaseInstructions::default(), dynamic_tools: Vec::new(), + selected_capability_roots: Vec::new(), multi_agent_version: None, + history_mode: ThreadHistoryMode::Legacy, initial_window_id: uuid::Uuid::now_v7().to_string(), metadata: ThreadPersistenceMetadata { cwd: None, @@ -192,6 +197,157 @@ mod tests { HashSet::from([child_thread_id, grandchild_thread_id]) ); } + + #[tokio::test] + async fn paginated_threads_allow_metadata_reads_and_reject_legacy_history_paths() { + let store = InMemoryThreadStore::default(); + let thread_id = ThreadId::default(); + let rollout_path = PathBuf::from("/tmp/paginated-thread.jsonl"); + + store + .create_thread(create_thread_params(thread_id, ThreadHistoryMode::Legacy)) + .await + .expect("create legacy thread"); + store + .resume_thread(ResumeThreadParams { + thread_id, + rollout_path: Some(rollout_path.clone()), + history: None, + include_archived: false, + metadata: thread_metadata(), + }) + .await + .expect("register rollout path"); + { + let mut state = store.state.lock().await; + state + .created_threads + .get_mut(&thread_id) + .expect("created thread") + .history_mode = ThreadHistoryMode::Paginated; + let Some(RolloutItem::SessionMeta(meta_line)) = state + .histories + .get_mut(&thread_id) + .and_then(|history| history.first_mut()) + else { + panic!("canonical session meta"); + }; + meta_line.meta.history_mode = ThreadHistoryMode::Paginated; + } + + let thread = store + .read_thread(ReadThreadParams { + thread_id, + include_archived: false, + include_history: false, + }) + .await + .expect("metadata read"); + assert_eq!(thread.history_mode, ThreadHistoryMode::Paginated); + assert!(thread.history.is_none()); + + let thread = store + .read_thread_by_rollout_path(ReadThreadByRolloutPathParams { + rollout_path, + include_archived: false, + include_history: false, + }) + .await + .expect("metadata path read"); + assert_eq!(thread.history_mode, ThreadHistoryMode::Paginated); + assert!(thread.history.is_none()); + + assert_paginated_threads_unsupported( + store + .read_thread(ReadThreadParams { + thread_id, + include_archived: false, + include_history: true, + }) + .await + .expect_err("full history read should fail"), + ); + assert_paginated_threads_unsupported( + store + .read_thread_by_rollout_path(ReadThreadByRolloutPathParams { + rollout_path: PathBuf::from("/tmp/paginated-thread.jsonl"), + include_archived: false, + include_history: true, + }) + .await + .expect_err("full history path read should fail"), + ); + assert_paginated_threads_unsupported( + store + .load_history(LoadThreadHistoryParams { + thread_id, + include_archived: false, + }) + .await + .expect_err("history load should fail"), + ); + assert_paginated_threads_unsupported( + store + .resume_thread(ResumeThreadParams { + thread_id, + rollout_path: None, + history: None, + include_archived: false, + metadata: thread_metadata(), + }) + .await + .expect_err("resume should fail"), + ); + assert_paginated_threads_unsupported( + store + .create_thread(create_thread_params( + ThreadId::default(), + ThreadHistoryMode::Paginated, + )) + .await + .expect_err("paginated create should fail"), + ); + } + + fn create_thread_params( + thread_id: ThreadId, + history_mode: ThreadHistoryMode, + ) -> CreateThreadParams { + CreateThreadParams { + session_id: thread_id.into(), + thread_id, + extra_config: None, + forked_from_id: None, + parent_thread_id: None, + source: SessionSource::Exec, + thread_source: None, + originator: "test_originator".to_string(), + base_instructions: BaseInstructions::default(), + dynamic_tools: Vec::new(), + selected_capability_roots: Vec::new(), + multi_agent_version: None, + history_mode, + initial_window_id: uuid::Uuid::now_v7().to_string(), + metadata: thread_metadata(), + } + } + + fn thread_metadata() -> ThreadPersistenceMetadata { + ThreadPersistenceMetadata { + cwd: None, + model_provider: "test-provider".to_string(), + memory_mode: ThreadMemoryMode::Enabled, + } + } + + fn assert_paginated_threads_unsupported(err: ThreadStoreError) { + assert!(matches!( + err, + ThreadStoreError::Unsupported { + operation: "paginated_threads" + } + )); + } } fn stores_guard() -> MutexGuard<'static, HashMap>> { @@ -264,6 +420,7 @@ impl InMemoryThreadStore { } async fn create_thread(&self, params: CreateThreadParams) -> ThreadStoreResult<()> { + reject_paginated_history_mode(params.history_mode)?; let mut state = self.state.lock().await; state.calls.create_thread += 1; let session_meta = SessionMeta { @@ -281,8 +438,10 @@ impl InMemoryThreadStore { model_provider: Some(params.metadata.model_provider.clone()), base_instructions: Some(params.base_instructions.clone()), dynamic_tools: (!params.dynamic_tools.is_empty()).then(|| params.dynamic_tools.clone()), + selected_capability_roots: params.selected_capability_roots.clone(), memory_mode: matches!(params.metadata.memory_mode, ThreadMemoryMode::Disabled) .then_some("disabled".to_string()), + history_mode: params.history_mode, multi_agent_version: params.multi_agent_version, context_window: Some(SessionContextWindow::new(params.initial_window_id.clone())), ..SessionMeta::default() @@ -302,6 +461,13 @@ impl InMemoryThreadStore { async fn resume_thread(&self, params: ResumeThreadParams) -> ThreadStoreResult<()> { let mut state = self.state.lock().await; state.calls.resume_thread += 1; + let history_mode = params + .history + .as_deref() + .map(Vec::as_slice) + .map(canonical_history_mode_from_rollout_items) + .unwrap_or_else(|| history_mode_from_state(&state, params.thread_id)); + reject_paginated_history_mode(history_mode)?; if let Some(history) = params.history { state .histories @@ -336,14 +502,18 @@ impl InMemoryThreadStore { ) -> ThreadStoreResult { let mut state = self.state.lock().await; state.calls.load_history += 1; - let items = state.histories.get(¶ms.thread_id).cloned().ok_or( - ThreadStoreError::ThreadNotFound { - thread_id: params.thread_id, - }, - )?; + let items = + state + .histories + .get(¶ms.thread_id) + .ok_or(ThreadStoreError::ThreadNotFound { + thread_id: params.thread_id, + })?; + let history_mode = history_mode_from_state(&state, params.thread_id); + reject_paginated_history_mode(history_mode)?; Ok(StoredThreadHistory { thread_id: params.thread_id, - items, + items: items.clone(), }) } @@ -352,8 +522,10 @@ impl InMemoryThreadStore { state.calls.read_thread += 1; if params.include_history { state.calls.read_thread_with_history += 1; + reject_paginated_history_mode(history_mode_from_state(&state, params.thread_id))?; } - stored_thread_from_state(&state, params.thread_id, params.include_history) + let thread = stored_thread_from_state(&state, params.thread_id, params.include_history)?; + Ok(thread) } async fn read_thread_by_rollout_path( @@ -370,7 +542,11 @@ impl InMemoryThreadStore { ), }); }; - stored_thread_from_state(&state, thread_id, params.include_history) + if params.include_history { + reject_paginated_history_mode(history_mode_from_state(&state, thread_id))?; + } + let thread = stored_thread_from_state(&state, thread_id, params.include_history)?; + Ok(thread) } async fn list_threads(&self) -> ThreadStoreResult { @@ -613,6 +789,7 @@ fn stored_thread_from_state( source: metadata .and_then(|metadata| metadata.source.clone()) .unwrap_or_else(|| created.source.clone()), + history_mode: created.history_mode, thread_source: metadata .and_then(|metadata| metadata.thread_source.clone()) .unwrap_or_else(|| created.thread_source.clone()), @@ -632,6 +809,17 @@ fn stored_thread_from_state( }) } +fn history_mode_from_state( + state: &InMemoryThreadStoreState, + thread_id: ThreadId, +) -> ThreadHistoryMode { + state + .created_threads + .get(&thread_id) + .map(|thread| thread.history_mode) + .unwrap_or_default() +} + fn git_info_from_patch(patch: &ThreadMetadataPatch) -> Option { let git_info = patch.git_info.as_ref()?; let sha = git_info.sha.clone().flatten(); diff --git a/codex-rs/thread-store/src/local/create_thread.rs b/codex-rs/thread-store/src/local/create_thread.rs index eba6a5a6a9..9b370ef095 100644 --- a/codex-rs/thread-store/src/local/create_thread.rs +++ b/codex-rs/thread-store/src/local/create_thread.rs @@ -2,6 +2,7 @@ use super::LocalThreadStore; use crate::CreateThreadParams; use crate::ThreadStoreError; use crate::ThreadStoreResult; +use crate::error::reject_paginated_history_mode; use codex_protocol::protocol::ThreadMemoryMode; use codex_rollout::RolloutConfig; use codex_rollout::RolloutRecorder; @@ -11,6 +12,7 @@ pub(super) async fn create_thread( store: &LocalThreadStore, params: CreateThreadParams, ) -> ThreadStoreResult { + reject_paginated_history_mode(params.history_mode)?; let cwd = params .metadata .cwd @@ -38,7 +40,9 @@ pub(super) async fn create_thread( params.dynamic_tools, ) .with_session_id(params.session_id) + .with_selected_capability_roots(params.selected_capability_roots) .with_multi_agent_version(params.multi_agent_version) + .with_history_mode(params.history_mode) .with_initial_window_id(params.initial_window_id), ) .await diff --git a/codex-rs/thread-store/src/local/helpers.rs b/codex-rs/thread-store/src/local/helpers.rs index f48a0136c0..d883bd7874 100644 --- a/codex-rs/thread-store/src/local/helpers.rs +++ b/codex-rs/thread-store/src/local/helpers.rs @@ -142,6 +142,7 @@ pub(super) fn stored_thread_from_rollout_item( cwd: item.cwd.unwrap_or_default(), cli_version: item.cli_version.unwrap_or_default(), source, + history_mode: item.history_mode, thread_source: None, agent_nickname: item.agent_nickname, agent_role: item.agent_role, diff --git a/codex-rs/thread-store/src/local/list_threads.rs b/codex-rs/thread-store/src/local/list_threads.rs index 2619fe68aa..f514231333 100644 --- a/codex-rs/thread-store/src/local/list_threads.rs +++ b/codex-rs/thread-store/src/local/list_threads.rs @@ -219,6 +219,7 @@ mod tests { use chrono::Utc; use codex_protocol::ThreadId; use codex_protocol::protocol::SessionSource; + use codex_protocol::protocol::ThreadHistoryMode; use pretty_assertions::assert_eq; use std::fs; use tempfile::TempDir; @@ -243,6 +244,7 @@ mod tests { Uuid::from_u128(102), "Hello from user", /*model_provider*/ None, + ThreadHistoryMode::Legacy, ) .expect("session file"); diff --git a/codex-rs/thread-store/src/local/live_writer.rs b/codex-rs/thread-store/src/local/live_writer.rs index 3e7541b36b..74e707bb58 100644 --- a/codex-rs/thread-store/src/local/live_writer.rs +++ b/codex-rs/thread-store/src/local/live_writer.rs @@ -16,6 +16,8 @@ use crate::ReadThreadParams; use crate::ResumeThreadParams; use crate::ThreadStoreError; use crate::ThreadStoreResult; +use crate::error::reject_paginated_history_mode; +use crate::types::canonical_history_mode_from_rollout_items; const ROLLOUT_SIZE_BYTES_METRIC: &str = "codex.rollout.size_bytes"; @@ -24,9 +26,12 @@ pub(super) async fn create_thread( params: CreateThreadParams, ) -> ThreadStoreResult<()> { let thread_id = params.thread_id; + let history_mode = params.history_mode; store.ensure_live_recorder_absent(thread_id).await?; let recorder = create_thread::create_thread(store, params).await?; - store.insert_live_recorder(thread_id, recorder).await + store + .insert_live_recorder(thread_id, recorder, history_mode) + .await } pub(super) async fn resume_thread( @@ -34,6 +39,30 @@ pub(super) async fn resume_thread( params: ResumeThreadParams, ) -> ThreadStoreResult<()> { store.ensure_live_recorder_absent(params.thread_id).await?; + let history_mode = if let Some(history) = params.history.as_deref() { + canonical_history_mode_from_rollout_items(history) + } else if let Some(rollout_path) = params.rollout_path.as_ref() { + super::read_thread::read_thread_by_rollout_path( + store, + rollout_path.clone(), + params.include_archived, + /*include_history*/ false, + ) + .await? + .history_mode + } else { + super::read_thread::read_thread( + store, + ReadThreadParams { + thread_id: params.thread_id, + include_archived: params.include_archived, + include_history: false, + }, + ) + .await? + .history_mode + }; + reject_paginated_history_mode(history_mode)?; let rollout_path = match (params.rollout_path, params.history) { (Some(rollout_path), _history) => rollout_path, (None, history) => { @@ -46,7 +75,6 @@ pub(super) async fn resume_thread( }, ) .await?; - thread .rollout_path .ok_or_else(|| ThreadStoreError::Internal { @@ -73,7 +101,9 @@ pub(super) async fn resume_thread( .map_err(|err| ThreadStoreError::Internal { message: format!("failed to resume local thread recorder: {err}"), })?; - store.insert_live_recorder(params.thread_id, recorder).await + store + .insert_live_recorder(params.thread_id, recorder, history_mode) + .await } #[tracing::instrument( @@ -166,6 +196,7 @@ pub(super) async fn rollout_path( .await .get(&thread_id) .ok_or(ThreadStoreError::ThreadNotFound { thread_id })? + .recorder .rollout_path() .to_path_buf()) } diff --git a/codex-rs/thread-store/src/local/mod.rs b/codex-rs/thread-store/src/local/mod.rs index 51e622fc5b..5939ea606e 100644 --- a/codex-rs/thread-store/src/local/mod.rs +++ b/codex-rs/thread-store/src/local/mod.rs @@ -13,6 +13,7 @@ mod update_thread_metadata; mod test_support; use codex_protocol::ThreadId; +use codex_protocol::protocol::ThreadHistoryMode; use codex_rollout::RolloutRecorder; use codex_rollout::StateDbHandle; use std::collections::HashMap; @@ -57,10 +58,18 @@ use crate::UpdateThreadMetadataParams; #[derive(Clone)] pub struct LocalThreadStore { pub(super) config: LocalThreadStoreConfig, - live_recorders: Arc>>, + live_recorders: Arc>>, state_db: Option, } +struct LiveRecorderEntry { + recorder: RolloutRecorder, + // Local rollout files are materialized lazily, but metadata updates can arrive before the + // canonical SessionMeta is durable. Retain the mode captured when live persistence was opened + // so missing SQLite rows can still be seeded. + history_mode: ThreadHistoryMode, +} + /// Process-scoped configuration for local thread storage. /// /// This describes where local storage lives. New-thread rollout metadata such @@ -135,7 +144,7 @@ impl LocalThreadStore { .lock() .await .get(&thread_id) - .cloned() + .map(|entry| entry.recorder.clone()) .ok_or(ThreadStoreError::ThreadNotFound { thread_id }) } @@ -155,13 +164,17 @@ impl LocalThreadStore { &self, thread_id: ThreadId, recorder: RolloutRecorder, + history_mode: ThreadHistoryMode, ) -> ThreadStoreResult<()> { match self.live_recorders.lock().await.entry(thread_id) { Entry::Occupied(entry) => Err(ThreadStoreError::InvalidRequest { message: format!("thread {} already has a live local writer", entry.key()), }), Entry::Vacant(entry) => { - entry.insert(recorder); + entry.insert(LiveRecorderEntry { + recorder, + history_mode, + }); Ok(()) } } @@ -321,6 +334,7 @@ mod tests { use codex_protocol::protocol::EventMsg; use codex_protocol::protocol::RolloutItem; use codex_protocol::protocol::SessionSource; + use codex_protocol::protocol::ThreadHistoryMode; use codex_protocol::protocol::ThreadMemoryMode; use codex_protocol::protocol::TurnCompleteEvent; use codex_protocol::protocol::TurnStartedEvent; @@ -333,6 +347,7 @@ mod tests { use crate::local::test_support::test_config; use crate::local::test_support::write_archived_session_file; use crate::local::test_support::write_session_file; + use crate::local::test_support::write_session_file_with_history_mode; #[tokio::test] async fn live_writer_lifecycle_writes_and_closes() { @@ -578,6 +593,46 @@ mod tests { ); } + #[tokio::test] + async fn live_thread_memory_mode_update_before_rollout_materializes_keeps_history_mode() { + let home = TempDir::new().expect("temp dir"); + let config = test_config(home.path()); + let runtime = codex_state::StateRuntime::init( + config.sqlite_home.clone(), + config.default_model_provider_id.clone(), + ) + .await + .expect("state db should initialize"); + let store = Arc::new(LocalThreadStore::new(config, Some(runtime.clone()))); + let thread_id = ThreadId::default(); + let live_thread = LiveThread::create(store.clone(), create_thread_params(thread_id)) + .await + .expect("create live thread"); + + live_thread + .update_memory_mode(ThreadMemoryMode::Disabled, /*include_archived*/ false) + .await + .expect("update memory mode"); + + assert_eq!( + runtime + .get_thread(thread_id) + .await + .expect("sqlite metadata read") + .expect("sqlite metadata") + .history_mode, + ThreadHistoryMode::Legacy + ); + assert_eq!( + runtime + .get_thread_memory_mode(thread_id) + .await + .expect("thread memory mode should be readable") + .as_deref(), + Some("disabled") + ); + } + #[tokio::test] async fn live_thread_shutdown_with_buffered_items_materializes_before_metadata_read() { let home = TempDir::new().expect("temp dir"); @@ -1109,18 +1164,108 @@ mod tests { .expect("read thread by rollout path"); assert_eq!(thread.thread_id, thread_id); + assert_eq!(thread.history_mode, ThreadHistoryMode::Legacy); assert_eq!( thread .history + .as_ref() .expect("history") .items - .into_iter() + .iter() .filter(|item| matches!(item, RolloutItem::EventMsg(EventMsg::UserMessage(_)))) .count(), 1 ); } + #[tokio::test] + async fn paginated_threads_allow_metadata_reads_and_reject_legacy_history_paths() { + let home = TempDir::new().expect("temp dir"); + let store = LocalThreadStore::new(test_config(home.path()), /*state_db*/ None); + let uuid = uuid::Uuid::from_u128(408); + let thread_id = ThreadId::from_string(&uuid.to_string()).expect("valid thread id"); + let rollout_path = write_session_file_with_history_mode( + home.path(), + "2025-01-04T12-00-00", + uuid, + ThreadHistoryMode::Paginated, + ) + .expect("session file"); + + let thread = store + .read_thread(ReadThreadParams { + thread_id, + include_archived: false, + include_history: false, + }) + .await + .expect("metadata read"); + assert_eq!(thread.history_mode, ThreadHistoryMode::Paginated); + assert!(thread.history.is_none()); + + let thread = store + .read_thread_by_rollout_path( + rollout_path.clone(), + /*include_archived*/ true, + /*include_history*/ false, + ) + .await + .expect("metadata path read"); + assert_eq!(thread.history_mode, ThreadHistoryMode::Paginated); + assert!(thread.history.is_none()); + + assert_paginated_threads_unsupported( + store + .read_thread(ReadThreadParams { + thread_id, + include_archived: false, + include_history: true, + }) + .await + .expect_err("full history read should fail"), + ); + assert_paginated_threads_unsupported( + store + .read_thread_by_rollout_path( + rollout_path.clone(), + /*include_archived*/ true, + /*include_history*/ true, + ) + .await + .expect_err("full history path read should fail"), + ); + assert_paginated_threads_unsupported( + store + .load_history(LoadThreadHistoryParams { + thread_id, + include_archived: false, + }) + .await + .expect_err("history load should fail"), + ); + assert_paginated_threads_unsupported( + store + .resume_thread(ResumeThreadParams { + thread_id, + rollout_path: Some(rollout_path), + history: None, + include_archived: false, + metadata: thread_metadata(), + }) + .await + .expect_err("resume should fail"), + ); + + let mut create_params = create_thread_params(ThreadId::default()); + create_params.history_mode = ThreadHistoryMode::Paginated; + assert_paginated_threads_unsupported( + store + .create_thread(create_params) + .await + .expect_err("paginated create should fail"), + ); + } + fn create_thread_params(thread_id: ThreadId) -> CreateThreadParams { CreateThreadParams { session_id: thread_id.into(), @@ -1133,12 +1278,23 @@ mod tests { originator: "test_originator".to_string(), base_instructions: BaseInstructions::default(), dynamic_tools: Vec::new(), + selected_capability_roots: Vec::new(), multi_agent_version: None, + history_mode: ThreadHistoryMode::Legacy, initial_window_id: uuid::Uuid::now_v7().to_string(), metadata: thread_metadata(), } } + fn assert_paginated_threads_unsupported(err: ThreadStoreError) { + assert!(matches!( + err, + ThreadStoreError::Unsupported { + operation: "paginated_threads" + } + )); + } + fn thread_metadata() -> ThreadPersistenceMetadata { ThreadPersistenceMetadata { cwd: Some(std::env::current_dir().expect("cwd")), diff --git a/codex-rs/thread-store/src/local/read_thread.rs b/codex-rs/thread-store/src/local/read_thread.rs index 1271e4fcf5..27ce3076e9 100644 --- a/codex-rs/thread-store/src/local/read_thread.rs +++ b/codex-rs/thread-store/src/local/read_thread.rs @@ -25,6 +25,7 @@ use crate::StoredThread; use crate::StoredThreadHistory; use crate::ThreadStoreError; use crate::ThreadStoreResult; +use crate::error::reject_paginated_history_mode; pub(super) async fn read_thread( store: &LocalThreadStore, @@ -47,7 +48,7 @@ pub(super) async fn read_thread( .await) { let metadata_sandbox_policy = metadata.sandbox_policy.clone(); - let mut thread = stored_thread_from_sqlite_metadata(store, metadata).await; + let mut thread = stored_thread_from_sqlite_metadata(store, metadata).await?; if !params.include_history && let Some(rollout_path) = thread.rollout_path.clone() && let Ok(mut rollout_thread) = read_thread_from_rollout_path(store, rollout_path).await @@ -66,6 +67,7 @@ pub(super) async fn read_thread( ); thread = rollout_thread; } + reject_paginated_history(&thread, params.include_history)?; attach_history_if_requested(&mut thread, params.include_history).await?; return Ok(thread); } @@ -82,6 +84,7 @@ pub(super) async fn read_thread( message: format!("thread {} is archived", thread.thread_id), }); } + reject_paginated_history(&thread, params.include_history)?; attach_history_if_requested(&mut thread, params.include_history).await?; Ok(thread) } @@ -132,10 +135,18 @@ pub(super) async fn read_thread_by_rollout_path( metadata.git_origin_url.or(fallback_origin_url), ); } + reject_paginated_history(&thread, include_history)?; attach_history_if_requested(&mut thread, include_history).await?; Ok(thread) } +fn reject_paginated_history(thread: &StoredThread, include_history: bool) -> ThreadStoreResult<()> { + if include_history { + reject_paginated_history_mode(thread.history_mode)?; + } + Ok(()) +} + async fn resolve_requested_rollout_path( store: &LocalThreadStore, rollout_path: std::path::PathBuf, @@ -261,16 +272,16 @@ async fn read_thread_from_rollout_path( message: format!("failed to read thread id from {}", path.display()), })?; thread.rollout_path = Some(codex_rollout::plain_rollout_path(path.as_path())); - if let Ok(meta_line) = read_session_meta_line(path.as_path()).await { - thread.forked_from_id = meta_line.meta.forked_from_id; - thread.parent_thread_id = meta_line.meta.parent_thread_id; - if let Some(model_provider) = meta_line - .meta - .model_provider - .filter(|provider| !provider.is_empty()) - { - thread.model_provider = model_provider; - } + let meta_line = read_required_session_meta_line(path.as_path()).await?; + thread.forked_from_id = meta_line.meta.forked_from_id; + thread.parent_thread_id = meta_line.meta.parent_thread_id; + thread.history_mode = meta_line.meta.history_mode; + if let Some(model_provider) = meta_line + .meta + .model_provider + .filter(|provider| !provider.is_empty()) + { + thread.model_provider = model_provider; } if let Ok(Some(title)) = find_thread_name_by_id(store.config.codex_home.as_path(), &thread.thread_id).await @@ -302,7 +313,7 @@ async fn read_sqlite_metadata( async fn stored_thread_from_sqlite_metadata( store: &LocalThreadStore, metadata: ThreadMetadata, -) -> StoredThread { +) -> ThreadStoreResult { let name = match distinct_thread_metadata_title(&metadata) { Some(title) => Some(title), None => find_thread_name_by_id(store.config.codex_home.as_path(), &metadata.id) @@ -311,13 +322,32 @@ async fn stored_thread_from_sqlite_metadata( .flatten() .filter(|title| !title.trim().is_empty()), }; - let session_meta = read_session_meta_line(metadata.rollout_path.as_path()) - .await - .ok() - .map(|meta_line| meta_line.meta); + let session_meta = match read_required_session_meta_line(metadata.rollout_path.as_path()).await + { + Ok(meta_line) => Some(meta_line.meta), + Err(_) + if codex_rollout::existing_rollout_path(metadata.rollout_path.as_path()) + .await + .is_none() => + { + None + } + Err(err) => { + return Err(ThreadStoreError::Internal { + message: format!( + "failed to read session metadata {}: {err}", + metadata.rollout_path.display() + ), + }); + } + }; let rollout_path = codex_rollout::plain_rollout_path(metadata.rollout_path.as_path()); let forked_from_id = session_meta.as_ref().and_then(|meta| meta.forked_from_id); let parent_thread_id = session_meta.as_ref().and_then(|meta| meta.parent_thread_id); + let history_mode = session_meta + .as_ref() + .map(|meta| meta.history_mode) + .unwrap_or(metadata.history_mode); let preview = metadata .preview .clone() @@ -325,7 +355,7 @@ async fn stored_thread_from_sqlite_metadata( .unwrap_or_default(); let permission_profile = permission_profile_from_metadata_value(&metadata.sandbox_policy, metadata.cwd.as_path()); - StoredThread { + Ok(StoredThread { thread_id: metadata.id, extra_config: None, rollout_path: Some(rollout_path), @@ -347,6 +377,7 @@ async fn stored_thread_from_sqlite_metadata( cwd: metadata.cwd, cli_version: metadata.cli_version, source: parse_session_source(&metadata.source), + history_mode, thread_source: metadata.thread_source, agent_nickname: metadata.agent_nickname, agent_role: metadata.agent_role, @@ -361,24 +392,30 @@ async fn stored_thread_from_sqlite_metadata( token_usage: None, first_user_message: metadata.first_user_message, history: None, - } + }) } async fn stored_thread_from_session_meta( store: &LocalThreadStore, path: std::path::PathBuf, ) -> ThreadStoreResult { - let meta_line = read_session_meta_line(path.as_path()) - .await - .map_err(|err| ThreadStoreError::Internal { - message: format!("failed to read thread {}: {err}", path.display()), - })?; + let meta_line = read_required_session_meta_line(path.as_path()).await?; let archived = rollout_path_is_archived(store.config.codex_home.as_path(), path.as_path()); Ok(stored_thread_from_meta_line( store, meta_line, path, archived, )) } +async fn read_required_session_meta_line( + path: &std::path::Path, +) -> ThreadStoreResult { + read_session_meta_line(path) + .await + .map_err(|err| ThreadStoreError::Internal { + message: format!("failed to read session metadata {}: {err}", path.display()), + }) +} + fn stored_thread_from_meta_line( store: &LocalThreadStore, meta_line: SessionMetaLine, @@ -414,6 +451,7 @@ fn stored_thread_from_meta_line( cwd: meta_line.meta.cwd, cli_version: meta_line.meta.cli_version, source: meta_line.meta.source, + history_mode: meta_line.meta.history_mode, thread_source: meta_line.meta.thread_source, agent_nickname: meta_line.meta.agent_nickname, agent_role: meta_line.meta.agent_role, @@ -457,6 +495,7 @@ mod tests { use codex_protocol::ThreadId; use codex_protocol::protocol::SandboxPolicy; use codex_protocol::protocol::SessionSource; + use codex_protocol::protocol::ThreadHistoryMode; use codex_state::ThreadMetadataBuilder; use pretty_assertions::assert_eq; use tempfile::TempDir; @@ -665,6 +704,7 @@ mod tests { "Forked user message", Some("test-provider"), Some(parent_uuid), + ThreadHistoryMode::Legacy, ) .expect("forked session file"); diff --git a/codex-rs/thread-store/src/local/test_support.rs b/codex-rs/thread-store/src/local/test_support.rs index 10a55c4fd6..a930067fc4 100644 --- a/codex-rs/thread-store/src/local/test_support.rs +++ b/codex-rs/thread-store/src/local/test_support.rs @@ -3,6 +3,7 @@ use std::io::Write; use std::path::Path; use std::path::PathBuf; +use codex_protocol::protocol::ThreadHistoryMode; use codex_rollout::ARCHIVED_SESSIONS_SUBDIR; use uuid::Uuid; @@ -17,6 +18,15 @@ pub(super) fn test_config(codex_home: &Path) -> LocalThreadStoreConfig { } pub(super) fn write_session_file(root: &Path, ts: &str, uuid: Uuid) -> std::io::Result { + write_session_file_with_history_mode(root, ts, uuid, ThreadHistoryMode::Legacy) +} + +pub(super) fn write_session_file_with_history_mode( + root: &Path, + ts: &str, + uuid: Uuid, + history_mode: ThreadHistoryMode, +) -> std::io::Result { write_session_file_with( root, root.join("sessions/2025/01/03"), @@ -24,6 +34,7 @@ pub(super) fn write_session_file(root: &Path, ts: &str, uuid: Uuid) -> std::io:: uuid, "Hello from user", Some("test-provider"), + history_mode, ) } @@ -39,6 +50,7 @@ pub(super) fn write_archived_session_file( uuid, "Archived user message", Some("test-provider"), + ThreadHistoryMode::Legacy, ) } @@ -49,6 +61,7 @@ pub(super) fn write_session_file_with( uuid: Uuid, first_user_message: &str, model_provider: Option<&str>, + history_mode: ThreadHistoryMode, ) -> std::io::Result { write_session_file_with_fork( root, @@ -58,9 +71,11 @@ pub(super) fn write_session_file_with( first_user_message, model_provider, /*forked_from_id*/ None, + history_mode, ) } +#[allow(clippy::too_many_arguments)] pub(super) fn write_session_file_with_fork( root: &Path, day_dir: PathBuf, @@ -69,6 +84,7 @@ pub(super) fn write_session_file_with_fork( first_user_message: &str, model_provider: Option<&str>, forked_from_id: Option, + history_mode: ThreadHistoryMode, ) -> std::io::Result { fs::create_dir_all(&day_dir)?; let path = day_dir.join(format!("rollout-{ts}-{uuid}.jsonl")); @@ -86,6 +102,7 @@ pub(super) fn write_session_file_with_fork( "cli_version": "test_version", "source": "cli", "model_provider": model_provider, + "history_mode": history_mode, "git": { "commit_hash": "abcdef", "branch": "main", diff --git a/codex-rs/thread-store/src/local/update_thread_metadata.rs b/codex-rs/thread-store/src/local/update_thread_metadata.rs index 5f78b2e1f1..580df1dc61 100644 --- a/codex-rs/thread-store/src/local/update_thread_metadata.rs +++ b/codex-rs/thread-store/src/local/update_thread_metadata.rs @@ -6,6 +6,7 @@ use codex_protocol::ThreadId; use codex_protocol::protocol::GitInfo; use codex_protocol::protocol::RolloutItem; use codex_protocol::protocol::SessionSource; +use codex_protocol::protocol::ThreadHistoryMode; use codex_protocol::protocol::ThreadMemoryMode; use codex_rollout::ARCHIVED_SESSIONS_SUBDIR; use codex_rollout::append_rollout_item_to_path; @@ -27,6 +28,7 @@ use crate::ThreadMetadataPatch; use crate::ThreadStoreError; use crate::ThreadStoreResult; use crate::UpdateThreadMetadataParams; +use crate::error::reject_paginated_history_mode; use crate::local::read_thread; struct ResolvedRolloutPath { @@ -53,6 +55,20 @@ pub(super) async fn update_thread_metadata( } let needs_rollout_compat = needs_rollout_compatibility_update(&patch); + if needs_rollout_compat { + // These explicit patches still write legacy rollout/name-index state after the + // SQLite update. Paginated threads must fail before either side is mutated. + let thread = read_thread::read_thread( + store, + ReadThreadParams { + thread_id, + include_archived: params.include_archived, + include_history: false, + }, + ) + .await?; + reject_paginated_history_mode(thread.history_mode)?; + } let require_sqlite_write = sqlite_write_failure_should_block(&patch); let updated = apply_metadata_update( store, @@ -210,30 +226,27 @@ async fn apply_metadata_update( rollout_path_archived = resolved.archived; rollout_path = Some(resolved.path); } - let mut metadata = existing.clone().unwrap_or_else(|| { - let created_at = patch - .created_at - .or(patch.updated_at) - .unwrap_or_else(Utc::now); - let mut builder = ThreadMetadataBuilder::new( - thread_id, - rollout_path.clone().unwrap_or_default(), - created_at, - patch.source.clone().unwrap_or(SessionSource::Unknown), - ); - builder.model_provider = patch.model_provider.clone(); - builder.thread_source = patch.thread_source.clone().flatten(); - builder.agent_nickname = patch.agent_nickname.clone().flatten(); - builder.agent_role = patch.agent_role.clone().flatten(); - builder.agent_path = patch.agent_path.clone().flatten(); - builder.cwd = patch.cwd.clone().map(normalize_cwd).unwrap_or_default(); - builder.cli_version = patch.cli_version.clone(); - let mut metadata = builder.build(store.config.default_model_provider_id.as_str()); - if rollout_path_archived { - metadata.archived_at = Some(metadata.updated_at); + let mut metadata = match existing.clone() { + Some(metadata) => metadata, + None => { + let rollout_path = + rollout_path + .as_deref() + .ok_or_else(|| ThreadStoreError::Internal { + message: format!( + "thread metadata missing rollout path for {thread_id}" + ), + })?; + metadata_for_missing_sqlite_row( + store, + thread_id, + rollout_path, + rollout_path_archived, + &patch, + ) + .await? } - metadata - }); + }; if let Some(rollout_path) = rollout_path { metadata.rollout_path = rollout_path; } @@ -370,6 +383,78 @@ async fn apply_metadata_update( .await } +async fn metadata_for_missing_sqlite_row( + store: &LocalThreadStore, + thread_id: ThreadId, + rollout_path: &Path, + rollout_path_archived: bool, + patch: &ThreadMetadataPatch, +) -> ThreadStoreResult { + let created_at = patch + .created_at + .or(patch.updated_at) + .unwrap_or_else(Utc::now); + let mut builder = ThreadMetadataBuilder::new( + thread_id, + rollout_path.to_path_buf(), + created_at, + patch.source.clone().unwrap_or(SessionSource::Unknown), + ); + builder.model_provider = patch.model_provider.clone(); + builder.history_mode = canonical_history_mode(store, thread_id, rollout_path).await?; + builder.thread_source = patch.thread_source.clone().flatten(); + builder.agent_nickname = patch.agent_nickname.clone().flatten(); + builder.agent_role = patch.agent_role.clone().flatten(); + builder.agent_path = patch.agent_path.clone().flatten(); + builder.cwd = patch.cwd.clone().map(normalize_cwd).unwrap_or_default(); + builder.cli_version = patch.cli_version.clone(); + let mut metadata = builder.build(store.config.default_model_provider_id.as_str()); + if rollout_path_archived { + metadata.archived_at = Some(metadata.updated_at); + } + Ok(metadata) +} + +async fn canonical_history_mode( + store: &LocalThreadStore, + thread_id: ThreadId, + rollout_path: &Path, +) -> ThreadStoreResult { + let session_meta = match read_session_meta_line(rollout_path).await { + Ok(session_meta) => session_meta, + Err(err) => { + if codex_rollout::existing_rollout_path(rollout_path) + .await + .is_none() + && let Some(history_mode) = store + .live_recorders + .lock() + .await + .get(&thread_id) + .map(|entry| entry.history_mode) + { + // The live writer retains the canonical mode selected before its deferred + // SessionMeta reaches JSONL. + return Ok(history_mode); + } + return Err(ThreadStoreError::Internal { + message: format!( + "failed to read canonical session metadata for {thread_id}: {err}" + ), + }); + } + }; + if session_meta.meta.id != thread_id { + return Err(ThreadStoreError::Internal { + message: format!( + "failed to rebuild thread metadata: rollout session metadata id mismatch: expected {thread_id}, found {}", + session_meta.meta.id + ), + }); + } + Ok(session_meta.meta.history_mode) +} + fn needs_rollout_compatibility_update(patch: &ThreadMetadataPatch) -> bool { if patch.name.is_some() { return true; @@ -632,6 +717,7 @@ fn rollout_path_is_archived(store: &LocalThreadStore, path: &Path) -> bool { #[cfg(test)] mod tests { use codex_protocol::models::PermissionProfile; + use codex_protocol::protocol::ThreadHistoryMode; use pretty_assertions::assert_eq; use serde_json::Value; use serde_json::json; @@ -651,6 +737,7 @@ mod tests { use crate::local::test_support::test_config; use crate::local::test_support::write_archived_session_file; use crate::local::test_support::write_session_file; + use crate::local::test_support::write_session_file_with_history_mode; #[tokio::test] async fn update_thread_metadata_sets_name_on_active_rollout_and_indexes_name() { @@ -719,6 +806,55 @@ mod tests { assert_eq!(memory_mode.as_deref(), Some("disabled")); } + #[tokio::test] + async fn update_thread_metadata_rejects_paginated_rollout_compatibility_writes() { + let home = TempDir::new().expect("temp dir"); + let config = test_config(home.path()); + let uuid = Uuid::from_u128(303); + let thread_id = ThreadId::from_string(&uuid.to_string()).expect("valid thread id"); + let path = write_session_file_with_history_mode( + home.path(), + "2025-01-03T14-35-00", + uuid, + ThreadHistoryMode::Paginated, + ) + .expect("session file"); + let runtime = codex_state::StateRuntime::init( + home.path().to_path_buf(), + config.default_model_provider_id.clone(), + ) + .await + .expect("state db should initialize"); + let store = LocalThreadStore::new(config, Some(runtime.clone())); + + assert!(matches!( + store + .update_thread_metadata(UpdateThreadMetadataParams { + thread_id, + patch: ThreadMetadataPatch { + memory_mode: Some(ThreadMemoryMode::Disabled), + ..Default::default() + }, + include_archived: false, + }) + .await + .expect_err("paginated rollout compatibility write should fail"), + ThreadStoreError::Unsupported { + operation: "paginated_threads" + } + )); + + assert_eq!(last_rollout_item(path.as_path())["type"], "event_msg"); + assert_eq!( + runtime + .get_thread_memory_mode(thread_id) + .await + .expect("thread memory mode should be readable") + .as_deref(), + Some("enabled") + ); + } + #[tokio::test] async fn update_thread_metadata_preserves_memory_mode_when_updating_git_info() { let home = TempDir::new().expect("temp dir"); @@ -1404,6 +1540,51 @@ mod tests { assert!(metadata.is_none()); } + #[tokio::test] + async fn observed_metadata_rebuilds_history_mode_from_canonical_session_meta() { + let home = TempDir::new().expect("temp dir"); + let config = test_config(home.path()); + let uuid = Uuid::from_u128(317); + let thread_id = ThreadId::from_string(&uuid.to_string()).expect("valid thread id"); + write_session_file_with_history_mode( + home.path(), + "2025-01-03T19-15-00", + uuid, + ThreadHistoryMode::Paginated, + ) + .expect("session file"); + let runtime = codex_state::StateRuntime::init( + home.path().to_path_buf(), + config.default_model_provider_id.clone(), + ) + .await + .expect("state db should initialize"); + let store = LocalThreadStore::new(config, Some(runtime.clone())); + + let thread = store + .update_thread_metadata(UpdateThreadMetadataParams { + thread_id, + patch: ThreadMetadataPatch { + preview: Some("Paginated preview".to_string()), + ..Default::default() + }, + include_archived: false, + }) + .await + .expect("update paginated thread without sqlite row"); + + assert_eq!(thread.history_mode, ThreadHistoryMode::Paginated); + assert_eq!( + runtime + .get_thread(thread_id) + .await + .expect("sqlite metadata read") + .expect("sqlite metadata") + .history_mode, + ThreadHistoryMode::Paginated + ); + } + #[tokio::test] async fn update_thread_metadata_recreates_missing_archived_sqlite_row_as_archived() { let home = TempDir::new().expect("temp dir"); diff --git a/codex-rs/thread-store/src/store.rs b/codex-rs/thread-store/src/store.rs index 65ad1ce8a8..67ee603f49 100644 --- a/codex-rs/thread-store/src/store.rs +++ b/codex-rs/thread-store/src/store.rs @@ -1,4 +1,5 @@ use codex_protocol::ThreadId; +use codex_protocol::protocol::ThreadHistoryMode; use std::any::Any; use std::future::Future; use std::pin::Pin; @@ -33,6 +34,14 @@ pub trait ThreadStore: Any + Send + Sync { /// Return this store as [`Any`] for implementation-owned escape hatches. fn as_any(&self) -> &dyn Any; + /// Returns the history mode to use when history does not carry a persisted mode. + /// + /// The default is legacy so existing stores stay compatible. Stores whose durable contract is + /// already paginated should override this instead of relying on core to infer storage behavior. + fn default_history_mode(&self) -> ThreadHistoryMode { + ThreadHistoryMode::Legacy + } + /// Creates a new live thread. fn create_thread(&self, params: CreateThreadParams) -> ThreadStoreFuture<'_, ()>; diff --git a/codex-rs/thread-store/src/types.rs b/codex-rs/thread-store/src/types.rs index 1f761cea93..4e688eca6f 100644 --- a/codex-rs/thread-store/src/types.rs +++ b/codex-rs/thread-store/src/types.rs @@ -5,6 +5,7 @@ use chrono::DateTime; use chrono::Utc; use codex_protocol::SessionId; use codex_protocol::ThreadId; +use codex_protocol::capabilities::SelectedCapabilityRoot; use codex_protocol::dynamic_tools::DynamicToolSpec; use codex_protocol::models::BaseInstructions; use codex_protocol::models::PermissionProfile; @@ -14,6 +15,7 @@ use codex_protocol::protocol::GitInfo; use codex_protocol::protocol::MultiAgentVersion; use codex_protocol::protocol::RolloutItem; use codex_protocol::protocol::SessionSource; +use codex_protocol::protocol::ThreadHistoryMode; use codex_protocol::protocol::ThreadMemoryMode as MemoryMode; use codex_protocol::protocol::ThreadSource; use codex_protocol::protocol::TokenUsage; @@ -85,8 +87,13 @@ pub struct CreateThreadParams { pub base_instructions: BaseInstructions, /// Dynamic tools available to the thread at startup. pub dynamic_tools: Vec, + /// Environment-qualified capability roots selected for this thread. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub selected_capability_roots: Vec, /// Multi-agent runtime selected when the thread was created. pub multi_agent_version: Option, + /// Persisted thread history contract selected when the thread was created. + pub history_mode: ThreadHistoryMode, /// Initial context-window identity captured when the thread was created. pub initial_window_id: String, /// Metadata captured for the newly created thread. @@ -108,6 +115,20 @@ pub struct ResumeThreadParams { pub metadata: ThreadPersistenceMetadata, } +pub(crate) fn canonical_history_mode_from_rollout_items( + items: &[RolloutItem], +) -> ThreadHistoryMode { + // Forked rollouts keep copied source SessionMeta items after the new thread's + // canonical SessionMeta, so the thread contract comes from the first one. + items + .iter() + .find_map(|item| match item { + RolloutItem::SessionMeta(meta_line) => Some(meta_line.meta.history_mode), + _ => None, + }) + .unwrap_or_default() +} + /// Parameters for appending rollout items to a live thread. #[derive(Clone, Debug)] pub struct AppendThreadItemsParams { @@ -426,6 +447,8 @@ pub struct StoredThread { pub cli_version: String, /// Runtime source for the thread. pub source: SessionSource, + /// Persisted thread history contract selected when this thread was created. + pub history_mode: ThreadHistoryMode, /// Optional analytics source classification for this thread. pub thread_source: Option, /// Optional random nickname for thread-spawn sub-agents. @@ -781,6 +804,17 @@ mod tests { assert!(decoded.is_empty()); } + #[test] + fn canonical_history_mode_uses_first_session_meta() { + assert_eq!( + canonical_history_mode_from_rollout_items(&[ + session_meta(ThreadHistoryMode::Legacy), + session_meta(ThreadHistoryMode::Paginated), + ]), + ThreadHistoryMode::Legacy + ); + } + #[test] fn thread_metadata_patch_merge_uses_presence_semantics() { let mut current = ThreadMetadataPatch { @@ -818,4 +852,14 @@ mod tests { }) ); } + + fn session_meta(history_mode: ThreadHistoryMode) -> RolloutItem { + RolloutItem::SessionMeta(codex_protocol::protocol::SessionMetaLine { + meta: codex_protocol::protocol::SessionMeta { + history_mode, + ..Default::default() + }, + git: None, + }) + } } diff --git a/codex-rs/tools/src/tool_config_tests.rs b/codex-rs/tools/src/tool_config_tests.rs index 7a4738772c..af340af6d2 100644 --- a/codex-rs/tools/src/tool_config_tests.rs +++ b/codex-rs/tools/src/tool_config_tests.rs @@ -27,6 +27,7 @@ fn model_with_shell_type(shell_type: ConfigShellToolType) -> ModelInfo { upgrade: None, base_instructions: String::new(), model_messages: None, + include_skills_usage_instructions: false, supports_reasoning_summaries: false, default_reasoning_summary: Default::default(), support_verbosity: false, diff --git a/codex-rs/tools/src/tool_payload.rs b/codex-rs/tools/src/tool_payload.rs index 71af457743..101d95a278 100644 --- a/codex-rs/tools/src/tool_payload.rs +++ b/codex-rs/tools/src/tool_payload.rs @@ -3,7 +3,7 @@ use std::borrow::Cow; use codex_protocol::models::SearchToolCallParams; /// Canonical payload shapes accepted by model-visible tool runtimes. -#[derive(Clone, Debug)] +#[derive(Clone, Debug, PartialEq)] pub enum ToolPayload { Function { arguments: String }, ToolSearch { arguments: SearchToolCallParams }, diff --git a/codex-rs/tui/src/app.rs b/codex-rs/tui/src/app.rs index b7f2b9df6a..554df264e2 100644 --- a/codex-rs/tui/src/app.rs +++ b/codex-rs/tui/src/app.rs @@ -51,6 +51,7 @@ use crate::legacy_core::config::ConfigBuilder; use crate::legacy_core::config::ConfigOverrides; use crate::legacy_core::config::PermissionProfileSnapshot; use crate::legacy_core::config::edit::ConfigEditsBuilder; +use crate::managed_new_thread_defaults::apply_managed_new_thread_defaults; use crate::model_catalog::ModelCatalog; use crate::model_migration::ModelMigrationOutcome; use crate::model_migration::migration_copy_for_models; @@ -794,7 +795,18 @@ impl App { None => app_server.bootstrap(&config).await?, }; let bootstrap_ms = bootstrap.duration.as_millis(); - let mut model = bootstrap.default_model; + if matches!( + &session_selection, + SessionSelection::StartFresh | SessionSelection::Exit + ) { + apply_managed_new_thread_defaults( + &mut config, + app_server.managed_new_thread_defaults(), + &cli_kv_overrides, + &harness_overrides, + ); + } + let mut model = config.model.clone().unwrap_or(bootstrap.default_model); let available_models = bootstrap.available_models; let remote_connection = crate::status::remote_connection::remote_connection_status_value( &app_server_target, diff --git a/codex-rs/tui/src/app/app_server_event_targets.rs b/codex-rs/tui/src/app/app_server_event_targets.rs index 212c45b4be..6c55850e4d 100644 --- a/codex-rs/tui/src/app/app_server_event_targets.rs +++ b/codex-rs/tui/src/app/app_server_event_targets.rs @@ -285,6 +285,7 @@ mod tests { name: "sentry".to_string(), status: McpServerStartupState::Failed, error: Some("sentry is not logged in".to_string()), + failure_reason: None, }); let target = server_notification_thread_target(¬ification); @@ -300,6 +301,7 @@ mod tests { name: "sentry".to_string(), status: McpServerStartupState::Failed, error: Some("sentry is not logged in".to_string()), + failure_reason: None, }); let target = server_notification_thread_target(¬ification); diff --git a/codex-rs/tui/src/app/background_requests.rs b/codex-rs/tui/src/app/background_requests.rs index 9391f4b3e3..9162b2c868 100644 --- a/codex-rs/tui/src/app/background_requests.rs +++ b/codex-rs/tui/src/app/background_requests.rs @@ -798,7 +798,10 @@ pub(super) async fn consume_rate_limit_reset_credit_request( request_handle .request_typed(ClientRequest::ConsumeAccountRateLimitResetCredit { request_id, - params: ConsumeAccountRateLimitResetCreditParams { idempotency_key }, + params: ConsumeAccountRateLimitResetCreditParams { + idempotency_key, + credit_id: None, + }, }) .await .wrap_err("account/rateLimitResetCredit/consume failed in TUI") diff --git a/codex-rs/tui/src/app/loaded_threads.rs b/codex-rs/tui/src/app/loaded_threads.rs index 8e790170e6..9482453e92 100644 --- a/codex-rs/tui/src/app/loaded_threads.rs +++ b/codex-rs/tui/src/app/loaded_threads.rs @@ -134,6 +134,7 @@ mod tests { parent_thread_id: None, preview: String::new(), ephemeral: false, + history_mode: Default::default(), model_provider: "openai".to_string(), created_at: 0, updated_at: 0, diff --git a/codex-rs/tui/src/app/plugin_mentions.rs b/codex-rs/tui/src/app/plugin_mentions.rs index f12dee72cc..ae2d25a4dc 100644 --- a/codex-rs/tui/src/app/plugin_mentions.rs +++ b/codex-rs/tui/src/app/plugin_mentions.rs @@ -165,6 +165,7 @@ mod tests { PluginSummary { id: format!("{name}@server-marketplace"), remote_plugin_id: Some(format!("plugins~{name}")), + version: None, local_version: None, name: name.to_string(), share_context: None, diff --git a/codex-rs/tui/src/app/session_lifecycle.rs b/codex-rs/tui/src/app/session_lifecycle.rs index a961658c80..b4bbbd231a 100644 --- a/codex-rs/tui/src/app/session_lifecycle.rs +++ b/codex-rs/tui/src/app/session_lifecycle.rs @@ -539,7 +539,13 @@ impl App { self.refresh_in_memory_config_from_disk_best_effort("starting a new thread") .await; let model = self.chat_widget.current_model().to_string(); - let config = self.fresh_session_config(); + let mut config = self.fresh_session_config(); + apply_managed_new_thread_defaults( + &mut config, + app_server.managed_new_thread_defaults(), + &self.cli_kv_overrides, + &self.harness_overrides, + ); let summary = session_summary( self.chat_widget.token_usage(), self.chat_widget.thread_id(), diff --git a/codex-rs/tui/src/app/tests.rs b/codex-rs/tui/src/app/tests.rs index 21aca66bcd..d8d63575bb 100644 --- a/codex-rs/tui/src/app/tests.rs +++ b/codex-rs/tui/src/app/tests.rs @@ -2880,6 +2880,7 @@ async fn inactive_thread_started_notification_initializes_replay_session() -> Re parent_thread_id: None, preview: "agent thread".to_string(), ephemeral: false, + history_mode: Default::default(), model_provider: "agent-provider".to_string(), created_at: 1, updated_at: 2, @@ -2974,6 +2975,7 @@ async fn inactive_thread_started_notification_preserves_primary_model_when_path_ parent_thread_id: None, preview: "agent thread".to_string(), ephemeral: false, + history_mode: Default::default(), model_provider: "agent-provider".to_string(), created_at: 1, updated_at: 2, @@ -3035,6 +3037,7 @@ async fn thread_read_session_state_does_not_reuse_primary_permission_profile() { parent_thread_id: None, preview: "read thread".to_string(), ephemeral: false, + history_mode: Default::default(), model_provider: "read-provider".to_string(), created_at: 1, updated_at: 2, @@ -3496,6 +3499,7 @@ async fn primary_thread_ignores_child_mcp_startup_notifications() { name: "sentry".to_string(), status: McpServerStartupState::Failed, error: Some("sentry is not logged in".to_string()), + failure_reason: None, }), ), ) @@ -3567,6 +3571,7 @@ async fn app_scoped_mcp_startup_notifications_do_not_render_in_active_thread() { name: "sentry".to_string(), status: McpServerStartupState::Failed, error: Some("sentry is not logged in".to_string()), + failure_reason: None, }), ), ) @@ -3631,6 +3636,7 @@ async fn active_side_thread_renders_live_mcp_startup_notifications() { status, error: matches!(status, McpServerStartupState::Failed) .then(|| "sentry is not logged in".to_string()), + failure_reason: None, }), ), ) @@ -5649,6 +5655,7 @@ async fn thread_rollback_response_discards_queued_active_thread_events() { parent_thread_id: None, preview: String::new(), ephemeral: false, + history_mode: Default::default(), model_provider: "openai".to_string(), created_at: 0, updated_at: 0, diff --git a/codex-rs/tui/src/app/thread_events.rs b/codex-rs/tui/src/app/thread_events.rs index 00b7fcdedb..8d41f1c052 100644 --- a/codex-rs/tui/src/app/thread_events.rs +++ b/codex-rs/tui/src/app/thread_events.rs @@ -619,6 +619,7 @@ mod tests { name: "sentry".to_string(), status: codex_app_server_protocol::McpServerStartupState::Failed, error: Some("sentry is not logged in".to_string()), + failure_reason: None, }, ); let mut store = ThreadEventStore::new(/*capacity*/ 8); diff --git a/codex-rs/tui/src/app/thread_session_state.rs b/codex-rs/tui/src/app/thread_session_state.rs index 58412bff08..602c5b2101 100644 --- a/codex-rs/tui/src/app/thread_session_state.rs +++ b/codex-rs/tui/src/app/thread_session_state.rs @@ -415,6 +415,7 @@ mod tests { parent_thread_id: None, preview: "read thread".to_string(), ephemeral: false, + history_mode: Default::default(), model_provider: "read-provider".to_string(), created_at: 1, updated_at: 2, diff --git a/codex-rs/tui/src/app_server_session.rs b/codex-rs/tui/src/app_server_session.rs index 7181d1a560..aeca17091e 100644 --- a/codex-rs/tui/src/app_server_session.rs +++ b/codex-rs/tui/src/app_server_session.rs @@ -24,6 +24,7 @@ use codex_app_server_protocol::AskForApproval; use codex_app_server_protocol::AuthMode; use codex_app_server_protocol::ClientRequest; use codex_app_server_protocol::ConfigBatchWriteParams; +use codex_app_server_protocol::ConfigRequirementsReadResponse; use codex_app_server_protocol::ConfigWriteResponse; use codex_app_server_protocol::ExternalAgentConfigDetectParams; use codex_app_server_protocol::ExternalAgentConfigDetectResponse; @@ -39,6 +40,7 @@ use codex_app_server_protocol::MemoryResetResponse; use codex_app_server_protocol::Model as ApiModel; use codex_app_server_protocol::ModelListParams; use codex_app_server_protocol::ModelListResponse; +use codex_app_server_protocol::NewThreadModelDefaults; use codex_app_server_protocol::RateLimitSnapshot; use codex_app_server_protocol::RequestId; use codex_app_server_protocol::ReviewDelivery; @@ -178,6 +180,7 @@ pub(crate) struct AppServerSession { thread_settings_update_supported: bool, default_model: Option, available_models: Vec, + managed_new_thread_defaults: Option, external_agent_config_import_completion_pending: AtomicBool, } @@ -222,6 +225,7 @@ impl AppServerSession { thread_settings_update_supported: true, default_model: None, available_models: Vec::new(), + managed_new_thread_defaults: None, external_agent_config_import_completion_pending: AtomicBool::new(false), } } @@ -260,6 +264,21 @@ impl AppServerSession { pub(crate) async fn bootstrap(&mut self, config: &Config) -> Result { let started_at = Instant::now(); let account = self.read_account().await?; + let requirements_request_id = self.next_request_id(); + let requirements: ConfigRequirementsReadResponse = self + .client + .request_typed(ClientRequest::ConfigRequirementsRead { + request_id: requirements_request_id, + params: None, + }) + .await + .map_err(|err| { + bootstrap_request_error("configRequirements/read failed during TUI bootstrap", err) + })?; + self.managed_new_thread_defaults = requirements + .requirements + .and_then(|requirements| requirements.models) + .and_then(|models| models.new_thread); let model_request_id = self.next_request_id(); let models: ModelListResponse = self .client @@ -350,6 +369,10 @@ impl AppServerSession { }) } + pub(crate) fn managed_new_thread_defaults(&self) -> Option<&NewThreadModelDefaults> { + self.managed_new_thread_defaults.as_ref() + } + /// Fetches the current account info without refreshing the auth token. /// /// Used by both `bootstrap` (to populate the initial UI) and `get_login_status` @@ -2326,6 +2349,7 @@ mod tests { parent_thread_id: None, preview: "hello".to_string(), ephemeral: false, + history_mode: Default::default(), model_provider: "openai".to_string(), created_at: 1, updated_at: 2, diff --git a/codex-rs/tui/src/bottom_pane/chat_composer.rs b/codex-rs/tui/src/bottom_pane/chat_composer.rs index e568436b40..5ba8d88ff5 100644 --- a/codex-rs/tui/src/bottom_pane/chat_composer.rs +++ b/codex-rs/tui/src/bottom_pane/chat_composer.rs @@ -1826,42 +1826,10 @@ impl ChatComposer { }; let sel_path = sel.to_string_lossy().to_string(); - if Self::is_image_path(&sel_path) { - let path_buf = PathBuf::from(&sel_path); - match image::image_dimensions(&path_buf) { - Ok((width, height)) => { - tracing::debug!("selected image dimensions={}x{}", width, height); - let cursor_offset = self.draft.textarea.cursor(); - let text = self.draft.textarea.text(); - let safe_cursor = Self::clamp_to_char_boundary(text, cursor_offset); - let before_cursor = &text[..safe_cursor]; - let after_cursor = &text[safe_cursor..]; - - let start_idx = before_cursor - .char_indices() - .rfind(|(_, c)| c.is_whitespace()) - .map(|(idx, c)| idx + c.len_utf8()) - .unwrap_or(0); - let end_rel_idx = after_cursor - .char_indices() - .find(|(_, c)| c.is_whitespace()) - .map(|(idx, _)| idx) - .unwrap_or(after_cursor.len()); - let end_idx = safe_cursor + end_rel_idx; - - self.draft.textarea.replace_range(start_idx..end_idx, ""); - self.draft.textarea.set_cursor(start_idx); - - self.attach_image(path_buf); - self.draft.textarea.insert_str(" "); - } - Err(err) => { - tracing::trace!("image dimensions lookup failed: {err}"); - self.insert_selected_path(&sel_path); - } - } - } else { - self.insert_selected_path(&sel_path); + if let Some((token_range, _)) = + self.current_editable_at_token_range_with_options(/*allow_empty*/ false) + { + self.insert_selected_file_path(token_range, &sel_path); } self.popups.active = ActivePopup::None; (InputResult::None, true) @@ -1935,8 +1903,10 @@ impl ChatComposer { }; if close_popup { - if let Some((insert_text, path)) = selected_mention { - self.insert_selected_mention(&insert_text, path.as_deref()); + if let Some((insert_text, path)) = selected_mention + && let Some((token_range, _)) = self.current_mention_token_range() + { + self.insert_selected_mention(token_range, &insert_text, path.as_deref()); } self.popups.active = ActivePopup::None; } @@ -2040,13 +2010,19 @@ impl ChatComposer { }; if close_popup { - if let Some(selected) = selected { + let token_range = self + .current_editable_at_token_range_with_options(/*allow_empty*/ true) + .map(|(range, _)| range); + if let (Some(selected), Some(token_range)) = (selected, token_range) { match selected { MentionV2Selection::File(path) => { - self.insert_selected_file_path(path.to_string_lossy().as_ref()); + self.insert_selected_file_path( + token_range, + path.to_string_lossy().as_ref(), + ); } MentionV2Selection::Tool { insert_text, path } => { - self.insert_selected_mention(&insert_text, path.as_deref()); + self.insert_selected_mention(token_range, &insert_text, path.as_deref()); } } } @@ -2068,42 +2044,25 @@ impl ChatComposer { || lower.ends_with(".webp") } - fn insert_selected_file_path(&mut self, selected_path: &str) { + fn insert_selected_file_path(&mut self, token_range: Range, selected_path: &str) { if Self::is_image_path(selected_path) { let path_buf = PathBuf::from(selected_path); match image::image_dimensions(&path_buf) { Ok((width, height)) => { tracing::debug!("selected image dimensions={}x{}", width, height); - let cursor_offset = self.draft.textarea.cursor(); - let text = self.draft.textarea.text(); - let safe_cursor = Self::clamp_to_char_boundary(text, cursor_offset); - let before_cursor = &text[..safe_cursor]; - let after_cursor = &text[safe_cursor..]; - - let start_idx = before_cursor - .char_indices() - .rfind(|(_, c)| c.is_whitespace()) - .map(|(idx, c)| idx + c.len_utf8()) - .unwrap_or(0); - let end_rel_idx = after_cursor - .char_indices() - .find(|(_, c)| c.is_whitespace()) - .map(|(idx, _)| idx) - .unwrap_or(after_cursor.len()); - let end_idx = safe_cursor + end_rel_idx; - - self.draft.textarea.replace_range(start_idx..end_idx, ""); + let start_idx = token_range.start; + self.draft.textarea.replace_range(token_range, ""); self.draft.textarea.set_cursor(start_idx); self.attach_image(path_buf); self.draft.textarea.insert_str(" "); } Err(err) => { tracing::trace!("image dimensions lookup failed: {err}"); - self.insert_selected_path(selected_path); + self.insert_selected_path(token_range, selected_path); } } } else { - self.insert_selected_path(selected_path); + self.insert_selected_path(token_range, selected_path); } } @@ -2372,7 +2331,10 @@ impl ChatComposer { Self::current_prefixed_token(textarea, '@', /*allow_empty*/ false) } - fn current_editable_at_token_with_options(&self, allow_empty: bool) -> Option { + fn current_editable_at_token_range_with_options( + &self, + allow_empty: bool, + ) -> Option<(Range, String)> { let (range, token) = Self::current_prefixed_token_range(&self.draft.textarea, '@', allow_empty)?; if self @@ -2402,55 +2364,43 @@ impl ChatComposer { return None; } - Some(token) + Some((range, token)) + } + + fn current_editable_at_token_with_options(&self, allow_empty: bool) -> Option { + self.current_editable_at_token_range_with_options(allow_empty) + .map(|(_, token)| token) } fn current_editable_at_token(&self) -> Option { self.current_editable_at_token_with_options(/*allow_empty*/ false) } - fn current_mentions_v2_token(&self) -> Option { + fn current_mentions_v2_token_range(&self) -> Option<(Range, String)> { if !self.mentions_v2_enabled { return None; } - self.current_editable_at_token_with_options(/*allow_empty*/ true) + self.current_editable_at_token_range_with_options(/*allow_empty*/ true) } - fn current_mention_token(&self) -> Option { + fn current_mentions_v2_token(&self) -> Option { + self.current_mentions_v2_token_range() + .map(|(_, token)| token) + } + + fn current_mention_token_range(&self) -> Option<(Range, String)> { if !self.mentions_enabled() { return None; } - Self::current_prefixed_token(&self.draft.textarea, '$', /*allow_empty*/ true) + Self::current_prefixed_token_range(&self.draft.textarea, '$', /*allow_empty*/ true) + } + + fn current_mention_token(&self) -> Option { + self.current_mention_token_range().map(|(_, token)| token) } /// Replace the active `@token` (the one under the cursor) with `path`. - /// - /// The algorithm mirrors `current_at_token` so replacement works no matter - /// where the cursor is within the token and regardless of how many - /// `@tokens` exist in the line. - fn insert_selected_path(&mut self, path: &str) { - let cursor_offset = self.draft.textarea.cursor(); - let text = self.draft.textarea.text(); - // Clamp to a valid char boundary to avoid panics when slicing. - let safe_cursor = Self::clamp_to_char_boundary(text, cursor_offset); - - let before_cursor = &text[..safe_cursor]; - let after_cursor = &text[safe_cursor..]; - - // Determine token boundaries. - let start_idx = before_cursor - .char_indices() - .rfind(|(_, c)| c.is_whitespace()) - .map(|(idx, c)| idx + c.len_utf8()) - .unwrap_or(0); - - let end_rel_idx = after_cursor - .char_indices() - .find(|(_, c)| c.is_whitespace()) - .map(|(idx, _)| idx) - .unwrap_or(after_cursor.len()); - let end_idx = safe_cursor + end_rel_idx; - + fn insert_selected_path(&mut self, token_range: Range, path: &str) { // If the path contains whitespace, wrap it in double quotes so the // local prompt arg parser treats it as a single argument. Avoid adding // quotes when the path already contains one to keep behavior simple. @@ -2463,36 +2413,23 @@ impl ChatComposer { // Replace just the active `@token` so unrelated text elements, such as // large-paste placeholders, remain atomic and can still expand on submit. + let start_idx = token_range.start; self.draft .textarea - .replace_range(start_idx..end_idx, &format!("{inserted} ")); + .replace_range(token_range, &format!("{inserted} ")); let new_cursor = start_idx.saturating_add(inserted.len()).saturating_add(1); self.draft.textarea.set_cursor(new_cursor); } - fn insert_selected_mention(&mut self, insert_text: &str, path: Option<&str>) { - let cursor_offset = self.draft.textarea.cursor(); - let text = self.draft.textarea.text(); - let safe_cursor = Self::clamp_to_char_boundary(text, cursor_offset); - - let before_cursor = &text[..safe_cursor]; - let after_cursor = &text[safe_cursor..]; - - let start_idx = before_cursor - .char_indices() - .rfind(|(_, c)| c.is_whitespace()) - .map(|(idx, c)| idx + c.len_utf8()) - .unwrap_or(0); - - let end_rel_idx = after_cursor - .char_indices() - .find(|(_, c)| c.is_whitespace()) - .map(|(idx, _)| idx) - .unwrap_or(after_cursor.len()); - let end_idx = safe_cursor + end_rel_idx; - + fn insert_selected_mention( + &mut self, + token_range: Range, + insert_text: &str, + path: Option<&str>, + ) { // Remove the active token and insert the selected mention as an atomic element. - self.draft.textarea.replace_range(start_idx..end_idx, ""); + let start_idx = token_range.start; + self.draft.textarea.replace_range(token_range, ""); self.draft.textarea.set_cursor(start_idx); let id = self.draft.textarea.insert_element(insert_text); diff --git a/codex-rs/tui/src/chatwidget/model_popups.rs b/codex-rs/tui/src/chatwidget/model_popups.rs index 3042ff6c71..1194fc15c6 100644 --- a/codex-rs/tui/src/chatwidget/model_popups.rs +++ b/codex-rs/tui/src/chatwidget/model_popups.rs @@ -505,6 +505,7 @@ impl ChatWidget { ReasoningEffortConfig::Medium => "Medium".to_string(), ReasoningEffortConfig::High => "High".to_string(), ReasoningEffortConfig::XHigh => "Extra high".to_string(), + ReasoningEffortConfig::Max => "Max".to_string(), ReasoningEffortConfig::Ultra => "Ultra".to_string(), ReasoningEffortConfig::Custom(value) => value.clone(), } diff --git a/codex-rs/tui/src/chatwidget/plugin_catalog.rs b/codex-rs/tui/src/chatwidget/plugin_catalog.rs index cb92ea7d31..40dfca0ce5 100644 --- a/codex-rs/tui/src/chatwidget/plugin_catalog.rs +++ b/codex-rs/tui/src/chatwidget/plugin_catalog.rs @@ -1474,6 +1474,12 @@ fn plugin_source_summary(plugin: &PluginDetail) -> String { Some(ref_name) => format!("Git · {url}@{ref_name}"), None => format!("Git · {url}"), }, + PluginSource::Npm { + package, version, .. + } => match version { + Some(version) => format!("npm · {package}@{version}"), + None => format!("npm · {package}"), + }, PluginSource::Remote => { let marketplace_label = MarketplaceProduct::from_marketplace_name(&plugin.marketplace_name) diff --git a/codex-rs/tui/src/chatwidget/reasoning_shortcuts.rs b/codex-rs/tui/src/chatwidget/reasoning_shortcuts.rs index 404bf87029..0873bda6e0 100644 --- a/codex-rs/tui/src/chatwidget/reasoning_shortcuts.rs +++ b/codex-rs/tui/src/chatwidget/reasoning_shortcuts.rs @@ -225,7 +225,7 @@ mod tests { #[test] fn next_reasoning_effort_uses_advertised_order_for_custom_levels() { - let custom_effort = ReasoningEffortConfig::Custom("max".to_string()); + let custom_effort = ReasoningEffortConfig::Custom("future".to_string()); let choices = vec![ ReasoningEffortConfig::High, ReasoningEffortConfig::Low, diff --git a/codex-rs/tui/src/chatwidget/safety_buffering.rs b/codex-rs/tui/src/chatwidget/safety_buffering.rs index 389048fb0b..649432f21c 100644 --- a/codex-rs/tui/src/chatwidget/safety_buffering.rs +++ b/codex-rs/tui/src/chatwidget/safety_buffering.rs @@ -4,15 +4,16 @@ use super::*; use codex_app_server_protocol::ModelSafetyBufferingUpdatedNotification; const SAFETY_BUFFERING_PROMPT_VIEW_ID: &str = "safety-buffering-prompt"; +const SAFETY_BUFFERING_LEARN_MORE_URL: &str = "https://help.openai.com/en/articles/20001326"; -const SAFETY_BUFFERING_MESSAGE_WITH_RETRY: &str = "This request requires additional safety checks, which can take extra time. You can retry with a faster model for a quicker response, though it may be less capable of handling complex requests."; +const SAFETY_BUFFERING_MESSAGE_WITH_RETRY: &str = "This request requires additional safety checks, which can take extra time. Hang tight or retry with a faster model for a quicker response, though it may be less capable of handling complex requests."; const SAFETY_BUFFERING_MESSAGE_WITHOUT_RETRY: &str = "This request requires additional safety checks, which can take extra time."; #[derive(Debug)] struct ActiveSafetyBuffering { turn_id: String, - retry_prompt_shown: bool, + last_prompt_had_retry: bool, agent_message_started: bool, } @@ -119,22 +120,18 @@ impl ChatWidget { .map(|(_, turn)| turn.clone()); let thread_id = self.thread_id; let can_offer_retry = faster_model.is_some() && retry_turn.is_some() && thread_id.is_some(); - if !can_offer_retry { - self.bottom_pane - .dismiss_view_by_id(SAFETY_BUFFERING_PROMPT_VIEW_ID); - } let previous_active = self .safety_buffering .active .as_ref() .filter(|active| active.turn_id == turn_id); - let retry_prompt_shown = previous_active.is_some_and(|active| active.retry_prompt_shown); - let should_show_retry_prompt = can_offer_retry && !retry_prompt_shown; + let should_show_prompt = + previous_active.is_none_or(|active| active.last_prompt_had_retry != can_offer_retry); let agent_message_started = previous_active.is_some_and(|active| active.agent_message_started); self.safety_buffering.active = Some(ActiveSafetyBuffering { turn_id: turn_id.clone(), - retry_prompt_shown: retry_prompt_shown || should_show_retry_prompt, + last_prompt_had_retry: can_offer_retry, agent_message_started, }); @@ -151,45 +148,54 @@ impl ChatWidget { /*details_max_lines*/ 6, ); - let (Some(faster_model), Some(turn), Some(thread_id)) = - (faster_model, retry_turn, thread_id) - else { - return; - }; - if !should_show_retry_prompt { + if !should_show_prompt { return; } + self.bottom_pane + .dismiss_view_by_id(SAFETY_BUFFERING_PROMPT_VIEW_ID); let header = ColumnRenderable::with(vec![ Box::new(Line::from("Additional safety checks").bold()) as Box, - Box::new( - Paragraph::new(Line::from(SAFETY_BUFFERING_MESSAGE_WITH_RETRY).dim()) - .wrap(Wrap { trim: false }), - ), + Box::new(Paragraph::new(Line::from(message).dim()).wrap(Wrap { trim: false })), + ]); + let mut items = Vec::new(); + if let (Some(faster_model), Some(turn), Some(thread_id)) = + (faster_model, retry_turn, thread_id) + { + items.push(SelectionItem { + name: "Retry with a faster model".to_string(), + actions: vec![Box::new(move |tx| { + tx.send(AppEvent::RetrySafetyBufferedTurn { + thread_id, + turn_id: turn_id.clone(), + model: faster_model.clone(), + turn: turn.clone(), + }); + })], + dismiss_on_select: true, + ..Default::default() + }); + } + items.extend([ + SelectionItem { + name: "Keep waiting".to_string(), + dismiss_on_select: true, + ..Default::default() + }, + SelectionItem { + name: "Learn more".to_string(), + actions: vec![Box::new(|tx| { + tx.send(AppEvent::OpenUrlInBrowser { + url: SAFETY_BUFFERING_LEARN_MORE_URL.to_string(), + }); + })], + ..Default::default() + }, ]); self.bottom_pane.show_selection_view(SelectionViewParams { view_id: Some(SAFETY_BUFFERING_PROMPT_VIEW_ID), header: Box::new(header), - items: vec![ - SelectionItem { - name: "Retry with a faster model".to_string(), - actions: vec![Box::new(move |tx| { - tx.send(AppEvent::RetrySafetyBufferedTurn { - thread_id, - turn_id: turn_id.clone(), - model: faster_model.clone(), - turn: turn.clone(), - }); - })], - dismiss_on_select: true, - ..Default::default() - }, - SelectionItem { - name: "Keep waiting".to_string(), - dismiss_on_select: true, - ..Default::default() - }, - ], + items, ..Default::default() }); } diff --git a/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__long_hook_context_is_truncated_with_transcript_hint.snap b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__long_hook_context_is_truncated_with_transcript_hint.snap new file mode 100644 index 0000000000..4b88bc9e1b --- /dev/null +++ b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__long_hook_context_is_truncated_with_transcript_hint.snap @@ -0,0 +1,10 @@ +--- +source: tui/src/chatwidget/tests/status_and_layout.rs +expression: history +--- +• SessionStart hook (stopped) + hook context: This hook context is intentionally long enough to wrap across + several terminal rows while keeping the complete value available in the + … +2 lines (ctrl + t to view transcript) + stop: The hook stopped this turn for an important reason. + This second line must remain visible in full. diff --git a/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__model_reasoning_selection_popup.snap b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__model_reasoning_selection_popup.snap index a436136570..6027772004 100644 --- a/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__model_reasoning_selection_popup.snap +++ b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__model_reasoning_selection_popup.snap @@ -6,7 +6,7 @@ expression: popup 1. Low Fast responses with lighter reasoning 2. Medium (default) Balances speed and reasoning depth for everyday tasks - 3. max Maximum available reasoning + 3. Max Maximum available reasoning › 4. High (current) Greater reasoning depth for complex problems 5. Extra high Extra high reasoning depth for complex problems diff --git a/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__plugin_detail_popup_npm_source.snap b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__plugin_detail_popup_npm_source.snap new file mode 100644 index 0000000000..cb07253718 --- /dev/null +++ b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__plugin_detail_popup_npm_source.snap @@ -0,0 +1,20 @@ +--- +source: tui/src/chatwidget/tests/popups_and_settings.rs +expression: strip_osc8_for_snapshot(&popup) +--- + Plugins + Figma · Can be installed · ChatGPT Marketplace + Data shared with this app is subject to the app's terms of service and privacy policy. Learn + more. + Turn Figma files into implementation context. + +› 1. Back to plugins Return to the plugin list. + 2. Install plugin Install this plugin now. + Source npm · @acme/figma-plugin@^1.2.0 + Auth Auth on install + Skills design-review, extract-copy + Hooks PreToolUse (1), Stop (2) + Apps Figma, Slack + MCP Servers figma-mcp, docs-mcp + + Press esc to close. diff --git a/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__safety_buffering_retry_prompt.snap b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__safety_buffering_retry_prompt.snap index 60fd4d38c3..6517d73336 100644 --- a/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__safety_buffering_retry_prompt.snap +++ b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__safety_buffering_retry_prompt.snap @@ -4,10 +4,11 @@ expression: popup --- Additional safety checks This request requires additional safety checks, which can take extra time. - You can retry with a faster model for a quicker response, though it may be - less capable of handling complex requests. + Hang tight or retry with a faster model for a quicker response, though it + may be less capable of handling complex requests. › 1. Retry with a faster model 2. Keep waiting + 3. Learn more Press enter to confirm or esc to go back diff --git a/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__safety_buffering_status_without_retry.snap b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__safety_buffering_status_without_retry.snap index a719f937cd..f3645db4fc 100644 --- a/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__safety_buffering_status_without_retry.snap +++ b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__safety_buffering_status_without_retry.snap @@ -1,11 +1,11 @@ --- source: tui/src/chatwidget/tests/app_server.rs -expression: "render_bottom_popup(&chat, 80)" +expression: popup --- -• Working (0s • esc to interrupt) - └ This request requires additional safety checks, which can take extra time. + Additional safety checks + This request requires additional safety checks, which can take extra time. +› 1. Keep waiting + 2. Learn more -› Ask Codex to do anything - - gpt-5.5 default · /tmp/project + Press enter to confirm or esc to go back diff --git a/codex-rs/tui/src/chatwidget/tests/app_server.rs b/codex-rs/tui/src/chatwidget/tests/app_server.rs index 2e6d7f8f7f..952aff9f95 100644 --- a/codex-rs/tui/src/chatwidget/tests/app_server.rs +++ b/codex-rs/tui/src/chatwidget/tests/app_server.rs @@ -125,6 +125,21 @@ async fn safety_buffering_offers_one_retry_with_app_wording() { let popup = render_bottom_popup(&chat, /*width*/ 80); assert_chatwidget_snapshot!("safety_buffering_retry_prompt", popup); + chat.handle_key_event(KeyEvent::new(KeyCode::Down, KeyModifiers::NONE)); + chat.handle_key_event(KeyEvent::new(KeyCode::Down, KeyModifiers::NONE)); + chat.handle_key_event(KeyEvent::new(KeyCode::Enter, KeyModifiers::NONE)); + let opened_url = loop { + match rx.try_recv() { + Ok(AppEvent::OpenUrlInBrowser { url }) => break url, + Ok(_) => continue, + Err(err) => panic!("expected learn-more URL event: {err}"), + } + }; + assert_eq!(opened_url, "https://help.openai.com/en/articles/20001326"); + assert!(render_bottom_popup(&chat, /*width*/ 80).contains("Additional safety checks")); + + chat.handle_key_event(KeyEvent::new(KeyCode::Up, KeyModifiers::NONE)); + chat.handle_key_event(KeyEvent::new(KeyCode::Up, KeyModifiers::NONE)); chat.handle_key_event(KeyEvent::new(KeyCode::Enter, KeyModifiers::NONE)); let (event_thread_id, event_turn_id, model, turn) = loop { match rx.try_recv() { @@ -146,7 +161,7 @@ async fn safety_buffering_offers_one_retry_with_app_wording() { } #[tokio::test] -async fn safety_buffering_stops_retrying_after_agent_message_starts() { +async fn safety_buffering_remains_visible_until_turn_completes() { let (mut chat, _rx, mut op_rx) = make_chatwidget_manual(/*model_override*/ None).await; let (thread_id, turn_id, _) = start_safety_buffering_test_turn(&mut chat, &mut op_rx); chat.handle_server_notification( @@ -162,6 +177,11 @@ async fn safety_buffering_stops_retrying_after_agent_message_starts() { chat.on_agent_message_delta("Visible response".to_string()); assert!(!chat.can_retry_safety_buffered_turn(turn_id)); + assert!(render_bottom_popup(&chat, /*width*/ 80).contains("Additional safety checks")); + + handle_turn_completed(&mut chat, turn_id, /*duration_ms*/ None); + + assert!(!render_bottom_popup(&chat, /*width*/ 80).contains("Additional safety checks")); } #[tokio::test] @@ -196,6 +216,9 @@ async fn safety_buffering_without_retry_shows_short_app_message() { Some(ReplayKind::ThreadSnapshot), ); assert_eq!(render_popup(&chat), popup); + + chat.handle_key_event(KeyEvent::new(KeyCode::Enter, KeyModifiers::NONE)); + assert!(!render_bottom_popup(&chat, /*width*/ 80).contains("Additional safety checks")); } #[tokio::test] @@ -1151,8 +1174,8 @@ async fn live_app_server_cyber_policy_error_renders_dedicated_notice() { let cells = drain_insert_history(&mut rx); assert_eq!(cells.len(), 1); let rendered = lines_to_single_string(&cells[0]); - assert!(rendered.contains("This chat was flagged for possible cybersecurity risk")); - assert!(rendered.contains("Trusted Access for Cyber")); + assert!(rendered.contains("This content can't be shown")); + assert!(rendered.contains("extra caution with cybersecurity requests")); assert!(!rendered.contains("server fallback message")); assert!(!chat.bottom_pane.is_task_running()); } @@ -1171,6 +1194,7 @@ async fn app_server_safety_access_errors_render_dedicated_notice() { assert_eq!(cells.len(), 1); let rendered = lines_to_single_string(&cells[0]); assert!(rendered.contains("This content can't be shown")); + assert!(rendered.contains("biological research")); assert!(!rendered.contains("Invalid prompt:")); } } diff --git a/codex-rs/tui/src/chatwidget/tests/helpers.rs b/codex-rs/tui/src/chatwidget/tests/helpers.rs index 5e23e9f96a..1f9374bbc3 100644 --- a/codex-rs/tui/src/chatwidget/tests/helpers.rs +++ b/codex-rs/tui/src/chatwidget/tests/helpers.rs @@ -1331,6 +1331,7 @@ pub(super) fn plugins_test_summary( PluginSummary { id: id.to_string(), remote_plugin_id: None, + version: None, local_version: None, name: name.to_string(), share_context: None, @@ -1361,6 +1362,7 @@ pub(super) fn plugins_test_remote_summary( PluginSummary { id: remote_plugin_id.to_string(), remote_plugin_id: Some(remote_plugin_id.to_string()), + version: None, local_version: None, name: name.to_string(), share_context: None, diff --git a/codex-rs/tui/src/chatwidget/tests/mcp_startup.rs b/codex-rs/tui/src/chatwidget/tests/mcp_startup.rs index 60efbe1353..e2992585a4 100644 --- a/codex-rs/tui/src/chatwidget/tests/mcp_startup.rs +++ b/codex-rs/tui/src/chatwidget/tests/mcp_startup.rs @@ -8,6 +8,7 @@ fn notify_mcp_status(chat: &mut ChatWidget, name: &str, status: McpServerStartup name: name.to_string(), status, error: None, + failure_reason: None, }), /*replay_kind*/ None, ); @@ -20,6 +21,7 @@ fn notify_mcp_status_error(chat: &mut ChatWidget, name: &str, error: &str) { name: name.to_string(), status: McpServerStartupState::Failed, error: Some(error.to_string()), + failure_reason: None, }), /*replay_kind*/ None, ); @@ -51,6 +53,7 @@ async fn mcp_startup_ignores_status_for_other_thread() { status, error: matches!(status, McpServerStartupState::Failed) .then(|| "sentry is not logged in".to_string()), + failure_reason: None, }), /*replay_kind*/ None, ); diff --git a/codex-rs/tui/src/chatwidget/tests/popups_and_settings.rs b/codex-rs/tui/src/chatwidget/tests/popups_and_settings.rs index 7d162d41e4..07cfc7dbbb 100644 --- a/codex-rs/tui/src/chatwidget/tests/popups_and_settings.rs +++ b/codex-rs/tui/src/chatwidget/tests/popups_and_settings.rs @@ -664,6 +664,51 @@ async fn plugin_detail_popup_snapshot_labels_personal_marketplace_as_local() { ); } +#[tokio::test] +async fn plugin_detail_popup_snapshot_shows_npm_source() { + let (mut chat, _rx, _op_rx) = make_chatwidget_manual(/*model_override*/ None).await; + chat.set_feature_enabled(Feature::Plugins, /*enabled*/ true); + + let mut summary = plugins_test_summary( + "plugin-figma", + "figma", + Some("Figma"), + Some("Design handoff."), + /*installed*/ false, + /*enabled*/ true, + PluginInstallPolicy::Available, + ); + summary.source = PluginSource::Npm { + package: "@acme/figma-plugin".to_string(), + version: Some("^1.2.0".to_string()), + registry: Some("https://npm.example.com".to_string()), + }; + let response = plugins_test_response(vec![plugins_test_curated_marketplace(vec![ + summary.clone(), + ])]); + let cwd = chat.config.cwd.clone(); + chat.on_plugins_loaded(cwd.to_path_buf(), Ok(response)); + chat.add_plugins_output(); + let plugin = plugins_test_detail( + summary, + Some("Turn Figma files into implementation context."), + &["design-review", "extract-copy"], + &[ + (codex_app_server_protocol::HookEventName::PreToolUse, 1), + (codex_app_server_protocol::HookEventName::Stop, 2), + ], + &["Figma", "Slack"], + &["figma-mcp", "docs-mcp"], + ); + chat.on_plugin_detail_loaded(cwd.to_path_buf(), Ok(PluginReadResponse { plugin })); + + let popup = render_bottom_popup(&chat, /*width*/ 100); + assert_chatwidget_snapshot!( + "plugin_detail_popup_npm_source", + strip_osc8_for_snapshot(&popup) + ); +} + #[tokio::test] async fn plugin_detail_popup_distinguishes_admin_installed_from_enabled() { let (mut chat, _rx, _op_rx) = make_chatwidget_manual(/*model_override*/ None).await; @@ -1155,9 +1200,10 @@ async fn plugins_popup_admin_disabled_available_plugin_has_view_only_hint() { } #[tokio::test] -async fn plugins_popup_remote_section_fallback_states_snapshot() { +async fn plugins_popup_remote_section_fallback_states_when_remote_plugin_disabled_snapshot() { let (mut chat, _rx, _op_rx) = make_chatwidget_manual(/*model_override*/ None).await; chat.set_feature_enabled(Feature::Plugins, /*enabled*/ true); + chat.set_feature_enabled(Feature::RemotePlugin, /*enabled*/ false); let select_tab_containing = |chat: &mut ChatWidget, visible_text: &str| -> String { for _ in 0..8 { @@ -1221,7 +1267,6 @@ async fn plugins_popup_remote_section_fallback_states_snapshot() { let (mut remote_chat, _rx, _op_rx) = make_chatwidget_manual(/*model_override*/ None).await; remote_chat.set_feature_enabled(Feature::Plugins, /*enabled*/ true); - remote_chat.set_feature_enabled(Feature::RemotePlugin, /*enabled*/ true); remote_chat.add_plugins_output(); let remote_cwd = remote_chat.config.cwd.clone(); remote_chat.on_plugins_loaded( @@ -3146,7 +3191,7 @@ async fn model_reasoning_selection_popup_snapshot() { preset.supported_reasoning_efforts.insert( 2, ReasoningEffortPreset { - effort: ReasoningEffortConfig::Custom("max".to_string()), + effort: ReasoningEffortConfig::Max, description: "Maximum available reasoning".to_string(), }, ); @@ -3159,7 +3204,7 @@ async fn model_reasoning_selection_popup_snapshot() { #[tokio::test] async fn model_reasoning_selection_popup_applies_custom_effort() { let (mut chat, mut rx, _op_rx) = make_chatwidget_manual(Some("gpt-5.4")).await; - let custom_effort = ReasoningEffortConfig::Custom("max".to_string()); + let custom_effort = ReasoningEffortConfig::Custom("future".to_string()); chat.set_reasoning_effort(Some(ReasoningEffortConfig::XHigh)); let mut preset = get_available_model(&chat, "gpt-5.4"); diff --git a/codex-rs/tui/src/chatwidget/tests/status_and_layout.rs b/codex-rs/tui/src/chatwidget/tests/status_and_layout.rs index 329d249340..d118191390 100644 --- a/codex-rs/tui/src/chatwidget/tests/status_and_layout.rs +++ b/codex-rs/tui/src/chatwidget/tests/status_and_layout.rs @@ -54,8 +54,8 @@ async fn app_server_cyber_policy_error_renders_dedicated_notice() { let cells = drain_insert_history(&mut rx); assert_eq!(cells.len(), 1); let rendered = lines_to_single_string(&cells[0]); - assert!(rendered.contains("This chat was flagged for possible cybersecurity risk")); - assert!(rendered.contains("Trusted Access for Cyber")); + assert!(rendered.contains("This content can't be shown")); + assert!(rendered.contains("extra caution with cybersecurity requests")); assert!(!rendered.contains("server fallback message")); } @@ -3888,6 +3888,41 @@ async fn hook_completed_before_reveal_renders_completed_without_running_flash() ); } +#[tokio::test] +async fn long_hook_context_is_truncated_with_transcript_hint_snapshot() { + let (mut chat, mut rx, _op_rx) = make_chatwidget_manual(/*model_override*/ None).await; + + handle_hook_completed( + &mut chat, + hook_completed_run( + "session-start:0:/tmp/hooks.json", + codex_app_server_protocol::HookEventName::SessionStart, + codex_app_server_protocol::HookRunStatus::Stopped, + vec![ + codex_app_server_protocol::HookOutputEntry { + kind: codex_app_server_protocol::HookOutputEntryKind::Context, + text: "This hook context is intentionally long enough to wrap across several terminal rows while keeping the complete value available in the transcript overlay. The main conversation should stay compact even when a hook injects a large block of instructions for the model." + .to_string(), + }, + codex_app_server_protocol::HookOutputEntry { + kind: codex_app_server_protocol::HookOutputEntryKind::Stop, + text: "The hook stopped this turn for an important reason.\nThis second line must remain visible in full." + .to_string(), + }, + ], + ), + ); + + let history = drain_insert_history(&mut rx) + .iter() + .map(|lines| lines_to_single_string(lines)) + .collect::(); + assert_chatwidget_snapshot!( + "long_hook_context_is_truncated_with_transcript_hint", + history + ); +} + #[tokio::test] async fn session_start_hook_events_render_snapshot() { assert_hook_events_snapshot( diff --git a/codex-rs/tui/src/chatwidget/tests/usage.rs b/codex-rs/tui/src/chatwidget/tests/usage.rs index c05b4a62ff..22908b1112 100644 --- a/codex-rs/tui/src/chatwidget/tests/usage.rs +++ b/codex-rs/tui/src/chatwidget/tests/usage.rs @@ -14,7 +14,10 @@ async fn usage_command_opens_menu_when_reset_is_available_snapshot() { assert!(chat.finish_rate_limit_reset_hint_refresh( request_id, Vec::new(), - Ok(RateLimitResetCreditsSummary { available_count: 2 }), + Ok(RateLimitResetCreditsSummary { + available_count: 2, + credits: None + }), )); chat.dispatch_command(SlashCommand::Usage); @@ -35,7 +38,10 @@ async fn usage_command_disables_reset_after_cached_zero_snapshot() { assert!(chat.finish_rate_limit_reset_hint_refresh( request_id, Vec::new(), - Ok(RateLimitResetCreditsSummary { available_count: 0 }), + Ok(RateLimitResetCreditsSummary { + available_count: 0, + credits: None + }), )); chat.dispatch_command(SlashCommand::Usage); @@ -63,7 +69,10 @@ async fn usage_menu_refresh_enables_newly_available_reset() { assert!(chat.finish_rate_limit_reset_hint_refresh( request_id, Vec::new(), - Ok(RateLimitResetCreditsSummary { available_count: 0 }), + Ok(RateLimitResetCreditsSummary { + available_count: 0, + credits: None + }), )); chat.dispatch_command(SlashCommand::Usage); @@ -76,7 +85,10 @@ async fn usage_menu_refresh_enables_newly_available_reset() { chat.finish_usage_menu_rate_limit_refresh( /*request_id*/ 1, Vec::new(), - Ok(RateLimitResetCreditsSummary { available_count: 1 }), + Ok(RateLimitResetCreditsSummary { + available_count: 1, + credits: None, + }), ); chat.handle_key_event(KeyEvent::new(KeyCode::Down, KeyModifiers::NONE)); chat.handle_key_event(KeyEvent::new(KeyCode::Enter, KeyModifiers::NONE)); @@ -92,7 +104,10 @@ async fn usage_menu_refresh_failure_preserves_disabled_known_zero() { assert!(chat.finish_rate_limit_reset_hint_refresh( request_id, Vec::new(), - Ok(RateLimitResetCreditsSummary { available_count: 0 }), + Ok(RateLimitResetCreditsSummary { + available_count: 0, + credits: None + }), )); chat.dispatch_command(SlashCommand::Usage); @@ -122,7 +137,10 @@ async fn account_update_invalidates_usage_menu_refresh_when_visible_state_is_unc assert!(chat.finish_rate_limit_reset_hint_refresh( startup_request_id, Vec::new(), - Ok(RateLimitResetCreditsSummary { available_count: 0 }), + Ok(RateLimitResetCreditsSummary { + available_count: 0, + credits: None + }), )); chat.dispatch_command(SlashCommand::Usage); assert_matches!( @@ -139,7 +157,10 @@ async fn account_update_invalidates_usage_menu_refresh_when_visible_state_is_unc chat.finish_usage_menu_rate_limit_refresh( /*request_id*/ 1, vec![snapshot(/*percent*/ 92.0)], - Ok(RateLimitResetCreditsSummary { available_count: 2 }), + Ok(RateLimitResetCreditsSummary { + available_count: 2, + credits: None, + }), ); assert_eq!(chat.available_rate_limit_reset_credits, None); @@ -185,7 +206,10 @@ async fn usage_menu_rate_limit_reset_entry_opens_reset_flow() { assert!(chat.finish_rate_limit_reset_hint_refresh( request_id, Vec::new(), - Ok(RateLimitResetCreditsSummary { available_count: 2 }), + Ok(RateLimitResetCreditsSummary { + available_count: 2, + credits: None + }), )); chat.dispatch_command(SlashCommand::Usage); @@ -206,7 +230,10 @@ async fn rate_limit_reset_popup_states_snapshot() { assert!(chat.finish_rate_limit_reset_credits_refresh( loading_request_id, Vec::new(), - Ok(RateLimitResetCreditsSummary { available_count: 2 }), + Ok(RateLimitResetCreditsSummary { + available_count: 2, + credits: None + }), )); record_popup(&chat, &mut states); @@ -215,7 +242,10 @@ async fn rate_limit_reset_popup_states_snapshot() { assert!(chat.finish_rate_limit_reset_credits_refresh( empty_request_id, Vec::new(), - Ok(RateLimitResetCreditsSummary { available_count: 0 }), + Ok(RateLimitResetCreditsSummary { + available_count: 0, + credits: None + }), )); record_popup(&chat, &mut states); @@ -270,7 +300,10 @@ async fn rate_limit_reset_popup_states_snapshot() { assert!(chat.finish_post_consume_reset_credits_refresh( success_request_id, Vec::new(), - Ok(RateLimitResetCreditsSummary { available_count: 1 }), + Ok(RateLimitResetCreditsSummary { + available_count: 1, + credits: None + }), )); record_popup(&chat, &mut states); @@ -288,7 +321,10 @@ async fn usage_limit_reset_confirmation_uses_monthly_copy_for_monthly_limits_sna assert!(chat.finish_rate_limit_reset_credits_refresh( free_request_id, Vec::new(), - Ok(RateLimitResetCreditsSummary { available_count: 1 }), + Ok(RateLimitResetCreditsSummary { + available_count: 1, + credits: None + }), )); states.push(format!( "Free:\n{}", @@ -301,7 +337,10 @@ async fn usage_limit_reset_confirmation_uses_monthly_copy_for_monthly_limits_sna assert!(chat.finish_rate_limit_reset_credits_refresh( go_request_id, Vec::new(), - Ok(RateLimitResetCreditsSummary { available_count: 1 }), + Ok(RateLimitResetCreditsSummary { + available_count: 1, + credits: None + }), )); states.push(format!("Go:\n{}", render_bottom_popup(&chat, /*width*/ 80))); @@ -317,7 +356,10 @@ async fn usage_limit_reset_confirmation_uses_monthly_copy_for_monthly_limits_sna assert!(chat.finish_rate_limit_reset_credits_refresh( business_request_id, vec![monthly_business_snapshot], - Ok(RateLimitResetCreditsSummary { available_count: 1 }), + Ok(RateLimitResetCreditsSummary { + available_count: 1, + credits: None + }), )); states.push(format!( "Business with monthly window:\n{}", @@ -337,7 +379,10 @@ async fn rate_limit_reset_confirmation_selects_cancel_by_default() { assert!(chat.finish_rate_limit_reset_credits_refresh( request_id, Vec::new(), - Ok(RateLimitResetCreditsSummary { available_count: 1 }), + Ok(RateLimitResetCreditsSummary { + available_count: 1, + credits: None + }), )); chat.handle_key_event(KeyEvent::new(KeyCode::Enter, KeyModifiers::NONE)); @@ -353,7 +398,10 @@ async fn rate_limit_reset_confirmation_can_use_reset() { assert!(chat.finish_rate_limit_reset_credits_refresh( request_id, Vec::new(), - Ok(RateLimitResetCreditsSummary { available_count: 1 }), + Ok(RateLimitResetCreditsSummary { + available_count: 1, + credits: None + }), )); chat.handle_key_event(KeyEvent::new(KeyCode::Up, KeyModifiers::NONE)); @@ -393,7 +441,10 @@ async fn no_credit_outcome_disables_reset_entry_in_usage_menu() { assert!(chat.finish_rate_limit_reset_hint_refresh( startup_request_id, Vec::new(), - Ok(RateLimitResetCreditsSummary { available_count: 1 }), + Ok(RateLimitResetCreditsSummary { + available_count: 1, + credits: None + }), )); let consume_request_id = chat.show_rate_limit_reset_consuming_popup(); assert!(!finish_reset_consume_outcome( @@ -438,7 +489,10 @@ async fn rate_limit_reset_redemption_cannot_be_dismissed_while_in_flight() { assert!(chat.finish_post_consume_reset_credits_refresh( request_id, Vec::new(), - Ok(RateLimitResetCreditsSummary { available_count: 1 }), + Ok(RateLimitResetCreditsSummary { + available_count: 1, + credits: None + }), )); dismiss_popup(&mut chat); assert!(chat.bottom_pane.no_modal_or_popup_active()); @@ -469,7 +523,10 @@ async fn already_redeemed_is_an_idempotent_success() { assert!(chat.finish_post_consume_reset_credits_refresh( request_id, Vec::new(), - Ok(RateLimitResetCreditsSummary { available_count: 0 }), + Ok(RateLimitResetCreditsSummary { + available_count: 0, + credits: None + }), )); assert!( render_bottom_popup(&chat, /*width*/ 80) @@ -485,7 +542,10 @@ async fn failed_post_consume_refresh_does_not_keep_stale_reset_count() { assert!(chat.finish_rate_limit_reset_hint_refresh( startup_request_id, Vec::new(), - Ok(RateLimitResetCreditsSummary { available_count: 2 }), + Ok(RateLimitResetCreditsSummary { + available_count: 2, + credits: None + }), )); let consume_request_id = chat.show_rate_limit_reset_consuming_popup(); assert!(finish_reset_consume_outcome( @@ -522,7 +582,10 @@ async fn account_change_invalidates_pending_reset_requests() { assert!(!chat.finish_rate_limit_reset_credits_refresh( request_id, Vec::new(), - Ok(RateLimitResetCreditsSummary { available_count: 2 }), + Ok(RateLimitResetCreditsSummary { + available_count: 2, + credits: None + }), )); assert!(chat.bottom_pane.no_modal_or_popup_active()); } @@ -536,7 +599,10 @@ async fn clearing_pending_reset_hint_preserves_in_flight_redemption() { assert!(chat.finish_rate_limit_reset_hint_refresh( hint_request_id, Vec::new(), - Ok(RateLimitResetCreditsSummary { available_count: 2 }), + Ok(RateLimitResetCreditsSummary { + available_count: 2, + credits: None + }), )); chat.clear_pending_rate_limit_reset_hint(); @@ -559,7 +625,10 @@ async fn rate_limit_reset_load_result_updates_popup_beneath_overlay() { assert!(chat.finish_rate_limit_reset_credits_refresh( request_id, Vec::new(), - Ok(RateLimitResetCreditsSummary { available_count: 2 }), + Ok(RateLimitResetCreditsSummary { + available_count: 2, + credits: None + }), )); assert_eq!( chat.bottom_pane.active_view_id(), @@ -588,7 +657,10 @@ async fn rate_limit_reset_success_updates_popup_beneath_overlay() { assert!(chat.finish_post_consume_reset_credits_refresh( request_id, Vec::new(), - Ok(RateLimitResetCreditsSummary { available_count: 1 }), + Ok(RateLimitResetCreditsSummary { + available_count: 1, + credits: None + }), )); assert_eq!( chat.bottom_pane.active_view_id(), @@ -631,7 +703,10 @@ async fn startup_check_shows_available_reset_hint_snapshot() { assert!(chat.finish_rate_limit_reset_hint_refresh( hint_request_id, Vec::new(), - Ok(RateLimitResetCreditsSummary { available_count: 2 }), + Ok(RateLimitResetCreditsSummary { + available_count: 2, + credits: None + }), )); let rendered = lines_to_single_string( &chat @@ -654,7 +729,10 @@ async fn startup_reset_hint_waits_for_active_output_snapshot() { assert!(chat.finish_rate_limit_reset_hint_refresh( hint_request_id, Vec::new(), - Ok(RateLimitResetCreditsSummary { available_count: 2 }), + Ok(RateLimitResetCreditsSummary { + available_count: 2, + credits: None + }), )); assert!(chat.usage_history_insertion_blocked()); @@ -685,7 +763,10 @@ async fn opening_rate_limit_reset_flow_invalidates_in_flight_startup_hint() { assert!(!chat.finish_rate_limit_reset_hint_refresh( hint_request_id, Vec::new(), - Ok(RateLimitResetCreditsSummary { available_count: 2 }), + Ok(RateLimitResetCreditsSummary { + available_count: 2, + credits: None + }), )); assert!(chat.pending_rate_limit_reset_hint().is_none()); } @@ -698,7 +779,10 @@ async fn starting_rate_limit_reset_redemption_clears_deferred_startup_hint() { assert!(chat.finish_rate_limit_reset_hint_refresh( hint_request_id, Vec::new(), - Ok(RateLimitResetCreditsSummary { available_count: 2 }), + Ok(RateLimitResetCreditsSummary { + available_count: 2, + credits: None + }), )); assert!(chat.pending_rate_limit_reset_hint().is_some()); @@ -716,7 +800,10 @@ async fn startup_check_omits_reset_hint_when_none_are_available() { assert!(chat.finish_rate_limit_reset_hint_refresh( hint_request_id, Vec::new(), - Ok(RateLimitResetCreditsSummary { available_count: 0 }), + Ok(RateLimitResetCreditsSummary { + available_count: 0, + credits: None + }), )); assert!(chat.pending_rate_limit_reset_hint().is_none()); } @@ -731,7 +818,10 @@ async fn startup_check_shows_reset_hint_for_workspace_account_with_credit() { assert!(chat.finish_rate_limit_reset_hint_refresh( hint_request_id, Vec::new(), - Ok(RateLimitResetCreditsSummary { available_count: 2 }), + Ok(RateLimitResetCreditsSummary { + available_count: 2, + credits: None + }), )); assert!(chat.pending_rate_limit_reset_hint().is_some()); assert_eq!(chat.available_rate_limit_reset_credits, Some(2)); diff --git a/codex-rs/tui/src/chatwidget/turn_runtime.rs b/codex-rs/tui/src/chatwidget/turn_runtime.rs index 8af43550e3..fdfad7f6d8 100644 --- a/codex-rs/tui/src/chatwidget/turn_runtime.rs +++ b/codex-rs/tui/src/chatwidget/turn_runtime.rs @@ -170,6 +170,7 @@ impl ChatWidget { self.input_queue.user_turn_pending_start = false; self.clear_active_hook_cell(); self.turn_lifecycle.finish(); + self.clear_safety_buffering(); self.update_task_running_state(); self.running_commands.clear(); self.suppressed_exec_calls.clear(); diff --git a/codex-rs/tui/src/debug_config.rs b/codex-rs/tui/src/debug_config.rs index ed9aeffddb..c02d4c8261 100644 --- a/codex-rs/tui/src/debug_config.rs +++ b/codex-rs/tui/src/debug_config.rs @@ -699,7 +699,7 @@ mod tests { mcp_servers: Some(Sourced::new( BTreeMap::from([( "docs".to_string(), - McpServerRequirement { + McpServerRequirement::Identity { identity: McpServerIdentity::Command { command: "codex-mcp".to_string(), }, @@ -778,7 +778,7 @@ mod tests { hooks: None, mcp_servers: Some(BTreeMap::from([( "docs".to_string(), - McpServerRequirement { + McpServerRequirement::Identity { identity: McpServerIdentity::Command { command: "codex-mcp".to_string(), }, @@ -791,6 +791,7 @@ mod tests { enforce_residency: Some(ResidencyRequirement::Us), network: None, permissions: None, + models: None, }; let user_file = if cfg!(windows) { @@ -1149,6 +1150,7 @@ approval_policy = "never" enforce_residency: None, network: None, permissions: None, + models: None, }; let stack = ConfigLayerStack::new(Vec::new(), requirements, requirements_toml) diff --git a/codex-rs/tui/src/exec_cell/render.rs b/codex-rs/tui/src/exec_cell/render.rs index 882683ad28..bf17277b91 100644 --- a/codex-rs/tui/src/exec_cell/render.rs +++ b/codex-rs/tui/src/exec_cell/render.rs @@ -12,6 +12,7 @@ use crate::motion::activity_indicator; use crate::render::highlight::highlight_bash_to_lines; use crate::render::line_utils::prefix_lines; use crate::render::line_utils::push_owned_lines; +use crate::ui_consts::TRANSCRIPT_HINT; use crate::wrapping::RtOptions; use crate::wrapping::adaptive_wrap_line; use crate::wrapping::adaptive_wrap_lines; @@ -32,7 +33,6 @@ use unicode_width::UnicodeWidthStr; pub(crate) const TOOL_CALL_MAX_LINES: usize = 5; const USER_SHELL_TOOL_CALL_MAX_LINES: usize = 50; const MAX_INTERACTION_PREVIEW_CHARS: usize = 80; -const TRANSCRIPT_HINT: &str = "ctrl + t to view transcript"; pub(crate) struct OutputLinesParams { pub(crate) line_limit: usize, diff --git a/codex-rs/tui/src/history_cell/hook_cell.rs b/codex-rs/tui/src/history_cell/hook_cell.rs index c3214aeb16..51191ea103 100644 --- a/codex-rs/tui/src/history_cell/hook_cell.rs +++ b/codex-rs/tui/src/history_cell/hook_cell.rs @@ -12,11 +12,16 @@ //! 4. Completed runs only persist when they have output or a non-success status. use super::HistoryCell; use super::plain_lines; +use crate::line_truncation::truncate_line_with_ellipsis_if_overflow; use crate::motion::MotionMode; use crate::motion::ReducedMotionIndicator; use crate::motion::activity_indicator; use crate::motion::shimmer_text; +use crate::render::line_utils::push_owned_lines; use crate::render::renderable::Renderable; +use crate::ui_consts::TRANSCRIPT_HINT; +use crate::wrapping::RtOptions; +use crate::wrapping::word_wrap_line; use codex_app_server_protocol::HookEventName; use codex_app_server_protocol::HookOutputEntry; use codex_app_server_protocol::HookOutputEntryKind; @@ -50,6 +55,7 @@ const QUIET_HOOK_MIN_VISIBLE: Duration = Duration::from_millis(600); const HOOK_OUTPUT_INDENT: &str = " "; const HOOK_OUTPUT_BODY_INDENT: &str = " "; +const HOOK_CONTEXT_MAX_DISPLAY_ROWS: usize = 3; #[derive(Debug)] struct HookRunCell { @@ -294,11 +300,9 @@ impl HookCell { run.reveal_running_after_delayed_redraw_for_test(now); } } -} -impl HistoryCell for HookCell { - /// Builds viewport lines while coalescing adjacent visible-running hooks. - fn display_lines(&self, _width: u16) -> Vec> { + /// Builds hook lines for either the bounded main viewport or the full transcript overlay. + fn output_lines(&self, width: u16, render_full_context: bool) -> Vec> { let mut lines = Vec::new(); let mut running_group: Option = None; for run in &self.runs { @@ -313,7 +317,12 @@ impl HistoryCell for HookCell { push_running_hook_group(&mut lines, &group, self.animations_enabled); } push_hook_line_separator(&mut lines); - run.push_display_lines(&mut lines, self.animations_enabled); + run.push_display_lines( + &mut lines, + self.animations_enabled, + width, + render_full_context, + ); continue; }; @@ -338,14 +347,21 @@ impl HistoryCell for HookCell { } lines } +} - /// Hook transcript output matches viewport output. +impl HistoryCell for HookCell { + /// Builds viewport lines while coalescing adjacent visible-running hooks. + fn display_lines(&self, width: u16) -> Vec> { + self.output_lines(width, /*render_full_context*/ false) + } + + /// The transcript overlay preserves complete hook context hidden by the viewport preview. fn transcript_lines(&self, width: u16) -> Vec> { - self.display_lines(width) + self.output_lines(width, /*render_full_context*/ true) } fn raw_lines(&self) -> Vec> { - plain_lines(self.display_lines(u16::MAX)) + plain_lines(self.output_lines(u16::MAX, /*render_full_context*/ true)) } /// Produces a coarse cache key for transcript overlays while hook animations are active. @@ -420,7 +436,13 @@ impl HookRunCell { } /// Appends the lines for a single, ungrouped hook run. - fn push_display_lines(&self, lines: &mut Vec>, animations_enabled: bool) { + fn push_display_lines( + &self, + lines: &mut Vec>, + animations_enabled: bool, + width: u16, + render_full_context: bool, + ) { let label = hook_event_label(self.event_name); match &self.state { HookRunState::VisibleRunning { start_time, .. } @@ -446,17 +468,10 @@ impl HookRunCell { .into(), ); for entry in entries { - let prefix = hook_output_prefix(entry.kind); - let mut output_lines = entry.text.split('\n'); - if let Some(first_line) = output_lines.next() { - lines.push(format!("{HOOK_OUTPUT_INDENT}{prefix}{first_line}").into()); - } - for line in output_lines { - if line.is_empty() { - lines.push("".into()); - } else { - lines.push(format!("{HOOK_OUTPUT_BODY_INDENT}{line}").into()); - } + if !render_full_context && entry.kind == HookOutputEntryKind::Context { + lines.extend(hook_context_preview_lines(&entry.text, width)); + } else { + push_full_hook_output_entry(lines, entry); } } } @@ -465,6 +480,80 @@ impl HookRunCell { } } +fn push_full_hook_output_entry(lines: &mut Vec>, entry: &HookOutputEntry) { + let prefix = hook_output_prefix(entry.kind); + let mut output_lines = entry.text.split('\n'); + if let Some(first_line) = output_lines.next() { + lines.push(format!("{HOOK_OUTPUT_INDENT}{prefix}{first_line}").into()); + } + for line in output_lines { + if line.is_empty() { + lines.push("".into()); + } else { + lines.push(format!("{HOOK_OUTPUT_BODY_INDENT}{line}").into()); + } + } +} + +fn hook_context_preview_lines(text: &str, width: u16) -> Vec> { + let width = usize::from(width.max(1)); + let mut wrapped = Vec::new(); + let mut source_lines = text.split('\n'); + let first_line = source_lines.next().unwrap_or_default(); + push_wrapped_hook_context_line( + &mut wrapped, + first_line, + width, + Line::from(format!( + "{HOOK_OUTPUT_INDENT}{}", + hook_output_prefix(HookOutputEntryKind::Context) + )), + ); + for line in source_lines { + if line.is_empty() { + wrapped.push("".into()); + } else { + push_wrapped_hook_context_line( + &mut wrapped, + line, + width, + Line::from(HOOK_OUTPUT_BODY_INDENT), + ); + } + } + + if wrapped.len() <= HOOK_CONTEXT_MAX_DISPLAY_ROWS { + return wrapped; + } + + let retained_rows = HOOK_CONTEXT_MAX_DISPLAY_ROWS - 1; + let omitted_rows = wrapped.len() - retained_rows; + wrapped.truncate(retained_rows); + let hint = vec![ + HOOK_OUTPUT_BODY_INDENT.into(), + format!("… +{omitted_rows} lines ({TRANSCRIPT_HINT})").dim(), + ] + .into(); + wrapped.push(truncate_line_with_ellipsis_if_overflow(hint, width)); + wrapped +} + +fn push_wrapped_hook_context_line( + output: &mut Vec>, + text: &str, + width: usize, + initial_indent: Line<'static>, +) { + let line = Line::from(text.to_string()); + let wrapped = word_wrap_line( + &line, + RtOptions::new(width) + .initial_indent(initial_indent) + .subsequent_indent(Line::from(HOOK_OUTPUT_BODY_INDENT)), + ); + push_owned_lines(&wrapped, output); +} + impl HookRunState { /// Creates the hidden initial state for a live hook run. fn pending(start_time: Instant) -> Self { @@ -760,14 +849,13 @@ mod tests { } #[test] - fn completed_hook_multiline_context_preserves_display_and_raw_lines() { + fn completed_hook_short_multiline_context_preserves_display_transcript_and_raw_lines() { let cell = completed_hook_cell( HookEventName::SessionStart, HookRunStatus::Completed, vec![HookOutputEntry { kind: HookOutputEntryKind::Context, - text: "## Working Memory Recall\n\nSource: Codex compaction\nScope: Durable workspace memory" - .to_string(), + text: "## Working Memory Recall\n\nSource: Codex compaction".to_string(), }], ); let expected = vec![ @@ -775,13 +863,87 @@ mod tests { " hook context: ## Working Memory Recall".to_string(), "".to_string(), " Source: Codex compaction".to_string(), - " Scope: Durable workspace memory".to_string(), ]; assert_eq!(line_texts(&cell.display_lines(/*width*/ 80)), expected); + assert_eq!(line_texts(&cell.transcript_lines(/*width*/ 80)), expected); assert_eq!(line_texts(&cell.raw_lines()), expected); } + #[test] + fn completed_hook_long_single_line_context_is_truncated_only_in_display() { + let full_context = format!( + "{}tail-marker", + "context words that should wrap across the terminal width ".repeat(8) + ); + let cell = completed_hook_cell( + HookEventName::SessionStart, + HookRunStatus::Completed, + vec![HookOutputEntry { + kind: HookOutputEntryKind::Context, + text: full_context.clone(), + }], + ); + + let display_lines = cell.display_lines(/*width*/ 80); + let display = line_texts(&display_lines); + assert_eq!(display.len(), 4); + assert_eq!( + Paragraph::new(Text::from(display_lines[1..].to_vec())) + .wrap(Wrap { trim: false }) + .line_count(/*width*/ 80), + HOOK_CONTEXT_MAX_DISPLAY_ROWS + ); + assert!( + display + .iter() + .any(|line| line.contains("ctrl + t to view transcript")), + "expected truncated context to advertise the transcript: {display:?}" + ); + assert!(display.iter().all(|line| !line.contains("tail-marker"))); + + let expected_full = vec![ + "• SessionStart hook (completed)".to_string(), + format!(" hook context: {full_context}"), + ]; + assert_eq!( + line_texts(&cell.transcript_lines(/*width*/ 80)), + expected_full + ); + assert_eq!(line_texts(&cell.raw_lines()), expected_full); + } + + #[test] + fn completed_hook_non_context_entries_are_not_truncated() { + for kind in [ + HookOutputEntryKind::Warning, + HookOutputEntryKind::Stop, + HookOutputEntryKind::Feedback, + HookOutputEntryKind::Error, + ] { + let cell = completed_hook_cell( + HookEventName::UserPromptSubmit, + HookRunStatus::Stopped, + vec![HookOutputEntry { + kind, + text: "first\nsecond\nthird\nfourth\nfifth".to_string(), + }], + ); + + let display = line_texts(&cell.display_lines(/*width*/ 20)); + assert!( + display.iter().any(|line| line == " fifth"), + "expected {kind:?} output to remain complete: {display:?}" + ); + assert!( + display + .iter() + .all(|line| !line.contains("ctrl + t to view transcript")), + "did not expect a transcript hint for {kind:?}: {display:?}" + ); + } + } + #[test] fn completed_hook_multiline_warning_prefixes_first_line_only() { let cell = completed_hook_cell( diff --git a/codex-rs/tui/src/history_cell/notices.rs b/codex-rs/tui/src/history_cell/notices.rs index c4cc79cf70..cda68aa839 100644 --- a/codex-rs/tui/src/history_cell/notices.rs +++ b/codex-rs/tui/src/history_cell/notices.rs @@ -86,91 +86,76 @@ pub(crate) fn new_warning_event(message: String) -> PrefixedWrappedHistoryCell { } #[derive(Debug)] -pub(crate) struct SafetyAccessBlockCell; +pub(crate) struct SafetyAccessBlockCell { + body: &'static str, + trusted_access_url: &'static str, +} const SAFETY_ACCESS_BLOCK_TITLE: &str = "This content can't be shown"; -const SAFETY_ACCESS_BLOCK_BODY: &str = "We've limited access to this content for safety reasons. This type of information may be used to benefit or to harm people."; +const SAFETY_ACCESS_BLOCK_LEARN_MORE_URL: &str = "https://help.openai.com/en/articles/20001326"; pub(crate) fn new_safety_access_block_event() -> SafetyAccessBlockCell { - SafetyAccessBlockCell + SafetyAccessBlockCell { + body: "We take extra caution with requests involving biological research and applications that could pose safety risks. Eligible researchers can apply for Trusted Access.", + trusted_access_url: "https://www.openai.com/form/trusted-access-for-biology-research/", + } +} + +pub(crate) fn new_cyber_policy_error_event() -> SafetyAccessBlockCell { + SafetyAccessBlockCell { + body: "We take extra caution with cybersecurity requests. If you’re a security professional, you may be able to apply for Trusted Access.", + trusted_access_url: "https://openai.com/form/enterprise-trusted-access-for-cyber/", + } } impl HistoryCell for SafetyAccessBlockCell { fn display_lines(&self, width: u16) -> Vec> { - let mut lines = vec![vec!["ⓘ ".cyan(), SAFETY_ACCESS_BLOCK_TITLE.bold()].into()]; - let body = Line::from(vec![" ".into(), SAFETY_ACCESS_BLOCK_BODY.dim()]); - let wrap_width = width.saturating_sub(2).max(1) as usize; - let wrapped = adaptive_wrap_line( - &body, - RtOptions::new(wrap_width).subsequent_indent(" ".into()), - ); - push_owned_lines(&wrapped, &mut lines); - lines - } - - fn raw_lines(&self) -> Vec> { - vec![ - Line::from(SAFETY_ACCESS_BLOCK_TITLE), - Line::from(SAFETY_ACCESS_BLOCK_BODY), - ] - } -} - -const TRUSTED_ACCESS_FOR_CYBER_URL: &str = "https://chatgpt.com/cyber"; - -#[derive(Debug)] -pub(crate) struct CyberPolicyNoticeCell; - -pub(crate) fn new_cyber_policy_error_event() -> CyberPolicyNoticeCell { - CyberPolicyNoticeCell -} - -impl HistoryCell for CyberPolicyNoticeCell { - fn display_lines(&self, width: u16) -> Vec> { - let mut lines: Vec> = Vec::new(); - lines.push( - vec![ - "ⓘ ".cyan(), - "This chat was flagged for possible cybersecurity risk".bold(), - ] - .into(), - ); - - let wrap_width = width.saturating_sub(2).max(1) as usize; - let body = Line::from(vec![ - " If this seems wrong, try rephrasing your request. To get authorized for security work, join the " - .dim(), - "Trusted Access for Cyber".cyan().underlined(), - " program.".dim(), - ]); - let wrapped = adaptive_wrap_line( - &body, - RtOptions::new(wrap_width).subsequent_indent(" ".into()), - ); - push_owned_lines(&wrapped, &mut lines); - lines.push( - vec![ - " ".into(), - TRUSTED_ACCESS_FOR_CYBER_URL.cyan().underlined(), - ] - .into(), - ); - - lines - } - - fn raw_lines(&self) -> Vec> { - vec![ - Line::from("This chat was flagged for possible cybersecurity risk"), - Line::from( - "If this seems wrong, try rephrasing your request. To get authorized for security work, join the Trusted Access for Cyber program.", - ), - Line::from(TRUSTED_ACCESS_FOR_CYBER_URL), - ] + visible_lines(self.display_hyperlink_lines(width)) } fn display_hyperlink_lines(&self, width: u16) -> Vec { - crate::terminal_hyperlinks::annotate_web_urls(self.display_lines(width)) + let mut lines = vec![HyperlinkLine::new( + vec!["ⓘ ".cyan(), SAFETY_ACCESS_BLOCK_TITLE.bold()].into(), + )]; + let body = Line::from(vec![" ".into(), self.body.dim()]); + let wrap_width = width.saturating_sub(2).max(1) as usize; + let wrapped = adaptive_wrap_line( + &body, + RtOptions::new(wrap_width).subsequent_indent(" ".into()), + ); + let mut wrapped_body = Vec::new(); + push_owned_lines(&wrapped, &mut wrapped_body); + lines.extend(plain_hyperlink_lines(wrapped_body)); + + for (label, url) in [ + ("Trusted Access", self.trusted_access_url), + ("Learn more", SAFETY_ACCESS_BLOCK_LEARN_MORE_URL), + ] { + let source = crate::terminal_hyperlinks::annotate_web_urls_in_line( + vec![format!(" {label}: ").dim(), url.cyan().underlined()].into(), + ); + let wrapped = crate::wrapping::word_wrap_line( + &source.line, + RtOptions::new(wrap_width).subsequent_indent(" ".into()), + ); + let mut wrapped_links = Vec::new(); + push_owned_lines(&wrapped, &mut wrapped_links); + lines.extend(crate::terminal_hyperlinks::remap_wrapped_line( + &source, + wrapped_links, + )); + } + lines + } + + fn raw_lines(&self) -> Vec> { + let trusted_access_url = self.trusted_access_url; + vec![ + Line::from(SAFETY_ACCESS_BLOCK_TITLE), + Line::from(self.body), + Line::from(format!("Trusted Access: {trusted_access_url}")), + Line::from(format!("Learn more: {SAFETY_ACCESS_BLOCK_LEARN_MORE_URL}")), + ] } fn transcript_hyperlink_lines(&self, width: u16) -> Vec { diff --git a/codex-rs/tui/src/history_cell/snapshots/codex_tui__history_cell__tests__cyber_policy_error_event_narrow_snapshot.snap b/codex-rs/tui/src/history_cell/snapshots/codex_tui__history_cell__tests__cyber_policy_error_event_narrow_snapshot.snap index 9039b196ee..17d6bf7ce6 100644 --- a/codex-rs/tui/src/history_cell/snapshots/codex_tui__history_cell__tests__cyber_policy_error_event_narrow_snapshot.snap +++ b/codex-rs/tui/src/history_cell/snapshots/codex_tui__history_cell__tests__cyber_policy_error_event_narrow_snapshot.snap @@ -1,12 +1,16 @@ --- -source: tui/src/history_cell.rs -assertion_line: 3312 +source: tui/src/history_cell/tests.rs expression: rendered --- -ⓘ This chat was flagged for possible cybersecurity risk - If this seems wrong, try - rephrasing your request. To get - authorized for security work, - join the Trusted Access for - Cyber program. - https://chatgpt.com/cyber +ⓘ This content can't be shown + We take extra caution with + cybersecurity requests. If + you’re a security professional, + you may be able to apply for + Trusted Access. + Trusted Access: https:// + openai.com/form/enterprise- + trusted-access-for-cyber/ + Learn more: https:// + help.openai.com/en/ + articles/20001326 diff --git a/codex-rs/tui/src/history_cell/snapshots/codex_tui__history_cell__tests__cyber_policy_error_event_snapshot.snap b/codex-rs/tui/src/history_cell/snapshots/codex_tui__history_cell__tests__cyber_policy_error_event_snapshot.snap index 888a10c37e..ca800e50de 100644 --- a/codex-rs/tui/src/history_cell/snapshots/codex_tui__history_cell__tests__cyber_policy_error_event_snapshot.snap +++ b/codex-rs/tui/src/history_cell/snapshots/codex_tui__history_cell__tests__cyber_policy_error_event_snapshot.snap @@ -3,7 +3,8 @@ source: tui/src/history_cell.rs assertion_line: 3305 expression: rendered --- -ⓘ This chat was flagged for possible cybersecurity risk - If this seems wrong, try rephrasing your request. To get authorized for - security work, join the Trusted Access for Cyber program. - https://chatgpt.com/cyber +ⓘ This content can't be shown + We take extra caution with cybersecurity requests. If you’re a security + professional, you may be able to apply for Trusted Access. + Trusted Access: https://openai.com/form/enterprise-trusted-access-for-cyber/ + Learn more: https://help.openai.com/en/articles/20001326 diff --git a/codex-rs/tui/src/history_cell/snapshots/codex_tui__history_cell__tests__safety_access_block_event_snapshot.snap b/codex-rs/tui/src/history_cell/snapshots/codex_tui__history_cell__tests__safety_access_block_event_snapshot.snap index f512697888..29e55178c4 100644 --- a/codex-rs/tui/src/history_cell/snapshots/codex_tui__history_cell__tests__safety_access_block_event_snapshot.snap +++ b/codex-rs/tui/src/history_cell/snapshots/codex_tui__history_cell__tests__safety_access_block_event_snapshot.snap @@ -3,5 +3,9 @@ source: tui/src/history_cell/tests.rs expression: rendered --- ⓘ This content can't be shown - We've limited access to this content for safety reasons. This type of - information may be used to benefit or to harm people. + We take extra caution with requests involving biological research and + applications that could pose safety risks. Eligible researchers can apply + for Trusted Access. + Trusted Access: https://www.openai.com/form/trusted-access-for-biology- + research/ + Learn more: https://help.openai.com/en/articles/20001326 diff --git a/codex-rs/tui/src/lib.rs b/codex-rs/tui/src/lib.rs index 897f1ec83f..dc8262f7db 100644 --- a/codex-rs/tui/src/lib.rs +++ b/codex-rs/tui/src/lib.rs @@ -3,7 +3,6 @@ // alternate‑screen mode starts; that file opts‑out locally via `allow`. #![deny(clippy::print_stdout, clippy::print_stderr)] #![deny(clippy::disallowed_methods)] -use crate::legacy_core::check_execpolicy_for_warnings; use crate::legacy_core::config::Config; use crate::legacy_core::config::ConfigBuilder; use crate::legacy_core::config::ConfigOverrides; @@ -13,7 +12,6 @@ use crate::legacy_core::config::resolve_bootstrap_auth_keyring_backend_kind; use crate::legacy_core::config::resolve_bootstrap_auth_route_config; use crate::legacy_core::config::resolve_oss_provider; use crate::legacy_core::config::resolve_profile_v2_config_path; -use crate::legacy_core::format_exec_policy_error_with_source; use crate::session_resume::ResolveCwdOutcome; use crate::session_resume::resolve_cwd_for_resume_or_fork; pub use crate::startup_error::LocalStateDbStartupError; @@ -137,6 +135,7 @@ mod line_truncation; pub(crate) mod live_wrap; pub use live_wrap::RowBuilder; mod local_chatgpt_auth; +mod managed_new_thread_defaults; mod markdown; mod markdown_render; mod markdown_stream; @@ -1140,18 +1139,6 @@ pub async fn run_main( .as_table() .is_some_and(|table| table.contains_key("log_dir")); - #[allow(clippy::print_stderr)] - match check_execpolicy_for_warnings(&config.config_layer_stack).await { - Ok(None) => {} - Ok(Some(err)) | Err(err) => { - eprintln!( - "Error loading rules:\n{}", - format_exec_policy_error_with_source(&err) - ); - std::process::exit(1); - } - } - set_default_client_residency_requirement(config.enforce_residency.value()); if let Some(warning) = add_dir_warning_message( diff --git a/codex-rs/tui/src/managed_new_thread_defaults.rs b/codex-rs/tui/src/managed_new_thread_defaults.rs new file mode 100644 index 0000000000..ad9ad30b8e --- /dev/null +++ b/codex-rs/tui/src/managed_new_thread_defaults.rs @@ -0,0 +1,51 @@ +use crate::legacy_core::config::Config; +use crate::legacy_core::config::ConfigOverrides; +use codex_app_server_protocol::NewThreadModelDefaults; +use codex_protocol::config_types::ServiceTier; +use toml::Value as TomlValue; + +pub(crate) fn apply_managed_new_thread_defaults( + config: &mut Config, + defaults: Option<&NewThreadModelDefaults>, + cli_kv_overrides: &[(String, TomlValue)], + harness_overrides: &ConfigOverrides, +) { + let Some(defaults) = defaults else { + return; + }; + // Managed values are defaults rather than enforcement. Preserve explicit launch choices from + // dedicated flags such as `-m` (`harness_overrides`) and generic `-c key=value` settings + // (`cli_kv_overrides`), then fill only the fields that were not selected for this invocation. + // Model and reasoning effort are a compatibility-sensitive pair, so an explicit override of + // either opts out of both managed values. For example, `codex -m gpt-5.4` keeps that model and + // its existing/default effort, while `-c model_reasoning_effort=low` does not switch to the + // managed model. Service tier remains independent and is resolved against the selected model + // before the thread starts. + let has_cli_override = |key: &str| cli_kv_overrides.iter().any(|(path, _value)| path == key); + let has_explicit_model_settings = harness_overrides.model.is_some() + || has_cli_override("model") + || has_cli_override("model_reasoning_effort"); + + if !has_explicit_model_settings && let Some(model) = defaults.model.as_ref() { + config.model = Some(model.clone()); + } + if !has_explicit_model_settings + && let Some(reasoning_effort) = defaults.model_reasoning_effort.as_ref() + { + config.model_reasoning_effort = Some(reasoning_effort.clone()); + } + if harness_overrides.service_tier.is_none() + && !has_cli_override("service_tier") + && let Some(service_tier) = defaults.service_tier.as_ref() + { + config.service_tier = Some( + ServiceTier::from_request_value(service_tier) + .map(|tier| tier.request_value().to_string()) + .unwrap_or_else(|| service_tier.clone()), + ); + } +} + +#[cfg(test)] +#[path = "managed_new_thread_defaults_tests.rs"] +mod tests; diff --git a/codex-rs/tui/src/managed_new_thread_defaults_tests.rs b/codex-rs/tui/src/managed_new_thread_defaults_tests.rs new file mode 100644 index 0000000000..d73d6459d6 --- /dev/null +++ b/codex-rs/tui/src/managed_new_thread_defaults_tests.rs @@ -0,0 +1,110 @@ +use super::*; +use crate::legacy_core::config::ConfigBuilder; +use codex_protocol::openai_models::ReasoningEffort; +use pretty_assertions::assert_eq; + +async fn test_config() -> Config { + let codex_home = tempfile::tempdir().expect("tempdir").keep(); + ConfigBuilder::default() + .codex_home(codex_home) + .build() + .await + .expect("config") +} + +fn defaults() -> NewThreadModelDefaults { + NewThreadModelDefaults { + model: Some("managed-model".to_string()), + model_reasoning_effort: Some(ReasoningEffort::High), + service_tier: Some("fast".to_string()), + } +} + +#[tokio::test] +async fn applies_managed_defaults_to_a_new_thread_config() { + let mut actual = test_config().await; + actual.model = Some("configured-model".to_string()); + actual.model_reasoning_effort = Some(ReasoningEffort::Low); + actual.service_tier = Some("flex".to_string()); + let mut expected = actual.clone(); + expected.model = Some("managed-model".to_string()); + expected.model_reasoning_effort = Some(ReasoningEffort::High); + expected.service_tier = Some(ServiceTier::Fast.request_value().to_string()); + + apply_managed_new_thread_defaults( + &mut actual, + Some(&defaults()), + &[], + &ConfigOverrides::default(), + ); + + assert_eq!(actual, expected); +} + +#[tokio::test] +async fn explicit_model_skips_managed_model_and_reasoning_effort() { + let mut actual = test_config().await; + actual.model = Some("explicit-model".to_string()); + actual.model_reasoning_effort = None; + actual.service_tier = Some("flex".to_string()); + let mut expected = actual.clone(); + expected.service_tier = Some(ServiceTier::Fast.request_value().to_string()); + let harness_overrides = ConfigOverrides { + model: Some("explicit-model".to_string()), + ..ConfigOverrides::default() + }; + + apply_managed_new_thread_defaults(&mut actual, Some(&defaults()), &[], &harness_overrides); + + assert_eq!(actual, expected); +} + +#[tokio::test] +async fn explicit_reasoning_effort_skips_managed_model_and_reasoning_effort() { + let mut actual = test_config().await; + actual.model = Some("configured-model".to_string()); + actual.model_reasoning_effort = Some(ReasoningEffort::Low); + actual.service_tier = Some("flex".to_string()); + let mut expected = actual.clone(); + expected.service_tier = Some(ServiceTier::Fast.request_value().to_string()); + let cli_kv_overrides = vec![( + "model_reasoning_effort".to_string(), + TomlValue::String("low".to_string()), + )]; + + apply_managed_new_thread_defaults( + &mut actual, + Some(&defaults()), + &cli_kv_overrides, + &ConfigOverrides::default(), + ); + + assert_eq!(actual, expected); +} + +#[tokio::test] +async fn explicit_launch_overrides_take_precedence() { + let mut actual = test_config().await; + actual.model = Some("explicit-model".to_string()); + actual.model_reasoning_effort = Some(ReasoningEffort::Low); + actual.service_tier = Some("flex".to_string()); + let expected = actual.clone(); + let cli_kv_overrides = vec![( + "model_reasoning_effort".to_string(), + TomlValue::String("low".to_string()), + )]; + let harness_overrides = ConfigOverrides { + model: Some("explicit-model".to_string()), + service_tier: Some(Some("flex".to_string())), + ..ConfigOverrides::default() + }; + + apply_managed_new_thread_defaults( + &mut actual, + Some(&defaults()), + &cli_kv_overrides, + &harness_overrides, + ); + + assert_eq!(actual, expected); +} diff --git a/codex-rs/tui/src/resume_picker.rs b/codex-rs/tui/src/resume_picker.rs index d74017d8d0..e0e3907246 100644 --- a/codex-rs/tui/src/resume_picker.rs +++ b/codex-rs/tui/src/resume_picker.rs @@ -5727,6 +5727,7 @@ session_picker_view = "dense" parent_thread_id: None, preview: String::from("remote thread"), ephemeral: false, + history_mode: Default::default(), model_provider: String::from("openai"), created_at: 1, updated_at: 2, @@ -5764,6 +5765,7 @@ session_picker_view = "dense" parent_thread_id: None, preview: String::from("preview"), ephemeral: false, + history_mode: Default::default(), model_provider: String::from("openai"), created_at: 1, updated_at: 2, @@ -5835,6 +5837,7 @@ session_picker_view = "dense" parent_thread_id: None, preview: String::from("preview"), ephemeral: false, + history_mode: Default::default(), model_provider: String::from("openai"), created_at: 1, updated_at: 2, @@ -5895,6 +5898,7 @@ session_picker_view = "dense" parent_thread_id: None, preview: String::from("preview"), ephemeral: false, + history_mode: Default::default(), model_provider: String::from("openai"), created_at: 1, updated_at: 2, diff --git a/codex-rs/tui/src/ui_consts.rs b/codex-rs/tui/src/ui_consts.rs index ebbe4ce366..f72dc64263 100644 --- a/codex-rs/tui/src/ui_consts.rs +++ b/codex-rs/tui/src/ui_consts.rs @@ -9,3 +9,4 @@ /// - User history lines account for this many columns (e.g., "▌ ") when wrapping. pub(crate) const LIVE_PREFIX_COLS: u16 = 2; pub(crate) const FOOTER_INDENT_COLS: usize = LIVE_PREFIX_COLS as usize; +pub(crate) const TRANSCRIPT_HINT: &str = "ctrl + t to view transcript"; diff --git a/justfile b/justfile index 03f87391ac..d4a0e6566c 100644 --- a/justfile +++ b/justfile @@ -31,6 +31,10 @@ tui-with-exec-server *args: file-search *args: cargo run --bin codex-file-search -- {args} +# Run the standalone code-mode host from source. +code-mode-host *args: + cargo run --bin codex-code-mode-host -- {args} + # Build the CLI and run the app-server test client app-server-test-client *args: cargo build -p codex-cli @@ -107,6 +111,16 @@ bazel-codex *args: bazel-codex *args: bazel run //codex-rs/cli:codex --run_under='cd /d "{{ invocation_directory_native() }}" &&' -- @($args | Select-Object -Skip 1) +# Build and run the standalone code-mode host from source using Bazel. +[no-cd] +[unix] +bazel-code-mode-host *args: + bazel run //codex-rs/code-mode-host:codex-code-mode-host --run_under="cd $PWD &&" -- "$@" + +[windows] +bazel-code-mode-host *args: + bazel run //codex-rs/code-mode-host:codex-code-mode-host --run_under='cd /d "{{ invocation_directory_native() }}" &&' -- @($args | Select-Object -Skip 1) + [no-cd] bazel-lock-update: bazel mod deps --lockfile_mode=update diff --git a/scripts/codex_package/README.md b/scripts/codex_package/README.md index 323a3ce5ba..b76c18c3c2 100644 --- a/scripts/codex_package/README.md +++ b/scripts/codex_package/README.md @@ -73,4 +73,6 @@ The patched zsh fork used by `shell_zsh_fork` is fetched from the DotSlash manifest at `scripts/codex_package/codex-zsh` when the selected target has a matching prebuilt artifact. Downloaded archives are cached under `$TMPDIR/codex-package/-zsh` and installed at -`codex-resources/zsh/bin/zsh`. +`codex-resources/zsh/bin/zsh`. Pass `--zsh-manifest` to use a different +DotSlash manifest, such as the manifest published with a standalone zsh +artifact release. diff --git a/scripts/codex_package/cli.py b/scripts/codex_package/cli.py index b7d919e4ff..1dd276827e 100644 --- a/scripts/codex_package/cli.py +++ b/scripts/codex_package/cli.py @@ -90,6 +90,14 @@ def parse_args() -> argparse.Namespace: "targets, bwrap is built with Cargo." ), ) + parser.add_argument( + "--zsh-manifest", + type=Path, + help=( + "Optional DotSlash manifest for the patched zsh fork instead of " + "scripts/codex_package/codex-zsh." + ), + ) parser.add_argument( "--codex-command-runner-bin", type=Path, @@ -160,7 +168,7 @@ def main() -> int: inputs = PackageInputs( entrypoint_bin=source_outputs.entrypoint_bin, rg_bin=resolve_rg_bin(spec, args.rg_bin), - zsh_bin=resolve_zsh_bin(spec), + zsh_bin=resolve_zsh_bin(spec, args.zsh_manifest), bwrap_bin=source_outputs.bwrap_bin, codex_command_runner_bin=source_outputs.codex_command_runner_bin, codex_windows_sandbox_setup_bin=source_outputs.codex_windows_sandbox_setup_bin, diff --git a/scripts/codex_package/test_zsh.py b/scripts/codex_package/test_zsh.py new file mode 100644 index 0000000000..b4f536cfc2 --- /dev/null +++ b/scripts/codex_package/test_zsh.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 + +import hashlib +import json +from pathlib import Path +import sys +import tarfile +import tempfile +import unittest +from unittest.mock import patch + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from codex_package.targets import TARGET_SPECS +from codex_package.zsh import resolve_zsh_bin + + +class ResolveZshBinTest(unittest.TestCase): + def test_uses_manifest_override(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + archive = root / "codex-zsh.tar.gz" + source = root / "zsh" + source.write_bytes(b"standalone zsh") + with tarfile.open(archive, "w:gz") as tar: + tar.add(source, arcname="codex-zsh/bin/zsh") + + manifest = root / "codex-zsh" + manifest.write_text( + json.dumps( + { + "platforms": { + "linux-x86_64": { + "size": archive.stat().st_size, + "hash": "sha256", + "digest": hashlib.sha256( + archive.read_bytes() + ).hexdigest(), + "format": "tar.gz", + "path": "codex-zsh/bin/zsh", + "providers": [{"url": archive.as_uri()}], + } + } + } + ), + encoding="utf-8", + ) + + with patch( + "codex_package.dotslash.default_cache_root", + return_value=root / "cache", + ): + zsh_bin = resolve_zsh_bin( + TARGET_SPECS["x86_64-unknown-linux-musl"], manifest + ) + + self.assertIsNotNone(zsh_bin) + assert zsh_bin is not None + self.assertEqual(zsh_bin.read_bytes(), b"standalone zsh") + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/codex_package/zsh.py b/scripts/codex_package/zsh.py index f4ee7e19dd..4ec2db502c 100644 --- a/scripts/codex_package/zsh.py +++ b/scripts/codex_package/zsh.py @@ -11,10 +11,13 @@ ZSH_MANIFEST = REPO_ROOT / "scripts" / "codex_package" / "codex-zsh" ZSH_RESOURCE_PATH = Path("zsh") / "bin" / "zsh" -def resolve_zsh_bin(spec: TargetSpec) -> Path | None: +def resolve_zsh_bin( + spec: TargetSpec, + manifest_path: Path | None = None, +) -> Path | None: return fetch_dotslash_executable( spec, - manifest_path=ZSH_MANIFEST, + manifest_path=manifest_path or ZSH_MANIFEST, artifact_label="codex-zsh", cache_key=f"{spec.target}-zsh", dest_name="zsh", diff --git a/scripts/install/install.ps1 b/scripts/install/install.ps1 index 6973d482e2..986f6c6566 100644 --- a/scripts/install/install.ps1 +++ b/scripts/install/install.ps1 @@ -79,11 +79,10 @@ function Assert-ValidReleaseVersion { function Find-ReleaseAssetMetadata { param( [string]$AssetName, - [string]$ResolvedVersion + [object]$ReleaseMetadata ) - $release = Invoke-RestMethod -Uri "https://api.github.com/repos/openai/codex/releases/tags/rust-v$ResolvedVersion" - $asset = $release.assets | Where-Object { $_.name -eq $AssetName } | Select-Object -First 1 + $asset = $ReleaseMetadata.assets | Where-Object { $_.name -eq $AssetName } | Select-Object -First 1 if ($null -eq $asset) { return $null } @@ -99,20 +98,6 @@ function Find-ReleaseAssetMetadata { } } -function Get-ReleaseAssetMetadata { - param( - [string]$AssetName, - [string]$ResolvedVersion - ) - - $metadata = Find-ReleaseAssetMetadata -AssetName $AssetName -ResolvedVersion $ResolvedVersion - if ($null -eq $metadata) { - throw "Could not find release asset $AssetName for Codex $ResolvedVersion." - } - - return $metadata -} - function Test-ArchiveDigest { param( [string]$ArchivePath, @@ -216,22 +201,38 @@ function Remove-StaleInstallArtifacts { } } -function Resolve-Version { +function Resolve-Release { $normalizedVersion = Normalize-Version -RawVersion $Release Assert-ValidReleaseVersion -Version $normalizedVersion - if ($normalizedVersion -ne "latest") { - return $normalizedVersion + + if ($normalizedVersion -eq "latest") { + $requestedRelease = "latest" + $metadataUri = "https://api.github.com/repos/openai/codex/releases/latest" + } else { + $resolvedVersion = $normalizedVersion + $requestedRelease = $resolvedVersion + $metadataUri = "https://api.github.com/repos/openai/codex/releases/tags/rust-v$resolvedVersion" } - $release = Invoke-RestMethod -Uri "https://api.github.com/repos/openai/codex/releases/latest" - if (-not $release.tag_name) { - Write-Error "Failed to resolve the latest Codex release version." - exit 1 + try { + $releaseMetadata = Invoke-RestMethod -Uri $metadataUri + } catch { + throw "Could not fetch GitHub release metadata for Codex $requestedRelease. GitHub API may be unavailable or rate limited. $($_.Exception.Message)" } - $resolvedVersion = Normalize-Version -RawVersion $release.tag_name - Assert-ValidReleaseVersion -Version $resolvedVersion - return $resolvedVersion + if ($normalizedVersion -eq "latest") { + if (-not $releaseMetadata.tag_name) { + throw "Failed to resolve the latest Codex release version." + } + + $resolvedVersion = Normalize-Version -RawVersion $releaseMetadata.tag_name + Assert-ValidReleaseVersion -Version $resolvedVersion + } + + return [PSCustomObject]@{ + Version = $resolvedVersion + Metadata = $releaseMetadata + } } function Get-VersionFromBinary { @@ -746,7 +747,9 @@ if ([string]::IsNullOrWhiteSpace($env:CODEX_INSTALL_DIR)) { } $currentVersion = Get-CurrentInstalledVersion -StandaloneCurrentDir $currentDir -$resolvedVersion = Resolve-Version +$resolvedRelease = Resolve-Release +$resolvedVersion = $resolvedRelease.Version +$releaseMetadata = $resolvedRelease.Metadata $releaseName = "$resolvedVersion-$target" $releaseDir = Join-Path $releasesDir $releaseName @@ -765,12 +768,12 @@ $oldStandaloneBackup = $null $packageAsset = "codex-package-$target.tar.gz" $checksumAsset = "codex-package_SHA256SUMS" -$packageMetadata = Find-ReleaseAssetMetadata -AssetName $packageAsset -ResolvedVersion $resolvedVersion -$checksumMetadata = Find-ReleaseAssetMetadata -AssetName $checksumAsset -ResolvedVersion $resolvedVersion +$packageMetadata = Find-ReleaseAssetMetadata -AssetName $packageAsset -ReleaseMetadata $releaseMetadata +$checksumMetadata = Find-ReleaseAssetMetadata -AssetName $checksumAsset -ReleaseMetadata $releaseMetadata $installLayout = "Package" if ($null -eq $packageMetadata -or $null -eq $checksumMetadata) { $packageAsset = "codex-npm-$npmTag-$resolvedVersion.tgz" - $packageMetadata = Find-ReleaseAssetMetadata -AssetName $packageAsset -ResolvedVersion $resolvedVersion + $packageMetadata = Find-ReleaseAssetMetadata -AssetName $packageAsset -ReleaseMetadata $releaseMetadata if ($null -ne $packageMetadata) { $installLayout = "LegacyPlatformNpm" } else { diff --git a/scripts/install/install.sh b/scripts/install/install.sh index 7513efc429..d44b353860 100755 --- a/scripts/install/install.sh +++ b/scripts/install/install.sh @@ -138,10 +138,36 @@ release_metadata_url() { printf 'https://api.github.com/repos/openai/codex/releases/tags/rust-v%s\n' "$resolved_version" } +resolve_release() { + normalized_version="$(normalize_version "$RELEASE")" + validate_version "$normalized_version" + + if [ "$normalized_version" = "latest" ]; then + requested_release="latest" + metadata_url="https://api.github.com/repos/openai/codex/releases/latest" + else + resolved_version="$normalized_version" + requested_release="$resolved_version" + metadata_url="$(release_metadata_url "$resolved_version")" + fi + + if ! release_json="$(download_text "$metadata_url")"; then + echo "Could not fetch GitHub release metadata for Codex $requested_release. GitHub API may be unavailable or rate limited." >&2 + exit 1 + fi + + if [ "$normalized_version" = "latest" ]; then + resolved_version="$(printf '%s\n' "$release_json" | sed -n 's/.*"tag_name":[[:space:]]*"rust-v\([^"]*\)".*/\1/p' | head -n 1)" + if [ -z "$resolved_version" ]; then + echo "Failed to resolve the latest Codex release version." >&2 + exit 1 + fi + validate_version "$resolved_version" + fi +} + release_asset_digest_or_empty() { asset="$1" - resolved_version="$2" - release_json="$(download_text "$(release_metadata_url "$resolved_version")")" digest="$(printf '%s\n' "$release_json" | awk -v asset="$asset" ' /"name":[[:space:]]*"[^"]+"/ { @@ -190,16 +216,14 @@ release_asset_digest_or_empty() { release_asset_exists() { asset="$1" - resolved_version="$2" - release_asset_digest_or_empty "$asset" "$resolved_version" >/dev/null 2>&1 + release_asset_digest_or_empty "$asset" >/dev/null 2>&1 } release_asset_digest() { asset="$1" - resolved_version="$2" - digest="$(release_asset_digest_or_empty "$asset" "$resolved_version" || true)" + digest="$(release_asset_digest_or_empty "$asset" || true)" if [ -z "$digest" ]; then echo "Could not find SHA-256 digest for release asset $asset." >&2 exit 1 @@ -275,27 +299,6 @@ require_command() { fi } -resolve_version() { - normalized_version="$(normalize_version "$RELEASE")" - validate_version "$normalized_version" - - if [ "$normalized_version" != "latest" ]; then - printf '%s\n' "$normalized_version" - return - fi - - release_json="$(download_text "https://api.github.com/repos/openai/codex/releases/latest")" - resolved="$(printf '%s\n' "$release_json" | sed -n 's/.*"tag_name":[[:space:]]*"rust-v\([^"]*\)".*/\1/p' | head -n 1)" - - if [ -z "$resolved" ]; then - echo "Failed to resolve the latest Codex release version." >&2 - exit 1 - fi - - validate_version "$resolved" - printf '%s\n' "$resolved" -} - pick_profile() { # Use the same shell-specific split Homebrew documents because there is no # universal startup file across macOS/Linux login and interactive shells. @@ -839,14 +842,14 @@ else fi fi -resolved_version="$(resolve_version)" +resolve_release package_asset="codex-package-$vendor_target.tar.gz" checksum_asset="codex-package_SHA256SUMS" -if release_asset_exists "$package_asset" "$resolved_version" && - release_asset_exists "$checksum_asset" "$resolved_version"; then +if release_asset_exists "$package_asset" && + release_asset_exists "$checksum_asset"; then install_layout="package" asset="$package_asset" -elif release_asset_exists "codex-npm-$npm_tag-$resolved_version.tgz" "$resolved_version"; then +elif release_asset_exists "codex-npm-$npm_tag-$resolved_version.tgz"; then install_layout="legacy-platform-npm" asset="codex-npm-$npm_tag-$resolved_version.tgz" else @@ -893,12 +896,12 @@ if ! release_dir_is_complete "$release_dir" "$resolved_version" "$vendor_target" step "Downloading Codex CLI" if [ "$install_layout" = "package" ]; then - checksum_digest="$(release_asset_digest "$checksum_asset" "$resolved_version")" + checksum_digest="$(release_asset_digest "$checksum_asset")" download_file "$checksum_url" "$checksum_path" verify_archive_digest "$checksum_path" "$checksum_digest" expected_digest="$(package_archive_digest "$asset" "$checksum_path")" else - expected_digest="$(release_asset_digest "$asset" "$resolved_version")" + expected_digest="$(release_asset_digest "$asset")" fi download_file "$download_url" "$archive_path" verify_archive_digest "$archive_path" "$expected_digest" diff --git a/scripts/install/test_install_sh.py b/scripts/install/test_install_sh.py new file mode 100644 index 0000000000..556c6213bb --- /dev/null +++ b/scripts/install/test_install_sh.py @@ -0,0 +1,159 @@ +#!/usr/bin/env python3 + +import json +import os +from pathlib import Path +import subprocess +import tempfile +import textwrap +import unittest + + +INSTALL_SCRIPT = Path(__file__).with_name("install.sh") +VERSION = "0.142.5" + + +class InstallShTest(unittest.TestCase): + def test_metadata_fetch_failure_is_not_reported_as_missing_assets(self) -> None: + result, requests = run_installer(VERSION, metadata_failure=True) + + self.assertNotEqual(result.returncode, 0) + self.assertEqual( + requests, + [ + "https://api.github.com/repos/openai/codex/releases/tags/" + f"rust-v{VERSION}" + ], + ) + self.assertIn( + f"Could not fetch GitHub release metadata for Codex {VERSION}", + result.stderr, + ) + self.assertNotIn("Could not find Codex package", result.stderr) + + def test_exact_release_fetches_metadata_once(self) -> None: + result, requests = run_installer(VERSION) + + self.assertNotEqual(result.returncode, 0) + self.assertEqual( + requests, + [ + "https://api.github.com/repos/openai/codex/releases/tags/" + f"rust-v{VERSION}", + "https://github.com/openai/codex/releases/download/" + f"rust-v{VERSION}/codex-package_SHA256SUMS", + ], + ) + self.assertIn(f"Resolved version: {VERSION}", result.stdout) + + def test_latest_release_reuses_version_metadata(self) -> None: + result, requests = run_installer("latest") + + self.assertNotEqual(result.returncode, 0) + self.assertEqual( + requests, + [ + "https://api.github.com/repos/openai/codex/releases/latest", + "https://github.com/openai/codex/releases/download/" + f"rust-v{VERSION}/codex-package_SHA256SUMS", + ], + ) + self.assertIn(f"Resolved version: {VERSION}", result.stdout) + + +def run_installer( + release: str, *, metadata_failure: bool = False +) -> tuple[subprocess.CompletedProcess[str], list[str]]: + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + bin_dir = root / "bin" + bin_dir.mkdir() + request_log = root / "requests.log" + fake_curl = bin_dir / "curl" + fake_curl.write_text( + textwrap.dedent( + """\ + #!/bin/sh + url="" + for arg in "$@"; do + case "$arg" in + https://*) url="$arg" ;; + esac + done + printf '%s\n' "$url" >>"$CODEX_TEST_REQUEST_LOG" + + case "$url" in + https://api.github.com/*) + if [ "$CODEX_TEST_METADATA_FAILURE" = "1" ]; then + echo "curl: (22) The requested URL returned error: 403" >&2 + exit 22 + fi + printf '%s\n' "$CODEX_TEST_METADATA_JSON" + ;; + *) + exit 22 + ;; + esac + """ + ), + encoding="utf-8", + ) + fake_curl.chmod(0o755) + + env = os.environ.copy() + env.update( + { + "CODEX_HOME": str(root / "codex-home"), + "CODEX_INSTALL_DIR": str(root / "install-bin"), + "CODEX_NON_INTERACTIVE": "1", + "CODEX_RELEASE": release, + "CODEX_TEST_METADATA_FAILURE": "1" if metadata_failure else "0", + "CODEX_TEST_METADATA_JSON": release_metadata(), + "CODEX_TEST_REQUEST_LOG": str(request_log), + "HOME": str(root / "home"), + "PATH": f"{bin_dir}:/usr/bin:/bin", + "SHELL": "/bin/sh", + } + ) + result = subprocess.run( + ["/bin/sh", str(INSTALL_SCRIPT)], + capture_output=True, + check=False, + env=env, + text=True, + ) + requests = ( + request_log.read_text(encoding="utf-8").splitlines() + if request_log.exists() + else [] + ) + return result, requests + + +def release_metadata() -> str: + assets = [ + { + "name": f"codex-package-{target}.tar.gz", + "digest": f"sha256:{'a' * 64}", + } + for target in ( + "aarch64-apple-darwin", + "x86_64-apple-darwin", + "aarch64-unknown-linux-musl", + "x86_64-unknown-linux-musl", + ) + ] + assets.append( + { + "name": "codex-package_SHA256SUMS", + "digest": f"sha256:{'b' * 64}", + } + ) + return json.dumps( + {"tag_name": f"rust-v{VERSION}", "assets": assets}, + indent=2, + ) + + +if __name__ == "__main__": + unittest.main()