From d38c809843f737bd1719a2f84eb6ae887af31029 Mon Sep 17 00:00:00 2001 From: jif-oai Date: Tue, 4 Aug 2026 13:15:28 +0100 Subject: [PATCH] Backport alpha.7 source to the alpha.6.1 hotfix line --- .github/scripts/publish_r2_release.py | 239 +- .github/workflows/r2-release.yml | 7 +- .github/workflows/rust-release.yml | 20 +- codex-rs/Cargo.lock | 25 +- codex-rs/Cargo.toml | 2 + codex-rs/code-mode-host/Cargo.toml | 1 + codex-rs/code-mode-host/src/host_tests.rs | 113 +- codex-rs/code-mode-host/src/lib.rs | 178 +- codex-rs/code-mode-host/src/peer.rs | 19 +- codex-rs/code-mode-host/src/transport.rs | 101 + .../code-mode-host/src/transport_tests.rs | 25 + codex-rs/code-mode-host/tests/websocket.rs | 497 ++ .../code-mode-protocol/src/host/host_tests.rs | 125 + .../code-mode-protocol/src/host/message.rs | 64 + codex-rs/code-mode-protocol/src/host/mod.rs | 15 +- .../src/remote_session/connection.rs | 341 +- .../src/remote_session/connection/driver.rs | 20 +- .../connection/driver/commands.rs | 9 +- .../connection/driver/delegate_runtime.rs | 8 +- .../connection/driver/request_tracker.rs | 9 + .../connection/driver/responses.rs | 68 +- .../connection/driver/session_registry.rs | 6 + .../remote_session/connection/driver_tests.rs | 402 +- .../src/remote_session/connection/reader.rs | 7 + .../code-mode/src/remote_session_tests.rs | 89 + codex-rs/core-skills/src/loader.rs | 220 +- .../core-skills/src/loader/environment.rs | 145 +- .../core-skills/tests/environment_loader.rs | 402 +- codex-rs/core/Cargo.toml | 2 +- codex-rs/core/src/agents_md_tests.rs | 4 + .../context/approved_command_prefix_saved.rs | 8 +- codex-rs/core/src/context/mod.rs | 1 + .../core/src/context/token_budget_context.rs | 8 +- .../world_state/compact_permissions.rs | 59 + .../world_state/compact_permissions_tests.rs | 64 + codex-rs/core/src/context/world_state/mod.rs | 2 + .../src/context/world_state/permissions.rs | 77 +- .../context/world_state/permissions_tests.rs | 157 + ...dered_without_reinjecting_permissions.snap | 16 + ..._state__permissions__tests__snapshots.snap | 10 +- codex-rs/core/src/context_manager/history.rs | 2 +- .../core/src/context_manager/history_tests.rs | 28 + codex-rs/core/src/environment_selection.rs | 65 +- codex-rs/core/src/event_mapping.rs | 2 + codex-rs/core/src/event_mapping_tests.rs | 2 +- codex-rs/core/src/lib.rs | 1 - codex-rs/core/src/mcp_openai_file.rs | 5 + codex-rs/core/src/mcp_tool_call_tests.rs | 4 + codex-rs/core/src/session/handlers.rs | 31 +- codex-rs/core/src/session/mod.rs | 23 +- codex-rs/core/src/session/session.rs | 5 + codex-rs/core/src/session/tests.rs | 36 +- codex-rs/core/src/session/turn_context.rs | 10 + codex-rs/core/src/session/world_state.rs | 4 + codex-rs/core/src/thread_manager.rs | 423 +- codex-rs/core/src/tools/approvals.rs | 25 +- .../core/src/tools/code_mode/execute_spec.rs | 2 + codex-rs/core/src/tools/code_mode/mod.rs | 2 +- .../src/tools/handlers/apply_patch_spec.rs | 1 + .../tools/handlers/apply_patch_spec_tests.rs | 1 + .../src/tools/handlers/extension_tools.rs | 31 +- .../src/tools/handlers/shell/shell_command.rs | 5 +- .../core/src/tools/handlers/shell_tests.rs | 15 +- .../handlers/unified_exec/exec_command.rs | 2 +- .../src/tools/handlers/unified_exec_tests.rs | 43 + .../core/src/tools/handlers/view_image.rs | 1 + codex-rs/core/src/tools/orchestrator.rs | 17 +- codex-rs/core/src/tools/router_tests.rs | 1 + .../src/tools/runtimes/apply_patch_tests.rs | 4 + .../core/src/tools/runtimes/shell_tests.rs | 4 + .../core/src/tools/runtimes/unified_exec.rs | 4 + codex-rs/core/src/tools/sandboxing.rs | 1 + codex-rs/core/src/tools/spec_plan.rs | 25 +- codex-rs/core/src/tools/spec_plan_tests.rs | 72 +- codex-rs/core/tests/suite/approvals.rs | 121 + codex-rs/core/tests/suite/code_mode.rs | 314 ++ ..._new_context_window_tool_full_context.snap | 2 +- codex-rs/core/tests/suite/token_budget.rs | 38 +- codex-rs/core/tests/suite/unified_exec.rs | 52 + codex-rs/exec/src/lib.rs | 20 +- codex-rs/exec/tests/suite/resume.rs | 190 + codex-rs/ext/image-generation/src/tests.rs | 4 +- codex-rs/ext/skills/Cargo.toml | 5 + codex-rs/ext/skills/src/lib.rs | 1 + codex-rs/ext/skills/src/loader/discovery.rs | 167 + codex-rs/ext/skills/src/loader/environment.rs | 382 ++ .../skills/src/loader/environment_io_tests.rs | 414 ++ .../skills/src/loader/environment_tests.rs | 151 + codex-rs/ext/skills/src/loader/metadata.rs | 176 + codex-rs/ext/skills/src/loader/mod.rs | 22 + codex-rs/ext/skills/src/loader/namespace.rs | 172 + codex-rs/ext/skills/src/provider/executor.rs | 4 +- .../tests/executor_file_system_authority.rs | 409 ++ ...rity__pre_discovered_executor_catalog.snap | 30 + codex-rs/otel/src/metrics/config.rs | 9 +- codex-rs/otel/src/provider.rs | 4 + .../otel/tests/suite/otlp_http_loopback.rs | 16 + codex-rs/skills/Cargo.toml | 5 + codex-rs/skills/src/lib.rs | 4 + codex-rs/skills/src/parser.rs | 225 + codex-rs/skills/src/parser_tests.rs | 43 + codex-rs/terminal-detection/src/lib.rs | 3 +- .../terminal-detection/src/terminal_tests.rs | 19 + codex-rs/tools/src/code_mode.rs | 25 + codex-rs/tools/src/code_mode_tests.rs | 41 + codex-rs/tools/src/responses_api.rs | 5 + codex-rs/tools/src/responses_api_tests.rs | 30 + codex-rs/tools/src/tool_search.rs | 24 +- codex-rs/tools/src/tool_search_tests.rs | 60 + codex-rs/tools/src/tool_spec_tests.rs | 53 +- codex-rs/tui/src/lib.rs | 2 +- codex-rs/tui/src/session_archive_commands.rs | 150 +- .../tui/src/session_archive_commands_tests.rs | 373 ++ codex-rs/tui/src/tui/keyboard_modes.rs | 17 +- codex-rs/utils/audio/BUILD.bazel | 6 + codex-rs/utils/audio/Cargo.toml | 23 + .../audio}/src/audio_preparation_tests.rs | 0 .../audio/src/lib.rs} | 10 +- pnpm-lock.yaml | 253 +- pnpm-workspace.yaml | 4 +- scripts/mcp_conformance/README.md | 121 + .../codex_conformance_adapter.py | 976 ++++ .../mcp_conformance/official_conformance.py | 518 ++ .../regression-baseline-v1.json | 4208 +++++++++++++++++ .../review-regression-baseline-v1.json | 1039 ++++ scripts/mcp_conformance/review_regressions.py | 1294 +++++ .../mcp_conformance/run_codex_compliance.py | 2201 +++++++++ scripts/mcp_conformance/server.py | 1744 +++++++ .../mcp_conformance/test_codex_compliance.py | 1202 +++++ .../test_official_conformance.py | 685 +++ .../test_review_regressions.py | 1093 +++++ scripts/mcp_conformance/test_server.py | 1590 +++++++ .../tests/test_mcp_conformance_fixtures.py | 37 + sdk/typescript/package.json | 1 + sdk/typescript/tests/mcpConformance.test.ts | 158 + 135 files changed, 23925 insertions(+), 1482 deletions(-) create mode 100644 codex-rs/core/src/context/world_state/compact_permissions.rs create mode 100644 codex-rs/core/src/context/world_state/compact_permissions_tests.rs create mode 100644 codex-rs/core/src/context/world_state/snapshots/codex_core__context__world_state__permissions__tests__approved_prefix_is_rendered_without_reinjecting_permissions.snap create mode 100644 codex-rs/ext/skills/src/loader/discovery.rs create mode 100644 codex-rs/ext/skills/src/loader/environment.rs create mode 100644 codex-rs/ext/skills/src/loader/environment_io_tests.rs create mode 100644 codex-rs/ext/skills/src/loader/environment_tests.rs create mode 100644 codex-rs/ext/skills/src/loader/metadata.rs create mode 100644 codex-rs/ext/skills/src/loader/mod.rs create mode 100644 codex-rs/ext/skills/src/loader/namespace.rs create mode 100644 codex-rs/ext/skills/tests/snapshots/executor_file_system_authority__pre_discovered_executor_catalog.snap create mode 100644 codex-rs/skills/src/parser.rs create mode 100644 codex-rs/skills/src/parser_tests.rs create mode 100644 codex-rs/tui/src/session_archive_commands_tests.rs create mode 100644 codex-rs/utils/audio/BUILD.bazel create mode 100644 codex-rs/utils/audio/Cargo.toml rename codex-rs/{core => utils/audio}/src/audio_preparation_tests.rs (100%) rename codex-rs/{core/src/audio_preparation.rs => utils/audio/src/lib.rs} (95%) create mode 100644 scripts/mcp_conformance/README.md create mode 100644 scripts/mcp_conformance/codex_conformance_adapter.py create mode 100644 scripts/mcp_conformance/official_conformance.py create mode 100644 scripts/mcp_conformance/regression-baseline-v1.json create mode 100644 scripts/mcp_conformance/review-regression-baseline-v1.json create mode 100644 scripts/mcp_conformance/review_regressions.py create mode 100644 scripts/mcp_conformance/run_codex_compliance.py create mode 100644 scripts/mcp_conformance/server.py create mode 100644 scripts/mcp_conformance/test_codex_compliance.py create mode 100644 scripts/mcp_conformance/test_official_conformance.py create mode 100644 scripts/mcp_conformance/test_review_regressions.py create mode 100644 scripts/mcp_conformance/test_server.py create mode 100644 sdk/python/tests/test_mcp_conformance_fixtures.py create mode 100644 sdk/typescript/tests/mcpConformance.test.ts diff --git a/.github/scripts/publish_r2_release.py b/.github/scripts/publish_r2_release.py index 59825358aa..e7d351ae75 100755 --- a/.github/scripts/publish_r2_release.py +++ b/.github/scripts/publish_r2_release.py @@ -21,6 +21,7 @@ import re import subprocess import sys import tempfile +from concurrent.futures import ThreadPoolExecutor, as_completed from pathlib import Path from typing import Any, NamedTuple, NoReturn from urllib.parse import quote @@ -30,6 +31,7 @@ PREFIX = "codex" REPOSITORY = "openai/codex" RELEASE_METADATA_NAME = "release.json" INSTALLER_NAMES = ("install.sh", "install.ps1") +MAX_UPLOAD_WORKERS = 8 # Keep this pattern in sync with release-tag validation in # .github/workflows/rust-release.yml. VERSION_RE = re.compile( @@ -38,6 +40,7 @@ VERSION_RE = re.compile( ) CRC64_RE = re.compile(r"^[A-Za-z0-9+/]{11}=$") SHA256_RE = re.compile(r"^sha256:([0-9a-f]{64})$") +MISSING_OBJECT_RE = re.compile(r"\((?:404|NoSuchKey|NotFound)\)") class PublishError(RuntimeError): @@ -65,7 +68,7 @@ def run_command(args: list[str]) -> str: return result.stdout or "" -def download_assets(tag: str, directory: Path) -> list[ReleaseAsset]: +def get_release_metadata(tag: str, directory: Path, stage: str) -> list[ReleaseAsset]: try: metadata = json.loads( run_command( @@ -83,21 +86,9 @@ def download_assets(tag: str, directory: Path) -> list[ReleaseAsset]: ] ) ) - run_command( - [ - "gh", - "release", - "download", - tag, - "--repo", - REPOSITORY, - "--dir", - str(directory), - ] - ) except (OSError, subprocess.CalledProcessError) as error: raise PublishError( - f"GitHub release download failed for {tag}: {error}" + f"GitHub release metadata request failed for {tag}: {error}" ) from error except json.JSONDecodeError as error: raise PublishError( @@ -112,6 +103,9 @@ def download_assets(tag: str, directory: Path) -> list[ReleaseAsset]: raise PublishError( f"GitHub returned invalid release metadata for {tag}: {asset!r}" ) + # DotSlash runs concurrently, so defer its in-progress assets to finalization. + if stage == "assets" and asset.get("state") != "uploaded": + continue name = asset.get("name") size = asset.get("size") digest = asset.get("digest") @@ -131,14 +125,37 @@ def download_assets(tag: str, directory: Path) -> list[ReleaseAsset]: ) expected[name] = ReleaseAsset(directory / name, size, match.group(1)) - assets = sorted(directory.iterdir(), key=lambda path: path.name) - if not assets: + if not expected: raise PublishError(f"GitHub Release {tag} has no assets") - if any(not path.is_file() for path in assets) or { - path.name for path in assets - } != set(expected): + return sorted(expected.values(), key=lambda asset: asset.path.name) + + +def download_assets(tag: str, directory: Path, assets: list[ReleaseAsset]) -> None: + if not assets: + return + + command = [ + "gh", + "release", + "download", + tag, + "--repo", + REPOSITORY, + "--dir", + str(directory), + ] + for asset in assets: + command.extend(["--pattern", asset.path.name]) + + try: + run_command(command) + except (OSError, subprocess.CalledProcessError) as error: + raise PublishError( + f"GitHub release download failed for {tag}: {error}" + ) from error + + if any(not asset.path.is_file() for asset in assets): raise PublishError("GitHub returned invalid release assets") - return [expected[path.name] for path in assets] def stream_digest(source: Any) -> tuple[int, str]: @@ -150,6 +167,17 @@ def stream_digest(source: Any) -> tuple[int, str]: return size, digest.hexdigest() +def validate_asset(asset: ReleaseAsset) -> None: + with asset.path.open("rb") as source: + size, sha256 = stream_digest(source) + if size != asset.size or sha256 != asset.sha256: + raise PublishError( + f"GitHub asset mismatch for {asset.path.name}: expected " + f"size={asset.size} sha256={asset.sha256}, got " + f"size={size} sha256={sha256}" + ) + + def raise_s3( action: str, key: str, error: Exception, detail: str | None = None ) -> NoReturn: @@ -246,6 +274,7 @@ def publish_installers(endpoint: str, tag: str, assets: list[ReleaseAsset]) -> N ) for name in INSTALLER_NAMES: asset = installers[name] + validate_asset(asset) installer_key = f"{PREFIX}/{name}" put_object(endpoint, installer_key, asset.path, asset.sha256, extra_args=[]) verify_remote(endpoint, installer_key, asset.size, asset.sha256) @@ -256,11 +285,92 @@ def publish_installers(endpoint: str, tag: str, assets: list[ReleaseAsset]) -> N ) +def publish_asset( + endpoint: str, version: str, asset: ReleaseAsset, *, verify: bool = True +) -> dict[str, Any]: + validate_asset(asset) + key = f"{PREFIX}/releases/{version}/{asset.path.name}" + put_object(endpoint, key, asset.path, asset.sha256, extra_args=["--no-overwrite"]) + if verify: + verify_remote(endpoint, key, asset.size, asset.sha256) + status = "published and verified" if verify else "published" + print( + f"{status} s3://{BUCKET}/{key} size={asset.size} sha256={asset.sha256}", + file=sys.stderr, + ) + return { + "key": key, + "name": asset.path.name, + "sha256": asset.sha256, + "size": asset.size, + } + + +def publish_assets( + endpoint: str, version: str, assets: list[ReleaseAsset], *, verify: bool = True +) -> list[dict[str, Any]]: + if not assets: + return [] + + published = {} + with ThreadPoolExecutor( + max_workers=min(MAX_UPLOAD_WORKERS, len(assets)) + ) as executor: + futures = { + executor.submit( + publish_asset, endpoint, version, asset, verify=verify + ): asset + for asset in assets + } + for future in as_completed(futures): + asset = futures[future] + published[asset.path.name] = future.result() + return [published[asset.path.name] for asset in assets] + + +def find_published_assets( + endpoint: str, version: str, assets: list[ReleaseAsset] +) -> dict[str, dict[str, Any]]: + published = {} + with ThreadPoolExecutor( + max_workers=min(MAX_UPLOAD_WORKERS, len(assets)) + ) as executor: + futures = { + executor.submit( + verify_remote, + endpoint, + f"{PREFIX}/releases/{version}/{asset.path.name}", + asset.size, + asset.sha256, + ): asset + for asset in assets + } + for future in as_completed(futures): + asset = futures[future] + try: + future.result() + except PublishError as error: + cause = error.__cause__ + if isinstance(cause, subprocess.CalledProcessError) and ( + MISSING_OBJECT_RE.search(cause.stderr or "") + ): + continue + raise + published[asset.path.name] = { + "key": f"{PREFIX}/releases/{version}/{asset.path.name}", + "name": asset.path.name, + "sha256": asset.sha256, + "size": asset.size, + } + return published + + def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--tag", required=True) parser.add_argument("--make-latest", choices=("true", "false"), required=True) parser.add_argument("--prerelease", choices=("true", "false"), required=True) + parser.add_argument("--stage", choices=("assets", "finalize"), required=True) return parser.parse_args() @@ -276,50 +386,62 @@ def main() -> int: version = args.tag.removeprefix("rust-v") if args.tag == version or not VERSION_RE.fullmatch(version): raise PublishError(f"invalid rust release tag: {args.tag}") - published = [] - metadata_assets = [] with tempfile.TemporaryDirectory() as temp_dir: assets_directory = Path(temp_dir) / "assets" assets_directory.mkdir() - assets = download_assets(args.tag, assets_directory) - for asset in assets: - with asset.path.open("rb") as source: - size, sha256 = stream_digest(source) - if size != asset.size or sha256 != asset.sha256: - raise PublishError( - f"GitHub asset mismatch for {asset.path.name}: expected " - f"size={asset.size} sha256={asset.sha256}, got " - f"size={size} sha256={sha256}" - ) - for asset in assets: - path = asset.path - size = asset.size - sha256 = asset.sha256 - key = f"{PREFIX}/releases/{version}/{path.name}" - put_object(endpoint, key, path, sha256, extra_args=["--no-overwrite"]) - verify_remote(endpoint, key, size, sha256) + assets = get_release_metadata(args.tag, assets_directory, args.stage) + + if args.stage == "assets": + download_assets(args.tag, assets_directory, assets) + published = publish_assets(endpoint, version, assets, verify=False) print( - f"published and verified s3://{BUCKET}/{key} " - f"size={size} sha256={sha256}", - file=sys.stderr, + json.dumps( + { + "assetCount": len(published), + "assets": published, + "releasePrefix": f"{PREFIX}/releases/{version}/", + "stage": args.stage, + "tag": args.tag, + "version": version, + }, + sort_keys=True, + ) ) - published.append( - { - "key": key, - "sha256": sha256, - "size": size, - } - ) - metadata_assets.append( - { - "name": path.name, - "digest": f"sha256:{sha256}", - "browser_download_url": ( - f"https://releases.openai.com/{PREFIX}/releases/" - f"{version}/{quote(path.name, safe='')}" - ), - } + return 0 + + previously_published = find_published_assets(endpoint, version, assets) + remaining = [ + asset for asset in assets if asset.path.name not in previously_published + ] + required_downloads = list(remaining) + if args.prerelease == "false": + required_downloads.extend( + asset + for asset in assets + if asset.path.name in INSTALLER_NAMES + and asset.path.name in previously_published ) + download_assets(args.tag, assets_directory, required_downloads) + additionally_published = { + asset["name"]: asset + for asset in publish_assets(endpoint, version, remaining) + } + published = [ + previously_published.get(asset.path.name) + or additionally_published[asset.path.name] + for asset in assets + ] + metadata_assets = [ + { + "name": asset.path.name, + "digest": f"sha256:{asset.sha256}", + "browser_download_url": ( + f"https://releases.openai.com/{PREFIX}/releases/" + f"{version}/{quote(asset.path.name, safe='')}" + ), + } + for asset in assets + ] metadata_path = Path(temp_dir) / RELEASE_METADATA_NAME metadata_path.write_text( @@ -393,6 +515,7 @@ def main() -> int: "size": metadata_size, }, "releasePrefix": f"{PREFIX}/releases/{version}/", + "stage": args.stage, "tag": args.tag, "version": version, }, diff --git a/.github/workflows/r2-release.yml b/.github/workflows/r2-release.yml index 3abe94dde0..96a07685cf 100644 --- a/.github/workflows/r2-release.yml +++ b/.github/workflows/r2-release.yml @@ -12,6 +12,9 @@ on: prerelease: required: true type: boolean + stage: + required: true + type: string permissions: {} @@ -40,9 +43,11 @@ jobs: RELEASE_TAG: ${{ inputs.tag }} RELEASE_MAKE_LATEST: ${{ inputs.make_latest }} RELEASE_PRERELEASE: ${{ inputs.prerelease }} + RELEASE_STAGE: ${{ inputs.stage }} run: | set -euo pipefail python3 .github/scripts/publish_r2_release.py \ --tag "${RELEASE_TAG}" \ --make-latest "${RELEASE_MAKE_LATEST}" \ - --prerelease "${RELEASE_PRERELEASE}" + --prerelease "${RELEASE_PRERELEASE}" \ + --stage "${RELEASE_STAGE}" diff --git a/.github/workflows/rust-release.yml b/.github/workflows/rust-release.yml index 04ae92db0c..1b423273af 100644 --- a/.github/workflows/rust-release.yml +++ b/.github/workflows/rust-release.yml @@ -1356,9 +1356,9 @@ jobs: # (e.g. -alpha, -beta). Otherwise publish a normal release. prerelease: ${{ steps.release_name.outputs.prerelease }} - publish-r2: - name: publish-r2 - needs: [release, publish-dotslash] + publish-r2-assets: + name: publish-r2-assets + needs: release permissions: contents: read uses: ./.github/workflows/r2-release.yml @@ -1367,6 +1367,20 @@ jobs: tag: ${{ needs.release.outputs.tag }} make_latest: ${{ fromJSON(needs.release.outputs.make_latest) }} prerelease: ${{ fromJSON(needs.release.outputs.prerelease) }} + stage: assets + + publish-r2: + name: publish-r2 + needs: [release, publish-dotslash, publish-r2-assets] + permissions: + contents: read + uses: ./.github/workflows/r2-release.yml + secrets: inherit + with: + tag: ${{ needs.release.outputs.tag }} + make_latest: ${{ fromJSON(needs.release.outputs.make_latest) }} + prerelease: ${{ fromJSON(needs.release.outputs.prerelease) }} + stage: finalize publish-dotslash: name: publish-dotslash diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index 5194ab6059..8677251bb5 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -2571,6 +2571,7 @@ dependencies = [ "tokio-util", "tracing", "tracing-subscriber", + "uuid", ] [[package]] @@ -2761,6 +2762,7 @@ dependencies = [ "codex-thread-store", "codex-tools", "codex-utils-absolute-path", + "codex-utils-audio", "codex-utils-cache", "codex-utils-cargo-bin", "codex-utils-home-dir", @@ -2802,7 +2804,6 @@ dependencies = [ "sha1 0.10.6", "shlex", "similar", - "symphonia", "tempfile", "test-case", "test-log", @@ -4086,6 +4087,9 @@ dependencies = [ "codex-utils-absolute-path", "codex-utils-path-uri", "include_dir", + "pretty_assertions", + "serde", + "serde_yaml", "thiserror 2.0.18", ] @@ -4103,14 +4107,19 @@ dependencies = [ "codex-skills", "codex-tools", "codex-utils-absolute-path", + "codex-utils-cargo-bin", "codex-utils-path-uri", + "codex-utils-plugins", "codex-utils-string", "futures", + "insta", "opentelemetry_sdk", "pretty_assertions", "schemars 0.8.22", "serde", "serde_json", + "serde_yaml", + "tempfile", "tokio", "tracing", "url", @@ -4377,6 +4386,20 @@ dependencies = [ "codex-protocol", ] +[[package]] +name = "codex-utils-audio" +version = "0.0.0" +dependencies = [ + "base64 0.22.1", + "codex-protocol", + "codex-utils-cache", + "codex-utils-string", + "pretty_assertions", + "symphonia", + "thiserror 2.0.18", + "tracing", +] + [[package]] name = "codex-utils-cache" version = "0.0.0" diff --git a/codex-rs/Cargo.toml b/codex-rs/Cargo.toml index 2f5588034b..9f971f7e96 100644 --- a/codex-rs/Cargo.toml +++ b/codex-rs/Cargo.toml @@ -94,6 +94,7 @@ members = [ "v8-poc", "websocket-client", "utils/absolute-path", + "utils/audio", "utils/path-uri", "utils/cargo-bin", "git-utils", @@ -246,6 +247,7 @@ codex-tui = { path = "tui" } codex-uds = { path = "uds" } codex-utils-absolute-path = { path = "utils/absolute-path" } codex-utils-approval-presets = { path = "utils/approval-presets" } +codex-utils-audio = { path = "utils/audio" } codex-utils-cache = { path = "utils/cache" } codex-utils-cargo-bin = { path = "utils/cargo-bin" } codex-utils-cli = { path = "utils/cli" } diff --git a/codex-rs/code-mode-host/Cargo.toml b/codex-rs/code-mode-host/Cargo.toml index 5bad7553f3..2f1febc2fb 100644 --- a/codex-rs/code-mode-host/Cargo.toml +++ b/codex-rs/code-mode-host/Cargo.toml @@ -27,6 +27,7 @@ tokio = { workspace = true, features = ["io-std", "io-util", "macros", "net", "p tokio-util = { workspace = true, features = ["rt"] } tracing = { workspace = true } tracing-subscriber = { workspace = true } +uuid = { workspace = true, features = ["v4"] } [dev-dependencies] codex-code-mode = { workspace = true } diff --git a/codex-rs/code-mode-host/src/host_tests.rs b/codex-rs/code-mode-host/src/host_tests.rs index b3639e7cc0..3e78ae6911 100644 --- a/codex-rs/code-mode-host/src/host_tests.rs +++ b/codex-rs/code-mode-host/src/host_tests.rs @@ -12,6 +12,7 @@ use codex_code_mode_protocol::host::Capability; use codex_code_mode_protocol::host::CapabilitySet; use codex_code_mode_protocol::host::ClientHello; use codex_code_mode_protocol::host::ClientToHost; +use codex_code_mode_protocol::host::DUAL_WEBSOCKET_CAPABILITY; use codex_code_mode_protocol::host::EncodedFrame; use codex_code_mode_protocol::host::FramedReader; use codex_code_mode_protocol::host::FramedWriter; @@ -33,6 +34,7 @@ use tokio::sync::mpsc; use tokio::sync::oneshot; use tokio_util::sync::CancellationToken; use tokio_util::task::TaskTracker; +use uuid::Uuid; use super::HostState; use super::MAX_ACTIVE_CELLS; @@ -41,8 +43,12 @@ use super::MAX_RECENT_REQUEST_IDS; use super::RequestKind; use super::RequestRegistry; use super::SeenSessionIds; +use super::negotiate; use super::peer::HostPeer; use super::run; +use super::transport::BulkConnectionRegistry; +use super::transport::ConnectionReader; +use super::transport::ConnectionWriter; fn client_hello( versions: impl IntoIterator, @@ -250,6 +256,107 @@ impl AsyncWrite for BlockingWriter { } } +#[tokio::test] +async fn failed_host_hello_removes_the_bulk_pairing_reservation() { + let (host_reader, client_writer) = tokio::io::duplex(/*max_buf_size*/ 4096); + let capability = Capability::new(DUAL_WEBSOCKET_CAPABILITY).expect("dual websocket capability"); + let hello = ClientToHost::ClientHello( + ClientHello::new( + SupportedProtocolVersions::try_new([ProtocolVersion::V1]).expect("supported versions"), + CapabilitySet::empty(), + CapabilitySet::try_new([capability]).expect("optional capabilities"), + ) + .expect("client hello"), + ); + FramedWriter::new(client_writer) + .write(&hello) + .await + .expect("write client hello"); + + let bytes = Arc::new(Mutex::new(Vec::new())); + let registry = BulkConnectionRegistry::default(); + let mut reader = ConnectionReader::from_reader(host_reader); + let mut writer = ConnectionWriter::from_writer(FailingHandshakeWriter { + bytes: Arc::clone(&bytes), + }); + let result = negotiate(&mut reader, &mut writer, Some(®istry)).await; + assert!(result.is_err()); + + let message = EncodedFrame::decode_framed::( + &bytes.lock().unwrap_or_else(PoisonError::into_inner), + ) + .expect("decode failed host hello"); + let HostToClient::HostHello(hello) = message else { + panic!("expected a host hello"); + }; + let token = hello + .bulk_connection_token() + .expect("dual websocket pairing token"); + let token = Uuid::parse_str(token).expect("UUID pairing token"); + assert!(registry.remove(token).is_none()); +} + +struct FailingHandshakeWriter { + bytes: Arc>>, +} + +impl AsyncWrite for FailingHandshakeWriter { + fn poll_write( + self: Pin<&mut Self>, + _cx: &mut Context<'_>, + bytes: &[u8], + ) -> Poll> { + self.bytes + .lock() + .unwrap_or_else(PoisonError::into_inner) + .extend_from_slice(bytes); + Poll::Ready(Ok(bytes.len())) + } + + fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Err(std::io::Error::other("host hello write failed"))) + } + + fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } +} + +#[tokio::test] +async fn optional_dual_websocket_capability_falls_back_to_a_single_connection() { + let (host_stream, client_stream) = tokio::io::duplex(/*max_buf_size*/ 1024); + let (host_reader, host_writer) = tokio::io::split(host_stream); + let (client_reader, client_writer) = tokio::io::split(client_stream); + let host = tokio::spawn(run(host_reader, host_writer)); + let mut reader = FramedReader::new(client_reader); + let mut writer = FramedWriter::new(client_writer); + let capability = Capability::new(DUAL_WEBSOCKET_CAPABILITY).expect("dual websocket capability"); + + writer + .write(&ClientToHost::ClientHello( + ClientHello::new( + SupportedProtocolVersions::try_new([ProtocolVersion::V1]) + .expect("supported versions"), + CapabilitySet::empty(), + CapabilitySet::try_new([capability]).expect("optional capabilities"), + ) + .expect("client hello"), + )) + .await + .expect("write hello"); + assert_eq!( + reader.read::().await.expect("host hello"), + Some(HostToClient::HostHello(HostHello::new( + ProtocolVersion::V1, + CapabilitySet::empty(), + ))) + ); + + drop(writer); + drop(reader); + host.await.expect("host task").expect("host connection"); +} + #[tokio::test] async fn incompatible_or_invalid_handshake_is_rejected() { let (host_stream, client_stream) = tokio::io::duplex(/*max_buf_size*/ 1024); @@ -395,7 +502,7 @@ async fn session_id_cannot_be_reused_after_shutdown() { } #[test] -fn request_cancellation_tombstones_are_bounded() { +fn request_history_is_bounded() { let mut requests = RequestRegistry::default(); let duplicate = request_id(/*value*/ -1); requests @@ -411,10 +518,6 @@ fn request_cancellation_tombstones_are_bounded() { requests.cancel(id); requests.finish(id); } - for value in 10_000..20_000 { - requests.cancel(request_id(value)); - } - assert!(requests.active.is_empty()); assert_eq!(requests.recent.len(), MAX_RECENT_REQUEST_IDS); assert_eq!(requests.recent_order.len(), MAX_RECENT_REQUEST_IDS); diff --git a/codex-rs/code-mode-host/src/lib.rs b/codex-rs/code-mode-host/src/lib.rs index 83b545d7e7..88d2e161f5 100644 --- a/codex-rs/code-mode-host/src/lib.rs +++ b/codex-rs/code-mode-host/src/lib.rs @@ -10,18 +10,22 @@ use std::time::Duration; use anyhow::Context; use anyhow::Result; +use codex_code_mode_protocol::host::Capability; use codex_code_mode_protocol::host::CapabilitySet; use codex_code_mode_protocol::host::ClientToHost; +use codex_code_mode_protocol::host::DUAL_WEBSOCKET_CAPABILITY; use codex_code_mode_protocol::host::EncodedFrame; use codex_code_mode_protocol::host::HandshakeRejectReason; use codex_code_mode_protocol::host::HostHello; use codex_code_mode_protocol::host::HostRequest; use codex_code_mode_protocol::host::HostResponse; use codex_code_mode_protocol::host::HostToClient; +use codex_code_mode_protocol::host::MAX_PENDING_DELEGATE_CALLS; use codex_code_mode_protocol::host::ProtocolVersion; use codex_code_mode_protocol::host::RequestId; use codex_code_mode_protocol::host::SessionId; use codex_code_mode_protocol::host::SupportedProtocolVersions; +use codex_code_mode_protocol::host::TransportLane; use codex_code_mode_runtime::InProcessCodeModeSession; use tokio::io::AsyncRead; use tokio::io::AsyncWrite; @@ -32,6 +36,8 @@ use tokio_util::task::TaskTracker; use self::delegate::RemoteDelegate; use self::peer::HostPeer; +use self::transport::BulkConnectionRegistration; +use self::transport::BulkConnectionRegistry; use self::transport::ConnectionReader; use self::transport::ConnectionWriter; @@ -46,6 +52,14 @@ const MAX_ACTIVE_CELLS: usize = 128; const MAX_RECENT_REQUEST_IDS: usize = 4096; const MAX_RECENT_SESSION_IDS: usize = 4096; const SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(5); +const OUTGOING_CHANNEL_CAPACITY: usize = 128; +const BULK_PAIRING_TIMEOUT: Duration = Duration::from_secs(10); + +enum NegotiatedConnection { + Rejected, + Single, + Dual(BulkConnectionRegistration), +} struct HostLimits { request_permits: Arc, @@ -81,6 +95,7 @@ where ConnectionReader::from_reader(reader), ConnectionWriter::from_writer(writer), Arc::new(HostLimits::new()), + /*bulk_connections*/ None, ) .await } @@ -89,13 +104,39 @@ async fn run_connection( mut reader: ConnectionReader, mut writer: ConnectionWriter, limits: Arc, + bulk_connections: Option, ) -> Result<()> { - if !negotiate(&mut reader, &mut writer).await? { - return Ok(()); - } - - let (outgoing_tx, mut outgoing_rx) = mpsc::channel::(/*max_capacity*/ 128); - let peer = Arc::new(HostPeer::new(outgoing_tx)); + let negotiated = negotiate(&mut reader, &mut writer, bulk_connections.as_ref()).await?; + let bulk_connection = match negotiated { + NegotiatedConnection::Rejected => return Ok(()), + NegotiatedConnection::Single => None, + NegotiatedConnection::Dual(mut registration) => { + match tokio::time::timeout(BULK_PAIRING_TIMEOUT, registration.receive()).await { + Ok(Ok(connection)) => Some(connection), + Ok(Err(_)) => { + anyhow::bail!("code-mode host bulk websocket pairing was abandoned"); + } + Err(_) => { + anyhow::bail!("timed out pairing code-mode host bulk websocket"); + } + } + } + }; + let (mut bulk_reader, bulk_writer) = match bulk_connection { + Some(connection) => (Some(connection.reader), Some(connection.writer)), + None => (None, None), + }; + let (outgoing_tx, outgoing_rx) = mpsc::channel::(OUTGOING_CHANNEL_CAPACITY); + let (bulk_tx, bulk_rx) = if bulk_writer.is_some() { + let (sender, receiver) = mpsc::channel::(MAX_PENDING_DELEGATE_CALLS); + (Some(sender), Some(receiver)) + } else { + (None, None) + }; + let peer = match bulk_tx { + Some(sender) => Arc::new(HostPeer::new(outgoing_tx).with_bulk_sender(sender)), + None => Arc::new(HostPeer::new(outgoing_tx)), + }; let state = Arc::new(HostState { sessions: Mutex::new(HashMap::new()), seen_session_ids: Mutex::new(SeenSessionIds::default()), @@ -108,25 +149,14 @@ async fn run_connection( }); let writer_disconnected = peer.disconnection_token(); let writer_task = tokio::spawn(async move { - loop { - tokio::select! { - _ = writer_disconnected.cancelled() => return Ok::<(), anyhow::Error>(()), - frame = outgoing_rx.recv() => { - let Some(frame) = frame else { - return Ok(()); - }; - let result = tokio::select! { - _ = writer_disconnected.cancelled() => return Ok(()), - result = writer.write_frame(frame) => result, - }; - if let Err(err) = result { - return Err( - anyhow::Error::new(err) - .context("failed to write code-mode host message") - ); - } - } - } + if let (Some(bulk_writer), Some(bulk_rx)) = (bulk_writer, bulk_rx) { + tokio::try_join!( + drive_writer(writer, outgoing_rx, writer_disconnected.clone()), + drive_writer(bulk_writer, bulk_rx, writer_disconnected) + )?; + Ok(()) + } else { + drive_writer(writer, outgoing_rx, writer_disconnected).await } }); let writer_peer = Arc::clone(&peer); @@ -147,14 +177,30 @@ async fn run_connection( let input_result = async { loop { - let message = tokio::select! { + let (message, lane) = tokio::select! { + // Session operations and shutdown must make progress even when bulk callbacks are ready. + biased; _ = peer.disconnected() => break, - message = reader.read() => message - .context("failed to read code-mode client message")?, + message = reader.read() => ( + message.context("failed to read code-mode client control message")?, + TransportLane::Control, + ), + message = async { + match &mut bulk_reader { + Some(reader) => reader.read().await, + None => Ok(None), + } + }, if bulk_reader.is_some() => ( + message.context("failed to read code-mode client bulk message")?, + TransportLane::Bulk, + ), }; let Some(message) = message else { break; }; + if bulk_reader.is_some() && !message.allows_transport_lane(lane) { + anyhow::bail!("code-mode client sent a message on the wrong websocket lane"); + } match message { ClientToHost::ClientHello(_) => { anyhow::bail!("received a second code-mode client hello"); @@ -195,13 +241,40 @@ async fn run_connection( Ok(()) } -async fn negotiate(reader: &mut ConnectionReader, writer: &mut ConnectionWriter) -> Result { +async fn drive_writer( + mut writer: ConnectionWriter, + mut outgoing: mpsc::Receiver, + disconnected: CancellationToken, +) -> Result<()> { + loop { + tokio::select! { + _ = disconnected.cancelled() => return Ok(()), + frame = outgoing.recv() => { + let Some(frame) = frame else { + return Ok(()); + }; + tokio::select! { + _ = disconnected.cancelled() => return Ok(()), + result = writer.write_frame(frame) => { + result.context("failed to write code-mode host message")?; + } + } + } + } + } +} + +async fn negotiate( + reader: &mut ConnectionReader, + writer: &mut ConnectionWriter, + bulk_connections: Option<&BulkConnectionRegistry>, +) -> Result { let Some(first_message) = reader .read() .await .context("failed to read code-mode client hello")? else { - return Ok(false); + return Ok(NegotiatedConnection::Rejected); }; let ClientToHost::ClientHello(client_hello) = first_message else { writer @@ -212,7 +285,7 @@ async fn negotiate(reader: &mut ConnectionReader, writer: &mut ConnectionWriter) }) .await .context("failed to reject invalid code-mode client hello")?; - return Ok(false); + return Ok(NegotiatedConnection::Rejected); }; let supported_versions = SupportedProtocolVersions::try_new([ProtocolVersion::V1])?; @@ -226,10 +299,26 @@ async fn negotiate(reader: &mut ConnectionReader, writer: &mut ConnectionWriter) }) .await .context("failed to reject incompatible code-mode client")?; - return Ok(false); + return Ok(NegotiatedConnection::Rejected); } - let host_capabilities = CapabilitySet::empty(); + let dual_capability = Capability::new(DUAL_WEBSOCKET_CAPABILITY)?; + let dual_requested = client_hello + .required_capabilities() + .contains(&dual_capability) + || client_hello + .optional_capabilities() + .contains(&dual_capability); + let registration = if dual_requested { + bulk_connections.and_then(BulkConnectionRegistry::reserve) + } else { + None + }; + let host_capabilities = if registration.is_some() { + CapabilitySet::try_new([dual_capability])? + } else { + CapabilitySet::empty() + }; if let Some(capability) = client_hello .required_capabilities() .iter() @@ -243,17 +332,26 @@ async fn negotiate(reader: &mut ConnectionReader, writer: &mut ConnectionWriter) }) .await .context("failed to reject unsupported code-mode capability")?; - return Ok(false); + return Ok(NegotiatedConnection::Rejected); } + let (hello, negotiated) = if let Some(registration) = registration { + ( + HostHello::new(ProtocolVersion::V1, host_capabilities) + .with_bulk_connection_token(registration.token().to_string()), + NegotiatedConnection::Dual(registration), + ) + } else { + ( + HostHello::new(ProtocolVersion::V1, host_capabilities), + NegotiatedConnection::Single, + ) + }; writer - .write(&HostToClient::HostHello(HostHello::new( - ProtocolVersion::V1, - host_capabilities, - ))) + .write(&HostToClient::HostHello(hello)) .await .context("failed to write code-mode host hello")?; - Ok(true) + Ok(negotiated) } struct HostState { @@ -584,7 +682,7 @@ impl RequestRegistry { Ok(cancellation) } - fn cancel(&self, request_id: RequestId) { + fn cancel(&mut self, request_id: RequestId) { if let Some(request) = self.active.get(&request_id) && request.kind.is_cancellable() { diff --git a/codex-rs/code-mode-host/src/peer.rs b/codex-rs/code-mode-host/src/peer.rs index 7912051802..02b0c263dc 100644 --- a/codex-rs/code-mode-host/src/peer.rs +++ b/codex-rs/code-mode-host/src/peer.rs @@ -16,6 +16,7 @@ use codex_code_mode_protocol::host::HostToClient; use codex_code_mode_protocol::host::MAX_PENDING_DELEGATE_CALLS; use codex_code_mode_protocol::host::RequestId; use codex_code_mode_protocol::host::SessionId; +use codex_code_mode_protocol::host::TransportLane; use codex_code_mode_protocol::host::WireResult; use tokio::sync::Mutex; use tokio::sync::Notify; @@ -29,6 +30,7 @@ const CELL_MESSAGE_CAPACITY: usize = 128; pub(super) struct HostPeer { outgoing_tx: mpsc::Sender, + bulk_tx: Option>, pending: Mutex>, delegate_permits: Arc, cell_routes: StdMutex>, @@ -62,6 +64,7 @@ impl HostPeer { pub(super) fn new(outgoing_tx: mpsc::Sender) -> Self { Self { outgoing_tx, + bulk_tx: None, pending: Mutex::new(HashMap::new()), delegate_permits: Arc::new(Semaphore::new(MAX_PENDING_DELEGATE_CALLS)), cell_routes: StdMutex::new(HashMap::new()), @@ -72,10 +75,16 @@ impl HostPeer { } } + pub(super) fn with_bulk_sender(mut self, bulk_tx: mpsc::Sender) -> Self { + self.bulk_tx = Some(bulk_tx); + self + } + pub(super) fn send(&self, message: HostToClient) -> Result<(), PeerSendError> { let frame = EncodedFrame::encode(&message) .map_err(|err| PeerSendError::Payload(err.to_string()))?; - self.send_frame(frame) + let lane = message.transport_lane(); + self.send_frame(frame, lane) } pub(super) fn respond( @@ -389,8 +398,12 @@ impl HostPeer { }); } - fn send_frame(&self, frame: EncodedFrame) -> Result<(), PeerSendError> { - match self.outgoing_tx.try_send(frame) { + fn send_frame(&self, frame: EncodedFrame, lane: TransportLane) -> Result<(), PeerSendError> { + let sender = match lane { + TransportLane::Control => &self.outgoing_tx, + TransportLane::Bulk => self.bulk_tx.as_ref().unwrap_or(&self.outgoing_tx), + }; + match sender.try_send(frame) { Ok(()) => Ok(()), Err(mpsc::error::TrySendError::Full(_)) => { self.disconnect(); diff --git a/codex-rs/code-mode-host/src/transport.rs b/codex-rs/code-mode-host/src/transport.rs index 3342fc217c..c06fca40e3 100644 --- a/codex-rs/code-mode-host/src/transport.rs +++ b/codex-rs/code-mode-host/src/transport.rs @@ -1,13 +1,17 @@ +use std::collections::HashMap; use std::io; use std::io::Write as _; use std::net::SocketAddr; use std::sync::Arc; +use std::sync::Mutex; +use std::sync::PoisonError; use anyhow::Context; use anyhow::Result; use axum::Router; use axum::body::Body; use axum::extract::ConnectInfo; +use axum::extract::Path; use axum::extract::State; use axum::extract::ws::Message; use axum::extract::ws::WebSocket; @@ -34,15 +38,19 @@ use futures::stream::SplitStream; use tokio::io::AsyncRead; use tokio::io::AsyncWrite; use tokio::net::TcpListener; +use tokio::sync::oneshot; use tracing::info; use tracing::warn; +use uuid::Uuid; use crate::HostLimits; +use crate::MAX_IN_FLIGHT_REQUESTS; /// The default transport retains the standalone host's original stdio behavior. pub const DEFAULT_LISTEN_URL: &str = "stdio"; const MAX_WEBSOCKET_FRAME_BYTES: usize = MAX_FRAME_BYTES + std::mem::size_of::(); +const MAX_PENDING_BULK_CONNECTIONS: usize = MAX_IN_FLIGHT_REQUESTS; type BoxedReader = Box; type BoxedWriter = Box; @@ -66,6 +74,64 @@ pub(crate) enum ConnectionWriter { #[derive(Clone)] struct WebSocketListenerState { limits: Arc, + bulk_connections: BulkConnectionRegistry, +} + +pub(crate) struct BulkConnection { + pub(crate) reader: ConnectionReader, + pub(crate) writer: ConnectionWriter, +} + +#[derive(Clone, Default)] +pub(crate) struct BulkConnectionRegistry { + pending: Arc>>>, +} + +pub(crate) struct BulkConnectionRegistration { + registry: BulkConnectionRegistry, + token: Uuid, + receiver: oneshot::Receiver, +} + +impl BulkConnectionRegistry { + pub(crate) fn reserve(&self) -> Option { + let mut pending = self.pending.lock().unwrap_or_else(PoisonError::into_inner); + if pending.len() >= MAX_PENDING_BULK_CONNECTIONS { + return None; + } + + let token = Uuid::new_v4(); + let (sender, receiver) = oneshot::channel(); + pending.insert(token, sender); + Some(BulkConnectionRegistration { + registry: self.clone(), + token, + receiver, + }) + } + + pub(crate) fn remove(&self, token: Uuid) -> Option> { + self.pending + .lock() + .unwrap_or_else(PoisonError::into_inner) + .remove(&token) + } +} + +impl BulkConnectionRegistration { + pub(crate) fn token(&self) -> Uuid { + self.token + } + + pub(crate) async fn receive(&mut self) -> Result { + (&mut self.receiver).await + } +} + +impl Drop for BulkConnectionRegistration { + fn drop(&mut self) { + self.registry.remove(self.token); + } } impl ConnectionReader { @@ -165,6 +231,7 @@ async fn run_websocket_listener(bind_address: SocketAddr) -> Result<()> { .context("failed to read code-mode host websocket listen address")?; let state = WebSocketListenerState { limits: Arc::new(HostLimits::new()), + bulk_connections: BulkConnectionRegistry::default(), }; info!("codex-code-mode-host listening on ws://{local_addr}"); println!("ws://{local_addr}"); @@ -174,6 +241,7 @@ async fn run_websocket_listener(bind_address: SocketAddr) -> Result<()> { let router = Router::new() .route("/", any(websocket_upgrade_handler)) + .route("/bulk/{token}", any(bulk_websocket_upgrade_handler)) .route("/readyz", get(readiness_handler)) .layer(middleware::from_fn(reject_requests_with_origin_header)) .with_state(state); @@ -220,6 +288,7 @@ async fn websocket_upgrade_handler( ConnectionReader::WebSocket(reader), ConnectionWriter::WebSocket(writer), state.limits, + Some(state.bulk_connections), ) .await { @@ -228,6 +297,38 @@ async fn websocket_upgrade_handler( }) } +async fn bulk_websocket_upgrade_handler( + websocket: WebSocketUpgrade, + Path(token): Path, + ConnectInfo(peer_addr): ConnectInfo, + State(state): State, +) -> Response { + let Ok(token) = Uuid::parse_str(&token) else { + return StatusCode::NOT_FOUND.into_response(); + }; + let Some(pending) = state.bulk_connections.remove(token) else { + return StatusCode::NOT_FOUND.into_response(); + }; + + websocket + .max_frame_size(MAX_WEBSOCKET_FRAME_BYTES) + .max_message_size(MAX_WEBSOCKET_FRAME_BYTES) + .on_upgrade(move |stream| async move { + info!(%peer_addr, "code-mode host bulk websocket client connected"); + let (writer, reader) = stream.split(); + if pending + .send(BulkConnection { + reader: ConnectionReader::WebSocket(reader), + writer: ConnectionWriter::WebSocket(writer), + }) + .is_err() + { + warn!(%peer_addr, "code-mode host bulk websocket pairing expired"); + } + }) + .into_response() +} + #[cfg(test)] #[path = "transport_tests.rs"] mod tests; diff --git a/codex-rs/code-mode-host/src/transport_tests.rs b/codex-rs/code-mode-host/src/transport_tests.rs index b62b63b380..7b4955f488 100644 --- a/codex-rs/code-mode-host/src/transport_tests.rs +++ b/codex-rs/code-mode-host/src/transport_tests.rs @@ -2,9 +2,34 @@ use std::net::SocketAddr; use pretty_assertions::assert_eq; +use super::BulkConnectionRegistry; use super::ListenTransport; +use super::MAX_PENDING_BULK_CONNECTIONS; use super::parse_listen_url; +#[test] +fn bulk_connection_registration_cleans_up_when_dropped() { + let registry = BulkConnectionRegistry::default(); + let registration = registry.reserve().expect("bulk connection registration"); + let token = registration.token(); + + drop(registration); + + assert!(registry.remove(token).is_none()); +} + +#[test] +fn bulk_connection_registrations_are_bounded_and_released() { + let registry = BulkConnectionRegistry::default(); + let mut registrations = (0..MAX_PENDING_BULK_CONNECTIONS) + .map(|_| registry.reserve().expect("bulk connection registration")) + .collect::>(); + + assert!(registry.reserve().is_none()); + drop(registrations.pop()); + assert!(registry.reserve().is_some()); +} + #[test] fn parse_listen_url_accepts_stdio_transports() { assert_eq!( diff --git a/codex-rs/code-mode-host/tests/websocket.rs b/codex-rs/code-mode-host/tests/websocket.rs index a0f471e737..030f83432d 100644 --- a/codex-rs/code-mode-host/tests/websocket.rs +++ b/codex-rs/code-mode-host/tests/websocket.rs @@ -1,12 +1,27 @@ use std::process::Stdio; +use std::sync::Arc; use std::time::Duration; use anyhow::Context; use anyhow::Result; +use codex_code_mode::CellId; +use codex_code_mode::CodeModeNestedToolCall; +use codex_code_mode::CodeModeSessionDelegate; +use codex_code_mode::CodeModeSessionProvider; +use codex_code_mode::CodeModeToolKind; +use codex_code_mode::ExecuteRequest; +use codex_code_mode::FunctionCallOutputContentItem; +use codex_code_mode::NoopCodeModeSessionDelegate; +use codex_code_mode::NotificationFuture; +use codex_code_mode::RuntimeResponse; +use codex_code_mode::ToolDefinition; +use codex_code_mode::ToolInvocationFuture; +use codex_code_mode::WebSocketCodeModeSessionProvider; use codex_code_mode_protocol::host::Capability; use codex_code_mode_protocol::host::CapabilitySet; use codex_code_mode_protocol::host::ClientHello; use codex_code_mode_protocol::host::ClientToHost; +use codex_code_mode_protocol::host::DUAL_WEBSOCKET_CAPABILITY; use codex_code_mode_protocol::host::DelegateRequest; use codex_code_mode_protocol::host::DelegateResponse; use codex_code_mode_protocol::host::EncodedFrame; @@ -26,6 +41,8 @@ use codex_code_mode_protocol::host::WireRuntimeResponse; use codex_code_mode_protocol::host::WireToolDefinition; use codex_code_mode_protocol::host::WireToolKind; use codex_code_mode_protocol::host::WireToolName; +use codex_code_mode_protocol::host::WireWaitRequest; +use codex_protocol::ToolName; use futures::SinkExt; use futures::StreamExt; use pretty_assertions::assert_eq; @@ -37,6 +54,7 @@ use tokio::io::BufReader; use tokio::net::TcpStream; use tokio::process::Child; use tokio::process::Command; +use tokio::sync::Semaphore; use tokio::time::timeout; use tokio_tungstenite::MaybeTlsStream; use tokio_tungstenite::WebSocketStream; @@ -49,6 +67,8 @@ use tokio_tungstenite::tungstenite::http::HeaderValue; use tokio_tungstenite::tungstenite::http::StatusCode; use tokio_tungstenite::tungstenite::http::header::ORIGIN; use tokio_tungstenite::tungstenite::protocol::WebSocketConfig; +use tokio_util::sync::CancellationToken; +use uuid::Uuid; const TEST_TIMEOUT: Duration = Duration::from_secs(10); const MAX_WEBSOCKET_FRAME_BYTES: usize = MAX_FRAME_BYTES + std::mem::size_of::(); @@ -62,6 +82,43 @@ struct HostClient { websocket: WebSocketStream>, } +struct LargeToolResultDelegate { + started: Semaphore, + release: Semaphore, +} + +impl CodeModeSessionDelegate for LargeToolResultDelegate { + fn invoke_tool<'a>( + &'a self, + invocation: CodeModeNestedToolCall, + _cancellation_token: CancellationToken, + ) -> ToolInvocationFuture<'a> { + Box::pin(async move { + assert_eq!(invocation.tool_name, ToolName::plain("large")); + self.started.add_permits(1); + let permit = self + .release + .acquire() + .await + .map_err(|_| "large tool release closed".to_string())?; + permit.forget(); + Ok(json!({ "value": "x".repeat(8 * 1024 * 1024) })) + }) + } + + fn notify<'a>( + &'a self, + _call_id: String, + _cell_id: CellId, + _text: String, + _cancellation_token: CancellationToken, + ) -> NotificationFuture<'a> { + Box::pin(async { Ok(()) }) + } + + fn cell_closed(&self, _cell_id: &CellId) {} +} + impl HostHarness { async fn start() -> Result { let host_program = codex_utils_cargo_bin::cargo_bin("codex-code-mode-host")?; @@ -166,6 +223,34 @@ impl HostClient { Ok(()) } + async fn negotiate_dual(&mut self, websocket_url: &str) -> Result { + let capability = Capability::new(DUAL_WEBSOCKET_CAPABILITY)?; + let hello = ClientHello::new( + SupportedProtocolVersions::try_new([ProtocolVersion::V1])?, + CapabilitySet::empty(), + CapabilitySet::try_new([capability.clone()])?, + )?; + self.send(&ClientToHost::ClientHello(hello)).await?; + let HostToClient::HostHello(hello) = self.read().await? else { + anyhow::bail!("expected code-mode host hello"); + }; + assert!(hello.capabilities().contains(&capability)); + let token = hello + .bulk_connection_token() + .context("dual websocket handshake omitted its pairing token")?; + let bulk_url = format!("{}/bulk/{token}", websocket_url.trim_end_matches('/')); + let config = WebSocketConfig::default() + .max_frame_size(Some(MAX_WEBSOCKET_FRAME_BYTES)) + .max_message_size(Some(MAX_WEBSOCKET_FRAME_BYTES)); + let (websocket, _) = timeout( + TEST_TIMEOUT, + connect_async_with_config(bulk_url, Some(config), /*disable_nagle*/ false), + ) + .await + .context("timed out connecting to code-mode host bulk websocket")??; + Ok(HostClient { websocket }) + } + async fn open_session(&mut self, session_id: SessionId) -> Result<()> { let id = RequestId::new(/*value*/ 1); self.send(&ClientToHost::Request { @@ -317,6 +402,418 @@ async fn websocket_listener_executes_cells_and_forwards_tool_callbacks() -> Resu Ok(()) } +#[tokio::test] +async fn production_websocket_client_runs_nested_tools_while_other_sessions_progress() -> Result<()> +{ + let host = HostHarness::start().await?; + let provider = WebSocketCodeModeSessionProvider::new(host.websocket_url.clone()); + let delegate = Arc::new(LargeToolResultDelegate { + started: Semaphore::new(/*permits*/ 0), + release: Semaphore::new(/*permits*/ 0), + }); + let slow_session = provider + .create_session(delegate.clone()) + .await + .map_err(anyhow::Error::msg)?; + let fast_session = provider + .create_session(Arc::new(NoopCodeModeSessionDelegate)) + .await + .map_err(anyhow::Error::msg)?; + + let slow_cell = slow_session + .execute(ExecuteRequest { + tool_call_id: "large-tool".to_string(), + enabled_tools: vec![ToolDefinition { + name: "large".to_string(), + tool_name: ToolName::plain("large"), + description: String::new(), + kind: CodeModeToolKind::Function, + input_schema: None, + output_schema: None, + }], + source: r#"const result = await tools.large({ value: "request" }); text(String(result.value.length));"# + .to_string(), + yield_time_ms: Some(20_000), + max_output_tokens: Some(1_000), + }) + .await + .map_err(anyhow::Error::msg)?; + let started = timeout(TEST_TIMEOUT, delegate.started.acquire()) + .await + .context("large tool callback did not start")??; + started.forget(); + + let fast_response = timeout(TEST_TIMEOUT, async { + fast_session + .execute(ExecuteRequest { + tool_call_id: "fast-before-transfer".to_string(), + enabled_tools: Vec::new(), + source: r#"text("fast-before");"#.to_string(), + yield_time_ms: Some(5_000), + max_output_tokens: Some(1_000), + }) + .await + .map_err(anyhow::Error::msg)? + .initial_response() + .await + .map_err(anyhow::Error::msg) + }) + .await + .context("unrelated execution was blocked by the pending tool callback")??; + assert_eq!( + fast_response, + RuntimeResponse::Result { + cell_id: CellId::new("1".to_string()), + content_items: vec![FunctionCallOutputContentItem::InputText { + text: "fast-before".to_string(), + }], + error_text: None, + } + ); + + delegate.release.add_permits(1); + let slow_response = async { + timeout(TEST_TIMEOUT, slow_cell.initial_response()) + .await + .context("large tool result did not finish")? + .map_err(anyhow::Error::msg) + }; + let concurrent_response = async { + timeout(TEST_TIMEOUT, async { + fast_session + .execute(ExecuteRequest { + tool_call_id: "fast-during-transfer".to_string(), + enabled_tools: Vec::new(), + source: r#"text("fast-during");"#.to_string(), + yield_time_ms: Some(5_000), + max_output_tokens: Some(1_000), + }) + .await + .map_err(anyhow::Error::msg)? + .initial_response() + .await + .map_err(anyhow::Error::msg) + }) + .await + .context("unrelated execution was blocked by the large tool transfer")? + }; + tokio::pin!(slow_response); + tokio::pin!(concurrent_response); + let concurrent_response = tokio::select! { + response = &mut concurrent_response => response?, + response = &mut slow_response => { + response?; + anyhow::bail!("large tool result completed before the unrelated control response"); + } + }; + let slow_response = slow_response.await?; + assert_eq!( + slow_response, + RuntimeResponse::Result { + cell_id: CellId::new("1".to_string()), + content_items: vec![FunctionCallOutputContentItem::InputText { + text: "8388608".to_string(), + }], + error_text: None, + } + ); + assert_eq!( + concurrent_response, + RuntimeResponse::Result { + cell_id: CellId::new("2".to_string()), + content_items: vec![FunctionCallOutputContentItem::InputText { + text: "fast-during".to_string(), + }], + error_text: None, + } + ); + + slow_session.shutdown().await.map_err(anyhow::Error::msg)?; + fast_session.shutdown().await.map_err(anyhow::Error::msg)?; + Ok(()) +} + +#[tokio::test] +async fn websocket_dual_connections_route_notifications_and_tool_callbacks_to_separate_lanes() +-> Result<()> { + let host = HostHarness::start().await?; + let mut control = host.connect().await?; + let mut bulk = control.negotiate_dual(&host.websocket_url).await?; + let session_id = SessionId::new("dual-websocket-session")?; + control.open_session(session_id.clone()).await?; + + let execute_id = RequestId::new(/*value*/ 2); + control + .send(&ClientToHost::Request { + id: execute_id, + request: HostRequest::Execute { + session_id: session_id.clone(), + request: WireExecuteRequest { + tool_call_id: "dual-websocket-call".to_string(), + enabled_tools: vec![WireToolDefinition { + name: "echo".to_string(), + tool_name: WireToolName { + name: "echo".to_string(), + namespace: None, + }, + description: String::new(), + kind: WireToolKind::Function, + input_schema: None, + output_schema: None, + }], + source: r#"notify("important"); const result = await tools.echo({ value: "ping" }); text(result.value);"# + .to_string(), + yield_time_ms: Some(5_000), + max_output_tokens: Some(1_000), + }, + }, + }) + .await?; + + let started = control.read().await?; + let HostToClient::Response { + id, + result: + WireResult::Ok { + value: HostResponse::ExecutionStarted { cell_id }, + }, + } = started + else { + anyhow::bail!("expected execution-started response on control lane, got {started:?}"); + }; + assert_eq!(id, execute_id); + + let notification = control.read().await?; + let HostToClient::DelegateRequest { + id: notification_id, + .. + } = ¬ification + else { + anyhow::bail!("expected notification on control lane, got {notification:?}"); + }; + let notification_id = *notification_id; + assert_eq!( + notification, + HostToClient::DelegateRequest { + id: notification_id, + session_id: session_id.clone(), + request: DelegateRequest::Notify { + call_id: "dual-websocket-call".to_string(), + cell_id: cell_id.clone(), + text: "important".to_string(), + }, + } + ); + control + .send(&ClientToHost::DelegateResponse { + id: notification_id, + result: WireResult::Ok { + value: DelegateResponse::NotificationDelivered, + }, + }) + .await?; + + let callback = bulk.read().await?; + let HostToClient::DelegateRequest { + id: delegate_id, + session_id: callback_session_id, + request: DelegateRequest::InvokeTool { invocation }, + } = callback + else { + anyhow::bail!("expected tool callback on bulk lane, got {callback:?}"); + }; + assert_eq!(callback_session_id, session_id); + assert_eq!(invocation.input, Some(json!({ "value": "ping" }))); + + bulk.send(&ClientToHost::DelegateResponse { + id: delegate_id, + result: WireResult::Ok { + value: DelegateResponse::ToolResult { + result: json!({ "value": "pong" }), + }, + }, + }) + .await?; + + assert_eq!( + control.read().await?, + HostToClient::InitialResponse { + id: execute_id, + result: WireResult::Ok { + value: WireRuntimeResponse::Result { + cell_id: cell_id.clone(), + content_items: vec![WireContentItem::InputText { + text: "pong".to_string(), + }], + error_text: None, + }, + }, + } + ); + + assert_eq!( + control.read().await?, + HostToClient::CellClosed { + session_id: session_id.clone(), + cell_id, + } + ); + + let shutdown_id = RequestId::new(/*value*/ 3); + control + .send(&ClientToHost::Request { + id: shutdown_id, + request: HostRequest::ShutdownSession { + session_id: session_id.clone(), + }, + }) + .await?; + assert_eq!( + control.read().await?, + HostToClient::Response { + id: shutdown_id, + result: WireResult::Ok { + value: HostResponse::SessionClosed { session_id }, + }, + } + ); + Ok(()) +} + +#[tokio::test] +async fn websocket_control_operations_bypass_an_incomplete_bulk_frame() -> Result<()> { + let host = HostHarness::start().await?; + let mut control = host.connect().await?; + let mut bulk = control.negotiate_dual(&host.websocket_url).await?; + let session_id = SessionId::new("bulk-priority-session")?; + control.open_session(session_id.clone()).await?; + + let bulk_message = ClientToHost::DelegateResponse { + id: codex_code_mode_protocol::host::DelegateRequestId::new(/*value*/ 999), + result: WireResult::Ok { + value: DelegateResponse::ToolResult { + result: json!({ "image": "x".repeat(1024 * 1024) }), + }, + }, + }; + let payload = EncodedFrame::encode(&bulk_message)?.into_framed_bytes(); + let mask = [0x13_u8, 0x37, 0xc0, 0xde]; + let mut partial_frame = vec![0x82_u8, 0xff]; + partial_frame.extend_from_slice(&(payload.len() as u64).to_be_bytes()); + partial_frame.extend_from_slice(&mask); + partial_frame.extend( + payload[..64 * 1024] + .iter() + .enumerate() + .map(|(index, byte)| byte ^ mask[index % mask.len()]), + ); + bulk.websocket + .get_mut() + .write_all(&partial_frame) + .await + .context("failed to start the intentionally incomplete bulk frame")?; + + let request_id = RequestId::new(/*value*/ 42); + control + .send(&ClientToHost::Request { + id: request_id, + request: HostRequest::Wait { + session_id: session_id.clone(), + request: WireWaitRequest { + cell_id: codex_code_mode_protocol::host::WireCellId::new("missing-cell"), + yield_time_ms: 10, + }, + }, + }) + .await?; + let response = timeout(TEST_TIMEOUT, control.read()) + .await + .context("control wait was blocked behind the incomplete bulk frame")??; + assert!(matches!( + response, + HostToClient::Response { + id, + result: WireResult::Ok { + value: HostResponse::WaitCompleted { .. }, + }, + } if id == request_id + )); + + let execute_id = RequestId::new(/*value*/ 43); + control + .send(&ClientToHost::Request { + id: execute_id, + request: HostRequest::Execute { + session_id, + request: WireExecuteRequest { + tool_call_id: "control-execute".to_string(), + enabled_tools: Vec::new(), + source: r#"text("fast");"#.to_string(), + yield_time_ms: Some(5_000), + max_output_tokens: Some(1_000), + }, + }, + }) + .await?; + for _ in 0..2 { + let response = timeout(TEST_TIMEOUT, control.read()) + .await + .context("control execute was blocked behind the incomplete bulk frame")??; + assert!(matches!( + response, + HostToClient::Response { id, .. } | HostToClient::InitialResponse { id, .. } + if id == execute_id + )); + } + Ok(()) +} + +#[tokio::test] +async fn websocket_bulk_lane_rejects_control_messages() -> Result<()> { + let host = HostHarness::start().await?; + let mut control = host.connect().await?; + let mut bulk = control.negotiate_dual(&host.websocket_url).await?; + let session_id = SessionId::new("wrong-lane-session")?; + control.open_session(session_id.clone()).await?; + + bulk.send(&ClientToHost::Request { + id: RequestId::new(/*value*/ 42), + request: HostRequest::Wait { + session_id, + request: WireWaitRequest { + cell_id: codex_code_mode_protocol::host::WireCellId::new("missing-cell"), + yield_time_ms: 10, + }, + }, + }) + .await?; + + let result = timeout(TEST_TIMEOUT, control.websocket.next()) + .await + .context("wrong-lane control message did not disconnect the paired sockets")?; + assert!( + matches!(result, None | Some(Ok(Message::Close(_))) | Some(Err(_))), + "wrong-lane message unexpectedly returned {result:?}" + ); + Ok(()) +} + +#[tokio::test] +async fn websocket_bulk_pairing_rejects_unknown_tokens() -> Result<()> { + let host = HostHarness::start().await?; + let unknown_token = Uuid::new_v4(); + let url = format!("{}/bulk/{unknown_token}", host.websocket_url); + let error = match connect_async(url).await { + Ok(_) => anyhow::bail!("unknown bulk pairing token should be rejected"), + Err(error) => error, + }; + let WebSocketError::Http(response) = error else { + anyhow::bail!("bulk pairing failed unexpectedly: {error}"); + }; + assert_eq!(response.status(), StatusCode::NOT_FOUND); + Ok(()) +} + #[tokio::test] async fn websocket_listener_accepts_frames_larger_than_default_websocket_limit() -> Result<()> { let host = HostHarness::start().await?; diff --git a/codex-rs/code-mode-protocol/src/host/host_tests.rs b/codex-rs/code-mode-protocol/src/host/host_tests.rs index dde38e87c9..240028a570 100644 --- a/codex-rs/code-mode-protocol/src/host/host_tests.rs +++ b/codex-rs/code-mode-protocol/src/host/host_tests.rs @@ -22,6 +22,7 @@ use super::ProtocolVersion; use super::RequestId; use super::SessionId; use super::SupportedProtocolVersions; +use super::TransportLane; use super::WireCellId; use super::WireContentItem; use super::WireExecuteRequest; @@ -72,6 +73,130 @@ where ); } +#[test] +fn dual_websocket_hello_preserves_the_pairing_token() { + assert_wire_round_trip( + HostToClient::HostHello( + HostHello::new( + ProtocolVersion::V1, + CapabilitySet::try_new([capability("dual-websocket-v1")]) + .expect("valid capabilities"), + ) + .with_bulk_connection_token("pairing-token".to_string()), + ), + json!({ + "type": "connection/ready", + "selectedVersion": 1, + "capabilities": ["dual-websocket-v1"], + "bulkConnectionToken": "pairing-token", + }), + ); +} + +#[test] +fn message_families_use_dedicated_transport_lanes() { + for (message, lane) in [ + ( + ClientToHost::CancelRequest { + id: request_id(/*value*/ 1), + }, + TransportLane::Control, + ), + ( + ClientToHost::DelegateResponse { + id: delegate_request_id(/*value*/ 1), + result: WireResult::Ok { + value: DelegateResponse::NotificationDelivered, + }, + }, + TransportLane::Control, + ), + ( + ClientToHost::DelegateResponse { + id: delegate_request_id(/*value*/ 2), + result: WireResult::Ok { + value: DelegateResponse::ToolResult { + result: json!({ "value": "tool result" }), + }, + }, + }, + TransportLane::Bulk, + ), + ( + ClientToHost::DelegateResponse { + id: delegate_request_id(/*value*/ 3), + result: WireResult::Err { + message: "delegate failed".to_string(), + }, + }, + TransportLane::Bulk, + ), + ] { + assert_eq!(message.transport_lane(), lane); + assert!(message.allows_transport_lane(lane)); + assert!(!message.allows_transport_lane(match lane { + TransportLane::Control => TransportLane::Bulk, + TransportLane::Bulk => TransportLane::Control, + })); + } + + for (message, lane) in [ + ( + HostToClient::Response { + id: request_id(/*value*/ 1), + result: WireResult::Err { + message: "x".repeat(128 * 1024), + }, + }, + TransportLane::Control, + ), + ( + HostToClient::DelegateRequest { + id: delegate_request_id(/*value*/ 1), + session_id: session_id(), + request: DelegateRequest::Notify { + call_id: "call-1".to_string(), + cell_id: cell_id("cell-1"), + text: "important".to_string(), + }, + }, + TransportLane::Control, + ), + ( + HostToClient::DelegateRequest { + id: delegate_request_id(/*value*/ 2), + session_id: session_id(), + request: DelegateRequest::InvokeTool { + invocation: WireNestedToolCall { + cell_id: cell_id("cell-1"), + runtime_tool_call_id: "runtime-call-1".to_string(), + tool_name: WireToolName { + name: "tool".to_string(), + namespace: None, + }, + tool_kind: WireToolKind::Function, + input: None, + }, + }, + }, + TransportLane::Bulk, + ), + ( + HostToClient::CancelDelegateRequest { + id: delegate_request_id(/*value*/ 1), + }, + TransportLane::Bulk, + ), + ] { + assert_eq!(message.transport_lane(), lane); + assert!(message.allows_transport_lane(lane)); + assert!(!message.allows_transport_lane(match lane { + TransportLane::Control => TransportLane::Bulk, + TransportLane::Bulk => TransportLane::Control, + })); + } +} + fn execute_request() -> WireExecuteRequest { WireExecuteRequest { tool_call_id: "call-1".to_string(), diff --git a/codex-rs/code-mode-protocol/src/host/message.rs b/codex-rs/code-mode-protocol/src/host/message.rs index 0e83c866e4..e6918ea4fd 100644 --- a/codex-rs/code-mode-protocol/src/host/message.rs +++ b/codex-rs/code-mode-protocol/src/host/message.rs @@ -12,6 +12,7 @@ use super::ProtocolVersion; use super::RequestId; use super::SessionId; use super::SupportedProtocolVersions; +use super::TransportLane; use super::WireCellId; use super::WireExecuteRequest; use super::WireNestedToolCall; @@ -105,6 +106,8 @@ impl std::error::Error for ClientHelloError {} pub struct HostHello { selected_version: ProtocolVersion, capabilities: CapabilitySet, + #[serde(default, skip_serializing_if = "Option::is_none")] + bulk_connection_token: Option, } impl HostHello { @@ -112,9 +115,15 @@ impl HostHello { Self { selected_version, capabilities, + bulk_connection_token: None, } } + pub fn with_bulk_connection_token(mut self, token: String) -> Self { + self.bulk_connection_token = Some(token); + self + } + pub fn selected_version(&self) -> ProtocolVersion { self.selected_version } @@ -122,6 +131,10 @@ impl HostHello { pub fn capabilities(&self) -> &CapabilitySet { &self.capabilities } + + pub fn bulk_connection_token(&self) -> Option<&str> { + self.bulk_connection_token.as_deref() + } } /// Messages sent from a client to the code-mode host. @@ -141,6 +154,30 @@ pub enum ClientToHost { }, } +impl ClientToHost { + /// Keeps notification acknowledgments with control traffic and tool results on the bulk lane. + pub fn transport_lane(&self) -> TransportLane { + match self { + Self::DelegateResponse { + result: + WireResult::Ok { + value: DelegateResponse::NotificationDelivered, + }, + .. + } + | Self::ClientHello(_) + | Self::Request { .. } + | Self::CancelRequest { .. } => TransportLane::Control, + Self::DelegateResponse { .. } => TransportLane::Bulk, + } + } + + /// Validates the message families accepted by each paired socket. + pub fn allows_transport_lane(&self, lane: TransportLane) -> bool { + self.transport_lane() == lane + } +} + /// Messages sent from the code-mode host to a client. #[derive(Debug, Deserialize, PartialEq, Serialize)] #[serde(deny_unknown_fields, tag = "type", rename_all_fields = "camelCase")] @@ -174,6 +211,33 @@ pub enum HostToClient { }, } +impl HostToClient { + /// Keeps notifications with control traffic and nested-tool callbacks on the bulk lane. + pub fn transport_lane(&self) -> TransportLane { + match self { + Self::DelegateRequest { + request: DelegateRequest::InvokeTool { .. }, + .. + } + | Self::CancelDelegateRequest { .. } => TransportLane::Bulk, + Self::DelegateRequest { + request: DelegateRequest::Notify { .. }, + .. + } + | Self::HostHello(_) + | Self::HandshakeRejected { .. } + | Self::Response { .. } + | Self::InitialResponse { .. } + | Self::CellClosed { .. } => TransportLane::Control, + } + } + + /// Rejects messages received on the wrong paired socket. + pub fn allows_transport_lane(&self, lane: TransportLane) -> bool { + self.transport_lane() == lane + } +} + #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] #[serde(deny_unknown_fields, tag = "method", rename_all_fields = "camelCase")] pub enum HostRequest { diff --git a/codex-rs/code-mode-protocol/src/host/mod.rs b/codex-rs/code-mode-protocol/src/host/mod.rs index 5c81b1d4a0..6169c61a17 100644 --- a/codex-rs/code-mode-protocol/src/host/mod.rs +++ b/codex-rs/code-mode-protocol/src/host/mod.rs @@ -1,9 +1,8 @@ //! Messages and framing for the code-mode host boundary. //! //! Protocol version 1 multiplexes session operations and delegate callbacks by -//! request ID over one ordered connection. It defines no optional capabilities -//! yet; capability names provide an extension point for later versions without -//! weakening the v1 decoder. +//! request ID over one ordered connection. WebSocket peers can negotiate a +//! separate bulk connection without changing the existing inner messages. mod codec; mod error; @@ -14,6 +13,16 @@ mod types; /// Maximum number of unresolved delegate callbacks allowed per host connection. pub const MAX_PENDING_DELEGATE_CALLS: usize = 1_024; +/// Optional second WebSocket carrying delegate callbacks and their responses. +pub const DUAL_WEBSOCKET_CAPABILITY: &str = "dual-websocket-v1"; + +/// Selects one socket of a negotiated dual-WebSocket connection. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum TransportLane { + Control, + Bulk, +} + pub use codec::EncodedFrame; pub use codec::FramedReader; pub use codec::FramedWriter; diff --git a/codex-rs/code-mode/src/remote_session/connection.rs b/codex-rs/code-mode/src/remote_session/connection.rs index 033d24ab57..d08c8fbbc4 100644 --- a/codex-rs/code-mode/src/remote_session/connection.rs +++ b/codex-rs/code-mode/src/remote_session/connection.rs @@ -1,4 +1,5 @@ use std::fmt; +use std::future::Future; use std::io; use std::path::Path; use std::path::PathBuf; @@ -14,17 +15,21 @@ use codex_code_mode_protocol::ExecuteRequest; use codex_code_mode_protocol::StartedCell; use codex_code_mode_protocol::WaitOutcome; use codex_code_mode_protocol::WaitRequest; +use codex_code_mode_protocol::host::Capability; use codex_code_mode_protocol::host::CapabilitySet; use codex_code_mode_protocol::host::ClientHello; use codex_code_mode_protocol::host::ClientToHost; +use codex_code_mode_protocol::host::DUAL_WEBSOCKET_CAPABILITY; use codex_code_mode_protocol::host::EncodedFrame; use codex_code_mode_protocol::host::FramedReader; use codex_code_mode_protocol::host::FramedWriter; use codex_code_mode_protocol::host::HostToClient; use codex_code_mode_protocol::host::MAX_FRAME_BYTES; +use codex_code_mode_protocol::host::MAX_PENDING_DELEGATE_CALLS; use codex_code_mode_protocol::host::ProtocolVersion; use codex_code_mode_protocol::host::RequestId; use codex_code_mode_protocol::host::SupportedProtocolVersions; +use codex_code_mode_protocol::host::TransportLane; use codex_http_client::HttpClientFactory; use codex_websocket_client::WebSocketConnector; use futures::StreamExt; @@ -36,6 +41,7 @@ use tokio::sync::mpsc; use tokio::sync::oneshot; use tokio::task::JoinHandle; use tokio_tungstenite::tungstenite::client::IntoClientRequest; +use tokio_tungstenite::tungstenite::http::Uri; use tokio_tungstenite::tungstenite::protocol::WebSocketConfig; use tokio_util::sync::CancellationToken; use tracing::debug; @@ -57,6 +63,8 @@ mod transport; const IPC_CHANNEL_CAPACITY: usize = 128; const HOST_HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(10); +// TODO(anp) make this timeout configurable if 60 seconds is insufficient. +const DEFAULT_HOST_WAIT_TRANSPORT_TIMEOUT: Duration = Duration::from_secs(60); const MAX_WEBSOCKET_FRAME_BYTES: usize = MAX_FRAME_BYTES + std::mem::size_of::(); // Host spawn errors become model-visible tool output. Bound configured paths // while preserving the executable-bearing suffix needed to diagnose failures. @@ -68,6 +76,7 @@ pub(super) enum ConnectionError { host_program: PathBuf, error: io::Error, }, + BulkConnectionUnavailable(String), Other(String), } @@ -98,7 +107,9 @@ impl fmt::Display for ConnectionError { &host_program[suffix_start..] ) } - Self::Other(message) => formatter.write_str(message), + Self::BulkConnectionUnavailable(message) | Self::Other(message) => { + formatter.write_str(message) + } } } } @@ -132,6 +143,45 @@ enum ConnectionOwner { WebSocket, } +struct BulkConnectionOptions { + websocket_url: String, + http_client_factory: HttpClientFactory, +} + +impl BulkConnectionOptions { + async fn connect( + &self, + token: &str, + ) -> Result<(ConnectionReader, ConnectionWriter), ConnectionError> { + let control_uri = self.websocket_url.parse::().map_err(|error| { + ConnectionError::Other(format!( + "failed to build code-mode host bulk websocket URL: {error}" + )) + })?; + let bulk_path = format!("{}/bulk/{token}", control_uri.path().trim_end_matches('/')); + let bulk_path_and_query = match control_uri.query() { + Some(query) => format!("{bulk_path}?{query}"), + None => bulk_path, + }; + let mut bulk_uri_parts = control_uri.into_parts(); + bulk_uri_parts.path_and_query = Some(bulk_path_and_query.parse().map_err(|error| { + ConnectionError::Other(format!( + "failed to build code-mode host bulk websocket path: {error}" + )) + })?); + let bulk_url = Uri::from_parts(bulk_uri_parts) + .map_err(|error| { + ConnectionError::Other(format!( + "failed to build code-mode host bulk websocket URL: {error}" + )) + })? + .to_string(); + connect_websocket_transport(&bulk_url, &self.http_client_factory) + .await + .map_err(|error| ConnectionError::BulkConnectionUnavailable(error.to_string())) + } +} + impl CallerCancellation { fn new() -> Self { Self { @@ -202,6 +252,7 @@ impl Connection { ConnectionReader::Stdio(FramedReader::new(stdout)), ConnectionWriter::Stdio(FramedWriter::new(stdin)), ConnectionOwner::Process(Box::new(child)), + /*bulk_connection_options*/ None, ) .await } @@ -210,38 +261,17 @@ impl Connection { websocket_url: &str, http_client_factory: &HttpClientFactory, ) -> Result { - let request = websocket_url.into_client_request().map_err(|error| { - ConnectionError::Other(format!( - "failed to build code-mode host websocket request: {error}" - )) - })?; - let connector = WebSocketConnector::new(http_client_factory).map_err(|error| { - ConnectionError::Other(format!( - "failed to configure code-mode host websocket TLS: {error}" - )) - })?; - let websocket_config = WebSocketConfig::default() - .max_frame_size(Some(MAX_WEBSOCKET_FRAME_BYTES)) - .max_message_size(Some(MAX_WEBSOCKET_FRAME_BYTES)); - let (websocket, _) = tokio::time::timeout( - HOST_HANDSHAKE_TIMEOUT, - connector.connect(request, websocket_config), - ) - .await - .map_err(|_| { - ConnectionError::Other("timed out connecting to the code-mode host websocket".into()) - })? - .map_err(|error| { - ConnectionError::Other(format!( - "failed to connect to the code-mode host websocket: {error}" - )) - })?; - let (writer, reader) = websocket.split(); + let (reader, writer) = + connect_websocket_transport(websocket_url, http_client_factory).await?; Self::establish( - ConnectionReader::WebSocket(reader), - ConnectionWriter::WebSocket(writer), + reader, + writer, ConnectionOwner::WebSocket, + Some(BulkConnectionOptions { + websocket_url: websocket_url.to_string(), + http_client_factory: http_client_factory.clone(), + }), ) .await } @@ -250,13 +280,22 @@ impl Connection { mut reader: ConnectionReader, mut writer: ConnectionWriter, mut owner: ConnectionOwner, + bulk_connection_options: Option, ) -> Result { let handshake = async { + let dual_capability = + Capability::new(DUAL_WEBSOCKET_CAPABILITY).map_err(|error| error.to_string())?; + let optional_capabilities = if bulk_connection_options.is_some() { + CapabilitySet::try_new([dual_capability.clone()]) + .map_err(|error| error.to_string())? + } else { + CapabilitySet::empty() + }; let hello = ClientHello::new( SupportedProtocolVersions::try_new([ProtocolVersion::V1]) .map_err(|err| err.to_string())?, CapabilitySet::empty(), - CapabilitySet::empty(), + optional_capabilities, ) .map_err(|err| err.to_string())?; writer @@ -271,7 +310,20 @@ impl Connection { Some(HostToClient::HostHello(hello)) if hello.selected_version() == ProtocolVersion::V1 => { - Ok(()) + if hello.capabilities().contains(&dual_capability) { + hello + .bulk_connection_token() + .map(str::to_string) + .ok_or_else(|| { + "code-mode host advertised dual websockets without a pairing token" + .to_string() + }) + .map(Some) + } else if hello.bulk_connection_token().is_some() { + Err("code-mode host returned an unexpected bulk pairing token".to_string()) + } else { + Ok(None) + } } Some(HostToClient::HandshakeRejected { reason }) => { Err(format!("code-mode host rejected the handshake: {reason:?}")) @@ -292,51 +344,85 @@ impl Connection { )); } }; - if let Err(err) = handshake_result { - let _ = writer.close().await; - owner.close().await; - return Err(ConnectionError::Other(err)); - } + let bulk_token = match handshake_result { + Ok(token) => token, + Err(err) => { + let _ = writer.close().await; + owner.close().await; + return Err(ConnectionError::Other(err)); + } + }; + let (bulk_reader, bulk_writer) = if let Some(token) = bulk_token { + let Some(options) = bulk_connection_options else { + let _ = writer.close().await; + owner.close().await; + return Err(ConnectionError::Other( + "code-mode host negotiated an unsupported bulk websocket".to_string(), + )); + }; + match options.connect(&token).await { + Ok((reader, writer)) => (Some(reader), Some(writer)), + Err(error) => { + let _ = writer.close().await; + owner.close().await; + return Err(error); + } + } + } else { + (None, None) + }; let (command_tx, command_rx) = mpsc::channel(IPC_CHANNEL_CAPACITY); let (event_tx, event_rx) = mpsc::channel(IPC_CHANNEL_CAPACITY); - let (outgoing_tx, mut outgoing_rx) = mpsc::channel::(IPC_CHANNEL_CAPACITY); + let (outgoing_tx, outgoing_rx) = mpsc::channel::(IPC_CHANNEL_CAPACITY); + let (bulk_tx, bulk_rx) = if bulk_writer.is_some() { + let (sender, receiver) = mpsc::channel::(MAX_PENDING_DELEGATE_CALLS); + (Some(sender), Some(receiver)) + } else { + (None, None) + }; + let dual_websocket = bulk_writer.is_some(); let cancellation = CancellationToken::new(); let alive = Arc::new(AtomicBool::new(true)); let failure = Arc::new(std::sync::Mutex::new(None)); let writer_cancellation = cancellation.clone(); let writer_task = tokio::spawn(async move { - loop { - tokio::select! { - _ = writer_cancellation.cancelled() => { - return writer - .close() - .await - .map_err(|error| format!("failed to close code-mode host connection: {error}")); - } - frame = outgoing_rx.recv() => { - let Some(frame) = frame else { - return Err("code-mode host outgoing stream closed".to_string()); - }; - let result = tokio::select! { - _ = writer_cancellation.cancelled() => return Ok(()), - result = writer.write_frame(frame) => result, - }; - if let Err(err) = result { - return Err(format!("failed to write code-mode host message: {err}")); - } - } - } + if let (Some(bulk_writer), Some(bulk_rx)) = (bulk_writer, bulk_rx) { + tokio::try_join!( + drive_writer(writer, outgoing_rx, writer_cancellation.clone()), + drive_writer(bulk_writer, bulk_rx, writer_cancellation) + )?; + Ok(()) + } else { + drive_writer(writer, outgoing_rx, writer_cancellation).await } }); let reader_events = event_tx.clone(); let reader_cancellation = cancellation.clone(); - let reader_task = - tokio::spawn( - async move { drive_reader(reader, reader_events, reader_cancellation).await }, - ); + let reader_task = tokio::spawn(async move { + let lane = dual_websocket.then_some(TransportLane::Control); + if let Some(bulk_reader) = bulk_reader { + tokio::try_join!( + drive_reader( + reader, + reader_events.clone(), + reader_cancellation.clone(), + lane, + ), + drive_reader( + bulk_reader, + reader_events, + reader_cancellation, + Some(TransportLane::Bulk), + ) + )?; + Ok(()) + } else { + drive_reader(reader, reader_events, reader_cancellation, lane).await + } + }); let (driver, execute_claim_tx) = ConnectionDriver::new( command_rx, @@ -349,6 +435,10 @@ impl Connection { cancellation: cancellation.clone(), }, ); + let driver = match bulk_tx { + Some(sender) => driver.with_bulk_sender(sender), + None => driver, + }; let driver_task = tokio::spawn(driver.run()); tokio::spawn( ConnectionSupervisor { @@ -439,16 +529,23 @@ impl Connection { session: RemoteSession, request: WaitRequest, ) -> Result { + // Account for the runtime's one-second yield grace separately from transport. + let runtime_timeout = + Duration::from_millis(request.yield_time_ms).saturating_add(Duration::from_secs(1)); let cancellation = CallerCancellation::new(); let (response_tx, response_rx) = oneshot::channel(); - self.send(DriverCommand::Wait { - session, - request, - caller_cancellation: cancellation.token(), - response_tx, - }) - .await?; - let result = self.receive(response_rx).await; + let result = self + .with_transport_deadline(runtime_timeout, "wait", async { + self.send(DriverCommand::Wait { + session, + request, + caller_cancellation: cancellation.token(), + response_tx, + }) + .await?; + self.receive(response_rx).await + }) + .await; cancellation.disarm(); result } @@ -459,13 +556,16 @@ impl Connection { cell_id: CellId, ) -> Result { let (response_tx, response_rx) = oneshot::channel(); - self.send(DriverCommand::Terminate { - session, - cell_id, - response_tx, + self.with_transport_deadline(Duration::ZERO, "terminate", async { + self.send(DriverCommand::Terminate { + session, + cell_id, + response_tx, + }) + .await?; + self.receive(response_rx).await }) - .await?; - self.receive(response_rx).await + .await } pub(super) async fn shutdown_session(&self, session: RemoteSession) -> Result<(), String> { @@ -478,6 +578,26 @@ impl Connection { self.receive(response_rx).await } + async fn with_transport_deadline( + &self, + runtime_timeout: Duration, + request_type: &str, + request: impl Future>, + ) -> Result { + let deadline = runtime_timeout.saturating_add(DEFAULT_HOST_WAIT_TRANSPORT_TIMEOUT); + match tokio::time::timeout(deadline, request).await { + Ok(result) => result, + Err(_) => { + warn!(request_type, "code-mode host request exceeded its deadline"); + let reason = + format!("code-mode host timed out waiting for {request_type} response"); + mark_connection_dead(&self.alive, &self.failure, reason.clone()); + self.cancellation.cancel(); + Err(reason) + } + } + } + async fn send(&self, command: DriverCommand) -> Result<(), String> { if !self.is_alive() { return Err(self.failure_message()); @@ -504,6 +624,73 @@ impl Connection { } } +async fn connect_websocket_transport( + websocket_url: &str, + http_client_factory: &HttpClientFactory, +) -> Result<(ConnectionReader, ConnectionWriter), ConnectionError> { + let request = websocket_url.into_client_request().map_err(|error| { + ConnectionError::Other(format!( + "failed to build code-mode host websocket request: {error}" + )) + })?; + let connector = WebSocketConnector::new(http_client_factory).map_err(|error| { + ConnectionError::Other(format!( + "failed to configure code-mode host websocket TLS: {error}" + )) + })?; + let websocket_config = WebSocketConfig::default() + .max_frame_size(Some(MAX_WEBSOCKET_FRAME_BYTES)) + .max_message_size(Some(MAX_WEBSOCKET_FRAME_BYTES)); + let (websocket, _) = tokio::time::timeout( + HOST_HANDSHAKE_TIMEOUT, + connector.connect(request, websocket_config), + ) + .await + .map_err(|_| { + ConnectionError::Other("timed out connecting to the code-mode host websocket".into()) + })? + .map_err(|error| { + ConnectionError::Other(format!( + "failed to connect to the code-mode host websocket: {error}" + )) + })?; + let (writer, reader) = websocket.split(); + Ok(( + ConnectionReader::WebSocket(reader), + ConnectionWriter::WebSocket(writer), + )) +} + +async fn drive_writer( + mut writer: ConnectionWriter, + mut outgoing: mpsc::Receiver, + cancellation: CancellationToken, +) -> Result<(), String> { + loop { + tokio::select! { + _ = cancellation.cancelled() => { + return writer + .close() + .await + .map_err(|error| format!("failed to close code-mode host connection: {error}")); + } + frame = outgoing.recv() => { + let Some(frame) = frame else { + return Err("code-mode host outgoing stream closed".to_string()); + }; + tokio::select! { + _ = cancellation.cancelled() => return Ok(()), + result = writer.write_frame(frame) => { + result.map_err(|error| { + format!("failed to write code-mode host message: {error}") + })?; + } + } + } + } + } +} + impl Drop for Connection { fn drop(&mut self) { mark_connection_dead( diff --git a/codex-rs/code-mode/src/remote_session/connection/driver.rs b/codex-rs/code-mode/src/remote_session/connection/driver.rs index 102ad47581..667a435805 100644 --- a/codex-rs/code-mode/src/remote_session/connection/driver.rs +++ b/codex-rs/code-mode/src/remote_session/connection/driver.rs @@ -1,3 +1,4 @@ +use std::collections::VecDeque; use std::panic::AssertUnwindSafe; use std::sync::Arc; use std::sync::atomic::AtomicBool; @@ -6,7 +7,9 @@ use std::sync::atomic::Ordering; use codex_code_mode_protocol::CellId; use codex_code_mode_protocol::CodeModeSessionDelegate; use codex_code_mode_protocol::host::EncodedFrame; +use codex_code_mode_protocol::host::HostToClient; use codex_code_mode_protocol::host::RequestId; +use codex_code_mode_protocol::host::TransportLane; use tokio::sync::mpsc; use tokio_util::sync::CancellationToken; @@ -39,7 +42,9 @@ pub(super) struct ConnectionDriver { event_tx: mpsc::Sender, execute_claim_rx: mpsc::UnboundedReceiver, outgoing_tx: mpsc::Sender, + bulk_tx: Option>, requests: RequestTracker, + deferred_host_messages: VecDeque, sessions: SessionRegistry, delegates: DelegateRuntime, alive: Arc, @@ -64,7 +69,9 @@ impl ConnectionDriver { event_tx: event_tx.clone(), execute_claim_rx, outgoing_tx, + bulk_tx: None, requests: RequestTracker::new(), + deferred_host_messages: VecDeque::new(), sessions: SessionRegistry::new(), delegates: DelegateRuntime::new(event_tx), alive: lifecycle.alive, @@ -130,8 +137,17 @@ impl ConnectionDriver { } } - fn queue_frame(&mut self, frame: EncodedFrame) -> bool { - match self.outgoing_tx.try_send(frame) { + pub(super) fn with_bulk_sender(mut self, sender: mpsc::Sender) -> Self { + self.bulk_tx = Some(sender); + self + } + + fn queue_frame(&mut self, frame: EncodedFrame, lane: TransportLane) -> bool { + let sender = match lane { + TransportLane::Control => &self.outgoing_tx, + TransportLane::Bulk => self.bulk_tx.as_ref().unwrap_or(&self.outgoing_tx), + }; + match sender.try_send(frame) { Ok(()) => true, Err(mpsc::error::TrySendError::Full(_)) => { self.fail("code-mode host outgoing queue is full".to_string()); diff --git a/codex-rs/code-mode/src/remote_session/connection/driver/commands.rs b/codex-rs/code-mode/src/remote_session/connection/driver/commands.rs index c9cba9c32d..4bfb713625 100644 --- a/codex-rs/code-mode/src/remote_session/connection/driver/commands.rs +++ b/codex-rs/code-mode/src/remote_session/connection/driver/commands.rs @@ -105,7 +105,8 @@ impl ConnectionDriver { }, &self.event_tx, ); - self.queue_frame(frame) + let lane = message.transport_lane(); + self.queue_frame(frame, lane) } fn execute( @@ -164,7 +165,8 @@ impl ConnectionDriver { }, &self.event_tx, ); - self.queue_frame(frame) + let lane = message.transport_lane(); + self.queue_frame(frame, lane) } fn wait( @@ -293,6 +295,7 @@ impl ConnectionDriver { }; self.requests .insert_pending(request_id, pending, &self.event_tx); - self.queue_frame(frame) + let lane = message.transport_lane(); + self.queue_frame(frame, lane) } } diff --git a/codex-rs/code-mode/src/remote_session/connection/driver/delegate_runtime.rs b/codex-rs/code-mode/src/remote_session/connection/driver/delegate_runtime.rs index 94be7b1b64..fb01f8783e 100644 --- a/codex-rs/code-mode/src/remote_session/connection/driver/delegate_runtime.rs +++ b/codex-rs/code-mode/src/remote_session/connection/driver/delegate_runtime.rs @@ -261,10 +261,7 @@ impl ConnectionDriver { }; let target = match self.sessions.delegate_target(&session_id, wire_cell_id) { Ok(target) => target, - Err(err) => { - self.fail(err); - return false; - } + Err(err) => return self.send_delegate_response(id, Err(err)), }; match self.delegates.start(id, target, request) { Ok(()) => true, @@ -321,7 +318,8 @@ impl ConnectionDriver { } } }; - self.queue_frame(frame) + let lane = message.transport_lane(); + self.queue_frame(frame, lane) } pub(super) fn close_cell(&mut self, session_id: SessionId, cell_id: WireCellId) -> bool { diff --git a/codex-rs/code-mode/src/remote_session/connection/driver/request_tracker.rs b/codex-rs/code-mode/src/remote_session/connection/driver/request_tracker.rs index 0e7fc0881d..d39981acd0 100644 --- a/codex-rs/code-mode/src/remote_session/connection/driver/request_tracker.rs +++ b/codex-rs/code-mode/src/remote_session/connection/driver/request_tracker.rs @@ -52,6 +52,15 @@ impl RequestTracker { }) } + pub(super) fn has_pending_execute_for_session(&self, session_id: &SessionId) -> bool { + self.pending.values().any(|request| { + matches!( + request, + PendingRequest::Execute { session, .. } if session.id == *session_id + ) + }) + } + pub(super) fn allocate_id(&mut self) -> Result { let id = self.next_request_id; self.next_request_id = self diff --git a/codex-rs/code-mode/src/remote_session/connection/driver/responses.rs b/codex-rs/code-mode/src/remote_session/connection/driver/responses.rs index 19c78ec000..17f697362e 100644 --- a/codex-rs/code-mode/src/remote_session/connection/driver/responses.rs +++ b/codex-rs/code-mode/src/remote_session/connection/driver/responses.rs @@ -56,6 +56,62 @@ impl ConnectionDriver { } pub(super) fn handle_host_message(&mut self, message: HostToClient) -> bool { + if self.should_defer_host_message(&message) { + if self.deferred_host_messages.len() + >= codex_code_mode_protocol::host::MAX_PENDING_DELEGATE_CALLS + { + self.fail( + "code-mode host exceeded deferred cross-socket message limit".to_string(), + ); + return false; + } + self.deferred_host_messages.push_back(message); + return true; + } + if !self.dispatch_host_message(message) { + return false; + } + for _ in 0..self.deferred_host_messages.len() { + let Some(message) = self.deferred_host_messages.pop_front() else { + break; + }; + if self.should_defer_host_message(&message) { + self.deferred_host_messages.push_back(message); + } else if !self.dispatch_host_message(message) { + return false; + } + } + true + } + + fn should_defer_host_message(&self, message: &HostToClient) -> bool { + match message { + HostToClient::DelegateRequest { + session_id, + request, + .. + } => { + let cell_id = match request { + codex_code_mode_protocol::host::DelegateRequest::InvokeTool { invocation } => { + &invocation.cell_id + } + codex_code_mode_protocol::host::DelegateRequest::Notify { cell_id, .. } => { + cell_id + } + }; + !self.sessions.contains_cell(session_id, cell_id) + && self.requests.has_pending_execute_for_session(session_id) + } + HostToClient::Response { .. } + | HostToClient::InitialResponse { .. } + | HostToClient::CellClosed { .. } + | HostToClient::CancelDelegateRequest { .. } + | HostToClient::HostHello(_) + | HostToClient::HandshakeRejected { .. } => false, + } + } + + fn dispatch_host_message(&mut self, message: HostToClient) -> bool { match message { HostToClient::Response { id, result } => { self.complete_request(id, result.into_result()) @@ -69,6 +125,12 @@ impl ConnectionDriver { request, } => self.start_delegate(id, session_id, request), HostToClient::CancelDelegateRequest { id } => { + self.deferred_host_messages.retain(|message| { + !matches!( + message, + HostToClient::DelegateRequest { id: deferred_id, .. } if *deferred_id == id + ) + }); self.delegates.cancel(id); true } @@ -302,7 +364,8 @@ impl ConnectionDriver { } fn send_cancel_request(&mut self, id: RequestId) -> bool { - let frame = match EncodedFrame::encode(&ClientToHost::CancelRequest { id }) { + let message = ClientToHost::CancelRequest { id }; + let frame = match EncodedFrame::encode(&message) { Ok(frame) => frame, Err(err) => { self.fail(format!( @@ -311,7 +374,8 @@ impl ConnectionDriver { return false; } }; - self.queue_frame(frame) + let lane = message.transport_lane(); + self.queue_frame(frame, lane) } fn shutdown_abandoned_session(&mut self, session: RemoteSession) -> bool { diff --git a/codex-rs/code-mode/src/remote_session/connection/driver/session_registry.rs b/codex-rs/code-mode/src/remote_session/connection/driver/session_registry.rs index a3d1c82ae6..8668baa27e 100644 --- a/codex-rs/code-mode/src/remote_session/connection/driver/session_registry.rs +++ b/codex-rs/code-mode/src/remote_session/connection/driver/session_registry.rs @@ -61,6 +61,12 @@ impl SessionRegistry { self.records.contains_key(session_id) } + pub(super) fn contains_cell(&self, session_id: &SessionId, cell_id: &WireCellId) -> bool { + self.records + .get(session_id) + .is_some_and(|session| session.cells.contains_key(cell_id)) + } + pub(super) fn insert_ready( &mut self, session: RemoteSession, diff --git a/codex-rs/code-mode/src/remote_session/connection/driver_tests.rs b/codex-rs/code-mode/src/remote_session/connection/driver_tests.rs index 9f0f3f7625..2dc84eeeec 100644 --- a/codex-rs/code-mode/src/remote_session/connection/driver_tests.rs +++ b/codex-rs/code-mode/src/remote_session/connection/driver_tests.rs @@ -15,6 +15,7 @@ use codex_code_mode_protocol::WaitRequest; use codex_code_mode_protocol::host::ClientToHost; use codex_code_mode_protocol::host::DelegateRequest; use codex_code_mode_protocol::host::DelegateRequestId; +use codex_code_mode_protocol::host::DelegateResponse; use codex_code_mode_protocol::host::EncodedFrame; use codex_code_mode_protocol::host::HostResponse; use codex_code_mode_protocol::host::HostToClient; @@ -31,6 +32,8 @@ use tokio::sync::mpsc; use tokio::sync::oneshot; use tokio_util::sync::CancellationToken; +use super::super::Connection; +use super::super::DEFAULT_HOST_WAIT_TRANSPORT_TIMEOUT; use super::ConnectionDriver; use super::DriverCommand; use super::DriverEvent; @@ -45,6 +48,7 @@ struct DriverHarness { outgoing_rx: mpsc::Receiver, cancellation: CancellationToken, alive: Arc, + failure: Arc>>, driver_task: tokio::task::JoinHandle<()>, } @@ -55,6 +59,7 @@ impl DriverHarness { let (outgoing_tx, outgoing_rx) = mpsc::channel(/*max_capacity*/ 16); let cancellation = CancellationToken::new(); let alive = Arc::new(AtomicBool::new(true)); + let failure = Arc::new(StdMutex::new(None)); let (driver, execute_claim_tx) = ConnectionDriver::new( command_rx, event_rx, @@ -62,7 +67,7 @@ impl DriverHarness { outgoing_tx, DriverLifecycle { alive: Arc::clone(&alive), - failure: Arc::new(StdMutex::new(None)), + failure: Arc::clone(&failure), cancellation: cancellation.clone(), }, ); @@ -74,6 +79,7 @@ impl DriverHarness { outgoing_rx, cancellation, alive, + failure, driver_task, } } @@ -193,6 +199,11 @@ struct RecordingDelegate { struct PanickingDelegate; +struct LargeResultBurstDelegate { + started: AtomicUsize, + release: CancellationToken, +} + #[derive(Debug, Eq, PartialEq)] enum HeldDelegateEvent { Started, @@ -282,6 +293,35 @@ impl CodeModeSessionDelegate for PanickingDelegate { fn cell_closed(&self, _cell_id: &CellId) {} } +impl CodeModeSessionDelegate for LargeResultBurstDelegate { + fn invoke_tool<'a>( + &'a self, + _invocation: CodeModeNestedToolCall, + cancellation_token: CancellationToken, + ) -> ToolInvocationFuture<'a> { + self.started.fetch_add(1, Ordering::Release); + let release = self.release.clone(); + Box::pin(async move { + tokio::select! { + _ = cancellation_token.cancelled() => Err("cancelled".to_string()), + _ = release.cancelled() => Ok("x".repeat(256 * 1024).into()), + } + }) + } + + fn notify<'a>( + &'a self, + _call_id: String, + _cell_id: CellId, + _text: String, + _cancellation_token: CancellationToken, + ) -> NotificationFuture<'a> { + Box::pin(async { Ok(()) }) + } + + fn cell_closed(&self, _cell_id: &CellId) {} +} + impl CodeModeSessionDelegate for RecordingDelegate { fn invoke_tool<'a>( &'a self, @@ -330,6 +370,143 @@ async fn next_held_delegate_event( .expect("delegate event stream") } +#[tokio::test] +async fn deferred_delegates_follow_cell_readiness_and_cancellation() { + let mut harness = DriverHarness::start(); + let session = remote_session(); + let delegate = Arc::new(RecordingDelegate::default()); + harness.open(session.clone(), delegate.clone()).await; + + let (first_response_tx, first_response_rx) = oneshot::channel(); + let (second_response_tx, second_response_rx) = oneshot::channel(); + for (tool_call_id, response_tx) in [ + ("first-cell", first_response_tx), + ("second-cell", second_response_tx), + ] { + harness + .command_tx + .send(DriverCommand::Execute { + session: session.clone(), + request: ExecuteRequest { + tool_call_id: tool_call_id.to_string(), + enabled_tools: Vec::new(), + source: "text('done')".to_string(), + yield_time_ms: Some(/*yield_time_ms*/ 1), + max_output_tokens: None, + }, + caller_cancellation: CancellationToken::new(), + response_tx, + }) + .await + .expect("execute command"); + harness.outgoing_rx.recv().await.expect("execute frame"); + } + + let first_cell_id = CellId::new("first-cell".to_string()); + let second_cell_id = CellId::new("second-cell".to_string()); + let first_delegate_id = DelegateRequestId::new(/*value*/ 7); + let second_delegate_id = DelegateRequestId::new(/*value*/ 8); + let cancelled_delegate_id = DelegateRequestId::new(/*value*/ 9); + for (delegate_id, cell_id) in [ + (first_delegate_id, first_cell_id.clone()), + (second_delegate_id, second_cell_id.clone()), + (cancelled_delegate_id, first_cell_id.clone()), + ] { + harness + .event_tx + .send(DriverEvent::HostMessage(HostToClient::DelegateRequest { + id: delegate_id, + session_id: session.id.clone(), + request: DelegateRequest::Notify { + call_id: format!("notify-{}", cell_id.as_str()), + cell_id: (&cell_id).into(), + text: "hello".to_string(), + }, + })) + .await + .expect("early delegate request"); + } + + harness + .event_tx + .send(DriverEvent::HostMessage( + HostToClient::CancelDelegateRequest { + id: cancelled_delegate_id, + }, + )) + .await + .expect("deferred delegate cancellation"); + + harness + .event_tx + .send(DriverEvent::HostMessage(HostToClient::Response { + id: RequestId::new(/*value*/ 3), + result: WireResult::Ok { + value: HostResponse::ExecutionStarted { + cell_id: (&second_cell_id).into(), + }, + }, + })) + .await + .expect("second execution-started response"); + let _second_started = second_response_rx + .await + .expect("second execute response") + .expect("second started cell"); + let second_response = tokio::time::timeout(Duration::from_secs(1), harness.outgoing_rx.recv()) + .await + .expect("second delegate response timeout") + .expect("second delegate response frame"); + assert_eq!( + EncodedFrame::decode_framed::(&second_response.into_framed_bytes()) + .expect("decode second delegate response"), + ClientToHost::DelegateResponse { + id: second_delegate_id, + result: WireResult::Ok { + value: codex_code_mode_protocol::host::DelegateResponse::NotificationDelivered, + }, + } + ); + assert_eq!(delegate.notifications.load(Ordering::Relaxed), 1); + + harness + .event_tx + .send(DriverEvent::HostMessage(HostToClient::Response { + id: RequestId::new(/*value*/ 2), + result: WireResult::Ok { + value: HostResponse::ExecutionStarted { + cell_id: (&first_cell_id).into(), + }, + }, + })) + .await + .expect("first execution-started response"); + let _first_started = first_response_rx + .await + .expect("first execute response") + .expect("first started cell"); + let first_response = tokio::time::timeout(Duration::from_secs(1), harness.outgoing_rx.recv()) + .await + .expect("first delegate response timeout") + .expect("first delegate response frame"); + assert_eq!( + EncodedFrame::decode_framed::(&first_response.into_framed_bytes()) + .expect("decode first delegate response"), + ClientToHost::DelegateResponse { + id: first_delegate_id, + result: WireResult::Ok { + value: codex_code_mode_protocol::host::DelegateResponse::NotificationDelivered, + }, + } + ); + assert_eq!(delegate.notifications.load(Ordering::Relaxed), 2); + assert!(matches!( + harness.outgoing_rx.try_recv(), + Err(mpsc::error::TryRecvError::Empty) + )); + assert!(harness.alive.load(Ordering::Relaxed)); +} + #[tokio::test] async fn dropped_open_waiter_shuts_down_committed_session() { let mut harness = DriverHarness::start(); @@ -465,6 +642,97 @@ async fn delegate_cancel_is_best_effort_and_sends_no_late_response() { assert_eq!(delegate.notifications.load(Ordering::Relaxed), 0); } +#[tokio::test] +async fn concurrent_large_delegate_results_do_not_disconnect_a_backpressured_bulk_lane() { + const CONCURRENT_RESULTS: usize = 129; + + let (command_tx, command_rx) = mpsc::channel(/*max_capacity*/ 16); + let (event_tx, event_rx) = mpsc::channel(/*max_capacity*/ 16); + let (outgoing_tx, outgoing_rx) = mpsc::channel(/*max_capacity*/ 16); + let (bulk_tx, mut bulk_rx) = mpsc::channel(MAX_PENDING_DELEGATE_CALLS); + let cancellation = CancellationToken::new(); + let alive = Arc::new(AtomicBool::new(true)); + let failure = Arc::new(StdMutex::new(None)); + let (driver, execute_claim_tx) = ConnectionDriver::new( + command_rx, + event_rx, + event_tx.clone(), + outgoing_tx, + DriverLifecycle { + alive: Arc::clone(&alive), + failure: Arc::clone(&failure), + cancellation: cancellation.clone(), + }, + ); + let driver_task = tokio::spawn(driver.with_bulk_sender(bulk_tx).run()); + let mut harness = DriverHarness { + command_tx, + event_tx, + execute_claim_tx, + outgoing_rx, + cancellation, + alive, + failure, + driver_task, + }; + let session = remote_session(); + let delegate = Arc::new(LargeResultBurstDelegate { + started: AtomicUsize::new(0), + release: CancellationToken::new(), + }); + harness.open(session.clone(), delegate.clone()).await; + let _started = harness + .start_cell(session.clone(), /*request_id*/ 2, "1") + .await; + + for value in 1..=CONCURRENT_RESULTS { + harness + .start_tool_delegate(&session, DelegateRequestId::new(value as i64)) + .await; + } + tokio::time::timeout(Duration::from_secs(10), async { + while delegate.started.load(Ordering::Acquire) < CONCURRENT_RESULTS { + tokio::task::yield_now().await; + } + }) + .await + .expect("concurrent delegate calls should all start"); + + delegate.release.cancel(); + tokio::time::timeout(Duration::from_secs(10), async { + while bulk_rx.len() < CONCURRENT_RESULTS { + assert!( + harness.alive.load(Ordering::Acquire), + "bulk queue disconnected before accepting all concurrent tool results" + ); + tokio::task::yield_now().await; + } + }) + .await + .expect("concurrent large results should queue behind the blocked bulk writer"); + + let _unrelated = harness.start_cell(session, /*request_id*/ 3, "2").await; + assert!(harness.alive.load(Ordering::Acquire)); + + for _ in 0..CONCURRENT_RESULTS { + let frame = bulk_rx.recv().await.expect("queued bulk delegate result"); + let message = EncodedFrame::decode_framed::(&frame.into_framed_bytes()) + .expect("decode queued delegate result"); + let ClientToHost::DelegateResponse { + result: + WireResult::Ok { + value: DelegateResponse::ToolResult { result }, + }, + .. + } = message + else { + panic!("expected a successful large delegate result"); + }; + assert_eq!(result.as_str().map(str::len), Some(256 * 1024)); + } + assert!(harness.alive.load(Ordering::Acquire)); +} + #[tokio::test] async fn delegate_limit_returns_an_error_without_disconnecting() { let mut harness = DriverHarness::start(); @@ -796,16 +1064,17 @@ async fn delegate_task_panic_becomes_tool_error_without_killing_connection() { } #[tokio::test] -async fn delegate_for_unknown_cell_fails_connection_without_invocation() { +async fn delegate_for_unknown_cell_returns_error_without_invocation() { let mut harness = DriverHarness::start(); let session = remote_session(); let delegate = Arc::new(RecordingDelegate::default()); harness.open(session.clone(), delegate.clone()).await; + let id = DelegateRequestId::new(/*value*/ 7); harness .event_tx .send(DriverEvent::HostMessage(HostToClient::DelegateRequest { - id: DelegateRequestId::new(/*value*/ 7), + id, session_id: session.id, request: DelegateRequest::InvokeTool { invocation: WireNestedToolCall { @@ -819,14 +1088,28 @@ async fn delegate_for_unknown_cell_fails_connection_without_invocation() { })) .await .expect("delegate request"); - tokio::task::yield_now().await; + let response = tokio::time::timeout(Duration::from_secs(1), harness.outgoing_rx.recv()) + .await + .expect("delegate response timeout") + .expect("delegate response frame"); - assert!(!harness.alive.load(Ordering::Acquire)); + assert_eq!( + EncodedFrame::decode_framed::(&response.into_framed_bytes()) + .expect("decode delegate response"), + ClientToHost::DelegateResponse { + id, + result: WireResult::Err { + message: "code-mode host delegated for unknown cell missing in session session-1" + .to_string(), + }, + } + ); + assert!(harness.alive.load(Ordering::Acquire)); assert_eq!(delegate.invocations.load(Ordering::Relaxed), 0); } #[tokio::test] -async fn delegate_after_cell_close_fails_connection_without_invocation() { +async fn delegate_after_cell_close_returns_error_without_invocation() { let mut harness = DriverHarness::start(); let session = remote_session(); let delegate = Arc::new(RecordingDelegate::default()); @@ -842,10 +1125,11 @@ async fn delegate_after_cell_close_fails_connection_without_invocation() { })) .await .expect("cell close"); + let id = DelegateRequestId::new(/*value*/ 7); harness .event_tx .send(DriverEvent::HostMessage(HostToClient::DelegateRequest { - id: DelegateRequestId::new(/*value*/ 7), + id, session_id: session.id, request: DelegateRequest::Notify { call_id: "notify-1".to_string(), @@ -855,10 +1139,24 @@ async fn delegate_after_cell_close_fails_connection_without_invocation() { })) .await .expect("delegate request"); - tokio::task::yield_now().await; + let response = tokio::time::timeout(Duration::from_secs(1), harness.outgoing_rx.recv()) + .await + .expect("delegate response timeout") + .expect("delegate response frame"); - assert!(!harness.alive.load(Ordering::Acquire)); - assert_eq!(delegate.invocations.load(Ordering::Relaxed), 0); + assert_eq!( + EncodedFrame::decode_framed::(&response.into_framed_bytes()) + .expect("decode delegate response"), + ClientToHost::DelegateResponse { + id, + result: WireResult::Err { + message: "code-mode host delegated for unknown cell 1 in session session-1" + .to_string(), + }, + } + ); + assert!(harness.alive.load(Ordering::Acquire)); + assert_eq!(delegate.notifications.load(Ordering::Relaxed), 0); } #[tokio::test] @@ -1037,6 +1335,90 @@ async fn remote_wait_accepts_durations_longer_than_five_minutes() { ); } +/// A stalled remote wait must expire even when its request never reaches the host. +#[tokio::test(start_paused = true)] +async fn queued_remote_wait_times_out_and_invalidates_the_connection() { + let mut harness = DriverHarness::start(); + let session = remote_session(); + harness + .open(session.clone(), Arc::new(RecordingDelegate::default())) + .await; + let _started = harness + .start_cell(session.clone(), /*request_id*/ 2, "1") + .await; + let connection = Connection { + command_tx: harness.command_tx.clone(), + execute_claim_tx: harness.execute_claim_tx.clone(), + alive: Arc::clone(&harness.alive), + failure: Arc::clone(&harness.failure), + cancellation: harness.cancellation.clone(), + }; + let response = tokio::spawn(async move { + let result = connection + .wait( + session, + WaitRequest { + cell_id: CellId::new("1".to_string()), + yield_time_ms: 1, + }, + ) + .await; + (connection, result) + }); + while harness.outgoing_rx.is_empty() { + tokio::task::yield_now().await; + } + tokio::time::advance(DEFAULT_HOST_WAIT_TRANSPORT_TIMEOUT + Duration::from_secs(2)).await; + + let (connection, result) = response.await.expect("wait task"); + assert_eq!( + result, + Err("code-mode host timed out waiting for wait response".to_string()) + ); + assert!(!harness.alive.load(Ordering::Acquire)); + assert!(harness.cancellation.is_cancelled()); + drop(connection); +} + +/// A stalled termination must expire and invalidate the same connection as a stalled wait. +#[tokio::test(start_paused = true)] +async fn queued_remote_termination_times_out_and_invalidates_the_connection() { + let mut harness = DriverHarness::start(); + let session = remote_session(); + harness + .open(session.clone(), Arc::new(RecordingDelegate::default())) + .await; + let _started = harness + .start_cell(session.clone(), /*request_id*/ 2, "1") + .await; + let connection = Connection { + command_tx: harness.command_tx.clone(), + execute_claim_tx: harness.execute_claim_tx.clone(), + alive: Arc::clone(&harness.alive), + failure: Arc::clone(&harness.failure), + cancellation: harness.cancellation.clone(), + }; + let response = tokio::spawn(async move { + let result = connection + .terminate(session, CellId::new("1".to_string())) + .await; + (connection, result) + }); + while harness.outgoing_rx.is_empty() { + tokio::task::yield_now().await; + } + tokio::time::advance(DEFAULT_HOST_WAIT_TRANSPORT_TIMEOUT + Duration::from_secs(1)).await; + + let (connection, result) = response.await.expect("termination task"); + assert_eq!( + result, + Err("code-mode host timed out waiting for terminate response".to_string()) + ); + assert!(!harness.alive.load(Ordering::Acquire)); + assert!(harness.cancellation.is_cancelled()); + drop(connection); +} + #[tokio::test] async fn cancelled_wait_is_retired_before_next_wait_is_sent() { let mut harness = DriverHarness::start(); diff --git a/codex-rs/code-mode/src/remote_session/connection/reader.rs b/codex-rs/code-mode/src/remote_session/connection/reader.rs index 8fb7cd8b29..8bb218e67c 100644 --- a/codex-rs/code-mode/src/remote_session/connection/reader.rs +++ b/codex-rs/code-mode/src/remote_session/connection/reader.rs @@ -1,3 +1,4 @@ +use codex_code_mode_protocol::host::TransportLane; use tokio::sync::mpsc; use tokio_util::sync::CancellationToken; @@ -8,6 +9,7 @@ pub(super) async fn drive_reader( mut reader: ConnectionReader, events: mpsc::Sender, cancellation: CancellationToken, + lane: Option, ) -> Result<(), String> { loop { let message = tokio::select! { @@ -19,6 +21,11 @@ pub(super) async fn drive_reader( Ok(None) => return Err("code-mode host closed its stdout".to_string()), Err(err) => return Err(format!("failed to read code-mode host message: {err}")), }; + if let Some(lane) = lane + && !message.allows_transport_lane(lane) + { + return Err("code-mode host sent a message on the wrong websocket lane".to_string()); + } events .send(DriverEvent::HostMessage(message)) .await diff --git a/codex-rs/code-mode/src/remote_session_tests.rs b/codex-rs/code-mode/src/remote_session_tests.rs index 0429209157..32cd548003 100644 --- a/codex-rs/code-mode/src/remote_session_tests.rs +++ b/codex-rs/code-mode/src/remote_session_tests.rs @@ -7,8 +7,10 @@ use codex_code_mode_protocol::CodeModeSessionProvider; use codex_code_mode_protocol::ExecuteRequest; use codex_code_mode_protocol::FunctionCallOutputContentItem; use codex_code_mode_protocol::RuntimeResponse; +use codex_code_mode_protocol::host::Capability; use codex_code_mode_protocol::host::CapabilitySet; use codex_code_mode_protocol::host::ClientToHost; +use codex_code_mode_protocol::host::DUAL_WEBSOCKET_CAPABILITY; use codex_code_mode_protocol::host::EncodedFrame; use codex_code_mode_protocol::host::HostHello; use codex_code_mode_protocol::host::HostRequest; @@ -24,6 +26,8 @@ use codex_http_client::OutboundProxyPolicy; use futures::SinkExt; use futures::StreamExt; use pretty_assertions::assert_eq; +use tokio::io::AsyncReadExt; +use tokio::io::AsyncWriteExt; use tokio::net::TcpListener; use tokio::time::timeout; use tokio_tungstenite::accept_async; @@ -224,6 +228,91 @@ async fn websocket_provider_executes_over_shared_connector() { .expect("websocket test host task should succeed"); } +#[tokio::test] +async fn websocket_provider_fails_when_a_negotiated_bulk_connection_is_unavailable() { + let listener = TcpListener::bind("127.0.0.1:0") + .await + .expect("websocket test listener should bind"); + let websocket_url = format!( + "ws://{}/?access_token=shared-token", + listener + .local_addr() + .expect("websocket test listener should have an address") + ); + let server = tokio::spawn(async move { + let (stream, _) = listener + .accept() + .await + .expect("websocket test host should accept the first connection"); + let mut control = accept_async(stream) + .await + .expect("first control websocket should connect"); + let frame = control + .next() + .await + .expect("first client hello") + .expect("first websocket frame") + .into_data(); + let ClientToHost::ClientHello(hello) = + EncodedFrame::decode_framed(&frame).expect("decode first client hello") + else { + panic!("expected first client hello"); + }; + let capability = + Capability::new(DUAL_WEBSOCKET_CAPABILITY).expect("dual websocket capability"); + assert!(hello.optional_capabilities().contains(&capability)); + let hello = HostToClient::HostHello( + HostHello::new( + ProtocolVersion::V1, + CapabilitySet::try_new([capability.clone()]).expect("host capabilities"), + ) + .with_bulk_connection_token("fallback-token".to_string()), + ); + let frame = EncodedFrame::encode(&hello).expect("encode dual host hello"); + control + .send(Message::Binary(frame.into_framed_bytes().into())) + .await + .expect("send dual host hello"); + + let (mut bulk, _) = listener + .accept() + .await + .expect("websocket test host should accept the bulk connection"); + let mut request = [0_u8; 1024]; + let request_len = bulk + .read(&mut request) + .await + .expect("read bulk websocket handshake"); + let request = std::str::from_utf8(&request[..request_len]).expect("bulk HTTP request"); + assert!( + request.starts_with("GET /bulk/fallback-token?access_token=shared-token HTTP/1.1\r\n") + ); + bulk.write_all(b"HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\nConnection: close\r\n\r\n") + .await + .expect("reject unavailable bulk websocket"); + drop(bulk); + drop(control); + }); + + let provider = WebSocketCodeModeSessionProvider::new(websocket_url); + let error = match provider + .create_session(Arc::new(NoopCodeModeSessionDelegate)) + .await + { + Ok(_) => panic!("provider should reject an unavailable negotiated bulk websocket"), + Err(error) => error, + }; + assert!( + error.contains("404"), + "unexpected negotiated bulk websocket error: {error}" + ); + drop(provider); + timeout(Duration::from_secs(5), server) + .await + .expect("negotiated bulk websocket test host should disconnect promptly") + .expect("negotiated bulk websocket test host task should succeed"); +} + #[tokio::test] async fn provider_returns_missing_host_error() { let provider = ProcessOwnedCodeModeSessionProvider::with_host_program( diff --git a/codex-rs/core-skills/src/loader.rs b/codex-rs/core-skills/src/loader.rs index 18a80f7d7b..2651c6445a 100644 --- a/codex-rs/core-skills/src/loader.rs +++ b/codex-rs/core-skills/src/loader.rs @@ -4,9 +4,6 @@ mod namespace; pub use environment::EnvironmentSkillLoadOutcome; pub use environment::EnvironmentSkillMetadata; -pub use environment::EnvironmentSkillSnapshot; -pub use environment::EnvironmentSkillSnapshotOutcome; -pub use environment::load_environment_skills_from_discovery; pub use environment::load_environment_skills_from_root; use crate::model::SkillDependencies; @@ -26,6 +23,9 @@ use codex_exec_server::ExecutorFileSystem; use codex_exec_server::LOCAL_FS; use codex_protocol::protocol::Product; use codex_protocol::protocol::SkillScope; +use codex_skills::ParsedSkillFrontmatter; +use codex_skills::SkillParseError; +use codex_skills::parse_skill_frontmatter_metadata; use codex_utils_absolute_path::AbsolutePathBuf; use codex_utils_absolute_path::AbsolutePathBufGuard; use codex_utils_path_uri::PathUri; @@ -60,22 +60,6 @@ use tracing::error; // TODO(anp): Tune this eight-scan limit after revisiting byte-based backpressure. pub const MAX_CONCURRENT_ROOT_SCANS: usize = 8; -#[derive(Debug, Deserialize)] -struct SkillFrontmatter { - #[serde(default)] - name: Option, - #[serde(default)] - description: Option, - #[serde(default)] - metadata: SkillFrontmatterMetadata, -} - -#[derive(Debug, Default, Deserialize)] -struct SkillFrontmatterMetadata { - #[serde(default, rename = "short-description")] - short_description: Option, -} - #[derive(Debug, Default, Deserialize)] struct SkillMetadataFile { #[serde(default)] @@ -128,13 +112,6 @@ struct DependencyTool { url: Option, } -#[derive(Debug, Clone, PartialEq, Eq)] -struct ParsedSkillFrontmatter { - name: String, - description: String, - short_description: Option, -} - const SKILLS_FILENAME: &str = "SKILL.md"; const AGENTS_DIR_NAME: &str = ".agents"; const SKILLS_METADATA_DIR: &str = "agents"; @@ -165,31 +142,27 @@ struct ResolvedDiscoveredSkill { } #[derive(Debug)] -enum SkillParseError { +enum SkillLoadError { Read(std::io::Error), - MissingFrontmatter, - InvalidYaml(serde_yaml::Error), - MissingField(&'static str), - InvalidField { field: &'static str, reason: String }, + Parse(SkillParseError), } -impl fmt::Display for SkillParseError { +impl fmt::Display for SkillLoadError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { - SkillParseError::Read(e) => write!(f, "failed to read file: {e}"), - SkillParseError::MissingFrontmatter => { - write!(f, "missing YAML frontmatter delimited by ---") - } - SkillParseError::InvalidYaml(e) => write!(f, "invalid YAML: {e}"), - SkillParseError::MissingField(field) => write!(f, "missing field `{field}`"), - SkillParseError::InvalidField { field, reason } => { - write!(f, "invalid {field}: {reason}") - } + Self::Read(error) => write!(f, "failed to read file: {error}"), + Self::Parse(error) => write!(f, "{error}"), } } } -impl Error for SkillParseError {} +impl Error for SkillLoadError {} + +impl From for SkillLoadError { + fn from(error: SkillParseError) -> Self { + Self::Parse(error) + } +} pub struct SkillRoot { pub path: AbsolutePathBuf, @@ -693,7 +666,7 @@ async fn parse_skill_file( scope: SkillScope, plugin_identity: Option<&PluginIdentity>, plugin_root: Option<&AbsolutePathBuf>, -) -> Result { +) -> Result { let metadata_path = path_uri .parent() .and_then(|parent| parent.join(SKILLS_METADATA_DIR).ok()) @@ -708,12 +681,12 @@ async fn parse_skill_file( fs.read_file_text(path_uri, /*sandbox*/ None), load_skill_metadata(fs, path, &metadata, plugin_root), ); - let contents = contents.map_err(SkillParseError::Read)?; + let contents = contents.map_err(SkillLoadError::Read)?; let ParsedSkillFrontmatter { name: base_name, description, short_description, - } = parse_skill_frontmatter_metadata_inner(&contents, || default_skill_name(path))?; + } = parse_skill_frontmatter_metadata(&contents, || default_skill_name(path))?; let LoadedSkillMetadata { interface, dependencies, @@ -734,56 +707,6 @@ async fn parse_skill_file( }) } -fn parse_skill_frontmatter_metadata_inner( - contents: &str, - default_name: impl FnOnce() -> String, -) -> Result { - let frontmatter = extract_frontmatter(contents).ok_or(SkillParseError::MissingFrontmatter)?; - - let parsed: SkillFrontmatter = match serde_yaml::from_str(&frontmatter) { - Ok(parsed) => Ok(parsed), - Err(original_error) => match repair_frontmatter_scalar_fields(&frontmatter) { - // Some third-party skills use prose like `description: Build for AWS: ECS` - // or `argument-hint: `. Keep the repair line-oriented - // so unrelated invalid YAML still surfaces. - Some(repaired_frontmatter) => { - serde_yaml::from_str(&repaired_frontmatter).map_err(|_| original_error) - } - None => Err(original_error), - }, - } - .map_err(SkillParseError::InvalidYaml)?; - - let name = parsed - .name - .as_deref() - .map(sanitize_single_line) - .filter(|value| !value.is_empty()) - .unwrap_or_else(default_name); - let description = parsed - .description - .as_deref() - .map(sanitize_single_line) - .unwrap_or_default(); - let short_description = parsed - .metadata - .short_description - .as_deref() - .map(sanitize_single_line) - .filter(|value| !value.is_empty()); - - validate_len(&name, MAX_NAME_LEN, "name")?; - if description.is_empty() { - return Err(SkillParseError::MissingField("description")); - } - - Ok(ParsedSkillFrontmatter { - name, - description, - short_description, - }) -} - fn default_skill_name(path: &AbsolutePathBuf) -> String { path.parent() .and_then(|parent| { @@ -1067,91 +990,6 @@ fn sanitize_single_line(raw: &str) -> String { raw.split_whitespace().collect::>().join(" ") } -fn repair_frontmatter_scalar_fields(frontmatter: &str) -> Option { - let mut changed = false; - let mut block_scalar_indent: Option = None; - let mut repaired_lines: Vec = Vec::new(); - for line in frontmatter.lines() { - let indent = line - .chars() - .take_while(|character| *character == ' ') - .count(); - if let Some(block_indent) = block_scalar_indent { - if line.trim().is_empty() || indent > block_indent { - repaired_lines.push(line.to_string()); - continue; - } - block_scalar_indent = None; - } - - let Some((key, value)) = line.split_once(':') else { - repaired_lines.push(line.to_string()); - continue; - }; - if key.trim().is_empty() || !value.chars().next().is_none_or(char::is_whitespace) { - repaired_lines.push(line.to_string()); - continue; - } - - let trimmed_start = value.trim_start(); - let leading_whitespace = &value[..value.len() - trimmed_start.len()]; - let mut scalar = trimmed_start; - let mut comment = ""; - for (index, character) in trimmed_start.char_indices() { - if character == '#' - && (index == 0 - || trimmed_start[..index] - .chars() - .next_back() - .is_some_and(char::is_whitespace)) - { - let comment_start = trimmed_start[..index].trim_end().len(); - scalar = &trimmed_start[..comment_start]; - comment = &trimmed_start[comment_start..]; - break; - } - } - - let scalar = scalar.trim_end(); - let Some(first_char) = scalar.chars().next() else { - repaired_lines.push(line.to_string()); - continue; - }; - if matches!(first_char, '|' | '>') { - block_scalar_indent = Some(indent); - repaired_lines.push(line.to_string()); - continue; - } - if matches!(first_char, '\'' | '"') { - repaired_lines.push(line.to_string()); - continue; - } - let mut has_colon_separator = false; - let mut chars = scalar.chars().peekable(); - while let Some(character) = chars.next() { - if character == ':' - && matches!(chars.peek(), Some(next_character) if next_character.is_whitespace()) - { - has_colon_separator = true; - break; - } - } - let invalid_flow_like_scalar = matches!(first_char, '[' | '{' | '@' | '`') - && serde_yaml::from_str::(scalar).is_err(); - if !has_colon_separator && !invalid_flow_like_scalar { - repaired_lines.push(line.to_string()); - continue; - } - - let quoted_scalar = format!("'{}'", scalar.replace('\'', "''")); - repaired_lines.push(format!( - "{key}:{leading_whitespace}{quoted_scalar}{comment}" - )); - changed = true; - } - changed.then(|| repaired_lines.join("\n")) -} - fn validate_len( value: &str, max_len: usize, @@ -1211,28 +1049,6 @@ fn resolve_color_str(value: Option, field: &'static str) -> Option Option { - let mut lines = contents.lines(); - if !matches!(lines.next(), Some(line) if line.trim() == "---") { - return None; - } - - let mut frontmatter_lines: Vec<&str> = Vec::new(); - let mut found_closing = false; - for line in lines.by_ref() { - if line.trim() == "---" { - found_closing = true; - break; - } - frontmatter_lines.push(line); - } - - if frontmatter_lines.is_empty() || !found_closing { - return None; - } - - Some(frontmatter_lines.join("\n")) -} #[cfg(test)] pub(crate) async fn skill_roots_from_layer_stack( fs: Arc, diff --git a/codex-rs/core-skills/src/loader/environment.rs b/codex-rs/core-skills/src/loader/environment.rs index 0e881b6c24..fa950ec7b1 100644 --- a/codex-rs/core-skills/src/loader/environment.rs +++ b/codex-rs/core-skills/src/loader/environment.rs @@ -1,10 +1,10 @@ -use std::collections::HashMap; use std::io; -use codex_exec_server::CapabilityRootDiscovery; use codex_exec_server::ExecutorFileSystem; use codex_protocol::protocol::Product; pub use codex_skills::EnvironmentSkillMetadata; +use codex_skills::ParsedSkillFrontmatter; +use codex_skills::parse_skill_frontmatter_metadata; use codex_utils_path_uri::PathUri; use futures::StreamExt; @@ -12,7 +12,6 @@ use crate::model::SkillDependencies; use crate::model::SkillPolicy; use super::MAX_QUALIFIED_NAME_LEN; -use super::ParsedSkillFrontmatter; use super::SkillMetadataFile; use super::discovery::DirectorySymlinkPolicy; use super::discovery::DiscoveredSkill; @@ -22,7 +21,6 @@ use super::discovery::SkillDiscoveryOptions; use super::discovery::SkillMetadataDiscovery; use super::discovery::discover_skills; use super::namespace::SkillNamespaceResolver; -use super::parse_skill_frontmatter_metadata_inner; use super::resolve_dependencies; use super::resolve_policy; use super::sanitize_single_line; @@ -37,19 +35,6 @@ struct ParsedEnvironmentSkill { policy: Option, } -/// Parsed executor skill plus the instructions already materialized by capability discovery. -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct EnvironmentSkillSnapshot { - pub metadata: EnvironmentSkillMetadata, - pub instructions: String, -} - -#[derive(Debug, Default)] -pub struct EnvironmentSkillSnapshotOutcome { - pub skills: Vec, - pub warnings: Vec, -} - impl ParsedEnvironmentSkill { async fn load( file_system: &dyn ExecutorFileSystem, @@ -72,7 +57,7 @@ impl ParsedEnvironmentSkill { name: base_name, description, short_description, - } = parse_skill_frontmatter_metadata_inner(&contents, || default_skill_name(&skill.path)) + } = parse_skill_frontmatter_metadata(&contents, || default_skill_name(&skill.path)) .map_err(|err| err.to_string())?; let (dependencies, policy) = match &skill.metadata { SkillMetadataDiscovery::Present(_) | SkillMetadataDiscovery::Absent => { @@ -197,130 +182,6 @@ pub async fn load_environment_skills_from_root( outcome } -/// Parses an executor-produced manifest bundle without issuing additional filesystem requests. -pub fn load_environment_skills_from_discovery( - discovery: &CapabilityRootDiscovery, - restriction_product: Option, -) -> EnvironmentSkillSnapshotOutcome { - let mut outcome = EnvironmentSkillSnapshotOutcome { - warnings: discovery.warnings.clone(), - ..Default::default() - }; - if let Some(error) = &discovery.error { - outcome.warnings.push(error.clone()); - return outcome; - } - - let mut plugin_namespaces = HashMap::new(); - for (plugin_root, name) in discovery.namespace_manifests.iter().filter_map(|manifest| { - #[derive(serde::Deserialize)] - struct ManifestName { - #[serde(default)] - name: String, - } - - let plugin_root = manifest.path.parent()?.parent()?; - let parsed = serde_json::from_str::(&manifest.contents).ok()?; - let name = if parsed.name.trim().is_empty() { - plugin_root.basename()? - } else { - parsed.name - }; - Some((plugin_root, name)) - }) { - // Exec-server orders manifests by the same precedence as local discovery. Preserve the - // first manifest if an older or alternate server returns duplicates for one plugin root. - plugin_namespaces.entry(plugin_root).or_insert(name); - } - - for skill in &discovery.skills { - let plugin_namespace = - nearest_plugin_namespace(&skill.instructions.path, &plugin_namespaces); - let ParsedSkillFrontmatter { - name: base_name, - description, - short_description, - } = match parse_skill_frontmatter_metadata_inner(&skill.instructions.contents, || { - default_skill_name(&skill.instructions.path) - }) { - Ok(frontmatter) => frontmatter, - Err(error) => { - outcome.warnings.push(format!( - "Failed to load environment skill at {}: {error}", - skill.instructions.path - )); - continue; - } - }; - let name = plugin_namespace - .map(|namespace| format!("{namespace}:{base_name}")) - .unwrap_or(base_name); - if let Err(error) = validate_len(&name, MAX_QUALIFIED_NAME_LEN, "qualified name") { - outcome.warnings.push(format!( - "Failed to load environment skill at {}: {error}", - skill.instructions.path - )); - continue; - } - let (dependencies, policy) = skill - .metadata - .as_ref() - .and_then(|metadata| { - serde_yaml::from_str::(&metadata.contents) - .map_err(|error| { - tracing::warn!( - path = %metadata.path, - "ignoring invalid discovered skill metadata: {error}" - ); - }) - .ok() - }) - .map(|metadata| { - ( - resolve_dependencies(metadata.dependencies), - resolve_policy(metadata.policy), - ) - }) - .unwrap_or((None, None)); - let metadata = EnvironmentSkillMetadata { - path_to_skills_md: skill.instructions.path.clone(), - name, - description, - short_description, - dependencies, - policy, - }; - if metadata.matches_product_restriction(restriction_product) { - outcome.skills.push(EnvironmentSkillSnapshot { - metadata, - instructions: skill.instructions.contents.clone(), - }); - } - } - outcome.skills.sort_by(|left, right| { - left.metadata.name.cmp(&right.metadata.name).then_with(|| { - left.metadata - .path_to_skills_md - .to_string() - .cmp(&right.metadata.path_to_skills_md.to_string()) - }) - }); - outcome -} - -fn nearest_plugin_namespace<'a>( - skill_path: &PathUri, - plugin_namespaces: &'a HashMap, -) -> Option<&'a str> { - let mut ancestor = skill_path.parent(); - while let Some(path) = ancestor { - if let Some(namespace) = plugin_namespaces.get(&path) { - return Some(namespace); - } - ancestor = path.parent(); - } - None -} async fn read_skill_contents( file_system: &dyn ExecutorFileSystem, skill_path: &PathUri, diff --git a/codex-rs/core-skills/tests/environment_loader.rs b/codex-rs/core-skills/tests/environment_loader.rs index 88b26f2349..53cbac4abd 100644 --- a/codex-rs/core-skills/tests/environment_loader.rs +++ b/codex-rs/core-skills/tests/environment_loader.rs @@ -1,15 +1,10 @@ +#![cfg(unix)] + use std::fs; use std::sync::Mutex; -use std::sync::atomic::AtomicBool; use std::sync::atomic::AtomicUsize; use std::sync::atomic::Ordering; -use std::time::Duration; -use codex_core_skills::loader::EnvironmentSkillMetadata; -use codex_core_skills::loader::load_environment_skills_from_discovery; -use codex_core_skills::loader::load_environment_skills_from_root; -use codex_exec_server::CapabilityRootDiscoverRequest; -use codex_exec_server::CapabilityRootsDiscoverParams; use codex_exec_server::CopyOptions; use codex_exec_server::CreateDirectoryOptions; use codex_exec_server::ExecutorFileSystem; @@ -22,26 +17,15 @@ use codex_exec_server::ReadDirectoryEntry; use codex_exec_server::RemoveOptions; use codex_exec_server::WalkOptions; use codex_exec_server::WalkOutcome; -use codex_exec_server::discover_capability_roots; use codex_utils_path_uri::PathUri; use pretty_assertions::assert_eq; use tempfile::tempdir; -use tokio::sync::Notify; - -#[derive(Clone, Copy)] -enum ManifestMetadataBehavior { - Immediate, - WaitForSkillRead, -} struct RecordingFileSystem<'a> { inner: &'a dyn ExecutorFileSystem, read_files: Mutex>, metadata_files: Mutex>, walks: AtomicUsize, - manifest_metadata_behavior: ManifestMetadataBehavior, - skill_read_started: AtomicBool, - skill_read_started_notify: Notify, } #[derive(Debug, PartialEq, Eq)] @@ -52,18 +36,12 @@ struct FileSystemCalls { } impl<'a> RecordingFileSystem<'a> { - fn new( - inner: &'a dyn ExecutorFileSystem, - manifest_metadata_behavior: ManifestMetadataBehavior, - ) -> Self { + fn new(inner: &'a dyn ExecutorFileSystem) -> Self { Self { inner, read_files: Mutex::new(Vec::new()), metadata_files: Mutex::new(Vec::new()), walks: AtomicUsize::new(0), - manifest_metadata_behavior, - skill_read_started: AtomicBool::new(false), - skill_read_started_notify: Notify::new(), } } @@ -106,10 +84,6 @@ impl ExecutorFileSystem for RecordingFileSystem<'_> { .lock() .unwrap_or_else(std::sync::PoisonError::into_inner) .push(path.clone()); - if path.basename().as_deref() == Some("SKILL.md") { - self.skill_read_started.store(true, Ordering::Release); - self.skill_read_started_notify.notify_waiters(); - } self.inner.read_file(path, sandbox) } @@ -148,22 +122,6 @@ impl ExecutorFileSystem for RecordingFileSystem<'_> { .lock() .unwrap_or_else(std::sync::PoisonError::into_inner) .push(path.clone()); - if matches!( - self.manifest_metadata_behavior, - ManifestMetadataBehavior::WaitForSkillRead - ) && path.basename().as_deref() == Some("plugin.json") - { - return Box::pin(async move { - loop { - let notified = self.skill_read_started_notify.notified(); - if self.skill_read_started.load(Ordering::Acquire) { - break; - } - notified.await; - } - self.inner.get_metadata(path, sandbox).await - }); - } self.inner.get_metadata(path, sandbox) } @@ -206,216 +164,6 @@ impl ExecutorFileSystem for RecordingFileSystem<'_> { } } -#[tokio::test] -async fn loads_nearest_plugin_namespaces_without_reading_unused_sibling_manifests() { - let root = tempdir().expect("tempdir"); - let standalone_skill = root.path().join("standalone/SKILL.md"); - let outer_root = root.path().join("plugins/outer"); - let outer_skill = outer_root.join("skills/deploy/SKILL.md"); - let inner_root = outer_root.join("nested/inner"); - let inner_skill = inner_root.join("skills/audit/SKILL.md"); - let unused_root = root.path().join("plugins/unused"); - - for path in [&standalone_skill, &outer_skill, &inner_skill] { - fs::create_dir_all(path.parent().expect("skill parent")).expect("skill dir"); - } - for (plugin_root, name) in [ - (&outer_root, "outer"), - (&inner_root, "inner"), - (&unused_root, "unused"), - ] { - fs::create_dir_all(plugin_root.join(".codex-plugin")).expect("manifest dir"); - fs::write( - plugin_root.join(".codex-plugin/plugin.json"), - format!(r#"{{"name":"{name}"}}"#), - ) - .expect("manifest"); - } - for (path, name) in [ - (&standalone_skill, "standalone"), - (&outer_skill, "deploy"), - (&inner_skill, "audit"), - ] { - fs::write( - path, - format!("---\nname: {name}\ndescription: {name} skill.\n---\n"), - ) - .expect("skill"); - } - - let file_system = - RecordingFileSystem::new(LOCAL_FS.as_ref(), ManifestMetadataBehavior::Immediate); - let root_uri = PathUri::from_host_native_path(root.path()).expect("root URI"); - let outcome = load_environment_skills_from_root( - &file_system, - &root_uri, - /*restriction_product*/ None, - ) - .await; - - assert_eq!(outcome.warnings, Vec::::new()); - assert_eq!( - outcome.skills, - vec![ - EnvironmentSkillMetadata { - path_to_skills_md: PathUri::from_host_native_path(&inner_skill).unwrap(), - name: "inner:audit".to_string(), - description: "audit skill.".to_string(), - short_description: None, - dependencies: None, - policy: None, - }, - EnvironmentSkillMetadata { - path_to_skills_md: PathUri::from_host_native_path(&outer_skill).unwrap(), - name: "outer:deploy".to_string(), - description: "deploy skill.".to_string(), - short_description: None, - dependencies: None, - policy: None, - }, - EnvironmentSkillMetadata { - path_to_skills_md: PathUri::from_host_native_path(&standalone_skill).unwrap(), - name: "standalone".to_string(), - description: "standalone skill.".to_string(), - short_description: None, - dependencies: None, - policy: None, - }, - ] - ); - - let mut manifest_reads = file_system - .calls() - .read_files - .into_iter() - .filter(|path| path.basename().as_deref() == Some("plugin.json")) - .collect::>(); - manifest_reads.sort_by_key(ToString::to_string); - let mut expected_manifest_reads = [&outer_root, &inner_root] - .into_iter() - .map(|plugin_root| { - PathUri::from_host_native_path(plugin_root.join(".codex-plugin/plugin.json")).unwrap() - }) - .collect::>(); - expected_manifest_reads.sort_by_key(ToString::to_string); - assert_eq!(manifest_reads, expected_manifest_reads); -} - -#[tokio::test] -async fn reuses_walk_inventory_for_missing_skill_metadata() { - const SKILL_COUNT: usize = 66; - - let root = tempdir().expect("tempdir"); - let manifest_path = root.path().join(".codex-plugin/plugin.json"); - fs::create_dir_all(manifest_path.parent().expect("manifest parent")).expect("manifest dir"); - fs::write(&manifest_path, r#"{"name":"inventory"}"#).expect("manifest"); - - let mut skill_paths = Vec::new(); - for index in 0..SKILL_COUNT { - let name = format!("skill-{index}"); - let skill_path = root.path().join(&name).join("SKILL.md"); - fs::create_dir_all(skill_path.parent().expect("skill parent")).expect("skill dir"); - fs::write( - &skill_path, - format!("---\nname: {name}\ndescription: {name} skill.\n---\n"), - ) - .expect("skill"); - skill_paths.push(skill_path); - } - - let file_system = - RecordingFileSystem::new(LOCAL_FS.as_ref(), ManifestMetadataBehavior::Immediate); - let root_uri = PathUri::from_host_native_path(root.path()).expect("root URI"); - let outcome = load_environment_skills_from_root( - &file_system, - &root_uri, - /*restriction_product*/ None, - ) - .await; - - let mut expected_skills = skill_paths - .iter() - .enumerate() - .map(|(index, skill_path)| EnvironmentSkillMetadata { - path_to_skills_md: PathUri::from_host_native_path(skill_path).unwrap(), - name: format!("inventory:skill-{index}"), - description: format!("skill-{index} skill."), - short_description: None, - dependencies: None, - policy: None, - }) - .collect::>(); - expected_skills.sort_by(|left, right| { - left.name.cmp(&right.name).then_with(|| { - left.path_to_skills_md - .to_string() - .cmp(&right.path_to_skills_md.to_string()) - }) - }); - assert_eq!(outcome.skills, expected_skills); - assert_eq!(outcome.warnings, Vec::::new()); - - let mut expected_read_files = skill_paths - .iter() - .map(|path| PathUri::from_host_native_path(path).unwrap()) - .collect::>(); - let manifest_uri = PathUri::from_host_native_path(manifest_path).unwrap(); - expected_read_files.push(manifest_uri.clone()); - expected_read_files.sort_by_key(ToString::to_string); - assert_eq!( - file_system.calls(), - FileSystemCalls { - walks: 1, - read_files: expected_read_files, - metadata_files: vec![manifest_uri], - } - ); -} - -#[tokio::test] -async fn reads_skill_files_while_resolving_plugin_namespaces() { - let root = tempdir().expect("tempdir"); - let manifest_path = root.path().join(".codex-plugin/plugin.json"); - fs::create_dir_all(manifest_path.parent().expect("manifest parent")).expect("manifest dir"); - fs::write(&manifest_path, r#"{"name":"parallel"}"#).expect("manifest"); - let skill_path = root.path().join("demo/SKILL.md"); - fs::create_dir_all(skill_path.parent().expect("skill parent")).expect("skill dir"); - fs::write( - &skill_path, - "---\nname: demo\ndescription: demo skill.\n---\n", - ) - .expect("skill"); - - let file_system = RecordingFileSystem::new( - LOCAL_FS.as_ref(), - ManifestMetadataBehavior::WaitForSkillRead, - ); - let root_uri = PathUri::from_host_native_path(root.path()).expect("root URI"); - let outcome = tokio::time::timeout( - Duration::from_secs(5), - load_environment_skills_from_root( - &file_system, - &root_uri, - /*restriction_product*/ None, - ), - ) - .await - .expect("skill reads should start before namespace resolution finishes"); - - assert_eq!(outcome.warnings, Vec::::new()); - assert_eq!( - outcome.skills, - vec![EnvironmentSkillMetadata { - path_to_skills_md: PathUri::from_host_native_path(skill_path).unwrap(), - name: "parallel:demo".to_string(), - description: "demo skill.".to_string(), - short_description: None, - dependencies: None, - policy: None, - }] - ); -} - #[cfg(unix)] #[tokio::test] async fn host_loading_reuses_walk_inventory_for_symlinked_skill_pack() { @@ -459,10 +207,7 @@ async fn host_loading_reuses_walk_inventory_for_symlinked_skill_pack() { let linked_root = host_root.join("linked-plugin"); symlink(&skills_root, &linked_root).expect("skill pack symlink"); - let recording = Arc::new(RecordingFileSystem::new( - LOCAL_FS.as_ref(), - ManifestMetadataBehavior::Immediate, - )); + let recording = Arc::new(RecordingFileSystem::new(LOCAL_FS.as_ref())); let file_system: Arc = recording.clone(); let future = load_skills_from_roots( [SkillRoot { @@ -547,142 +292,3 @@ async fn host_loading_reuses_walk_inventory_for_symlinked_skill_pack() { 1 ); } - -#[tokio::test] -async fn executor_bundle_parser_matches_the_existing_environment_loader() { - let root = tempdir().expect("tempdir"); - let plugin_manifest = root.path().join(".codex-plugin/plugin.json"); - let nested_manifest = root.path().join("nested/.claude-plugin/plugin.json"); - let deploy_skill = root.path().join("skills/deploy/SKILL.md"); - let deploy_metadata = root.path().join("skills/deploy/agents/openai.yaml"); - let audit_skill = root.path().join("nested/skills/audit/SKILL.md"); - for (path, contents) in [ - (&plugin_manifest, r#"{"name":"demo"}"#), - (&nested_manifest, r#"{"name":"nested"}"#), - ( - &deploy_skill, - "---\nname: deploy\ndescription: Deploy the service.\n---\n\nDeploy.\n", - ), - ( - &deploy_metadata, - "policy:\n allow_implicit_invocation: false\n", - ), - ( - &audit_skill, - "---\nname: audit\ndescription: Audit the service.\n---\n\nAudit.\n", - ), - ] { - fs::create_dir_all(path.parent().expect("test file parent")).expect("test directory"); - fs::write(path, contents).expect("test file"); - } - - let root_uri = PathUri::from_host_native_path(root.path()).expect("root URI"); - let existing = load_environment_skills_from_root( - LOCAL_FS.as_ref(), - &root_uri, - /*restriction_product*/ None, - ) - .await; - let response = discover_capability_roots( - LOCAL_FS.as_ref(), - CapabilityRootsDiscoverParams { - roots: vec![CapabilityRootDiscoverRequest { - id: "demo@1".to_string(), - path: root_uri, - sandbox: None, - }], - }, - ) - .await - .expect("capability discovery"); - let bundled = load_environment_skills_from_discovery( - response.roots.first().expect("discovered root"), - /*restriction_product*/ None, - ); - - assert_eq!(bundled.warnings, existing.warnings); - assert_eq!( - bundled - .skills - .iter() - .map(|skill| skill.metadata.clone()) - .collect::>(), - existing.skills - ); - assert_eq!( - bundled - .skills - .iter() - .map(|skill| skill.instructions.as_str()) - .collect::>(), - vec![ - "---\nname: deploy\ndescription: Deploy the service.\n---\n\nDeploy.\n", - "---\nname: audit\ndescription: Audit the service.\n---\n\nAudit.\n", - ] - ); -} - -#[tokio::test] -async fn executor_bundle_preserves_parent_namespace_and_manifest_precedence() { - let plugin = tempdir().expect("tempdir"); - for (relative_path, name) in [ - (".codex-plugin/plugin.json", "codex-name"), - (".claude-plugin/plugin.json", "claude-name"), - (".cursor-plugin/plugin.json", "cursor-name"), - ] { - let manifest = plugin.path().join(relative_path); - fs::create_dir_all(manifest.parent().expect("manifest parent")) - .expect("manifest directory"); - fs::write(&manifest, format!(r#"{{"name":"{name}"}}"#)).expect("manifest"); - } - let skills_root = plugin.path().join("skills"); - let skill_path = skills_root.join("search/SKILL.md"); - fs::create_dir_all(skill_path.parent().expect("skill parent")).expect("skill directory"); - fs::write( - &skill_path, - "---\nname: search\ndescription: Search the project.\n---\n\nSearch.\n", - ) - .expect("skill"); - - let root_uri = PathUri::from_host_native_path(&skills_root).expect("skills root URI"); - let existing = load_environment_skills_from_root( - LOCAL_FS.as_ref(), - &root_uri, - /*restriction_product*/ None, - ) - .await; - let response = discover_capability_roots( - LOCAL_FS.as_ref(), - CapabilityRootsDiscoverParams { - roots: vec![CapabilityRootDiscoverRequest { - id: "skills-only".to_string(), - path: root_uri, - sandbox: None, - }], - }, - ) - .await - .expect("capability discovery"); - let discovery = response.roots.first().expect("discovered root"); - let bundled = - load_environment_skills_from_discovery(discovery, /*restriction_product*/ None); - - assert_eq!(discovery.plugin, None); - assert_eq!(discovery.namespace_manifests.len(), 1); - assert!( - discovery.namespace_manifests[0] - .path - .to_string() - .ends_with("/.codex-plugin/plugin.json") - ); - assert_eq!(bundled.warnings, existing.warnings); - assert_eq!( - bundled - .skills - .iter() - .map(|skill| skill.metadata.clone()) - .collect::>(), - existing.skills - ); - assert_eq!(bundled.skills[0].metadata.name, "codex-name:search"); -} diff --git a/codex-rs/core/Cargo.toml b/codex-rs/core/Cargo.toml index cb524b4a93..3c8ad1b9fb 100644 --- a/codex-rs/core/Cargo.toml +++ b/codex-rs/core/Cargo.toml @@ -69,6 +69,7 @@ codex-terminal-detection = { workspace = true } codex-thread-store = { workspace = true } codex-tools = { workspace = true } codex-utils-absolute-path = { workspace = true } +codex-utils-audio = { workspace = true } codex-utils-cache = { workspace = true } codex-utils-image = { workspace = true } codex-utils-home-dir = { workspace = true } @@ -103,7 +104,6 @@ serde_json = { workspace = true } sha1 = { workspace = true } shlex = { workspace = true } similar = { workspace = true } -symphonia = { workspace = true } tempfile = { workspace = true } thiserror = { workspace = true } tokio = { workspace = true, features = [ diff --git a/codex-rs/core/src/agents_md_tests.rs b/codex-rs/core/src/agents_md_tests.rs index 8bb628d9da..1ffeaf131b 100644 --- a/codex-rs/core/src/agents_md_tests.rs +++ b/codex-rs/core/src/agents_md_tests.rs @@ -3,6 +3,7 @@ use crate::config::ConfigBuilder; use crate::context::ContextualUserFragment; use crate::environment_selection::TurnEnvironmentSnapshot; use crate::environment_selection::TurnEnvironmentState; +use crate::session::turn_context::EnvironmentConfig; use crate::session::turn_context::TurnEnvironment; use codex_config::ConfigLayerEntry; use codex_config::ConfigLayerStack; @@ -337,6 +338,9 @@ fn resolved_local_environments( PathUri::from_abs_path(&cwd), Vec::new(), /*shell*/ None, + EnvironmentConfig { + allow_login_shell: true, + }, )) }) .collect(), diff --git a/codex-rs/core/src/context/approved_command_prefix_saved.rs b/codex-rs/core/src/context/approved_command_prefix_saved.rs index 3686741d1c..176df83035 100644 --- a/codex-rs/core/src/context/approved_command_prefix_saved.rs +++ b/codex-rs/core/src/context/approved_command_prefix_saved.rs @@ -1,5 +1,8 @@ use super::ContextualUserFragment; +pub(crate) const APPROVED_COMMAND_PREFIX_SAVED_MESSAGE_PREFIX: &str = + "Approved command prefix saved:"; + #[derive(Debug, Clone, PartialEq)] pub(crate) struct ApprovedCommandPrefixSaved { prefixes: String, @@ -27,6 +30,9 @@ impl ContextualUserFragment for ApprovedCommandPrefixSaved { } fn body(&self) -> String { - format!("Approved command prefix saved:\n{}", self.prefixes) + format!( + "{APPROVED_COMMAND_PREFIX_SAVED_MESSAGE_PREFIX}\n{}", + self.prefixes + ) } } diff --git a/codex-rs/core/src/context/mod.rs b/codex-rs/core/src/context/mod.rs index fc874cd6d2..2e69331475 100644 --- a/codex-rs/core/src/context/mod.rs +++ b/codex-rs/core/src/context/mod.rs @@ -34,6 +34,7 @@ mod user_instructions; mod user_shell_command; pub(crate) mod world_state; +pub(crate) use approved_command_prefix_saved::APPROVED_COMMAND_PREFIX_SAVED_MESSAGE_PREFIX; pub(crate) use approved_command_prefix_saved::ApprovedCommandPrefixSaved; pub(crate) use apps_instructions::AppsInstructions; pub(crate) use available_plugins_instructions::AvailablePluginsInstructions; diff --git a/codex-rs/core/src/context/token_budget_context.rs b/codex-rs/core/src/context/token_budget_context.rs index f5a4409821..601042569e 100644 --- a/codex-rs/core/src/context/token_budget_context.rs +++ b/codex-rs/core/src/context/token_budget_context.rs @@ -1,4 +1,5 @@ use super::ContextualUserFragment; +use codex_protocol::AgentPath; use codex_protocol::ThreadId; use codex_protocol::protocol::CONTEXT_WINDOW_CLOSE_TAG; use codex_protocol::protocol::CONTEXT_WINDOW_GUIDANCE_CLOSE_TAG; @@ -9,6 +10,7 @@ use uuid::Uuid; #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) struct TokenBudgetContext { thread_id: ThreadId, + agent_path: AgentPath, first_window_id: Uuid, previous_window_id: Option, window_id: Uuid, @@ -18,6 +20,7 @@ pub(crate) struct TokenBudgetContext { impl TokenBudgetContext { pub(crate) fn new( thread_id: ThreadId, + agent_path: AgentPath, first_window_id: Uuid, previous_window_id: Option, window_id: Uuid, @@ -25,6 +28,7 @@ impl TokenBudgetContext { ) -> Self { Self { thread_id, + agent_path, first_window_id, previous_window_id, window_id, @@ -47,11 +51,11 @@ impl ContextualUserFragment for TokenBudgetContext { } fn body(&self) -> String { - let thread_id = self.thread_id; + let agent_path = &self.agent_path; let first_window_id = self.first_window_id; let window_id = self.window_id; let mut lines = vec![ - format!("Thread id: {thread_id}"), + format!("Agent name: {agent_path}"), format!("First context window id: {first_window_id}"), format!("Current context window id: {window_id}"), ]; diff --git a/codex-rs/core/src/context/world_state/compact_permissions.rs b/codex-rs/core/src/context/world_state/compact_permissions.rs new file mode 100644 index 0000000000..f6822ef7bc --- /dev/null +++ b/codex-rs/core/src/context/world_state/compact_permissions.rs @@ -0,0 +1,59 @@ +use super::PreviousSectionState; +use super::WorldStateSection; +use crate::context::APPROVED_COMMAND_PREFIX_SAVED_MESSAGE_PREFIX; +use crate::context::ApprovedCommandPrefixSaved; +use crate::context::ContextualUserFragment; +use codex_execpolicy::Policy; +use codex_protocol::models::format_allow_prefixes; +use std::collections::BTreeSet; + +/// Newly approved command prefixes visible without the full permissions instructions. +#[derive(Clone, Debug)] +pub(crate) struct CompactPermissionsState { + prefixes: BTreeSet>, +} + +impl CompactPermissionsState { + pub(crate) fn new(exec_policy: &Policy) -> Self { + Self { + prefixes: exec_policy.get_allowed_prefixes().into_iter().collect(), + } + } +} + +impl WorldStateSection for CompactPermissionsState { + const ID: &'static str = "approved_command_prefixes"; + type Snapshot = BTreeSet>; + + fn snapshot(&self) -> Self::Snapshot { + self.prefixes.clone() + } + + fn matches_legacy_fragment(role: &str, text: &str) -> bool { + role == "developer" + && text + .trim_start() + .starts_with(APPROVED_COMMAND_PREFIX_SAVED_MESSAGE_PREFIX) + } + + fn render_diff( + &self, + previous: PreviousSectionState<'_, Self::Snapshot>, + ) -> Option> { + let added_prefixes = match previous { + PreviousSectionState::Known(previous) => self + .prefixes + .difference(previous) + .cloned() + .collect::>(), + PreviousSectionState::Absent | PreviousSectionState::Unknown => return None, + }; + format_allow_prefixes(added_prefixes) + .filter(|prefixes| !prefixes.is_empty()) + .map(|prefixes| Box::new(ApprovedCommandPrefixSaved::new(prefixes)) as _) + } +} + +#[cfg(test)] +#[path = "compact_permissions_tests.rs"] +mod tests; diff --git a/codex-rs/core/src/context/world_state/compact_permissions_tests.rs b/codex-rs/core/src/context/world_state/compact_permissions_tests.rs new file mode 100644 index 0000000000..8f777cb1b1 --- /dev/null +++ b/codex-rs/core/src/context/world_state/compact_permissions_tests.rs @@ -0,0 +1,64 @@ +use super::*; +use codex_execpolicy::Decision; +use pretty_assertions::assert_eq; + +#[test] +fn renders_only_newly_approved_prefixes() { + use PreviousSectionState::Known; + + let mut exec_policy = Policy::empty(); + exec_policy + .add_prefix_rule(&["git".to_string(), "pull".to_string()], Decision::Allow) + .expect("test prefix should be valid"); + let with_existing_prefix = CompactPermissionsState::new(&exec_policy); + exec_policy + .add_prefix_rule(&["cargo".to_string(), "test".to_string()], Decision::Allow) + .expect("test prefix should be valid"); + let with_new_prefix = CompactPermissionsState::new(&exec_policy); + let existing_snapshot = with_existing_prefix.snapshot(); + let current_snapshot = with_new_prefix.snapshot(); + + assert_eq!( + with_new_prefix + .render_diff(Known(&existing_snapshot)) + .map(|fragment| fragment.render()), + Some("Approved command prefix saved:\n- [\"cargo\", \"test\"]".to_string()) + ); + assert!( + with_new_prefix + .render_diff(Known(¤t_snapshot)) + .is_none() + ); +} + +#[test] +fn does_not_duplicate_a_retained_legacy_update() { + use PreviousSectionState::Unknown; + + let mut exec_policy = Policy::empty(); + exec_policy + .add_prefix_rule(&["touch".to_string()], Decision::Allow) + .expect("test prefix should be valid"); + let state = CompactPermissionsState::new(&exec_policy); + + assert_eq!( + state.render_diff(Unknown).map(|fragment| fragment.render()), + None + ); +} + +#[test] +fn does_not_render_existing_prefixes_without_a_previous_snapshot() { + use PreviousSectionState::Absent; + + let mut exec_policy = Policy::empty(); + exec_policy + .add_prefix_rule(&["touch".to_string()], Decision::Allow) + .expect("test prefix should be valid"); + + assert!( + CompactPermissionsState::new(&exec_policy) + .render_diff(Absent) + .is_none() + ); +} diff --git a/codex-rs/core/src/context/world_state/mod.rs b/codex-rs/core/src/context/world_state/mod.rs index 38611f29bb..28cb48c882 100644 --- a/codex-rs/core/src/context/world_state/mod.rs +++ b/codex-rs/core/src/context/world_state/mod.rs @@ -1,6 +1,7 @@ mod agents_md; mod apps_instructions; mod collaboration_mode; +mod compact_permissions; mod context_window_guidance; mod environment; mod environments_instructions; @@ -33,6 +34,7 @@ use std::fmt; pub(crate) use agents_md::AgentsMdState; pub(crate) use apps_instructions::AppsInstructionsState; pub(crate) use collaboration_mode::CollaborationModeState; +pub(crate) use compact_permissions::CompactPermissionsState; pub(crate) use context_window_guidance::ContextWindowGuidanceState; pub(crate) use environment::EnvironmentsState; pub(crate) use environments_instructions::EnvironmentsInstructionsState; diff --git a/codex-rs/core/src/context/world_state/permissions.rs b/codex-rs/core/src/context/world_state/permissions.rs index ae21469a7f..00a690fc60 100644 --- a/codex-rs/core/src/context/world_state/permissions.rs +++ b/codex-rs/core/src/context/world_state/permissions.rs @@ -2,20 +2,35 @@ use super::PreviousSectionState; use super::WorldStateHash; use super::WorldStateSection; use crate::context::ApprovalPromptContext; +use crate::context::ApprovedCommandPrefixSaved; use crate::context::ContextualUserFragment; use crate::context::PermissionsInstructions; use codex_execpolicy::Policy; use codex_protocol::models::PermissionProfile; +use codex_protocol::models::format_allow_prefixes; use codex_protocol::protocol::AskForApproval; +use serde::Deserialize; +use serde::Serialize; +use std::collections::BTreeSet; use std::path::Path; /// Permission instructions currently visible to the model. #[derive(Clone, Debug)] pub(crate) struct PermissionsState { - snapshot: WorldStateHash, + snapshot: PermissionsSnapshot, instructions: PermissionsInstructions, } +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(untagged)] +pub(crate) enum PermissionsSnapshot { + Current { + instructions: WorldStateHash, + approved_command_prefixes: BTreeSet>, + }, + Legacy(WorldStateHash), +} + impl PermissionsState { pub(crate) fn new( permission_profile: &PermissionProfile, @@ -26,16 +41,23 @@ impl PermissionsState { exec_permission_approvals_enabled: bool, request_permissions_tool_enabled: bool, ) -> Self { - let instructions = PermissionsInstructions::from_permission_profile( - permission_profile, - approval_policy, - approval_context, - exec_policy, - cwd, - exec_permission_approvals_enabled, - request_permissions_tool_enabled, - ); - let snapshot = WorldStateHash::from_fragment(&instructions); + let build_instructions = |exec_policy| { + PermissionsInstructions::from_permission_profile( + permission_profile, + approval_policy, + approval_context, + exec_policy, + cwd, + exec_permission_approvals_enabled, + request_permissions_tool_enabled, + ) + }; + let instructions = build_instructions(exec_policy); + let instructions_without_approved_prefixes = build_instructions(&Policy::empty()); + let snapshot = PermissionsSnapshot::Current { + instructions: WorldStateHash::from_fragment(&instructions_without_approved_prefixes), + approved_command_prefixes: exec_policy.get_allowed_prefixes().into_iter().collect(), + }; Self { snapshot, instructions, @@ -45,7 +67,7 @@ impl PermissionsState { impl WorldStateSection for PermissionsState { const ID: &'static str = "permissions"; - type Snapshot = WorldStateHash; + type Snapshot = PermissionsSnapshot; fn snapshot(&self) -> Self::Snapshot { self.snapshot.clone() @@ -67,8 +89,35 @@ impl WorldStateSection for PermissionsState { &self, previous: PreviousSectionState<'_, Self::Snapshot>, ) -> Option> { - if matches!(previous, PreviousSectionState::Known(previous) if previous == &self.snapshot) { - return None; + match (previous, &self.snapshot) { + ( + PreviousSectionState::Known(PermissionsSnapshot::Current { + instructions: previous_instructions, + approved_command_prefixes: previous_prefixes, + }), + PermissionsSnapshot::Current { + instructions, + approved_command_prefixes, + }, + ) if previous_instructions == instructions => { + if previous_prefixes == approved_command_prefixes { + return None; + } + if previous_prefixes.is_subset(approved_command_prefixes) { + let added_prefixes = approved_command_prefixes + .difference(previous_prefixes) + .cloned() + .collect(); + if let Some(prefixes) = format_allow_prefixes(added_prefixes) { + return Some(Box::new(ApprovedCommandPrefixSaved::new(prefixes))); + } + } + } + ( + PreviousSectionState::Known(PermissionsSnapshot::Legacy(previous)), + PermissionsSnapshot::Current { .. }, + ) if previous == &WorldStateHash::from_fragment(&self.instructions) => return None, + _ => {} } Some(Box::new(self.instructions.clone())) diff --git a/codex-rs/core/src/context/world_state/permissions_tests.rs b/codex-rs/core/src/context/world_state/permissions_tests.rs index dedeee0903..3f76ed934d 100644 --- a/codex-rs/core/src/context/world_state/permissions_tests.rs +++ b/codex-rs/core/src/context/world_state/permissions_tests.rs @@ -1,5 +1,6 @@ use super::*; use crate::context::world_state::test_support::render_section_cases; +use codex_execpolicy::Decision; use codex_protocol::config_types::ApprovalsReviewer; use codex_protocol::models::ContentItem; use codex_protocol::models::PermissionProfile; @@ -8,6 +9,7 @@ use codex_protocol::openai_models::ApprovalMessages; use codex_protocol::openai_models::PermissionMessages; use codex_protocol::protocol::AskForApproval; use pretty_assertions::assert_eq; +use serde_json::json; use std::path::Path; #[test] @@ -30,6 +32,145 @@ fn snapshots() { ])); } +#[test] +fn approved_prefix_is_rendered_without_reinjecting_permissions() { + use PreviousSectionState::Known; + + let without_approved_prefix = permissions_state_with_default_messages(&Policy::empty()); + let mut exec_policy = Policy::empty(); + exec_policy + .add_prefix_rule( + &["touch".to_string(), "allow-prefix.txt".to_string()], + Decision::Allow, + ) + .expect("test prefix should be valid"); + let with_approved_prefix = permissions_state_with_default_messages(&exec_policy); + let approved_prefix = r#"["touch", "allow-prefix.txt"]"#; + let without_snapshot = without_approved_prefix.snapshot(); + let with_snapshot = with_approved_prefix.snapshot(); + let rendered_update = with_approved_prefix + .render_diff(Known(&without_snapshot)) + .expect("approving a prefix should render a world-state update") + .render(); + + assert_ne!(without_snapshot, with_snapshot); + assert_ne!( + without_approved_prefix.instructions, + with_approved_prefix.instructions + ); + assert!( + !without_approved_prefix + .instructions + .body() + .contains(approved_prefix) + ); + assert!( + with_approved_prefix + .instructions + .body() + .contains(approved_prefix) + ); + assert_eq!( + rendered_update, + "Approved command prefix saved:\n- [\"touch\", \"allow-prefix.txt\"]" + ); + assert!(!rendered_update.contains("")); + + let before_approval_permissions = + permissions_state(PermissionProfile::read_only(), AskForApproval::OnRequest) + .instructions + .render(); + let model_visible_before_and_after = format!( + "BEFORE APPROVAL — PERMISSIONS ALREADY IN CONTEXT (CONDENSED)\n{before_approval_permissions}\n\nAFTER APPROVAL — WORLD-STATE DIFF APPENDS\n{rendered_update}\n\nFULL PERMISSIONS BLOCK APPENDED AFTER APPROVAL\nNone", + ); + insta::assert_snapshot!( + "approved_prefix_is_rendered_without_reinjecting_permissions", + model_visible_before_and_after + ); +} + +#[test] +fn renders_only_newly_approved_prefixes() { + use PreviousSectionState::Known; + + let mut exec_policy = Policy::empty(); + exec_policy + .add_prefix_rule(&["git".to_string(), "pull".to_string()], Decision::Allow) + .expect("test prefix should be valid"); + let with_existing_prefix = permissions_state_with_default_messages(&exec_policy); + exec_policy + .add_prefix_rule(&["cargo".to_string(), "test".to_string()], Decision::Allow) + .expect("test prefix should be valid"); + let with_new_prefix = permissions_state_with_default_messages(&exec_policy); + let existing_snapshot = with_existing_prefix.snapshot(); + let current_snapshot = with_new_prefix.snapshot(); + + assert_eq!( + with_new_prefix + .render_diff(Known(&existing_snapshot)) + .map(|fragment| fragment.render()), + Some("Approved command prefix saved:\n- [\"cargo\", \"test\"]".to_string()) + ); + assert!( + with_new_prefix + .render_diff(Known(¤t_snapshot)) + .is_none() + ); +} + +#[test] +fn legacy_snapshot_deserializes_and_only_suppresses_matching_full_permissions() { + use super::super::WorldState; + use super::super::WorldStateSnapshot; + + let without_approved_prefix = permissions_state_with_default_messages(&Policy::empty()); + let mut exec_policy = Policy::empty(); + exec_policy + .add_prefix_rule(&["touch".to_string()], Decision::Allow) + .expect("test prefix should be valid"); + let with_approved_prefix = permissions_state_with_default_messages(&exec_policy); + let matching_legacy: WorldStateSnapshot = serde_json::from_value(json!({ + "permissions": WorldStateHash::from_fragment(&with_approved_prefix.instructions), + })) + .expect("legacy world-state snapshot should deserialize"); + let stale_legacy: WorldStateSnapshot = serde_json::from_value(json!({ + "permissions": WorldStateHash::from_fragment(&without_approved_prefix.instructions), + })) + .expect("legacy world-state snapshot should deserialize"); + let expected_permissions = with_approved_prefix.instructions.render(); + let mut world_state = WorldState::default(); + world_state.add_section(with_approved_prefix); + + assert!(world_state.render_diff(&matching_legacy).is_empty()); + assert_eq!( + world_state + .render_diff(&stale_legacy) + .into_iter() + .map(|fragment| fragment.render()) + .collect::>(), + vec![expected_permissions] + ); +} + +#[test] +fn removing_an_approved_prefix_renders_full_permissions() { + use PreviousSectionState::Known; + + let mut exec_policy = Policy::empty(); + exec_policy + .add_prefix_rule(&["touch".to_string()], Decision::Allow) + .expect("test prefix should be valid"); + let with_approved_prefix = permissions_state_with_default_messages(&exec_policy); + let without_approved_prefix = permissions_state_with_default_messages(&Policy::empty()); + + let rendered = without_approved_prefix + .render_diff(Known(&with_approved_prefix.snapshot())) + .expect("removing a prefix should refresh permissions") + .render(); + + assert_eq!(rendered, without_approved_prefix.instructions.render()); +} + #[test] fn persisted_permissions_are_detected_inside_bundled_developer_messages() { let state = permissions_state(PermissionProfile::read_only(), AskForApproval::OnRequest); @@ -94,3 +235,19 @@ fn permissions_state( /*request_permissions_tool_enabled*/ false, ) } + +fn permissions_state_with_default_messages(exec_policy: &Policy) -> PermissionsState { + PermissionsState::new( + &PermissionProfile::read_only(), + AskForApproval::OnRequest, + ApprovalPromptContext::new( + ApprovalsReviewer::User, + /*messages*/ None, + /*permission_messages*/ None, + ), + exec_policy, + Path::new("/workspace"), + /*exec_permission_approvals_enabled*/ false, + /*request_permissions_tool_enabled*/ false, + ) +} diff --git a/codex-rs/core/src/context/world_state/snapshots/codex_core__context__world_state__permissions__tests__approved_prefix_is_rendered_without_reinjecting_permissions.snap b/codex-rs/core/src/context/world_state/snapshots/codex_core__context__world_state__permissions__tests__approved_prefix_is_rendered_without_reinjecting_permissions.snap new file mode 100644 index 0000000000..0d4eff502f --- /dev/null +++ b/codex-rs/core/src/context/world_state/snapshots/codex_core__context__world_state__permissions__tests__approved_prefix_is_rendered_without_reinjecting_permissions.snap @@ -0,0 +1,16 @@ +--- +source: core/src/context/world_state/permissions_tests.rs +expression: model_visible_before_and_after +--- +BEFORE APPROVAL — PERMISSIONS ALREADY IN CONTEXT (CONDENSED) + +Read only. +Ask for approval. + + +AFTER APPROVAL — WORLD-STATE DIFF APPENDS +Approved command prefix saved: +- ["touch", "allow-prefix.txt"] + +FULL PERMISSIONS BLOCK APPENDED AFTER APPROVAL +None diff --git a/codex-rs/core/src/context/world_state/snapshots/codex_core__context__world_state__permissions__tests__snapshots.snap b/codex-rs/core/src/context/world_state/snapshots/codex_core__context__world_state__permissions__tests__snapshots.snap index caa912ecce..efe6728413 100644 --- a/codex-rs/core/src/context/world_state/snapshots/codex_core__context__world_state__permissions__tests__snapshots.snap +++ b/codex-rs/core/src/context/world_state/snapshots/codex_core__context__world_state__permissions__tests__snapshots.snap @@ -5,28 +5,28 @@ expression: "render_section_cases(&[(Absent, Absent), (Absent, Known(&read_only) Absent -> Absent None -Absent -> "0ccde536dd8b4cfebb1df573f679dfe86b5718e3" (role - developer) +Absent -> {"approved_command_prefixes":[],"instructions":"0ccde536dd8b4cfebb1df573f679dfe86b5718e3"} (role - developer) Read only. Ask for approval. -"0ccde536dd8b4cfebb1df573f679dfe86b5718e3" -> "0ccde536dd8b4cfebb1df573f679dfe86b5718e3" +{"approved_command_prefixes":[],"instructions":"0ccde536dd8b4cfebb1df573f679dfe86b5718e3"} -> {"approved_command_prefixes":[],"instructions":"0ccde536dd8b4cfebb1df573f679dfe86b5718e3"} None -"0ccde536dd8b4cfebb1df573f679dfe86b5718e3" -> "7e17d9a0df61e5f0c032ed7f0bf1dda58c9d5e85" (role - developer) +{"approved_command_prefixes":[],"instructions":"0ccde536dd8b4cfebb1df573f679dfe86b5718e3"} -> {"approved_command_prefixes":[],"instructions":"7e17d9a0df61e5f0c032ed7f0bf1dda58c9d5e85"} (role - developer) Full access. Ask for approval. -"0ccde536dd8b4cfebb1df573f679dfe86b5718e3" -> "d96f228104cb899a8d3788b08e1d64b3bbb6ac84" (role - developer) +{"approved_command_prefixes":[],"instructions":"0ccde536dd8b4cfebb1df573f679dfe86b5718e3"} -> {"approved_command_prefixes":[],"instructions":"d96f228104cb899a8d3788b08e1d64b3bbb6ac84"} (role - developer) Read only. Approval policy is currently never. Do not provide the `sandbox_permissions` for any reason, commands will be rejected. -Unknown -> "0ccde536dd8b4cfebb1df573f679dfe86b5718e3" (role - developer) +Unknown -> {"approved_command_prefixes":[],"instructions":"0ccde536dd8b4cfebb1df573f679dfe86b5718e3"} (role - developer) Read only. Ask for approval. diff --git a/codex-rs/core/src/context_manager/history.rs b/codex-rs/core/src/context_manager/history.rs index 6f4c259feb..a3cf288ee5 100644 --- a/codex-rs/core/src/context_manager/history.rs +++ b/codex-rs/core/src/context_manager/history.rs @@ -1,4 +1,3 @@ -use crate::audio_preparation::estimate_audio_token_count; use crate::context::ContextualUserFragment; use crate::context::world_state::WorldState; use crate::context::world_state::WorldStateSnapshot; @@ -23,6 +22,7 @@ use codex_protocol::protocol::TokenUsage; use codex_protocol::protocol::TokenUsageInfo; use codex_protocol::protocol::TurnContextItem; use codex_protocol::protocol::WorldStateItem; +use codex_utils_audio::estimate_audio_token_count; use codex_utils_cache::BlockingLruCache; use codex_utils_cache::sha1_digest; use codex_utils_output_truncation::TruncationPolicy; diff --git a/codex-rs/core/src/context_manager/history_tests.rs b/codex-rs/core/src/context_manager/history_tests.rs index d1dac7d312..2f3ea18237 100644 --- a/codex-rs/core/src/context_manager/history_tests.rs +++ b/codex-rs/core/src/context_manager/history_tests.rs @@ -1,4 +1,5 @@ use super::*; +use crate::context::APPROVED_COMMAND_PREFIX_SAVED_MESSAGE_PREFIX; use crate::context::UserInstructions; use crate::context::world_state::WorldState; use crate::context::world_state::WorldStateSection; @@ -1102,6 +1103,33 @@ fn drop_last_n_user_turns_trims_context_updates_above_rolled_back_turn() { ); } +#[test] +fn drop_last_n_user_turns_trims_saved_prefix_update_above_rolled_back_turn() { + let items = vec![ + assistant_msg("session prefix item"), + user_input_text_msg("turn 1 user"), + assistant_msg("turn 1 assistant"), + developer_msg(&format!( + "{APPROVED_COMMAND_PREFIX_SAVED_MESSAGE_PREFIX}\n- [\"touch\"]" + )), + user_input_text_msg("turn 2 user"), + assistant_msg("turn 2 assistant"), + ]; + + let modalities = default_input_modalities(); + let mut history = create_history_with_items(items); + history.drop_last_n_user_turns(/*num_turns*/ 1); + + assert_eq!( + history.for_prompt(&modalities), + vec![ + assistant_msg("session prefix item"), + user_input_text_msg("turn 1 user"), + assistant_msg("turn 1 assistant"), + ] + ); +} + #[test] fn drop_last_n_user_turns_clears_reference_context_for_mixed_developer_context_bundles() { let items = vec![ diff --git a/codex-rs/core/src/environment_selection.rs b/codex-rs/core/src/environment_selection.rs index 433e50819a..1ebe44a749 100644 --- a/codex-rs/core/src/environment_selection.rs +++ b/codex-rs/core/src/environment_selection.rs @@ -22,6 +22,7 @@ use futures::future::BoxFuture; use futures::future::Shared; use tokio_util::task::AbortOnDropHandle; +use crate::session::turn_context::EnvironmentConfig; use crate::session::turn_context::TurnEnvironment; use crate::shell::Shell; use crate::shell_snapshot::ShellSnapshot; @@ -79,6 +80,8 @@ impl StartingTurnEnvironment { pub(crate) struct ThreadEnvironments { environment_manager: Arc, local_shell: Shell, + // Temporary thread-wide config source until environments supply their own config. + fallback_environment_config: EnvironmentConfig, shell_snapshot: ShellSnapshot, non_blocking_snapshots: bool, environments: ArcSwap>, @@ -89,6 +92,7 @@ impl ThreadEnvironments { pub(crate) fn new( environment_manager: Arc, local_shell: Shell, + fallback_environment_config: EnvironmentConfig, shell_snapshot: ShellSnapshot, current: TurnEnvironmentSnapshot, non_blocking_snapshots: bool, @@ -98,9 +102,11 @@ impl ThreadEnvironments { .environments .into_iter() .filter_map(|environment| { - let TurnEnvironmentState::Ready(environment) = environment else { + let TurnEnvironmentState::Ready(mut environment) = environment else { return None; }; + // Child threads can have a different shell policy from their parent. + environment.config = fallback_environment_config.clone(); let selection = environment.selection(); let selected_environment = Arc::clone(&environment.environment); let resolution: TurnEnvironmentResolution = @@ -116,6 +122,7 @@ impl ThreadEnvironments { Self { environment_manager, local_shell, + fallback_environment_config, shell_snapshot, non_blocking_snapshots, environments: ArcSwap::from_pointee(environments), @@ -166,6 +173,7 @@ impl ThreadEnvironments { selected_environment.clone(), Arc::clone(&environment), self.local_shell.clone(), + self.fallback_environment_config.clone(), self.shell_snapshot.clone(), ) .remote_handle(); @@ -265,6 +273,7 @@ impl ThreadEnvironments { selection: TurnEnvironmentSelection, environment: Arc, local_shell: Shell, + config: EnvironmentConfig, shell_snapshot: ShellSnapshot, ) -> BoxFuture<'static, TurnEnvironmentResult> { async move { @@ -300,6 +309,7 @@ impl ThreadEnvironments { selection.cwd, selection.workspace_roots, shell, + config, ); let task = shell_snapshot .build(turn_environment.clone()) @@ -506,6 +516,9 @@ mod tests { let turn_environments = Arc::new(ThreadEnvironments::new( environment_manager, crate::shell::default_user_shell(), + EnvironmentConfig { + allow_login_shell: true, + }, ShellSnapshot::disabled(), TurnEnvironmentSnapshot::default(), /*non_blocking_snapshots*/ false, @@ -649,7 +662,7 @@ url = "ws://127.0.0.1:8765" } #[tokio::test] - async fn local_environment_uses_configured_shell() { + async fn local_environment_uses_configured_shell_and_login_policy() { let cwd = AbsolutePathBuf::current_dir().expect("cwd"); let local_shell = Shell { shell_type: crate::shell::ShellType::Zsh, @@ -658,6 +671,9 @@ url = "ws://127.0.0.1:8765" let turn_environments = ThreadEnvironments::new( Arc::new(EnvironmentManager::default_for_tests()), local_shell.clone(), + EnvironmentConfig { + allow_login_shell: false, + }, ShellSnapshot::disabled(), TurnEnvironmentSnapshot::default(), /*non_blocking_snapshots*/ false, @@ -669,13 +685,10 @@ url = "ws://127.0.0.1:8765" }]); let snapshot = turn_environments.snapshot().await; + let environment = snapshot.primary().expect("local environment"); - assert_eq!( - snapshot - .primary() - .and_then(|environment| environment.shell.as_ref()), - Some(&local_shell) - ); + assert_eq!(environment.shell.as_ref(), Some(&local_shell)); + assert!(!environment.config.allow_login_shell); } #[tokio::test] @@ -797,6 +810,9 @@ url = "ws://127.0.0.1:8765" let environments = Arc::new(ThreadEnvironments::new( manager, crate::shell::default_user_shell(), + EnvironmentConfig { + allow_login_shell: true, + }, ShellSnapshot::disabled(), TurnEnvironmentSnapshot::default(), /*non_blocking_snapshots*/ false, @@ -850,6 +866,9 @@ url = "ws://127.0.0.1:8765" let turn_environments = ThreadEnvironments::new( manager, crate::shell::default_user_shell(), + EnvironmentConfig { + allow_login_shell: true, + }, ShellSnapshot::disabled(), TurnEnvironmentSnapshot::default(), /*non_blocking_snapshots*/ true, @@ -924,6 +943,9 @@ url = "ws://127.0.0.1:8765" let environments = ThreadEnvironments::new( Arc::clone(&manager), crate::shell::default_user_shell(), + EnvironmentConfig { + allow_login_shell: true, + }, ShellSnapshot::disabled(), TurnEnvironmentSnapshot::default(), /*non_blocking_snapshots*/ true, @@ -978,6 +1000,9 @@ url = "ws://127.0.0.1:8765" let environments = Arc::new(ThreadEnvironments::new( Arc::clone(&manager), crate::shell::default_user_shell(), + EnvironmentConfig { + allow_login_shell: true, + }, ShellSnapshot::disabled(), TurnEnvironmentSnapshot::default(), /*non_blocking_snapshots*/ true, @@ -1050,7 +1075,7 @@ url = "ws://127.0.0.1:8765" } #[tokio::test] - async fn inherited_environment_reuses_parent_handle() { + async fn inherited_environment_reuses_parent_handle_and_uses_child_login_policy() { let cwd = AbsolutePathBuf::current_dir().expect("cwd"); let selection = TurnEnvironmentSelection { environment_id: REMOTE_ENVIRONMENT_ID.to_string(), @@ -1067,6 +1092,9 @@ url = "ws://127.0.0.1:8765" selection.cwd.clone(), Vec::new(), /*shell*/ None, + EnvironmentConfig { + allow_login_shell: true, + }, ); let manager = Arc::new(environment_manager_without_environments()); manager @@ -1079,6 +1107,9 @@ url = "ws://127.0.0.1:8765" let environments = ThreadEnvironments::new( manager, crate::shell::default_user_shell(), + EnvironmentConfig { + allow_login_shell: false, + }, ShellSnapshot::disabled(), TurnEnvironmentSnapshot { environments: vec![TurnEnvironmentState::Ready(inherited)], @@ -1089,13 +1120,9 @@ url = "ws://127.0.0.1:8765" environments.update_selections(std::slice::from_ref(&selection)); let snapshot = environments.snapshot().await; - assert!(Arc::ptr_eq( - &snapshot - .primary() - .expect("inherited environment") - .environment, - &inherited_environment, - )); + let inherited = snapshot.primary().expect("inherited environment"); + assert!(Arc::ptr_eq(&inherited.environment, &inherited_environment)); + assert!(!inherited.config.allow_login_shell); } #[tokio::test] @@ -1124,6 +1151,9 @@ url = "ws://127.0.0.1:8765" cwd_uri.clone(), Vec::new(), /*shell*/ None, + EnvironmentConfig { + allow_login_shell: true, + }, ))], }; let multiple = TurnEnvironmentSnapshot { @@ -1135,6 +1165,9 @@ url = "ws://127.0.0.1:8765" cwd_uri, Vec::new(), /*shell*/ None, + EnvironmentConfig { + allow_login_shell: true, + }, )), ], }; diff --git a/codex-rs/core/src/event_mapping.rs b/codex-rs/core/src/event_mapping.rs index 7af4e5f99a..42bbcc2df6 100644 --- a/codex-rs/core/src/event_mapping.rs +++ b/codex-rs/core/src/event_mapping.rs @@ -32,12 +32,14 @@ use codex_protocol::user_input::UserInput; use tracing::warn; use uuid::Uuid; +use crate::context::APPROVED_COMMAND_PREFIX_SAVED_MESSAGE_PREFIX; use crate::context::is_contextual_user_fragment; use crate::context::parse_visible_hook_prompt_message; use crate::web_search::web_search_action_detail; const CONTEXTUAL_DEVELOPER_PREFIXES: &[&str] = &[ "", + APPROVED_COMMAND_PREFIX_SAVED_MESSAGE_PREFIX, "", APPS_INSTRUCTIONS_OPEN_TAG, COLLABORATION_MODE_OPEN_TAG, diff --git a/codex-rs/core/src/event_mapping_tests.rs b/codex-rs/core/src/event_mapping_tests.rs index ac8ed66c5e..972df5e400 100644 --- a/codex-rs/core/src/event_mapping_tests.rs +++ b/codex-rs/core/src/event_mapping_tests.rs @@ -49,7 +49,7 @@ fn recognizes_context_window_as_contextual_developer_content() { let content = vec![ContentItem::InputText { text: format!( r#"{CONTEXT_WINDOW_OPEN_TAG} -Thread id: 00000000-0000-0000-0000-000000000000 +Agent name: /root {CONTEXT_WINDOW_CLOSE_TAG}"# ), }]; diff --git a/codex-rs/core/src/lib.rs b/codex-rs/core/src/lib.rs index c21bdc8c8d..862d98e9ca 100644 --- a/codex-rs/core/src/lib.rs +++ b/codex-rs/core/src/lib.rs @@ -7,7 +7,6 @@ mod apply_patch; mod apps; -mod audio_preparation; mod client; mod client_common; mod realtime_context; diff --git a/codex-rs/core/src/mcp_openai_file.rs b/codex-rs/core/src/mcp_openai_file.rs index 9075923115..1cc2e72a71 100644 --- a/codex-rs/core/src/mcp_openai_file.rs +++ b/codex-rs/core/src/mcp_openai_file.rs @@ -217,6 +217,7 @@ mod tests { use super::*; use crate::environment_selection::TurnEnvironmentState; use crate::session::tests::make_session_and_context; + use crate::session::turn_context::EnvironmentConfig; use crate::session::turn_context::TurnContext; use crate::session::turn_context::TurnEnvironment; use codex_utils_absolute_path::AbsolutePathBuf; @@ -239,6 +240,7 @@ mod tests { PathUri::from_abs_path(&cwd), Vec::new(), primary.shell.clone(), + primary.config.clone(), ); } @@ -323,6 +325,9 @@ mod tests { let environments = crate::environment_selection::ThreadEnvironments::new( session.services.turn_environments.environment_manager(), crate::shell::default_user_shell(), + EnvironmentConfig { + allow_login_shell: turn_context.config.permissions.allow_login_shell, + }, crate::shell_snapshot::ShellSnapshot::disabled(), Default::default(), /*non_blocking_snapshots*/ true, diff --git a/codex-rs/core/src/mcp_tool_call_tests.rs b/codex-rs/core/src/mcp_tool_call_tests.rs index 4db1cdbd5c..c0d93364fd 100644 --- a/codex-rs/core/src/mcp_tool_call_tests.rs +++ b/codex-rs/core/src/mcp_tool_call_tests.rs @@ -6,6 +6,7 @@ use crate::session::step_context::StepContext; use crate::session::tests::make_session_and_context; use crate::session::tests::make_session_and_context_with_rx; use crate::session::tests::mcp_config_for_test; +use crate::session::turn_context::EnvironmentConfig; use crate::session::turn_context::TurnEnvironment; use crate::state::ActiveTurn; use crate::test_support::models_manager_with_provider; @@ -1102,6 +1103,9 @@ async fn mcp_sandbox_cwd_uses_matching_server_environment_uri() -> anyhow::Resul secondary_cwd.clone(), Vec::new(), /*shell*/ None, + EnvironmentConfig { + allow_login_shell: true, + }, ))); let step_context = StepContext::for_test(Arc::new(turn_context)); diff --git a/codex-rs/core/src/session/handlers.rs b/codex-rs/core/src/session/handlers.rs index dc8b06f02e..8ed7a7badd 100644 --- a/codex-rs/core/src/session/handlers.rs +++ b/codex-rs/core/src/session/handlers.rs @@ -381,29 +381,18 @@ pub async fn exec_approval( if let ReviewDecision::ApprovedExecpolicyAmendment { proposed_execpolicy_amendment, } = &decision - { - match sess + && let Err(err) = sess .persist_execpolicy_amendment(proposed_execpolicy_amendment) .await - { - Ok(()) => { - sess.record_execpolicy_amendment_message( - &event_turn_id, - proposed_execpolicy_amendment, - ) - .await; - } - Err(err) => { - let message = format!("Failed to apply execpolicy amendment: {err}"); - tracing::warn!("{message}"); - let warning = EventMsg::Warning(WarningEvent { message }); - sess.send_event_raw(Event { - id: event_turn_id.clone(), - msg: warning, - }) - .await; - } - } + { + let message = format!("Failed to apply execpolicy amendment: {err}"); + tracing::warn!("{message}"); + let warning = EventMsg::Warning(WarningEvent { message }); + sess.send_event_raw(Event { + id: event_turn_id.clone(), + msg: warning, + }) + .await; } match decision { ReviewDecision::Abort => { diff --git a/codex-rs/core/src/session/mod.rs b/codex-rs/core/src/session/mod.rs index 36d311255d..f515fa2787 100644 --- a/codex-rs/core/src/session/mod.rs +++ b/codex-rs/core/src/session/mod.rs @@ -17,12 +17,10 @@ use crate::agent::status::is_final; use crate::agent_communication::AgentCommunicationContext; use crate::agent_communication::AgentCommunicationKind; use crate::attestation::AttestationProvider; -use crate::audio_preparation::prepare_response_items as prepare_audio_response_items; use crate::compact; use crate::compact::CompactedHistoryMetadata; use crate::config::ManagedFeatures; use crate::config::resolve_tool_suggest_config_from_layer_stack; -use crate::context::ApprovedCommandPrefixSaved; use crate::context::ContextualUserFragment; use crate::context::ModelSwitchInstructions; use crate::context::NetworkRuleSaved; @@ -104,7 +102,6 @@ use codex_protocol::models::AdditionalPermissionProfile; use codex_protocol::models::BaseInstructions; use codex_protocol::models::PermissionProfile; use codex_protocol::models::SandboxEnforcement; -use codex_protocol::models::format_allow_prefixes; use codex_protocol::openai_models::ModelInfo; use codex_protocol::openai_models::ModelPreset; use codex_protocol::permissions::FileSystemSandboxPolicy; @@ -154,6 +151,7 @@ use codex_thread_store::ReadThreadParams; use codex_thread_store::ResumeThreadParams; use codex_thread_store::ThreadPersistenceMetadata; use codex_thread_store::ThreadStore; +use codex_utils_audio::prepare_response_items as prepare_audio_response_items; use codex_utils_path_uri::PathUri; use futures::future::BoxFuture; use futures::future::Shared; @@ -2164,22 +2162,6 @@ impl Session { )) } - pub(crate) async fn record_execpolicy_amendment_message( - &self, - sub_id: &str, - amendment: &ExecPolicyAmendment, - ) { - let Some(prefixes) = format_allow_prefixes(vec![amendment.command.clone()]) else { - warn!("execpolicy amendment for {sub_id} had no command prefix"); - return; - }; - let message: ResponseItem = - ContextualUserFragment::into(ApprovedCommandPrefixSaved::new(prefixes)); - let turn_context = self.turn_context_for_sub_id(sub_id).await; - self.inject_no_new_turn(vec![message], turn_context.as_deref()) - .await; - } - pub(crate) async fn persist_network_policy_amendment( &self, amendment: &NetworkPolicyAmendment, @@ -3511,6 +3493,9 @@ impl Session { developer_sections.push( crate::context::TokenBudgetContext::new( self.thread_id(), + session_source + .get_agent_path() + .unwrap_or_else(codex_protocol::AgentPath::root), auto_compact_window_ids.first_window_id, auto_compact_window_ids.previous_window_id, auto_compact_window_ids.window_id, diff --git a/codex-rs/core/src/session/session.rs b/codex-rs/core/src/session/session.rs index 4b40cbc2e7..e6f5627920 100644 --- a/codex-rs/core/src/session/session.rs +++ b/codex-rs/core/src/session/session.rs @@ -5,6 +5,7 @@ use crate::agents_md_manager::AgentsMdManager; use crate::config::ConstraintError; use crate::environment_selection::ThreadEnvironments; use crate::environment_selection::TurnEnvironmentSnapshot; +use crate::session::turn_context::EnvironmentConfig; use crate::shell_snapshot::ShellSnapshot; use crate::skills::SkillError; use crate::state::ActiveTurn; @@ -939,6 +940,10 @@ impl Session { let turn_environments = Arc::new(ThreadEnvironments::new( environment_manager, default_shell.clone(), + // Temporary: preserve thread-level behavior until environments supply config. + EnvironmentConfig { + allow_login_shell: config.permissions.allow_login_shell, + }, shell_snapshot, inherited_environments.unwrap_or_default(), config.features.enabled(Feature::DeferredExecutor), diff --git a/codex-rs/core/src/session/tests.rs b/codex-rs/core/src/session/tests.rs index c67aa14dd6..040138acc0 100644 --- a/codex-rs/core/src/session/tests.rs +++ b/codex-rs/core/src/session/tests.rs @@ -1,4 +1,5 @@ use super::mcp_refresh::McpRefresh; +use super::turn_context::EnvironmentConfig; use super::turn_context::TurnEnvironment; use super::*; use crate::agents_md_manager::AgentsMdManager; @@ -4869,6 +4870,12 @@ async fn resolved_environments_for_configuration( let turn_environments = ThreadEnvironments::new( Arc::clone(&environment_manager), default_user_shell(), + EnvironmentConfig { + allow_login_shell: session_configuration + .original_config_do_not_use + .permissions + .allow_login_shell, + }, ShellSnapshot::disabled(), TurnEnvironmentSnapshot::default(), /*non_blocking_snapshots*/ false, @@ -5667,6 +5674,9 @@ pub(crate) async fn make_session_and_context() -> (Session, TurnContext) { let turn_environments = Arc::new(ThreadEnvironments::new( environment_manager, default_user_shell(), + EnvironmentConfig { + allow_login_shell: config.permissions.allow_login_shell, + }, ShellSnapshot::disabled(), resolved_environments, /*non_blocking_snapshots*/ false, @@ -6470,6 +6480,7 @@ async fn request_permissions_tool_resolves_relative_paths_against_selected_envir PathUri::from_abs_path(&environment_cwd), Vec::new(), current_environment.shell, + current_environment.config, )); let call_id = "call-1".to_string(); @@ -7131,6 +7142,7 @@ async fn primary_environment_uses_first_turn_environment() { second_cwd_uri.clone(), Vec::new(), /*shell*/ None, + first_environment.config.clone(), ))); assert_eq!( @@ -7856,6 +7868,9 @@ where let turn_environments = Arc::new(ThreadEnvironments::new( environment_manager, default_user_shell(), + EnvironmentConfig { + allow_login_shell: config.permissions.allow_login_shell, + }, ShellSnapshot::disabled(), resolved_turn_environments.clone(), /*non_blocking_snapshots*/ false, @@ -8446,6 +8461,7 @@ async fn conflicting_ready_environment_root_ids_keep_first_location() { local_environment.cwd().clone(), local_environment.workspace_roots().to_vec(), local_environment.shell.clone(), + local_environment.config.clone(), )); } let environments = TurnEnvironmentSnapshot { @@ -8656,6 +8672,7 @@ async fn record_context_updates_emits_environment_item_for_cwd_changes() { PathUri::from_abs_path(&cwd), Vec::new(), environment.shell, + environment.config, )); let update_items = @@ -8721,6 +8738,7 @@ async fn record_context_updates_omits_environment_item_when_disabled() { PathUri::from_abs_path(&test_path_buf("/new-repo").abs()), Vec::new(), environment.shell, + environment.config, )); let update_items = @@ -9048,13 +9066,24 @@ async fn build_initial_context_adds_multi_agent_v2_subagent_usage_hint_as_develo .await .session_configuration .session_source = session_source.clone(); - Arc::get_mut(&mut turn_context) - .expect("thread settings should not be shared") - .session_source = session_source; + let turn_context_mut = + Arc::get_mut(&mut turn_context).expect("thread settings should not be shared"); + turn_context_mut.session_source = session_source; + Arc::make_mut(&mut turn_context_mut.config) + .features + .enable(Feature::TokenBudget) + .expect("test config should allow token budget"); let initial_context = build_initial_context(&session, &turn_context).await; let developer_messages = developer_message_texts(&initial_context); + assert!( + developer_messages + .iter() + .flatten() + .any(|text| text.contains("\nAgent name: /root/worker\n")), + "expected subagent context window to include its canonical name, got {developer_messages:?}" + ); assert!( developer_messages .iter() @@ -9172,6 +9201,7 @@ async fn turn_context_item_stores_local_cwd() { cwd, Vec::new(), environment.shell, + environment.config, )); #[allow(deprecated)] diff --git a/codex-rs/core/src/session/turn_context.rs b/codex-rs/core/src/session/turn_context.rs index 600f736d69..30ae7e2d81 100644 --- a/codex-rs/core/src/session/turn_context.rs +++ b/codex-rs/core/src/session/turn_context.rs @@ -27,6 +27,12 @@ use tracing::instrument; pub(crate) type ShellSnapshotTask = Shared>>>; +/// Effective per-environment config; fields move here as executor config is migrated. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct EnvironmentConfig { + pub(crate) allow_login_shell: bool, +} + #[derive(Clone)] pub(crate) struct TurnEnvironment { pub(crate) environment_id: String, @@ -34,6 +40,7 @@ pub(crate) struct TurnEnvironment { cwd: PathUri, workspace_roots: Vec, pub(crate) shell: Option, + pub(crate) config: EnvironmentConfig, pub(crate) shell_snapshot: ShellSnapshotTask, } @@ -44,6 +51,7 @@ impl TurnEnvironment { cwd: PathUri, workspace_roots: Vec, shell: Option, + config: EnvironmentConfig, ) -> Self { Self { environment_id, @@ -51,6 +59,7 @@ impl TurnEnvironment { cwd, workspace_roots, shell, + config, shell_snapshot: futures::future::ready(None).boxed().shared(), } } @@ -90,6 +99,7 @@ impl std::fmt::Debug for TurnEnvironment { .field("cwd", &self.cwd) .field("workspace_roots", &self.workspace_roots) .field("shell", &self.shell) + .field("config", &self.config) .finish_non_exhaustive() } } diff --git a/codex-rs/core/src/session/world_state.rs b/codex-rs/core/src/session/world_state.rs index 073dba8aec..f56b8baecd 100644 --- a/codex-rs/core/src/session/world_state.rs +++ b/codex-rs/core/src/session/world_state.rs @@ -7,6 +7,7 @@ use crate::context::ApprovalPromptContext; use crate::context::world_state::AgentsMdState; use crate::context::world_state::AppsInstructionsState; use crate::context::world_state::CollaborationModeState; +use crate::context::world_state::CompactPermissionsState; use crate::context::world_state::ContextWindowGuidanceState; use crate::context::world_state::EnvironmentsInstructionsState; use crate::context::world_state::EnvironmentsState; @@ -135,6 +136,9 @@ impl Session { .features .enabled(Feature::RequestPermissionsTool), )); + } else { + let exec_policy = self.services.exec_policy.current(); + world_state.add_section(CompactPermissionsState::new(exec_policy.as_ref())); } if turn_context.config.include_collaboration_mode_instructions { world_state.add_section(CollaborationModeState::from_collaboration_mode( diff --git a/codex-rs/core/src/thread_manager.rs b/codex-rs/core/src/thread_manager.rs index 037752bd5b..c636ace7cd 100644 --- a/codex-rs/core/src/thread_manager.rs +++ b/codex-rs/core/src/thread_manager.rs @@ -232,6 +232,38 @@ impl StartThreadOptions { } } +struct ThreadSpawnRequest { + options: StartThreadOptions, + auth_manager: Arc, + agent_control: AgentControl, + parent_thread_id: Option, + forked_from_thread_id: Option, + fork_persistence: ForkPersistence, + inherited_environments: Option, + inherited_exec_policy: Option>, + user_shell_override: Option, +} + +impl ThreadSpawnRequest { + fn new( + options: StartThreadOptions, + auth_manager: Arc, + agent_control: AgentControl, + ) -> Self { + Self { + options, + auth_manager, + agent_control, + parent_thread_id: None, + forked_from_thread_id: None, + fork_persistence: ForkPersistence::Copied, + inherited_environments: None, + inherited_exec_policy: None, + user_shell_override: None, + } + } +} + fn originator_from_service_name(service_name: Option<&str>) -> Option { let service_name = service_name?.trim(); for originator in [ @@ -777,46 +809,25 @@ impl ThreadManager { async fn start_thread_inner( &self, - options: StartThreadOptions, + mut options: StartThreadOptions, forked_from_thread_id: Option, ) -> CodexResult { - let environments = options.environments.unwrap_or_else(|| { - default_thread_environment_selections( - self.state.environment_manager.as_ref(), - &options.config.cwd, - &options.config.workspace_roots, - ) - }); let agent_control = self.agent_control_for_config(&options.config); let (resumed_session_source, resumed_thread_source) = options .initial_history .get_resumed_session_sources() .unwrap_or_else(|| (self.state.session_source.clone(), None)); - let session_source = options.session_source.unwrap_or(resumed_session_source); - let thread_source = options.thread_source.or(resumed_thread_source); - Box::pin(self.state.spawn_thread_with_source( - options.config, - options.initial_history, - options.history_mode, - options.allow_provider_model_fallback, - Arc::clone(&self.state.auth_manager), - agent_control, - session_source, - /*parent_thread_id*/ None, - forked_from_thread_id, - ForkPersistence::Copied, - thread_source, - options.dynamic_tools, - options.metrics_service_name, - /*inherited_environments*/ None, - /*inherited_exec_policy*/ None, - options.parent_trace, - environments, - options.thread_extension_init, - options.supports_openai_form_elicitation, - /*user_shell_override*/ None, - )) - .await + options.session_source = Some( + options + .session_source + .take() + .unwrap_or(resumed_session_source), + ); + options.thread_source = options.thread_source.take().or(resumed_thread_source); + let mut request = + ThreadSpawnRequest::new(options, Arc::clone(&self.state.auth_manager), agent_control); + request.forked_from_thread_id = forked_from_thread_id; + Box::pin(self.state.spawn_thread(request)).await } // TODO(jif) merge with fork_agent @@ -885,11 +896,6 @@ impl ThreadManager { supports_openai_form_elicitation: bool, ) -> CodexResult { let agent_control = self.agent_control_for_config(&config); - let environments = default_thread_environment_selections( - self.state.environment_manager.as_ref(), - &config.cwd, - &config.workspace_roots, - ); let (session_source, thread_source) = initial_history .get_resumed_session_sources() .unwrap_or_else(|| (self.state.session_source.clone(), None)); @@ -901,28 +907,19 @@ impl ThreadManager { .restore_v2_agent_metadata(&config, resumed.conversation_id) .await; } - Box::pin(self.state.spawn_thread_with_source( - config, + let options = StartThreadOptions { initial_history, - /*history_mode*/ None, - /*allow_provider_model_fallback*/ false, + session_source: Some(session_source), + thread_source, + parent_trace, + supports_openai_form_elicitation, + ..StartThreadOptions::new(config) + }; + Box::pin(self.state.spawn_thread(ThreadSpawnRequest::new( + options, auth_manager, agent_control, - session_source, - /*parent_thread_id*/ None, - /*forked_from_thread_id*/ None, - ForkPersistence::Copied, - thread_source, - Vec::new(), - /*metrics_service_name*/ None, - /*inherited_environments*/ None, - /*inherited_exec_policy*/ None, - parent_trace, - environments, - /*thread_extension_init*/ ExtensionDataInit::default(), - supports_openai_form_elicitation, - /*user_shell_override*/ None, - )) + ))) .await } @@ -933,29 +930,14 @@ impl ThreadManager { supports_openai_form_elicitation: bool, ) -> CodexResult { let agent_control = self.agent_control_for_config(&config); - let environments = default_thread_environment_selections( - self.state.environment_manager.as_ref(), - &config.cwd, - &config.workspace_roots, - ); - Box::pin(self.state.spawn_thread( - config, - InitialHistory::New, - Arc::clone(&self.state.auth_manager), - agent_control, - /*parent_thread_id*/ None, - /*forked_from_thread_id*/ None, - ForkPersistence::Copied, - /*thread_source*/ None, - Vec::new(), - /*metrics_service_name*/ None, - /*parent_trace*/ None, - environments, - /*thread_extension_init*/ ExtensionDataInit::default(), + let options = StartThreadOptions { supports_openai_form_elicitation, - /*user_shell_override*/ Some(user_shell_override), - )) - .await + ..StartThreadOptions::new(config) + }; + let mut request = + ThreadSpawnRequest::new(options, Arc::clone(&self.state.auth_manager), agent_control); + request.user_shell_override = Some(user_shell_override); + Box::pin(self.state.spawn_thread(request)).await } pub(crate) async fn resume_thread_from_rollout_with_user_shell_override_for_tests( @@ -968,37 +950,19 @@ impl ThreadManager { ) -> CodexResult { let agent_control = self.agent_control_for_config(&config); let initial_history = self.initial_history_from_rollout_path(rollout_path).await?; - let environments = default_thread_environment_selections( - self.state.environment_manager.as_ref(), - &config.cwd, - &config.workspace_roots, - ); let (session_source, thread_source) = initial_history .get_resumed_session_sources() .unwrap_or_else(|| (self.state.session_source.clone(), None)); - Box::pin(self.state.spawn_thread_with_source( - config, + let options = StartThreadOptions { initial_history, - /*history_mode*/ None, - /*allow_provider_model_fallback*/ false, - auth_manager, - agent_control, - session_source, - /*parent_thread_id*/ None, - /*forked_from_thread_id*/ None, - ForkPersistence::Copied, + session_source: Some(session_source), thread_source, - Vec::new(), - /*metrics_service_name*/ None, - /*inherited_environments*/ None, - /*inherited_exec_policy*/ None, - /*parent_trace*/ None, - environments, - /*thread_extension_init*/ ExtensionDataInit::default(), supports_openai_form_elicitation, - /*user_shell_override*/ Some(user_shell_override), - )) - .await + ..StartThreadOptions::new(config) + }; + let mut request = ThreadSpawnRequest::new(options, auth_manager, agent_control); + request.user_shell_override = Some(user_shell_override); + Box::pin(self.state.spawn_thread(request)).await } /// Removes the thread from the manager's internal map, though the thread is stored @@ -1200,30 +1164,19 @@ impl ThreadManager { let interrupted_marker = InterruptedTurnHistoryMarker::from_config_and_version(&config, multi_agent_version); let history = fork_history_from_snapshot(snapshot, history, interrupted_marker); - let environments = default_thread_environment_selections( - self.state.environment_manager.as_ref(), - &config.cwd, - &config.workspace_roots, - ); let agent_control = self.agent_control_for_config(&config); - Box::pin(self.state.spawn_thread( - config, - history, - Arc::clone(&self.state.auth_manager), - agent_control, - /*parent_thread_id*/ None, - source_thread_id, - fork_persistence, + let options = StartThreadOptions { + initial_history: history, thread_source, - Vec::new(), - /*metrics_service_name*/ None, parent_trace, - environments, - /*thread_extension_init*/ ExtensionDataInit::default(), supports_openai_form_elicitation, - /*user_shell_override*/ None, - )) - .await + ..StartThreadOptions::new(config) + }; + let mut request = + ThreadSpawnRequest::new(options, Arc::clone(&self.state.auth_manager), agent_control); + request.forked_from_thread_id = source_thread_id; + request.fork_persistence = fork_persistence; + Box::pin(self.state.spawn_thread(request)).await } pub(crate) fn agent_control(&self) -> AgentControl { @@ -1551,36 +1504,21 @@ impl ThreadManagerState { inherited_exec_policy: Option>, environments: Option>, ) -> CodexResult { - let environments = environments.unwrap_or_else(|| { - default_thread_environment_selections( - self.environment_manager.as_ref(), - &config.cwd, - &config.workspace_roots, - ) - }); - Box::pin(self.spawn_thread_with_source( - config, - InitialHistory::New, + let options = StartThreadOptions { history_mode, - /*allow_provider_model_fallback*/ false, - Arc::clone(&self.auth_manager), - agent_control, - session_source, - parent_thread_id, - forked_from_thread_id, - ForkPersistence::Copied, + session_source: Some(session_source), thread_source, - Vec::new(), metrics_service_name, - inherited_environments, - inherited_exec_policy, - /*parent_trace*/ None, environments, - /*thread_extension_init*/ ExtensionDataInit::default(), - /*supports_openai_form_elicitation*/ false, - /*user_shell_override*/ None, - )) - .await + ..StartThreadOptions::new(config) + }; + let mut request = + ThreadSpawnRequest::new(options, Arc::clone(&self.auth_manager), agent_control); + request.parent_thread_id = parent_thread_id; + request.forked_from_thread_id = forked_from_thread_id; + request.inherited_environments = inherited_environments; + request.inherited_exec_policy = inherited_exec_policy; + Box::pin(self.spawn_thread(request)).await } pub(crate) async fn resume_thread_with_history_with_source( @@ -1596,35 +1534,19 @@ impl ThreadManagerState { inherited_environments, inherited_exec_policy, } = options; - let environments = default_thread_environment_selections( - self.environment_manager.as_ref(), - &config.cwd, - &config.workspace_roots, - ); let thread_source = initial_history.get_resumed_thread_source(); - Box::pin(self.spawn_thread_with_source( - config, + let options = StartThreadOptions { initial_history, - /*history_mode*/ None, - /*allow_provider_model_fallback*/ false, - Arc::clone(&self.auth_manager), - agent_control, - session_source, - parent_thread_id, - /*forked_from_thread_id*/ None, - ForkPersistence::Copied, + session_source: Some(session_source), thread_source, - Vec::new(), - /*metrics_service_name*/ None, - inherited_environments, - inherited_exec_policy, - /*parent_trace*/ None, - environments, - /*thread_extension_init*/ ExtensionDataInit::default(), - /*supports_openai_form_elicitation*/ false, - /*user_shell_override*/ None, - )) - .await + ..StartThreadOptions::new(config) + }; + let mut request = + ThreadSpawnRequest::new(options, Arc::clone(&self.auth_manager), agent_control); + request.parent_thread_id = parent_thread_id; + request.inherited_environments = inherited_environments; + request.inherited_exec_policy = inherited_exec_policy; + Box::pin(self.spawn_thread(request)).await } #[allow(clippy::too_many_arguments)] @@ -1643,6 +1565,52 @@ impl ThreadManagerState { environments: Option>, thread_extension_init: ExtensionDataInit, ) -> CodexResult { + let options = StartThreadOptions { + initial_history, + history_mode, + session_source: Some(session_source), + thread_source, + environments, + thread_extension_init, + ..StartThreadOptions::new(config) + }; + let mut request = + ThreadSpawnRequest::new(options, Arc::clone(&self.auth_manager), agent_control); + request.parent_thread_id = parent_thread_id; + request.forked_from_thread_id = forked_from_thread_id; + request.inherited_environments = inherited_environments; + request.inherited_exec_policy = inherited_exec_policy; + Box::pin(self.spawn_thread(request)).await + } + + /// Spawn a new thread with optional history and register it with the manager. + async fn spawn_thread(&self, request: ThreadSpawnRequest) -> CodexResult { + let ThreadSpawnRequest { + options, + auth_manager, + agent_control, + parent_thread_id, + forked_from_thread_id, + fork_persistence, + inherited_environments, + inherited_exec_policy, + user_shell_override, + } = request; + let StartThreadOptions { + config, + allow_provider_model_fallback, + initial_history, + history_mode, + session_source, + thread_source, + dynamic_tools, + metrics_service_name, + parent_trace, + environments, + thread_extension_init, + supports_openai_form_elicitation, + } = options; + let session_source = session_source.unwrap_or_else(|| self.session_source.clone()); let environments = environments.unwrap_or_else(|| { default_thread_environment_selections( self.environment_manager.as_ref(), @@ -1650,109 +1618,6 @@ impl ThreadManagerState { &config.workspace_roots, ) }); - Box::pin(self.spawn_thread_with_source( - config, - initial_history, - history_mode, - /*allow_provider_model_fallback*/ false, - Arc::clone(&self.auth_manager), - agent_control, - session_source, - parent_thread_id, - forked_from_thread_id, - ForkPersistence::Copied, - thread_source, - Vec::new(), - /*metrics_service_name*/ None, - inherited_environments, - inherited_exec_policy, - /*parent_trace*/ None, - environments, - thread_extension_init, - /*supports_openai_form_elicitation*/ false, - /*user_shell_override*/ None, - )) - .await - } - - /// Spawn a new thread with optional history and register it with the manager. - #[allow(clippy::too_many_arguments)] - pub(crate) async fn spawn_thread( - &self, - config: Config, - initial_history: InitialHistory, - auth_manager: Arc, - agent_control: AgentControl, - parent_thread_id: Option, - forked_from_thread_id: Option, - fork_persistence: ForkPersistence, - thread_source: Option, - dynamic_tools: Vec, - metrics_service_name: Option, - parent_trace: Option, - environments: Vec, - thread_extension_init: ExtensionDataInit, - supports_openai_form_elicitation: bool, - user_shell_override: Option, - ) -> CodexResult { - Box::pin(self.spawn_thread_with_source( - config, - initial_history, - /*history_mode*/ None, - /*allow_provider_model_fallback*/ false, - auth_manager, - agent_control, - self.session_source.clone(), - parent_thread_id, - forked_from_thread_id, - fork_persistence, - thread_source, - dynamic_tools, - metrics_service_name, - /*inherited_environments*/ None, - /*inherited_exec_policy*/ None, - parent_trace, - environments, - thread_extension_init, - supports_openai_form_elicitation, - user_shell_override, - )) - .await - } - - #[allow(clippy::too_many_arguments)] - pub(crate) async fn spawn_thread_with_source( - &self, - config: Config, - initial_history: InitialHistory, - history_mode: Option, - allow_provider_model_fallback: bool, - auth_manager: Arc, - agent_control: AgentControl, - session_source: SessionSource, - parent_thread_id: Option, - forked_from_thread_id: Option, - fork_persistence: ForkPersistence, - thread_source: Option, - dynamic_tools: Vec, - metrics_service_name: Option, - inherited_environments: Option, - inherited_exec_policy: Option>, - parent_trace: Option, - environments: Vec, - thread_extension_init: ExtensionDataInit, - supports_openai_form_elicitation: bool, - user_shell_override: Option, - ) -> CodexResult { - let source_changed_during_startup = Arc::new(AtomicBool::new(false)); - { - let mut starting = self - .starting_mcp_runtimes - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - starting.retain(|runtime| runtime.strong_count() != 0); - starting.push(Arc::downgrade(&source_changed_during_startup)); - } let is_resumed_thread = matches!(&initial_history, InitialHistory::Resumed(_)); if let InitialHistory::Resumed(resumed) = &initial_history { let mut threads = self.threads.write().await; diff --git a/codex-rs/core/src/tools/approvals.rs b/codex-rs/core/src/tools/approvals.rs index 6b4bedd00d..693f37707a 100644 --- a/codex-rs/core/src/tools/approvals.rs +++ b/codex-rs/core/src/tools/approvals.rs @@ -9,7 +9,6 @@ use crate::sandboxing::SandboxPermissions; use crate::session::turn_context::TurnContext; use crate::tools::flat_tool_name; use crate::tools::sandboxing::ApprovalCtx; -use crate::tools::sandboxing::ToolCtx; use crate::tools::sandboxing::ToolError; use crate::tools::sandboxing::ToolRuntime; use codex_config::types::ApprovalsReviewer; @@ -188,14 +187,12 @@ impl ApprovalResolution { } } -pub(super) async fn resolve_tool_apporval( +pub(super) async fn resolve_tool_approval( tool: &mut T, req: &Rq, permission_request_run_id: &str, ctx: ApprovalCtx<'_>, - tool_ctx: &ToolCtx, reviewer: ApprovalReviewer, - otel: &codex_otel::SessionTelemetry, ) -> Result where T: ToolRuntime, @@ -214,7 +211,7 @@ where decision: ReviewDecision::Approved, source: ApprovalResolutionSource::Hook, }; - record_resolution(otel, tool_ctx, &resolution); + record_resolution(&ctx, &resolution); return resolution.into_tool_result(); } Some(PermissionRequestDecision::Deny { message }) => { @@ -222,7 +219,7 @@ where decision: ReviewDecision::denied(message), source: ApprovalResolutionSource::Hook, }; - record_resolution(otel, tool_ctx, &resolution); + record_resolution(&ctx, &resolution); return resolution.into_tool_result(); } None => {} @@ -245,7 +242,7 @@ where ), source: ApprovalResolutionSource::Guardian, }; - record_resolution(otel, tool_ctx, &resolution); + record_resolution(&ctx, &resolution); return resolution.into_tool_result(); } }; @@ -268,24 +265,20 @@ where decision: resolution, source, }; - record_resolution(otel, tool_ctx, &resolution); + record_resolution(&ctx, &resolution); resolution.into_tool_result() } -fn record_resolution( - otel: &codex_otel::SessionTelemetry, - tool_ctx: &ToolCtx, - resolution: &ApprovalResolution, -) { +fn record_resolution(ctx: &ApprovalCtx<'_>, resolution: &ApprovalResolution) { let source = match resolution.source { ApprovalResolutionSource::Hook => ToolDecisionSource::Config, ApprovalResolutionSource::Guardian => ToolDecisionSource::AutomatedReviewer, ApprovalResolutionSource::User => ToolDecisionSource::User, }; - let tool_name = flat_tool_name(&tool_ctx.tool_name); - otel.tool_decision( + let tool_name = flat_tool_name(ctx.tool_name); + ctx.turn.session_telemetry.tool_decision( tool_name.as_ref(), - &tool_ctx.call_id, + ctx.call_id, &resolution.decision, source, ); diff --git a/codex-rs/core/src/tools/code_mode/execute_spec.rs b/codex-rs/core/src/tools/code_mode/execute_spec.rs index 9987396556..b3476720c8 100644 --- a/codex-rs/core/src/tools/code_mode/execute_spec.rs +++ b/codex-rs/core/src/tools/code_mode/execute_spec.rs @@ -30,6 +30,7 @@ SOURCE: /[\s\S]+/ default_exec_yield_time_ms, code_mode_only, ), + defer_loading: None, format: FreeformToolFormat { r#type: "grammar".to_string(), syntax: "lark".to_string(), @@ -72,6 +73,7 @@ mod tests { codex_code_mode::DEFAULT_EXEC_YIELD_TIME_MS, /*code_mode_only*/ true, ), + defer_loading: None, format: FreeformToolFormat { r#type: "grammar".to_string(), syntax: "lark".to_string(), diff --git a/codex-rs/core/src/tools/code_mode/mod.rs b/codex-rs/core/src/tools/code_mode/mod.rs index 5054f68c9c..57f1e6725c 100644 --- a/codex-rs/core/src/tools/code_mode/mod.rs +++ b/codex-rs/core/src/tools/code_mode/mod.rs @@ -24,7 +24,6 @@ use serde_json::Value as JsonValue; use tokio::sync::OnceCell; use tokio_util::sync::CancellationToken; -use crate::audio_preparation::estimate_audio_token_count; use crate::function_tool::FunctionCallError; use crate::original_image_detail::can_request_original_image_detail; use crate::original_image_detail::sanitize_original_image_detail as sanitize_image_detail_items; @@ -41,6 +40,7 @@ use crate::tools::router::ToolCallSource; use crate::unified_exec::resolve_max_tokens; use codex_protocol::openai_models::ToolMode; use codex_tools::ToolName; +use codex_utils_audio::estimate_audio_token_count; use codex_utils_output_truncation::TruncationPolicy; use codex_utils_output_truncation::formatted_truncate_text_content_items_with_policy; use codex_utils_output_truncation::truncate_function_output_items_with_policy; diff --git a/codex-rs/core/src/tools/handlers/apply_patch_spec.rs b/codex-rs/core/src/tools/handlers/apply_patch_spec.rs index 8227605027..39956d208f 100644 --- a/codex-rs/core/src/tools/handlers/apply_patch_spec.rs +++ b/codex-rs/core/src/tools/handlers/apply_patch_spec.rs @@ -18,6 +18,7 @@ pub fn create_apply_patch_freeform_tool(include_environment_id: bool) -> ToolSpe ToolSpec::Freeform(FreeformTool { name: "apply_patch".to_string(), description: "The `apply_patch` tool can be used to edit files. This is a FREEFORM tool, so do not wrap the patch in JSON.".to_string(), + defer_loading: None, format: FreeformToolFormat { r#type: "grammar".to_string(), syntax: "lark".to_string(), diff --git a/codex-rs/core/src/tools/handlers/apply_patch_spec_tests.rs b/codex-rs/core/src/tools/handlers/apply_patch_spec_tests.rs index d6fe521c2f..6ea5ec7153 100644 --- a/codex-rs/core/src/tools/handlers/apply_patch_spec_tests.rs +++ b/codex-rs/core/src/tools/handlers/apply_patch_spec_tests.rs @@ -10,6 +10,7 @@ fn create_apply_patch_freeform_tool_matches_expected_spec() { description: "The `apply_patch` tool can be used to edit files. This is a FREEFORM tool, so do not wrap the patch in JSON." .to_string(), + defer_loading: None, format: FreeformToolFormat { r#type: "grammar".to_string(), syntax: "lark".to_string(), diff --git a/codex-rs/core/src/tools/handlers/extension_tools.rs b/codex-rs/core/src/tools/handlers/extension_tools.rs index 2f799e912a..52eb8da899 100644 --- a/codex-rs/core/src/tools/handlers/extension_tools.rs +++ b/codex-rs/core/src/tools/handlers/extension_tools.rs @@ -6,6 +6,7 @@ use codex_protocol::protocol::Event; use codex_protocol::protocol::EventMsg; use codex_tools::ConversationHistory; use codex_tools::ExtensionTurnItem; +use codex_tools::ResponsesApiNamespaceTool; use codex_tools::ToolCall as ExtensionToolCall; use codex_tools::ToolEnvironment; use codex_tools::ToolName; @@ -61,7 +62,23 @@ impl ToolExecutor for ExtensionToolAdapter { impl CoreToolRuntime for ExtensionToolAdapter { fn matches_kind(&self, payload: &ToolPayload) -> bool { - matches!(payload, ToolPayload::Function { .. }) + match payload { + ToolPayload::Function { .. } => true, + ToolPayload::Custom { .. } => match self.0.spec() { + ToolSpec::Freeform(_) => true, + ToolSpec::Namespace(namespace) => namespace.tools.iter().any(|tool| { + matches!( + tool, + ResponsesApiNamespaceTool::Custom(tool) + if tool.name == self.0.tool_name().name + ) + }), + ToolSpec::Function(_) + | ToolSpec::ToolSearch { .. } + | ToolSpec::WebSearch { .. } => false, + }, + ToolPayload::ToolSearch { .. } => false, + } } } @@ -286,6 +303,18 @@ mod tests { } } + #[test] + fn function_extensions_reject_custom_payloads() { + let handler = ExtensionToolAdapter::new(Arc::new(StubExtensionExecutor)); + + assert!(handler.matches_kind(&ToolPayload::Function { + arguments: "{}".to_string(), + })); + assert!(!handler.matches_kind(&ToolPayload::Custom { + input: "raw input".to_string(), + })); + } + #[tokio::test] async fn exposes_generic_hook_payloads() { let handler = ExtensionToolAdapter::new(Arc::new(StubExtensionExecutor)); diff --git a/codex-rs/core/src/tools/handlers/shell/shell_command.rs b/codex-rs/core/src/tools/handlers/shell/shell_command.rs index 2b4cc7e2ed..7b975e2f71 100644 --- a/codex-rs/core/src/tools/handlers/shell/shell_command.rs +++ b/codex-rs/core/src/tools/handlers/shell/shell_command.rs @@ -91,14 +91,14 @@ impl ShellCommandHandler { turn_context: &TurnContext, turn_environment: &TurnEnvironment, cwd: AbsolutePathBuf, - allow_login_shell: bool, ) -> Result { let session_shell = session.user_shell(); let shell = turn_environment .shell .as_ref() .unwrap_or(session_shell.as_ref()); - let use_login_shell = Self::resolve_use_login_shell(params.login, allow_login_shell)?; + let use_login_shell = + Self::resolve_use_login_shell(params.login, turn_environment.config.allow_login_shell)?; let command = Self::base_command(shell, ¶ms.command, use_login_shell); let mut env = create_env( @@ -214,7 +214,6 @@ impl ShellCommandHandler { turn.as_ref(), &turn_environment, cwd, - turn.config.permissions.allow_login_shell, )?; let shell_type = Some( turn_environment diff --git a/codex-rs/core/src/tools/handlers/shell_tests.rs b/codex-rs/core/src/tools/handlers/shell_tests.rs index f71f23ca1b..f7caa3a9e4 100644 --- a/codex-rs/core/src/tools/handlers/shell_tests.rs +++ b/codex-rs/core/src/tools/handlers/shell_tests.rs @@ -12,6 +12,7 @@ use crate::exec_env::inject_permission_profile_env; use crate::sandboxing::SandboxPermissions; use crate::session::step_context::StepContext; use crate::session::tests::make_session_and_context; +use crate::session::turn_context::EnvironmentConfig; use crate::session::turn_context::TurnEnvironment; use crate::shell::Shell; use crate::shell::ShellType; @@ -111,6 +112,9 @@ async fn shell_command_handler_to_exec_params_uses_selected_environment() { PathUri::from_abs_path(&selected_cwd), Vec::new(), Some(selected_shell), + EnvironmentConfig { + allow_login_shell: true, + }, ); let mut expected_env = create_env( &turn_context.config.permissions.shell_environment_policy, @@ -136,7 +140,6 @@ async fn shell_command_handler_to_exec_params_uses_selected_environment() { &turn_context, &selected_environment, expected_cwd.clone(), - /*allow_login_shell*/ true, ) .expect("login shells should be allowed"); @@ -192,10 +195,12 @@ fn shell_command_handler_respects_explicit_login_flag() { #[tokio::test] async fn shell_command_handler_defaults_to_non_login_when_disallowed() { let (session, turn_context) = make_session_and_context().await; - let turn_environment = turn_context + let mut turn_environment = turn_context .environments .primary() - .expect("primary environment"); + .expect("primary environment") + .clone(); + turn_environment.config.allow_login_shell = false; let cwd = turn_environment .cwd() .to_abs_path() @@ -215,9 +220,8 @@ async fn shell_command_handler_defaults_to_non_login_when_disallowed() { ¶ms, &session, &turn_context, - turn_environment, + &turn_environment, cwd, - /*allow_login_shell*/ false, ) .expect("non-login shells should still be allowed"); @@ -257,7 +261,6 @@ async fn shell_command_handler_rejects_justification_without_sandbox_permissions &turn_context, turn_environment, cwd, - /*allow_login_shell*/ false, ) .expect_err("justification without sandbox permissions should be rejected"); diff --git a/codex-rs/core/src/tools/handlers/unified_exec/exec_command.rs b/codex-rs/core/src/tools/handlers/unified_exec/exec_command.rs index 16eed42a80..d6d5ab699b 100644 --- a/codex-rs/core/src/tools/handlers/unified_exec/exec_command.rs +++ b/codex-rs/core/src/tools/handlers/unified_exec/exec_command.rs @@ -235,7 +235,7 @@ impl ExecCommandHandler { &args, shell, &shell_mode, - turn.config.permissions.allow_login_shell, + turn_environment.config.allow_login_shell, ) .map_err(FunctionCallError::RespondToModel)?; let command = resolved_command.command; diff --git a/codex-rs/core/src/tools/handlers/unified_exec_tests.rs b/codex-rs/core/src/tools/handlers/unified_exec_tests.rs index 9b107dd83d..626bb90414 100644 --- a/codex-rs/core/src/tools/handlers/unified_exec_tests.rs +++ b/codex-rs/core/src/tools/handlers/unified_exec_tests.rs @@ -9,6 +9,8 @@ use codex_utils_output_truncation::TruncationPolicy; use pretty_assertions::assert_eq; use std::sync::Arc; +use crate::environment_selection::TurnEnvironmentState; +use crate::function_tool::FunctionCallError; use crate::session::step_context::StepContext; use crate::session::tests::make_session_and_context; use crate::tools::context::ExecCommandToolOutput; @@ -17,6 +19,7 @@ use crate::tools::context::ToolInvocation; use crate::tools::context::ToolPayload; use crate::tools::hook_names::HookToolName; use crate::tools::registry::CoreToolRuntime; +use crate::tools::registry::ToolExecutor; use crate::turn_diff_tracker::TurnDiffTracker; use tokio::sync::Mutex; @@ -168,6 +171,46 @@ fn test_get_command_rejects_explicit_login_when_disallowed() -> anyhow::Result<( Ok(()) } +#[tokio::test] +async fn exec_command_rejects_login_when_selected_environment_disallows_it() { + let (session, mut turn) = make_session_and_context().await; + assert!(turn.config.permissions.allow_login_shell); + let TurnEnvironmentState::Ready(environment) = turn + .environments + .environments + .first_mut() + .expect("primary environment") + else { + panic!("primary environment should be ready"); + }; + environment.config.allow_login_shell = false; + + let turn = Arc::new(turn); + let invocation = ToolInvocation { + session: session.into(), + step_context: StepContext::for_test(Arc::clone(&turn)), + turn, + cancellation_token: tokio_util::sync::CancellationToken::new(), + tracker: Arc::new(Mutex::new(TurnDiffTracker::new())), + call_id: "login-disallowed".to_string(), + tool_name: codex_tools::ToolName::plain("exec_command"), + source: ToolCallSource::Direct, + payload: ToolPayload::Function { + arguments: serde_json::json!({ "cmd": "echo hello", "login": true }).to_string(), + }, + }; + + let Err(FunctionCallError::RespondToModel(message)) = + ExecCommandHandler::default().handle(invocation).await + else { + panic!("expected login-shell rejection"); + }; + assert_eq!( + message, + "login shell is disabled by config; omit `login` or set it to false." + ); +} + #[test] fn test_get_command_rejects_explicit_shell_in_zsh_fork_mode() -> anyhow::Result<()> { let json = r#"{"cmd": "echo hello", "shell": "/bin/bash"}"#; diff --git a/codex-rs/core/src/tools/handlers/view_image.rs b/codex-rs/core/src/tools/handlers/view_image.rs index 20efe1b549..675f409b72 100644 --- a/codex-rs/core/src/tools/handlers/view_image.rs +++ b/codex-rs/core/src/tools/handlers/view_image.rs @@ -273,6 +273,7 @@ mod tests { PathUri::from_abs_path(&cwd), Vec::new(), current.shell, + current.config, )); } diff --git a/codex-rs/core/src/tools/orchestrator.rs b/codex-rs/core/src/tools/orchestrator.rs index 073962db74..fbcbc596b5 100644 --- a/codex-rs/core/src/tools/orchestrator.rs +++ b/codex-rs/core/src/tools/orchestrator.rs @@ -7,7 +7,7 @@ retry with an escalated sandbox strategy on denial (no re‑approval thanks to caching). */ use super::approvals::ApprovalReviewer; -use super::approvals::resolve_tool_apporval; +use super::approvals::resolve_tool_approval; use crate::network_policy_decision::network_approval_context_from_payload; use crate::tools::flat_tool_name; use crate::tools::network_approval::ActiveNetworkApproval; @@ -171,17 +171,16 @@ impl ToolOrchestrator { session: &tool_ctx.session, turn: &tool_ctx.turn, call_id: &tool_ctx.call_id, + tool_name: &tool_ctx.tool_name, retry_reason: None, network_approval_context: None, }; - resolve_tool_apporval( + resolve_tool_approval( tool, req, tool_ctx.call_id.as_str(), approval_ctx, - tool_ctx, ApprovalReviewer::Guardian, - &otel, ) .await?; already_approved = true; @@ -202,21 +201,20 @@ impl ToolOrchestrator { session: &tool_ctx.session, turn: &tool_ctx.turn, call_id: &tool_ctx.call_id, + tool_name: &tool_ctx.tool_name, retry_reason: reason.clone(), network_approval_context: None, }; - resolve_tool_apporval( + resolve_tool_approval( tool, req, tool_ctx.call_id.as_str(), approval_ctx, - tool_ctx, if strict_auto_review { ApprovalReviewer::Guardian } else { ApprovalReviewer::for_turn(turn_ctx) }, - &otel, ) .await?; already_approved = true; @@ -397,23 +395,22 @@ impl ToolOrchestrator { session: &tool_ctx.session, turn: &tool_ctx.turn, call_id: &tool_ctx.call_id, + tool_name: &tool_ctx.tool_name, retry_reason: Some(retry_reason), network_approval_context: network_approval_context.clone(), }; let permission_request_run_id = format!("{}:retry", tool_ctx.call_id); - resolve_tool_apporval( + resolve_tool_approval( tool, req, &permission_request_run_id, approval_ctx, - tool_ctx, if strict_auto_review { ApprovalReviewer::Guardian } else { ApprovalReviewer::for_turn(turn_ctx) }, - &otel, ) .await?; } diff --git a/codex-rs/core/src/tools/router_tests.rs b/codex-rs/core/src/tools/router_tests.rs index 4549eac0b3..251d2825ed 100644 --- a/codex-rs/core/src/tools/router_tests.rs +++ b/codex-rs/core/src/tools/router_tests.rs @@ -582,6 +582,7 @@ fn namespace_function_names(specs: &[ToolSpec], namespace_name: &str) -> Vec tool.name.clone(), + ResponsesApiNamespaceTool::Custom(tool) => tool.name.clone(), }) .collect(), ), diff --git a/codex-rs/core/src/tools/runtimes/apply_patch_tests.rs b/codex-rs/core/src/tools/runtimes/apply_patch_tests.rs index 3a879de8c9..146e571a3b 100644 --- a/codex-rs/core/src/tools/runtimes/apply_patch_tests.rs +++ b/codex-rs/core/src/tools/runtimes/apply_patch_tests.rs @@ -1,4 +1,5 @@ use super::*; +use crate::session::turn_context::EnvironmentConfig; use crate::tools::sandboxing::SandboxAttempt; use codex_protocol::config_types::WindowsSandboxLevel; use codex_protocol::models::AdditionalPermissionProfile; @@ -21,6 +22,9 @@ fn test_turn_environment(environment_id: &str) -> crate::session::turn_context:: PathUri::from_abs_path(&std::env::temp_dir().abs()), Vec::new(), /*shell*/ None, + EnvironmentConfig { + allow_login_shell: true, + }, ) } diff --git a/codex-rs/core/src/tools/runtimes/shell_tests.rs b/codex-rs/core/src/tools/runtimes/shell_tests.rs index 7ae93ab918..b4f43b6a38 100644 --- a/codex-rs/core/src/tools/runtimes/shell_tests.rs +++ b/codex-rs/core/src/tools/runtimes/shell_tests.rs @@ -1,4 +1,5 @@ use super::*; +use crate::session::turn_context::EnvironmentConfig; use codex_exec_server::Environment; use codex_utils_path_uri::PathUri; use pretty_assertions::assert_eq; @@ -16,6 +17,9 @@ async fn approval_key_uses_path_uri_and_includes_environment_id() { PathUri::from_abs_path(&cwd), Vec::new(), /*shell*/ None, + EnvironmentConfig { + allow_login_shell: true, + }, ), shell_type: None, hook_command: "echo hello".to_string(), diff --git a/codex-rs/core/src/tools/runtimes/unified_exec.rs b/codex-rs/core/src/tools/runtimes/unified_exec.rs index a2781fb890..40bc938c22 100644 --- a/codex-rs/core/src/tools/runtimes/unified_exec.rs +++ b/codex-rs/core/src/tools/runtimes/unified_exec.rs @@ -549,6 +549,7 @@ impl<'a> ToolRuntime for UnifiedExecRunt mod tests { use super::*; use crate::exec::DEFAULT_EXEC_COMMAND_TIMEOUT_MS; + use crate::session::turn_context::EnvironmentConfig; use crate::tools::sandboxing::ToolRuntime; use codex_exec_server::Environment; use codex_exec_server::LOCAL_ENVIRONMENT_ID; @@ -566,6 +567,9 @@ mod tests { cwd, Vec::new(), /*shell*/ None, + EnvironmentConfig { + allow_login_shell: true, + }, ) } diff --git a/codex-rs/core/src/tools/sandboxing.rs b/codex-rs/core/src/tools/sandboxing.rs index 98332fd228..07bfeca90f 100644 --- a/codex-rs/core/src/tools/sandboxing.rs +++ b/codex-rs/core/src/tools/sandboxing.rs @@ -121,6 +121,7 @@ pub(crate) struct ApprovalCtx<'a> { pub session: &'a Arc, pub turn: &'a Arc, pub call_id: &'a str, + pub tool_name: &'a ToolName, pub retry_reason: Option, pub network_approval_context: Option, } diff --git a/codex-rs/core/src/tools/spec_plan.rs b/codex-rs/core/src/tools/spec_plan.rs index 36ab4758fa..20ef7a1249 100644 --- a/codex-rs/core/src/tools/spec_plan.rs +++ b/codex-rs/core/src/tools/spec_plan.rs @@ -713,11 +713,16 @@ fn merge_into_namespaces(specs: Vec) -> Vec { continue; }; - namespace.tools.sort_by(|left, right| match (left, right) { - ( - ResponsesApiNamespaceTool::Function(left), - ResponsesApiNamespaceTool::Function(right), - ) => left.name.cmp(&right.name), + namespace.tools.sort_by(|left, right| { + let left_name = match left { + ResponsesApiNamespaceTool::Function(tool) => &tool.name, + ResponsesApiNamespaceTool::Custom(tool) => &tool.name, + }; + let right_name = match right { + ResponsesApiNamespaceTool::Function(tool) => &tool.name, + ResponsesApiNamespaceTool::Custom(tool) => &tool.name, + }; + left_name.cmp(right_name) }); if namespace.description.trim().is_empty() { @@ -760,7 +765,7 @@ fn add_core_tool_sources(context: &CoreToolPlanContext<'_>, registry: &mut ToolR if environment_mode.has_environment() { let include_environment_id = matches!(environment_mode, ToolEnvironmentMode::Multiple); registry.add(ExecCommandHandler::new(ExecCommandHandlerOptions { - allow_login_shell: turn_context.config.permissions.allow_login_shell, + allow_login_shell: any_environment_allows_login_shell(context.environments), exec_permission_approvals_enabled: false, include_environment_id, include_shell_parameter: unified_exec_should_include_shell_parameter( @@ -800,6 +805,12 @@ fn tool_environment_mode(environments: &TurnEnvironmentSnapshot) -> ToolEnvironm ToolEnvironmentMode::from_count(environments.turn_environments().count()) } +fn any_environment_allows_login_shell(environments: &TurnEnvironmentSnapshot) -> bool { + environments + .turn_environments() + .any(|environment| environment.config.allow_login_shell) +} + #[instrument(level = "trace", skip_all)] fn add_shell_tools(context: &CoreToolPlanContext<'_>, registry: &mut ToolRegistry) { let turn_context = context.turn_context; @@ -809,7 +820,7 @@ fn add_shell_tools(context: &CoreToolPlanContext<'_>, registry: &mut ToolRegistr return; } - let allow_login_shell = turn_context.config.permissions.allow_login_shell; + let allow_login_shell = any_environment_allows_login_shell(context.environments); let exec_permission_approvals_enabled = features.enabled(Feature::ExecPermissionApprovals); let include_environment_id = matches!(environment_mode, ToolEnvironmentMode::Multiple); let supports_shell_command = context.environments.single_local_environment().is_some(); diff --git a/codex-rs/core/src/tools/spec_plan_tests.rs b/codex-rs/core/src/tools/spec_plan_tests.rs index 441eea6ed2..4bd8244ae7 100644 --- a/codex-rs/core/src/tools/spec_plan_tests.rs +++ b/codex-rs/core/src/tools/spec_plan_tests.rs @@ -84,6 +84,7 @@ impl ToolPlanProbe { .iter() .map(|tool| match tool { ResponsesApiNamespaceTool::Function(tool) => tool.name.clone(), + ResponsesApiNamespaceTool::Custom(tool) => tool.name.clone(), }) .collect::>(), )), @@ -661,6 +662,70 @@ async fn shell_family_registers_visible_unified_exec_and_hidden_legacy_shell() { assert!(has_parameter(plan.visible_spec("exec_command"), "shell")); } +#[tokio::test] +async fn login_shell_parameter_follows_selected_environment() { + for (tool_name, guardian) in [ + ("shell_command", false), + ("exec_command", false), + ("exec_command", true), + ] { + for allow_login_shell in [false, true] { + let plan = probe(|turn| { + set_feature(turn, Feature::ShellTool, /*enabled*/ true); + set_feature(turn, Feature::UnifiedExec, tool_name == "exec_command"); + set_feature(turn, Feature::ShellZshFork, /*enabled*/ false); + turn.model_info.shell_type = ConfigShellToolType::ShellCommand; + update_config(turn, |config| { + config.permissions.allow_login_shell = !allow_login_shell; + }); + let TurnEnvironmentState::Ready(environment) = turn + .environments + .environments + .first_mut() + .expect("primary environment") + else { + panic!("primary environment should be ready"); + }; + environment.config.allow_login_shell = allow_login_shell; + if guardian { + turn.session_source = codex_protocol::protocol::SessionSource::SubAgent( + codex_protocol::protocol::SubAgentSource::Other( + crate::guardian::GUARDIAN_REVIEWER_NAME.to_string(), + ), + ); + } + }) + .await; + + assert_eq!( + has_parameter(plan.visible_spec(tool_name), "login"), + allow_login_shell + ); + } + } +} + +#[tokio::test] +async fn login_shell_parameter_is_available_when_any_environment_allows_it() { + let plan = probe(|turn| { + set_features(turn, &[Feature::ShellTool, Feature::UnifiedExec]); + set_feature(turn, Feature::ShellZshFork, /*enabled*/ false); + update_config(turn, |config| { + config.permissions.allow_login_shell = false; + }); + duplicate_primary_environment(turn); + for (index, environment) in turn.environments.environments.iter_mut().enumerate() { + let TurnEnvironmentState::Ready(environment) = environment else { + panic!("environment should be ready"); + }; + environment.config.allow_login_shell = index == 1; + } + }) + .await; + + assert!(has_parameter(plan.visible_spec("exec_command"), "login")); +} + #[tokio::test] async fn shell_command_is_not_registered_without_a_single_local_environment() { let remote_environment = probe(|turn| { @@ -836,6 +901,9 @@ async fn zsh_fork_unified_exec_keeps_shell_parameter_when_remote_environment_ava remote_cwd, Vec::new(), /*shell*/ None, + crate::session::turn_context::EnvironmentConfig { + allow_login_shell: true, + }, ), )); }) @@ -1599,7 +1667,9 @@ async fn code_mode_only_exposes_configured_dynamic_namespace_directly() { let ToolSpec::Namespace(namespace) = plan.visible_spec("direct_only") else { panic!("expected direct-only namespace spec"); }; - let ResponsesApiNamespaceTool::Function(tool) = &namespace.tools[0]; + let ResponsesApiNamespaceTool::Function(tool) = &namespace.tools[0] else { + panic!("expected direct-only namespace function tool"); + }; assert_eq!(tool.defer_loading, None); let ToolSpec::Freeform(exec) = plan.visible_spec(codex_code_mode::PUBLIC_TOOL_NAME) else { panic!("expected code mode exec tool"); diff --git a/codex-rs/core/tests/suite/approvals.rs b/codex-rs/core/tests/suite/approvals.rs index 9e8ac16142..75a9fabc76 100644 --- a/codex-rs/core/tests/suite/approvals.rs +++ b/codex-rs/core/tests/suite/approvals.rs @@ -2139,6 +2139,127 @@ async fn approving_apply_patch_for_session_skips_future_prompts_for_same_file() Ok(()) } +#[tokio::test(flavor = "current_thread")] +#[cfg(unix)] +async fn approving_execpolicy_amendment_does_not_reinject_permissions_instructions() -> Result<()> { + assert_execpolicy_amendment_context(PermissionsInstructionsExpectation::Included).await +} + +#[tokio::test(flavor = "current_thread")] +#[cfg(unix)] +async fn approving_execpolicy_amendment_is_reported_when_permissions_are_disabled() -> Result<()> { + assert_execpolicy_amendment_context(PermissionsInstructionsExpectation::Omitted).await +} + +#[derive(Clone, Copy)] +enum PermissionsInstructionsExpectation { + Included, + Omitted, +} + +async fn assert_execpolicy_amendment_context( + expectation: PermissionsInstructionsExpectation, +) -> Result<()> { + let (include_permissions_instructions, expected_permissions_block_count) = match expectation { + PermissionsInstructionsExpectation::Included => (true, 1), + PermissionsInstructionsExpectation::Omitted => (false, 0), + }; + let server = start_mock_server().await; + let approval_policy = AskForApproval::OnRequest; + let sandbox_policy = SandboxPolicy::new_read_only_policy(); + let sandbox_policy_for_config = sandbox_policy.clone(); + let mut builder = test_codex() + .with_model_info_override("gpt-5.4", |model_info| { + model_info.model_messages = None; + }) + .with_config(move |config| { + config.include_permissions_instructions = include_permissions_instructions; + config.permissions.approval_policy = Constrained::allow_any(approval_policy); + config + .features + .disable(Feature::DeferredExecutor) + .expect("test config should allow feature update"); + config + .set_legacy_sandbox_policy(sandbox_policy_for_config) + .expect("set sandbox policy"); + }); + let test = builder.build(&server).await?; + + let call_id = "permissions-prefix"; + let (event, expected_command) = ActionKind::RunCommand { + command: "touch permissions-prefix.txt", + } + .prepare( + &test, + &server, + call_id, + SandboxPermissions::RequireEscalated, + ) + .await?; + let expected_command = + expected_command.expect("execpolicy amendment scenario should produce a shell command"); + let expected_execpolicy_amendment = ExecPolicyAmendment::new(vec![ + "touch".to_string(), + "permissions-prefix.txt".to_string(), + ]); + + let _ = mount_sse_once( + &server, + sse(vec![ + ev_response_created("resp-permissions-prefix-1"), + event, + ev_completed("resp-permissions-prefix-1"), + ]), + ) + .await; + let follow_up = mount_sse_once( + &server, + sse(vec![ + ev_assistant_message("msg-permissions-prefix", "done"), + ev_completed("resp-permissions-prefix-2"), + ]), + ) + .await; + + submit_turn(&test, "permissions-prefix", approval_policy, sandbox_policy).await?; + + let approval = expect_exec_approval(&test, expected_command.as_str()).await; + assert_eq!( + approval.proposed_execpolicy_amendment, + Some(expected_execpolicy_amendment.clone()) + ); + test.codex + .submit(Op::ExecApproval { + id: approval.effective_approval_id(), + turn_id: None, + decision: ReviewDecision::ApprovedExecpolicyAmendment { + proposed_execpolicy_amendment: expected_execpolicy_amendment, + }, + }) + .await?; + wait_for_completion(&test).await; + + let developer_messages = follow_up.single_request().message_input_texts("developer"); + let permissions_block_count = developer_messages + .iter() + .map(|message| message.matches("").count()) + .sum::(); + assert_eq!( + permissions_block_count, expected_permissions_block_count, + "saving an execpolicy amendment must respect the full permissions rendering mode: \ + {developer_messages:#?}" + ); + assert!( + developer_messages + .iter() + .any(|message| message.contains("Approved command prefix saved:") + && message.contains(r#"["touch", "permissions-prefix.txt"]"#)), + "expected developer message documenting saved rule, got: {developer_messages:?}" + ); + + Ok(()) +} + #[tokio::test(flavor = "current_thread")] #[cfg(unix)] async fn approving_execpolicy_amendment_persists_policy_and_skips_future_prompts() -> Result<()> { diff --git a/codex-rs/core/tests/suite/code_mode.rs b/codex-rs/core/tests/suite/code_mode.rs index 5ed90d2b62..5f8d545167 100644 --- a/codex-rs/core/tests/suite/code_mode.rs +++ b/codex-rs/core/tests/suite/code_mode.rs @@ -8,7 +8,9 @@ use codex_config::types::McpServerTransportConfig; use codex_core::StartThreadOptions; use codex_core::config::Config; use codex_core::config::CurrentTimeReminderConfig; +use codex_extension_api::ExtensionData; use codex_extension_api::ExtensionRegistryBuilder; +use codex_extension_api::ToolContributor; use codex_features::CurrentTimeSource; use codex_features::Feature; use codex_login::CodexAuth; @@ -21,11 +23,25 @@ use codex_protocol::dynamic_tools::DynamicToolNamespaceTool; use codex_protocol::dynamic_tools::DynamicToolResponse; use codex_protocol::dynamic_tools::DynamicToolSpec; use codex_protocol::models::PermissionProfile; +use codex_protocol::models::ResponseItem; use codex_protocol::openai_models::ToolMode; use codex_protocol::protocol::AskForApproval; use codex_protocol::protocol::EventMsg; use codex_protocol::protocol::Op; use codex_protocol::user_input::UserInput; +use codex_tools::FreeformTool; +use codex_tools::FreeformToolFormat; +use codex_tools::FunctionCallError; +use codex_tools::JsonToolOutput; +use codex_tools::ResponsesApiNamespace; +use codex_tools::ResponsesApiNamespaceTool; +use codex_tools::ToolCall; +use codex_tools::ToolExecutor; +use codex_tools::ToolExecutorFuture; +use codex_tools::ToolName; +use codex_tools::ToolOutput; +use codex_tools::ToolPayload; +use codex_tools::ToolSpec; use codex_web_search_extension::install as install_web_search_extension; use core_test_support::apps_test_server::AppsTestServer; use core_test_support::apps_test_server::AppsTestToolLoading; @@ -2053,6 +2069,140 @@ try { Ok(()) } +/// A stalled host wait must return its timeout to the model and reconnect for the next exec. +#[tokio::test(flavor = "current_thread")] +async fn code_mode_wait_timeout_reconnects_on_next_exec() -> Result<()> { + skip_if_no_network!(Ok(())); + + let server = responses::start_mock_server().await; + let mut builder = test_codex() + .with_model("test-gpt-5.1-codex") + .with_config(|config| { + config + .features + .enable(Feature::CodeMode) + .expect("code mode should be enabled"); + }); + let test = builder.build_with_auto_env(&server).await?; + + let first_turn = responses::mount_sse_sequence( + &server, + vec![ + sse(vec![ + ev_response_created("resp-1"), + ev_custom_tool_call( + "call-1", + "exec", + "yield_control(); await new Promise(() => {});", + ), + ev_completed("resp-1"), + ]), + sse(vec![ + ev_assistant_message("msg-1", "waiting"), + ev_completed("resp-2"), + ]), + ], + ) + .await; + + test.submit_turn("start a stalled code-mode cell").await?; + let first_request = first_turn + .last_request() + .expect("initial exec should be returned to the model"); + let first_items = custom_tool_output_items(&first_request, "call-1"); + let cell_id = extract_running_cell_id(text_item(&first_items, /*index*/ 0)); + + let timeout_completion = responses::mount_function_call_agent_response( + &server, + "call-2", + &serde_json::to_string(&serde_json::json!({ + "cell_id": cell_id, + "yield_time_ms": 60_000, + }))?, + "wait", + ) + .await + .completion; + + test.codex + .submit(Op::UserInput { + items: vec![UserInput::Text { + text: "wait for the stalled cell".to_string(), + text_elements: Vec::new(), + }], + final_output_json_schema: None, + responsesapi_client_metadata: None, + additional_context: Default::default(), + thread_settings: Default::default(), + }) + .await?; + wait_for_event_match(&test.codex, |event| match event { + EventMsg::RawResponseItem(raw) => match &raw.item { + ResponseItem::FunctionCall { call_id, .. } if call_id == "call-2" => Some(()), + _ => None, + }, + _ => None, + }) + .await; + + tokio::time::pause(); + for _ in 0..130 { + tokio::time::advance(Duration::from_secs(1)).await; + if timeout_completion + .function_call_output_text("call-2") + .is_some() + { + break; + } + } + tokio::time::resume(); + wait_for_event(&test.codex, |event| { + matches!(event, EventMsg::TurnComplete(_)) + }) + .await; + + let timeout_output = timeout_completion + .function_call_output_text("call-2") + .expect("timed-out wait should be returned to the model"); + assert!( + timeout_output.contains("code-mode host timed out waiting for wait response"), + "unexpected wait output: {timeout_output}" + ); + + let reconnect_turn = responses::mount_sse_sequence( + &server, + vec![ + sse(vec![ + ev_response_created("resp-5"), + ev_custom_tool_call( + "call-3", + "exec", + r#"text("reconnected"); yield_control(); await new Promise(() => {});"#, + ), + ev_completed("resp-5"), + ]), + sse(vec![ + ev_assistant_message("msg-3", "reconnected"), + ev_completed("resp-6"), + ]), + ], + ) + .await; + + test.submit_turn("run a cell after the timeout").await?; + let reconnect_request = reconnect_turn + .last_request() + .expect("replacement exec should be returned to the model"); + let reconnect_items = custom_tool_output_items(&reconnect_request, "call-3"); + assert_eq!( + extract_running_cell_id(text_item(&reconnect_items, /*index*/ 0)), + "g2:1" + ); + assert_eq!(text_item(&reconnect_items, /*index*/ 1), "reconnected"); + + Ok(()) +} + #[cfg_attr(windows, ignore = "no exec_command on Windows")] #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn code_mode_can_yield_and_resume_with_wait() -> Result<()> { @@ -3900,6 +4050,170 @@ text(JSON.stringify({ Ok(()) } +struct NamespacedCustomTool; + +impl ToolContributor for NamespacedCustomTool { + fn tools( + &self, + _session_store: &ExtensionData, + _thread_store: &ExtensionData, + ) -> Vec>> { + vec![Arc::new(Self)] + } +} + +impl ToolExecutor for NamespacedCustomTool { + fn tool_name(&self) -> ToolName { + ToolName::namespaced("editor", "apply_patch") + } + + fn spec(&self) -> ToolSpec { + ToolSpec::Namespace(ResponsesApiNamespace { + name: "editor".to_string(), + description: "Editing tools.".to_string(), + tools: vec![ResponsesApiNamespaceTool::Custom(FreeformTool { + name: "apply_patch".to_string(), + description: "Apply a raw editor patch.".to_string(), + defer_loading: None, + format: FreeformToolFormat { + r#type: "grammar".to_string(), + syntax: "lark".to_string(), + definition: "start: /.+/".to_string(), + }, + })], + }) + } + + fn handle(&self, call: ToolCall) -> ToolExecutorFuture<'_> { + Box::pin(async move { + let ToolPayload::Custom { input } = call.payload else { + return Err(FunctionCallError::Fatal( + "expected custom tool payload".to_string(), + )); + }; + Ok(Box::new(JsonToolOutput::new(serde_json::json!({ + "namespace": call.tool_name.namespace, + "name": call.tool_name.name, + "input": input, + }))) as Box) + }) + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn code_mode_exposes_and_dispatches_namespaced_custom_tools() -> Result<()> { + skip_if_no_network!(Ok(())); + + let server = responses::start_mock_server().await; + let mut extensions = ExtensionRegistryBuilder::::new(); + extensions.tool_contributor(Arc::new(NamespacedCustomTool)); + let mut builder = test_codex() + .with_model("test-gpt-5.1-codex") + .with_extensions(Arc::new(extensions.build())) + .with_config(|config| { + let _ = config.features.enable(Feature::CodeMode); + }); + let test = builder.build(&server).await?; + let code = r#" +const tool = ALL_TOOLS.find(({ name }) => name === "editor__apply_patch"); +const result = await tools.editor__apply_patch("nested patch"); +text(JSON.stringify({ + name: tool?.name ?? null, + description: tool?.description ?? null, + result, +})); +"#; + + let response_mock = responses::mount_sse_sequence( + &server, + vec![ + sse(vec![ + ev_response_created("resp-1"), + responses::ev_custom_tool_call_with_namespace( + "call-direct", + "editor", + "apply_patch", + "direct patch", + ), + ev_completed("resp-1"), + ]), + sse(vec![ + ev_response_created("resp-2"), + ev_custom_tool_call("call-exec", "exec", code), + ev_completed("resp-2"), + ]), + sse(vec![ + ev_assistant_message("msg-1", "done"), + ev_completed("resp-3"), + ]), + ], + ) + .await; + + test.submit_turn("call the namespaced custom editor tool directly and through exec") + .await?; + + let requests = response_mock.requests(); + assert_eq!(requests.len(), 3); + + let declaration = + "declare const tools: { editor__apply_patch(input: string): Promise; };"; + let description = + format!("Apply a raw editor patch.\n\nexec tool declaration:\n```ts\n{declaration}\n```"); + let first_body = requests[0].body_json(); + let namespaced_custom_tool = namespace_child_tool(&first_body, "editor", "apply_patch") + .expect("namespaced custom tool should be included in the model request"); + assert_eq!( + namespaced_custom_tool, + &serde_json::json!({ + "type": "custom", + "name": "apply_patch", + "description": description, + "format": { + "type": "grammar", + "syntax": "lark", + "definition": "start: /.+/", + }, + }) + ); + + let (direct_output, direct_success) = + custom_tool_output_body_and_success(&requests[1], "call-direct"); + assert_ne!(direct_success, Some(false)); + let direct_output = serde_json::from_str::(&direct_output).unwrap_or_else(|error| { + panic!("invalid direct custom tool output `{direct_output}`: {error}") + }); + assert_eq!( + direct_output, + serde_json::json!({ + "namespace": "editor", + "name": "apply_patch", + "input": "direct patch", + }) + ); + + let (exec_output, exec_success) = + custom_tool_output_body_and_success(&requests[2], "call-exec"); + assert_ne!(exec_success, Some(false)); + let exec_output = serde_json::from_str::(&exec_output).unwrap_or_else(|error| { + panic!("invalid code mode custom tool output `{exec_output}`: {error}") + }); + assert_eq!( + exec_output, + serde_json::json!({ + "name": "editor__apply_patch", + "description": format!("Editing tools.\n\n{description}"), + "result": { + "namespace": "editor", + "name": "apply_patch", + "input": "nested patch", + }, + }) + ); + + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn code_mode_exposes_namespaced_mcp_tools_on_global_tools_object() -> Result<()> { skip_if_no_network!(Ok(())); diff --git a/codex-rs/core/tests/suite/snapshots/all__suite__token_budget__token_budget_new_context_window_tool_full_context.snap b/codex-rs/core/tests/suite/snapshots/all__suite__token_budget__token_budget_new_context_window_tool_full_context.snap index 648353a1a4..46e90eeecd 100644 --- a/codex-rs/core/tests/suite/snapshots/all__suite__token_budget__token_budget_new_context_window_tool_full_context.snap +++ b/codex-rs/core/tests/suite/snapshots/all__suite__token_budget__token_budget_new_context_window_tool_full_context.snap @@ -6,7 +6,7 @@ Scenario: New context window tool installs fresh full context before the next fo ## Final Follow-Up Request 00:message/developer[3]: - [01] \nThread id: \nFirst context window id: \nCurrent context window id: \nPrevious context window id: \n + [01] \nAgent name: /root\nFirst context window id: \nCurrent context window id: \nPrevious context window id: \n [02] [03] 01:message/user:> diff --git a/codex-rs/core/tests/suite/token_budget.rs b/codex-rs/core/tests/suite/token_budget.rs index 088aedb562..e6721c3863 100644 --- a/codex-rs/core/tests/suite/token_budget.rs +++ b/codex-rs/core/tests/suite/token_budget.rs @@ -74,7 +74,7 @@ fn model_token_budget_config() -> ModelTokenBudgetConfig { } fn token_budget_contexts(request: &ResponsesRequest) -> Vec { - let context_window_prefix = format!("{CONTEXT_WINDOW_OPEN_TAG}\nThread id: "); + let context_window_prefix = format!("{CONTEXT_WINDOW_OPEN_TAG}\nAgent name: "); request .message_input_texts("developer") .into_iter() @@ -82,13 +82,10 @@ fn token_budget_contexts(request: &ResponsesRequest) -> Vec { .collect() } -fn token_budget_window_ids( - text: &str, - thread_id: codex_protocol::ThreadId, -) -> (String, Option, String) { +fn token_budget_window_ids(text: &str, agent_name: &str) -> (String, Option, String) { let captures = assert_regex_match( &format!( - r"^{CONTEXT_WINDOW_OPEN_TAG}\nThread id: {thread_id}\nFirst context window id: ([0-9a-f-]{{36}})\nCurrent context window id: ([0-9a-f-]{{36}})(?:\nPrevious context window id: ([0-9a-f-]{{36}}))?\n{CONTEXT_WINDOW_CLOSE_TAG}$" + r"^{CONTEXT_WINDOW_OPEN_TAG}\nAgent name: {agent_name}\nFirst context window id: ([0-9a-f-]{{36}})\nCurrent context window id: ([0-9a-f-]{{36}})(?:\nPrevious context window id: ([0-9a-f-]{{36}}))?\n{CONTEXT_WINDOW_CLOSE_TAG}$" ), text, ); @@ -223,11 +220,10 @@ async fn token_budget_context_is_only_emitted_with_full_context() -> Result<()> let requests = responses.requests(); assert_eq!(requests.len(), 2); - let thread_id = test.session_configured.thread_id; let initial_token_budget = token_budget_contexts(&requests[0]); assert_eq!(initial_token_budget.len(), 1); let (first_window_id, previous_window_id, window_id) = - token_budget_window_ids(&initial_token_budget[0], thread_id); + token_budget_window_ids(&initial_token_budget[0], "/root"); assert_eq!(previous_window_id, None); assert_eq!(first_window_id, window_id); assert_eq!( @@ -731,7 +727,7 @@ async fn token_budget_context_injects_plain_thread_hint_text() -> Result<()> { assert_eq!(token_budgets.len(), 1); let captures = assert_regex_match( &format!( - r"^{CONTEXT_WINDOW_OPEN_TAG}\nThread id: {thread_id}\nFirst context window id: ([0-9a-f-]{{36}})\nCurrent context window id: ([0-9a-f-]{{36}})\nmanual history hint for thread {thread_id}\nunstructured notes/thread_hint fixture result\n{CONTEXT_WINDOW_CLOSE_TAG}$" + r"^{CONTEXT_WINDOW_OPEN_TAG}\nAgent name: /root\nFirst context window id: ([0-9a-f-]{{36}})\nCurrent context window id: ([0-9a-f-]{{36}})\nmanual history hint for thread {thread_id}\nunstructured notes/thread_hint fixture result\n{CONTEXT_WINDOW_CLOSE_TAG}$" ), &token_budgets[0], ); @@ -915,11 +911,10 @@ async fn get_context_remaining_returns_token_budget_remaining_fragment() -> Resu "get_context_remaining should be exposed when token budget is enabled" ); - let thread_id = test.session_configured.thread_id; let remaining_context = "You have 6500 tokens left in this context window.".to_string(); let token_budgets = token_budget_contexts(&requests[1]); assert_eq!(token_budgets.len(), 1); - token_budget_window_ids(&token_budgets[0], thread_id); + token_budget_window_ids(&token_budgets[0], "/root"); assert_eq!( requests[2].function_call_output_content_and_success(call_id), Some((Some(remaining_context), None)) @@ -1099,18 +1094,17 @@ async fn token_budget_context_uses_new_window_after_compaction() -> Result<()> { "token budget compaction should not call server-side compaction" ); - let thread_id = test.session_configured.thread_id; let initial_token_budget = token_budget_contexts(&requests[0]); assert_eq!(initial_token_budget.len(), 1); let (initial_first_window_id, initial_previous_window_id, initial_window_id) = - token_budget_window_ids(&initial_token_budget[0], thread_id); + token_budget_window_ids(&initial_token_budget[0], "/root"); let post_compaction_token_budget = token_budget_contexts(&requests[1]); assert_eq!(post_compaction_token_budget.len(), 1); let ( post_compaction_first_window_id, post_compaction_previous_window_id, post_compaction_window_id, - ) = token_budget_window_ids(&post_compaction_token_budget[0], thread_id); + ) = token_budget_window_ids(&post_compaction_token_budget[0], "/root"); assert_eq!(initial_previous_window_id, None); assert_eq!(initial_first_window_id, initial_window_id); assert_eq!(post_compaction_first_window_id, initial_first_window_id); @@ -1264,15 +1258,14 @@ async fn token_budget_mid_turn_auto_compaction_resets_before_active_follow_up() "fresh token-budget windows should drop active tool output with the prior history" ); - let thread_id = test.session_configured.thread_id; let initial_token_budget = token_budget_contexts(&requests[0]); assert_eq!(initial_token_budget.len(), 1); let (initial_first_window_id, _, initial_window_id) = - token_budget_window_ids(&initial_token_budget[0], thread_id); + token_budget_window_ids(&initial_token_budget[0], "/root"); let follow_up_token_budget = token_budget_contexts(&requests[1]); assert_eq!(follow_up_token_budget.len(), 1); let (follow_up_first_window_id, follow_up_previous_window_id, follow_up_window_id) = - token_budget_window_ids(&follow_up_token_budget[0], thread_id); + token_budget_window_ids(&follow_up_token_budget[0], "/root"); assert_eq!(follow_up_first_window_id, initial_first_window_id); assert_eq!( follow_up_previous_window_id.as_deref(), @@ -1364,16 +1357,15 @@ async fn token_budget_auto_compact_fallback_uses_buffer_until_new_context() -> R fallback_request.function_call_output_text(trigger_call_id), Some("You have 0 tokens left in this context window.".to_string()) ); - let thread_id = test.session_configured.thread_id; let initial_context = token_budget_contexts(&requests[0]); assert_eq!(token_budget_contexts(fallback_request), initial_context); assert_eq!(token_budget_contexts(&requests[2]), initial_context); assert!(requests[2].body_contains_text(AUTO_COMPACT_FALLBACK_PROMPT)); assert!(requests[2].body_contains_text(fallback_call_id)); - let (_, _, initial_window_id) = token_budget_window_ids(&initial_context[0], thread_id); + let (_, _, initial_window_id) = token_budget_window_ids(&initial_context[0], "/root"); let follow_up_context = token_budget_contexts(&requests[3]); let (_, previous_window_id, follow_up_window_id) = - token_budget_window_ids(&follow_up_context[0], thread_id); + token_budget_window_ids(&follow_up_context[0], "/root"); assert_eq!( previous_window_id.as_deref(), Some(initial_window_id.as_str()) @@ -1523,15 +1515,14 @@ async fn new_context_tool_skips_auto_compact_fallback() -> Result<()> { .any(|name| name == "new_context"), "new_context should be exposed when token budget is enabled" ); - let thread_id = test.session_configured.thread_id; let initial_token_budget = token_budget_contexts(&requests[0]); assert_eq!(initial_token_budget.len(), 1); let (initial_first_window_id, _, initial_window_id) = - token_budget_window_ids(&initial_token_budget[0], thread_id); + token_budget_window_ids(&initial_token_budget[0], "/root"); let new_window_token_budget = token_budget_contexts(&requests[2]); assert_eq!(new_window_token_budget.len(), 1); let (new_first_window_id, new_previous_window_id, new_window_id) = - token_budget_window_ids(&new_window_token_budget[0], thread_id); + token_budget_window_ids(&new_window_token_budget[0], "/root"); assert_eq!(new_first_window_id, initial_first_window_id); assert_eq!( new_previous_window_id.as_deref(), @@ -1552,7 +1543,6 @@ async fn new_context_tool_skips_auto_compact_fallback() -> Result<()> { &ContextSnapshotOptions::default(), ); let snapshot = snapshot - .replace(&thread_id.to_string(), "") .replace(&new_first_window_id, "") .replace(&new_window_id, ""); insta::assert_snapshot!( diff --git a/codex-rs/core/tests/suite/unified_exec.rs b/codex-rs/core/tests/suite/unified_exec.rs index 23f644da57..4848080c00 100644 --- a/codex-rs/core/tests/suite/unified_exec.rs +++ b/codex-rs/core/tests/suite/unified_exec.rs @@ -242,6 +242,58 @@ async fn create_workspace_directory( Ok(abs_path.into_path_buf()) } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn exec_command_hides_and_rejects_login_when_disabled() -> Result<()> { + skip_if_no_network!(Ok(())); + + let builder = test_codex().with_model("gpt-5.4").with_config(|config| { + config.permissions.allow_login_shell = false; + config.use_experimental_unified_exec_tool = true; + config + .features + .enable(Feature::UnifiedExec) + .expect("test config should allow feature update"); + }); + let harness = TestCodexHarness::with_builder(builder).await?; + let call_id = "exec-command-login-disabled"; + let arguments = json!({ + "cmd": "echo should not run", + "login": true, + }); + let responses = mount_sse_sequence( + harness.server(), + vec![ + sse(vec![ + ev_response_created("resp-1"), + ev_function_call(call_id, "exec_command", &serde_json::to_string(&arguments)?), + ev_completed("resp-1"), + ]), + sse(vec![ + ev_assistant_message("msg-1", "done"), + ev_completed("resp-2"), + ]), + ], + ) + .await; + + harness.submit("run the command with login").await?; + + assert_eq!( + harness.function_call_stdout(call_id).await, + "login shell is disabled by config; omit `login` or set it to false." + ); + let request = responses.requests()[0].body_json(); + let exec_tool = request["tools"] + .as_array() + .expect("tools should be an array") + .iter() + .find(|tool| tool["name"] == "exec_command") + .expect("exec_command should be available"); + assert!(exec_tool["parameters"]["properties"].get("login").is_none()); + + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn unified_exec_intercepts_apply_patch_exec_command() -> Result<()> { skip_if_no_network!(Ok(())); diff --git a/codex-rs/exec/src/lib.rs b/codex-rs/exec/src/lib.rs index 097816cda9..e61e7df26a 100644 --- a/codex-rs/exec/src/lib.rs +++ b/codex-rs/exec/src/lib.rs @@ -72,6 +72,7 @@ use codex_core::config::resolve_profile_v2_config_path; use codex_core::find_thread_meta_by_name_str; use codex_core::format_exec_policy_error_with_source; use codex_core::path_utils; +use codex_core::read_session_meta_line; use codex_feedback::CodexFeedback; use codex_git_utils::get_git_repo_root; use codex_login::AuthConfig; @@ -1466,6 +1467,7 @@ async fn resolve_resume_thread_id( let model_providers = resume_lookup_model_providers(config, args); if args.last { + let mut use_state_db_only = state_db.is_some(); let mut cursor = None; loop { let response: ThreadListResponse = send_request_with_response( @@ -1484,7 +1486,7 @@ async fn resolve_resume_thread_id( parent_thread_id: None, ancestor_thread_id: None, cwd: None, - use_state_db_only: false, + use_state_db_only, search_term: None, }, }, @@ -1493,12 +1495,28 @@ async fn resolve_resume_thread_id( .await .map_err(anyhow::Error::msg)?; for thread in response.data { + if use_state_db_only && let Some(path) = thread.path.as_deref() { + let Ok(session_meta) = read_session_meta_line(path).await else { + continue; + }; + if session_meta.meta.id.to_string() != thread.id { + continue; + } + } let latest_cwd = latest_thread_cwd(&thread).await; if args.all || cwds_match(config.cwd.as_path(), latest_cwd.as_path()) { + // A usable SQLite candidate is authoritative. Scanning is reserved for a + // complete miss so successful `--last` lookups avoid auditing every rollout. return Ok(Some(thread.id)); } } let Some(next_cursor) = response.next_cursor else { + if use_state_db_only { + // Repair from rollouts before giving up on a missing SQLite match. + use_state_db_only = false; + cursor = None; + continue; + } return Ok(None); }; cursor = Some(next_cursor); diff --git a/codex-rs/exec/tests/suite/resume.rs b/codex-rs/exec/tests/suite/resume.rs index fc4720fc18..d6b1d3af1d 100644 --- a/codex-rs/exec/tests/suite/resume.rs +++ b/codex-rs/exec/tests/suite/resume.rs @@ -1,5 +1,8 @@ #![allow(clippy::unwrap_used)] use anyhow::Context; +use codex_core::config::ConfigBuilder; +use codex_core::init_state_db; +use codex_protocol::ThreadId; use core_test_support::responses; use core_test_support::skip_if_no_network; use core_test_support::test_codex_exec::test_codex_exec; @@ -202,6 +205,193 @@ async fn exec_resume_last_appends_to_existing_file() -> anyhow::Result<()> { Ok(()) } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn exec_resume_last_repairs_rollout_missing_from_state_db() -> anyhow::Result<()> { + skip_if_no_network!(Ok(())); + + let test = test_codex_exec(); + let server = MockServer::start().await; + let _response_mock = mount_exec_responses(&server, /*count*/ 2).await; + let repo_root = exec_repo_root()?; + + let marker = format!("resume-last-repair-{}", Uuid::new_v4()); + test.cmd_with_server(&server) + .arg("--skip-git-repo-check") + .arg("-C") + .arg(&repo_root) + .arg(format!("echo {marker}")) + .assert() + .success(); + + let sessions_dir = test.home_path().join("sessions"); + let path = find_session_file_containing_marker(&sessions_dir, &marker) + .expect("no session file found after first run"); + let thread_id = ThreadId::from_string(&extract_conversation_id(&path))?; + let config = ConfigBuilder::default() + .codex_home(test.home_path().to_path_buf()) + .build() + .await?; + let state_db = init_state_db(&config) + .await + .expect("state DB should initialize"); + assert_eq!(state_db.delete_thread(thread_id).await?, 1); + state_db + .mark_backfill_complete(/*last_watermark*/ None) + .await?; + + let resumed_marker = format!("resume-last-repaired-{}", Uuid::new_v4()); + test.cmd_with_server(&server) + .arg("--skip-git-repo-check") + .arg("-C") + .arg(&repo_root) + .arg("resume") + .arg("--last") + .arg(format!("echo {resumed_marker}")) + .assert() + .success(); + + let resumed_path = find_session_file_containing_marker(&sessions_dir, &resumed_marker) + .expect("no resumed session file after SQLite repair"); + assert_eq!(resumed_path, path); + assert!(state_db.get_thread(thread_id).await?.is_some()); + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn exec_resume_last_trusts_usable_state_db_candidate() -> anyhow::Result<()> { + skip_if_no_network!(Ok(())); + + let test = test_codex_exec(); + let server = MockServer::start().await; + let _response_mock = mount_exec_responses(&server, /*count*/ 3).await; + let repo_root = exec_repo_root()?; + let sessions_dir = test.home_path().join("sessions"); + + let older_marker = format!("resume-last-indexed-{}", Uuid::new_v4()); + test.cmd_with_server(&server) + .arg("--skip-git-repo-check") + .arg("-C") + .arg(&repo_root) + .arg(format!("echo {older_marker}")) + .assert() + .success(); + let older_path = find_session_file_containing_marker(&sessions_dir, &older_marker) + .expect("no indexed session file after first run"); + + let newer_marker = format!("resume-last-unindexed-{}", Uuid::new_v4()); + test.cmd_with_server(&server) + .arg("--skip-git-repo-check") + .arg("-C") + .arg(&repo_root) + .arg(format!("echo {newer_marker}")) + .assert() + .success(); + let newer_path = find_session_file_containing_marker(&sessions_dir, &newer_marker) + .expect("no unindexed session file after second run"); + let newer_thread_id = ThreadId::from_string(&extract_conversation_id(&newer_path))?; + + let config = ConfigBuilder::default() + .codex_home(test.home_path().to_path_buf()) + .build() + .await?; + let state_db = init_state_db(&config) + .await + .expect("state DB should initialize"); + assert_eq!(state_db.delete_thread(newer_thread_id).await?, 1); + state_db + .mark_backfill_complete(/*last_watermark*/ None) + .await?; + + let resumed_marker = format!("resume-last-authoritative-{}", Uuid::new_v4()); + test.cmd_with_server(&server) + .arg("--skip-git-repo-check") + .arg("-C") + .arg(&repo_root) + .arg("resume") + .arg("--last") + .arg(format!("echo {resumed_marker}")) + .assert() + .success(); + + let resumed_path = find_session_file_containing_marker(&sessions_dir, &resumed_marker) + .expect("no resumed session file after SQLite lookup"); + assert_eq!( + ( + resumed_path, + state_db + .get_thread(newer_thread_id) + .await? + .map(|metadata| metadata.id), + ), + (older_path, None), + ); + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn exec_resume_last_skips_mismatched_state_db_candidate() -> anyhow::Result<()> { + skip_if_no_network!(Ok(())); + + let test = test_codex_exec(); + let server = MockServer::start().await; + let _response_mock = mount_exec_responses(&server, /*count*/ 3).await; + let repo_root = exec_repo_root()?; + let sessions_dir = test.home_path().join("sessions"); + + let older_marker = format!("resume-last-valid-{}", Uuid::new_v4()); + test.cmd_with_server(&server) + .arg("--skip-git-repo-check") + .arg("-C") + .arg(&repo_root) + .arg(format!("echo {older_marker}")) + .assert() + .success(); + let older_path = find_session_file_containing_marker(&sessions_dir, &older_marker) + .expect("no valid session file after first run"); + + let newer_marker = format!("resume-last-mismatched-{}", Uuid::new_v4()); + test.cmd_with_server(&server) + .arg("--skip-git-repo-check") + .arg("-C") + .arg(&repo_root) + .arg(format!("echo {newer_marker}")) + .assert() + .success(); + let newer_path = find_session_file_containing_marker(&sessions_dir, &newer_marker) + .expect("no mismatched session file after second run"); + let newer_thread_id = ThreadId::from_string(&extract_conversation_id(&newer_path))?; + + let config = ConfigBuilder::default() + .codex_home(test.home_path().to_path_buf()) + .build() + .await?; + let state_db = init_state_db(&config) + .await + .expect("state DB should initialize"); + let mut mismatched = state_db + .get_thread(newer_thread_id) + .await? + .expect("newer thread should be indexed"); + mismatched.rollout_path = older_path.clone(); + state_db.upsert_thread(&mismatched).await?; + + let resumed_marker = format!("resume-last-valid-resumed-{}", Uuid::new_v4()); + test.cmd_with_server(&server) + .arg("--skip-git-repo-check") + .arg("-C") + .arg(&repo_root) + .arg("resume") + .arg("--last") + .arg(format!("echo {resumed_marker}")) + .assert() + .success(); + + let resumed_path = find_session_file_containing_marker(&sessions_dir, &resumed_marker) + .expect("no resumed session file after skipping mismatched SQLite row"); + assert_eq!(resumed_path, older_path); + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn exec_resume_last_accepts_prompt_after_flag_in_json_mode() -> anyhow::Result<()> { skip_if_no_network!(Ok(())); diff --git a/codex-rs/ext/image-generation/src/tests.rs b/codex-rs/ext/image-generation/src/tests.rs index d7543b33b3..7e6672e7cf 100644 --- a/codex-rs/ext/image-generation/src/tests.rs +++ b/codex-rs/ext/image-generation/src/tests.rs @@ -49,7 +49,9 @@ fn uses_reserved_image_gen_namespace() { panic!("imagegen should advertise a namespace tool"); }; assert_eq!(spec.name, IMAGE_GEN_NAMESPACE); - let ResponsesApiNamespaceTool::Function(function) = &spec.tools[0]; + let ResponsesApiNamespaceTool::Function(function) = &spec.tools[0] else { + panic!("imagegen should advertise a function tool"); + }; assert_eq!(function.name, IMAGEGEN_TOOL_NAME); } diff --git a/codex-rs/ext/skills/Cargo.toml b/codex-rs/ext/skills/Cargo.toml index 43518bd826..8c64028ac2 100644 --- a/codex-rs/ext/skills/Cargo.toml +++ b/codex-rs/ext/skills/Cargo.toml @@ -22,11 +22,13 @@ codex-protocol = { workspace = true } codex-skills = { workspace = true } codex-tools = { workspace = true } codex-utils-path-uri = { workspace = true } +codex-utils-plugins = { workspace = true } codex-utils-string = { workspace = true } futures = { workspace = true } schemars = { workspace = true } serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } +serde_yaml = { workspace = true } tokio = { workspace = true, features = ["sync", "time"] } tracing = { workspace = true } url = { workspace = true } @@ -34,6 +36,9 @@ url = { workspace = true } [dev-dependencies] codex-models-manager = { workspace = true } codex-utils-absolute-path = { workspace = true } +codex-utils-cargo-bin = { workspace = true } +insta = { workspace = true } opentelemetry_sdk = { workspace = true } pretty_assertions = { workspace = true } +tempfile = { workspace = true } tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } diff --git a/codex-rs/ext/skills/src/lib.rs b/codex-rs/ext/skills/src/lib.rs index 3893fd03ed..4b4c4e57ce 100644 --- a/codex-rs/ext/skills/src/lib.rs +++ b/codex-rs/ext/skills/src/lib.rs @@ -4,6 +4,7 @@ mod config; mod dynamic_skill_selector; mod extension; mod fragments; +mod loader; pub mod provider; mod render; mod render_observability; diff --git a/codex-rs/ext/skills/src/loader/discovery.rs b/codex-rs/ext/skills/src/loader/discovery.rs new file mode 100644 index 0000000000..3d1609b237 --- /dev/null +++ b/codex-rs/ext/skills/src/loader/discovery.rs @@ -0,0 +1,167 @@ +use std::collections::HashSet; +use std::io; + +use codex_exec_server::ExecutorFileSystem; +use codex_exec_server::WalkEntryKind; +use codex_exec_server::WalkOptions; +use codex_utils_path_uri::PathUri; +use codex_utils_plugins::DISCOVERABLE_PLUGIN_MANIFEST_PATHS; + +use super::MAX_SCAN_DEPTH; +use super::MAX_SKILLS_DIRS_PER_ROOT; +use super::SKILLS_FILENAME; +use super::SKILLS_METADATA_DIR; +use super::SKILLS_METADATA_FILENAME; + +const MAX_SKILLS_ENTRIES_PER_ROOT: usize = 20_000; +pub(super) const MAX_CONCURRENT_SKILL_LOADS: usize = 64; + +pub(super) struct SkillDiscovery { + pub skills: Vec, + pub plugin_roots: HashSet, + pub namespace_roots: HashSet, + pub warnings: Vec, +} + +pub(super) struct DiscoveredSkill { + pub path: PathUri, + pub metadata: SkillMetadataDiscovery, +} + +pub(super) enum SkillMetadataDiscovery { + Present(PathUri), + Absent, + Probe(PathUri), +} + +pub(super) async fn discover_skills( + file_system: &dyn ExecutorFileSystem, + root: &PathUri, +) -> SkillDiscovery { + let empty_discovery = || SkillDiscovery { + skills: Vec::new(), + plugin_roots: HashSet::new(), + namespace_roots: HashSet::new(), + warnings: Vec::new(), + }; + let walk = match file_system + .walk( + root, + WalkOptions { + max_depth: MAX_SCAN_DEPTH, + max_directories: MAX_SKILLS_DIRS_PER_ROOT, + max_entries: MAX_SKILLS_ENTRIES_PER_ROOT, + follow_directory_symlinks: true, + prune_hidden_directories: false, + }, + /*sandbox*/ None, + ) + .await + { + Ok(walk) => walk, + Err(error) if error.kind() == io::ErrorKind::NotFound => return empty_discovery(), + Err(error) => { + let mut discovery = empty_discovery(); + discovery + .warnings + .push(format!("failed to walk skills root {root}: {error:#}")); + return discovery; + } + }; + + let inventory_complete = !walk.truncated && walk.errors.is_empty(); + let mut warnings = walk + .errors + .into_iter() + .map(|error| { + format!( + "failed to scan skill path {}: {}", + error.path, error.message + ) + }) + .collect::>(); + if walk.truncated { + warnings.push(format!( + "skills scan reached its traversal limit (root: {root})" + )); + } + + let mut skill_files = Vec::new(); + let mut file_paths = HashSet::new(); + let mut metadata_directory_parents = HashSet::new(); + let mut plugin_roots = HashSet::new(); + for entry in walk.entries { + match entry.kind { + WalkEntryKind::Directory => { + if entry + .path + .basename() + .is_some_and(|name| name.eq_ignore_ascii_case(SKILLS_METADATA_DIR)) + && let Some(skill_dir) = entry.path.parent() + { + metadata_directory_parents.insert(skill_dir); + } + if DISCOVERABLE_PLUGIN_MANIFEST_PATHS + .iter() + .any(|path| path.split('/').next() == entry.path.basename().as_deref()) + && let Some(plugin_root) = entry.path.parent() + { + plugin_roots.insert(plugin_root); + } + } + WalkEntryKind::File => { + file_paths.insert(entry.path.clone()); + if entry.path.basename().as_deref() == Some(SKILLS_FILENAME) { + skill_files.push(entry.path); + } + } + } + } + let skills = skill_files + .into_iter() + .map(|path| DiscoveredSkill { + metadata: discover_skill_metadata( + &path, + &file_paths, + &metadata_directory_parents, + inventory_complete, + ), + path, + }) + .collect(); + + SkillDiscovery { + skills, + plugin_roots, + namespace_roots: HashSet::from([root.clone()]), + warnings, + } +} + +fn discover_skill_metadata( + skill_path: &PathUri, + file_paths: &HashSet, + metadata_directory_parents: &HashSet, + inventory_complete: bool, +) -> SkillMetadataDiscovery { + let Some(skill_dir) = skill_path.parent() else { + return SkillMetadataDiscovery::Absent; + }; + let Ok(metadata_dir) = skill_dir.join(SKILLS_METADATA_DIR) else { + return SkillMetadataDiscovery::Absent; + }; + let Ok(metadata_path) = metadata_dir.join(SKILLS_METADATA_FILENAME) else { + return SkillMetadataDiscovery::Absent; + }; + if file_paths.contains(&metadata_path) { + return SkillMetadataDiscovery::Present(metadata_path); + } + + if inventory_complete && !metadata_directory_parents.contains(&skill_dir) { + SkillMetadataDiscovery::Absent + } else { + // A complete walk proves ordinary absence, but keep a filesystem probe for case aliases, + // file symlinks omitted by the walk, and incomplete inventories. + SkillMetadataDiscovery::Probe(metadata_path) + } +} diff --git a/codex-rs/ext/skills/src/loader/environment.rs b/codex-rs/ext/skills/src/loader/environment.rs new file mode 100644 index 0000000000..dcf3ac15ce --- /dev/null +++ b/codex-rs/ext/skills/src/loader/environment.rs @@ -0,0 +1,382 @@ +use std::collections::HashMap; +use std::io; + +use codex_exec_server::CapabilityRootDiscovery; +use codex_exec_server::ExecutorFileSystem; +use codex_protocol::protocol::Product; +use codex_skills::EnvironmentSkillMetadata; +use codex_skills::ParsedSkillFrontmatter; +use codex_skills::SkillDependencies; +use codex_skills::SkillPolicy; +use codex_skills::parse_skill_frontmatter_metadata; +use codex_utils_path_uri::PathUri; +use futures::StreamExt; + +use super::MAX_QUALIFIED_NAME_LEN; +use super::discovery::DiscoveredSkill; +use super::discovery::MAX_CONCURRENT_SKILL_LOADS; +use super::discovery::SkillMetadataDiscovery; +use super::discovery::discover_skills; +use super::metadata::SkillMetadataFile; +use super::metadata::resolve_dependencies; +use super::metadata::resolve_policy; +use super::metadata::sanitize_single_line; +use super::metadata::validate_len; +use super::namespace::SkillNamespaceResolver; + +struct ParsedEnvironmentSkill { + path_to_skills_md: PathUri, + base_name: String, + description: String, + short_description: Option, + dependencies: Option, + policy: Option, +} + +/// Parsed executor skill plus the instructions already materialized by capability discovery. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct EnvironmentSkillSnapshot { + pub metadata: EnvironmentSkillMetadata, + pub instructions: String, +} + +#[derive(Debug, Default)] +pub struct EnvironmentSkillSnapshotOutcome { + pub skills: Vec, + pub warnings: Vec, +} + +impl ParsedEnvironmentSkill { + async fn load( + file_system: &dyn ExecutorFileSystem, + skill: &DiscoveredSkill, + ) -> Result { + let (contents, discovered_metadata) = match &skill.metadata { + SkillMetadataDiscovery::Present(metadata_path) => { + let (contents, metadata) = tokio::join!( + read_skill_contents(file_system, &skill.path), + read_skill_metadata(file_system, metadata_path), + ); + (contents?, metadata) + } + SkillMetadataDiscovery::Absent | SkillMetadataDiscovery::Probe(_) => ( + read_skill_contents(file_system, &skill.path).await?, + (None, None), + ), + }; + let ParsedSkillFrontmatter { + name: base_name, + description, + short_description, + } = parse_skill_frontmatter_metadata(&contents, || default_skill_name(&skill.path)) + .map_err(|err| err.to_string())?; + let (dependencies, policy) = match &skill.metadata { + SkillMetadataDiscovery::Present(_) | SkillMetadataDiscovery::Absent => { + discovered_metadata + } + SkillMetadataDiscovery::Probe(metadata_path) => { + probe_skill_metadata(file_system, metadata_path).await + } + }; + + Ok(Self { + path_to_skills_md: skill.path.clone(), + base_name, + description, + short_description, + dependencies, + policy, + }) + } +} + +#[derive(Debug, Default)] +pub struct EnvironmentSkillLoadOutcome { + pub skills: Vec, + pub warnings: Vec, +} + +/// Discovers skills without converting environment-owned paths to host paths. +#[tracing::instrument( + name = "skills.environment.load", + level = "info", + skip_all, + fields(skill_count = tracing::field::Empty) +)] +pub async fn load_environment_skills_from_root( + file_system: &dyn ExecutorFileSystem, + root: &PathUri, + restriction_product: Option, +) -> EnvironmentSkillLoadOutcome { + let mut outcome = EnvironmentSkillLoadOutcome::default(); + // Preserve environment discovery behavior by following directory aliases and including + // hidden directories exposed by the executor. + let discovery = discover_skills(file_system, root).await; + tracing::Span::current().record("skill_count", discovery.skills.len()); + outcome.warnings.extend(discovery.warnings); + if discovery.skills.is_empty() { + return outcome; + } + + let skill_paths = discovery + .skills + .iter() + .map(|skill| skill.path.clone()) + .collect::>(); + let namespace_resolver = SkillNamespaceResolver::discover( + file_system, + root, + &skill_paths, + discovery.plugin_roots, + discovery.namespace_roots, + ); + + // Remote executors can multiplex these independent per-skill reads, so polling a bounded + // number together allows the I/O for each skill and its metadata to happen concurrently. + let skill_results = futures::stream::iter(discovery.skills) + .map(|skill| { + let path = skill.path.clone(); + async move { + ( + path, + ParsedEnvironmentSkill::load(file_system, &skill).await, + ) + } + }) + .buffered(MAX_CONCURRENT_SKILL_LOADS) + .collect::>(); + let (namespace_resolver, skill_results) = tokio::join!(namespace_resolver, skill_results); + + for (path, result) in skill_results { + let result = result.and_then(|skill| { + let name = namespace_resolver + .for_skill(root, &skill.path_to_skills_md) + .qualify(&skill.base_name); + validate_len(&name, MAX_QUALIFIED_NAME_LEN, "qualified name") + .map_err(|err| err.to_string())?; + + Ok(EnvironmentSkillMetadata { + path_to_skills_md: skill.path_to_skills_md, + name, + description: skill.description, + short_description: skill.short_description, + dependencies: skill.dependencies, + policy: skill.policy, + }) + }); + match result { + Ok(skill) if skill.matches_product_restriction(restriction_product) => { + outcome.skills.push(skill); + } + Ok(_) => {} + Err(message) => outcome.warnings.push(format!( + "Failed to load environment skill at {path}: {message}" + )), + } + } + outcome.skills.sort_by(|left, right| { + left.name.cmp(&right.name).then_with(|| { + left.path_to_skills_md + .to_string() + .cmp(&right.path_to_skills_md.to_string()) + }) + }); + outcome +} + +/// Parses an executor-produced manifest bundle without issuing additional filesystem requests. +pub fn load_environment_skills_from_discovery( + discovery: &CapabilityRootDiscovery, + restriction_product: Option, +) -> EnvironmentSkillSnapshotOutcome { + let mut outcome = EnvironmentSkillSnapshotOutcome { + warnings: discovery.warnings.clone(), + ..Default::default() + }; + if let Some(error) = &discovery.error { + outcome.warnings.push(error.clone()); + return outcome; + } + + let mut plugin_namespaces = HashMap::new(); + for (plugin_root, name) in discovery.namespace_manifests.iter().filter_map(|manifest| { + #[derive(serde::Deserialize)] + struct ManifestName { + #[serde(default)] + name: String, + } + + let plugin_root = manifest.path.parent()?.parent()?; + let parsed = serde_json::from_str::(&manifest.contents).ok()?; + let name = if parsed.name.trim().is_empty() { + plugin_root.basename()? + } else { + parsed.name + }; + Some((plugin_root, name)) + }) { + // Exec-server orders manifests by the same precedence as local discovery. Preserve the + // first manifest if an older or alternate server returns duplicates for one plugin root. + plugin_namespaces.entry(plugin_root).or_insert(name); + } + + for skill in &discovery.skills { + let plugin_namespace = + nearest_plugin_namespace(&skill.instructions.path, &plugin_namespaces); + let ParsedSkillFrontmatter { + name: base_name, + description, + short_description, + } = match parse_skill_frontmatter_metadata(&skill.instructions.contents, || { + default_skill_name(&skill.instructions.path) + }) { + Ok(frontmatter) => frontmatter, + Err(error) => { + outcome.warnings.push(format!( + "Failed to load environment skill at {}: {error}", + skill.instructions.path + )); + continue; + } + }; + let name = plugin_namespace + .map(|namespace| format!("{namespace}:{base_name}")) + .unwrap_or(base_name); + if let Err(error) = validate_len(&name, MAX_QUALIFIED_NAME_LEN, "qualified name") { + outcome.warnings.push(format!( + "Failed to load environment skill at {}: {error}", + skill.instructions.path + )); + continue; + } + let (dependencies, policy) = skill + .metadata + .as_ref() + .and_then(|metadata| { + serde_yaml::from_str::(&metadata.contents) + .map_err(|error| { + tracing::warn!( + path = %metadata.path, + "ignoring invalid discovered skill metadata: {error}" + ); + }) + .ok() + }) + .map(|metadata| { + ( + resolve_dependencies(metadata.dependencies), + resolve_policy(metadata.policy), + ) + }) + .unwrap_or((None, None)); + let metadata = EnvironmentSkillMetadata { + path_to_skills_md: skill.instructions.path.clone(), + name, + description, + short_description, + dependencies, + policy, + }; + if metadata.matches_product_restriction(restriction_product) { + outcome.skills.push(EnvironmentSkillSnapshot { + metadata, + instructions: skill.instructions.contents.clone(), + }); + } + } + outcome.skills.sort_by(|left, right| { + left.metadata.name.cmp(&right.metadata.name).then_with(|| { + left.metadata + .path_to_skills_md + .to_string() + .cmp(&right.metadata.path_to_skills_md.to_string()) + }) + }); + outcome +} + +fn nearest_plugin_namespace<'a>( + skill_path: &PathUri, + plugin_namespaces: &'a HashMap, +) -> Option<&'a str> { + let mut ancestor = skill_path.parent(); + while let Some(path) = ancestor { + if let Some(namespace) = plugin_namespaces.get(&path) { + return Some(namespace); + } + ancestor = path.parent(); + } + None +} +async fn read_skill_contents( + file_system: &dyn ExecutorFileSystem, + skill_path: &PathUri, +) -> Result { + file_system + .read_file_text(skill_path, /*sandbox*/ None) + .await + .map_err(|err| format!("failed to read file: {err}")) +} + +async fn probe_skill_metadata( + file_system: &dyn ExecutorFileSystem, + metadata_path: &PathUri, +) -> (Option, Option) { + match file_system + .get_metadata(metadata_path, /*sandbox*/ None) + .await + { + Ok(metadata) if metadata.is_file => {} + Ok(_) => return (None, None), + Err(error) if error.kind() == io::ErrorKind::NotFound => return (None, None), + Err(error) => { + tracing::warn!("ignoring {metadata_path}: failed to stat metadata: {error}"); + return (None, None); + } + } + read_skill_metadata(file_system, metadata_path).await +} + +async fn read_skill_metadata( + file_system: &dyn ExecutorFileSystem, + metadata_path: &PathUri, +) -> (Option, Option) { + let contents = match file_system + .read_file_text(metadata_path, /*sandbox*/ None) + .await + { + Ok(contents) => contents, + Err(error) => { + tracing::warn!("ignoring {metadata_path}: failed to read metadata: {error}"); + return (None, None); + } + }; + let parsed: SkillMetadataFile = match serde_yaml::from_str(&contents) { + Ok(parsed) => parsed, + Err(error) => { + tracing::warn!("ignoring {metadata_path}: invalid metadata: {error}"); + return (None, None); + } + }; + + ( + resolve_dependencies(parsed.dependencies), + resolve_policy(parsed.policy), + ) +} + +fn default_skill_name(path: &PathUri) -> String { + path.parent() + .and_then(|parent| parent.basename()) + .map(|name| sanitize_single_line(&name)) + .filter(|name| !name.is_empty()) + .unwrap_or_else(|| "skill".to_string()) +} + +#[cfg(test)] +#[path = "environment_tests.rs"] +mod tests; + +#[cfg(test)] +#[path = "environment_io_tests.rs"] +mod io_tests; diff --git a/codex-rs/ext/skills/src/loader/environment_io_tests.rs b/codex-rs/ext/skills/src/loader/environment_io_tests.rs new file mode 100644 index 0000000000..0a02413dc0 --- /dev/null +++ b/codex-rs/ext/skills/src/loader/environment_io_tests.rs @@ -0,0 +1,414 @@ +use std::fs; +use std::sync::Mutex; +use std::sync::atomic::AtomicBool; +use std::sync::atomic::AtomicUsize; +use std::sync::atomic::Ordering; +use std::time::Duration; + +use codex_exec_server::CopyOptions; +use codex_exec_server::CreateDirectoryOptions; +use codex_exec_server::ExecutorFileSystem; +use codex_exec_server::ExecutorFileSystemFuture; +use codex_exec_server::FileMetadata; +use codex_exec_server::FileSystemReadStream; +use codex_exec_server::FileSystemSandboxContext; +use codex_exec_server::LOCAL_FS; +use codex_exec_server::ReadDirectoryEntry; +use codex_exec_server::RemoveOptions; +use codex_exec_server::WalkOptions; +use codex_exec_server::WalkOutcome; +use codex_skills::EnvironmentSkillMetadata; +use codex_utils_path_uri::PathUri; +use pretty_assertions::assert_eq; +use tempfile::tempdir; +use tokio::sync::Notify; + +use super::load_environment_skills_from_root; + +#[derive(Clone, Copy)] +enum ManifestMetadataBehavior { + Immediate, + WaitForSkillRead, +} + +struct RecordingFileSystem<'a> { + inner: &'a dyn ExecutorFileSystem, + read_files: Mutex>, + metadata_files: Mutex>, + walks: AtomicUsize, + manifest_metadata_behavior: ManifestMetadataBehavior, + skill_read_started: AtomicBool, + skill_read_started_notify: Notify, +} + +#[derive(Debug, PartialEq, Eq)] +struct FileSystemCalls { + walks: usize, + read_files: Vec, + metadata_files: Vec, +} + +impl<'a> RecordingFileSystem<'a> { + fn new( + inner: &'a dyn ExecutorFileSystem, + manifest_metadata_behavior: ManifestMetadataBehavior, + ) -> Self { + Self { + inner, + read_files: Mutex::new(Vec::new()), + metadata_files: Mutex::new(Vec::new()), + walks: AtomicUsize::new(0), + manifest_metadata_behavior, + skill_read_started: AtomicBool::new(false), + skill_read_started_notify: Notify::new(), + } + } + + fn calls(&self) -> FileSystemCalls { + let mut read_files = self + .read_files + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clone(); + read_files.sort_by_key(ToString::to_string); + let mut metadata_files = self + .metadata_files + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clone(); + metadata_files.sort_by_key(ToString::to_string); + FileSystemCalls { + walks: self.walks.load(Ordering::Relaxed), + read_files, + metadata_files, + } + } +} + +impl ExecutorFileSystem for RecordingFileSystem<'_> { + fn canonicalize<'a>( + &'a self, + path: &'a PathUri, + sandbox: Option<&'a FileSystemSandboxContext>, + ) -> ExecutorFileSystemFuture<'a, PathUri> { + self.inner.canonicalize(path, sandbox) + } + + fn read_file<'a>( + &'a self, + path: &'a PathUri, + sandbox: Option<&'a FileSystemSandboxContext>, + ) -> ExecutorFileSystemFuture<'a, Vec> { + self.read_files + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .push(path.clone()); + if path.basename().as_deref() == Some("SKILL.md") { + self.skill_read_started.store(true, Ordering::Release); + self.skill_read_started_notify.notify_waiters(); + } + self.inner.read_file(path, sandbox) + } + + fn read_file_stream<'a>( + &'a self, + path: &'a PathUri, + sandbox: Option<&'a FileSystemSandboxContext>, + ) -> ExecutorFileSystemFuture<'a, FileSystemReadStream> { + self.inner.read_file_stream(path, sandbox) + } + + fn write_file<'a>( + &'a self, + path: &'a PathUri, + contents: Vec, + sandbox: Option<&'a FileSystemSandboxContext>, + ) -> ExecutorFileSystemFuture<'a, ()> { + self.inner.write_file(path, contents, sandbox) + } + + fn create_directory<'a>( + &'a self, + path: &'a PathUri, + options: CreateDirectoryOptions, + sandbox: Option<&'a FileSystemSandboxContext>, + ) -> ExecutorFileSystemFuture<'a, ()> { + self.inner.create_directory(path, options, sandbox) + } + + fn get_metadata<'a>( + &'a self, + path: &'a PathUri, + sandbox: Option<&'a FileSystemSandboxContext>, + ) -> ExecutorFileSystemFuture<'a, FileMetadata> { + self.metadata_files + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .push(path.clone()); + if matches!( + self.manifest_metadata_behavior, + ManifestMetadataBehavior::WaitForSkillRead + ) && path.basename().as_deref() == Some("plugin.json") + { + return Box::pin(async move { + loop { + let notified = self.skill_read_started_notify.notified(); + if self.skill_read_started.load(Ordering::Acquire) { + break; + } + notified.await; + } + self.inner.get_metadata(path, sandbox).await + }); + } + self.inner.get_metadata(path, sandbox) + } + + fn read_directory<'a>( + &'a self, + path: &'a PathUri, + sandbox: Option<&'a FileSystemSandboxContext>, + ) -> ExecutorFileSystemFuture<'a, Vec> { + self.inner.read_directory(path, sandbox) + } + + fn walk<'a>( + &'a self, + path: &'a PathUri, + options: WalkOptions, + sandbox: Option<&'a FileSystemSandboxContext>, + ) -> ExecutorFileSystemFuture<'a, WalkOutcome> { + self.walks.fetch_add(1, Ordering::Relaxed); + self.inner.walk(path, options, sandbox) + } + + fn remove<'a>( + &'a self, + path: &'a PathUri, + options: RemoveOptions, + sandbox: Option<&'a FileSystemSandboxContext>, + ) -> ExecutorFileSystemFuture<'a, ()> { + self.inner.remove(path, options, sandbox) + } + + fn copy<'a>( + &'a self, + source_path: &'a PathUri, + destination_path: &'a PathUri, + options: CopyOptions, + sandbox: Option<&'a FileSystemSandboxContext>, + ) -> ExecutorFileSystemFuture<'a, ()> { + self.inner + .copy(source_path, destination_path, options, sandbox) + } +} + +#[tokio::test] +async fn loads_nearest_plugin_namespaces_without_reading_unused_sibling_manifests() { + let root = tempdir().expect("tempdir"); + let standalone_skill = root.path().join("standalone/SKILL.md"); + let outer_root = root.path().join("plugins/outer"); + let outer_skill = outer_root.join("skills/deploy/SKILL.md"); + let inner_root = outer_root.join("nested/inner"); + let inner_skill = inner_root.join("skills/audit/SKILL.md"); + let unused_root = root.path().join("plugins/unused"); + + for path in [&standalone_skill, &outer_skill, &inner_skill] { + fs::create_dir_all(path.parent().expect("skill parent")).expect("skill dir"); + } + for (plugin_root, name) in [ + (&outer_root, "outer"), + (&inner_root, "inner"), + (&unused_root, "unused"), + ] { + fs::create_dir_all(plugin_root.join(".codex-plugin")).expect("manifest dir"); + fs::write( + plugin_root.join(".codex-plugin/plugin.json"), + format!(r#"{{"name":"{name}"}}"#), + ) + .expect("manifest"); + } + for (path, name) in [ + (&standalone_skill, "standalone"), + (&outer_skill, "deploy"), + (&inner_skill, "audit"), + ] { + fs::write( + path, + format!("---\nname: {name}\ndescription: {name} skill.\n---\n"), + ) + .expect("skill"); + } + + let file_system = + RecordingFileSystem::new(LOCAL_FS.as_ref(), ManifestMetadataBehavior::Immediate); + let root_uri = PathUri::from_host_native_path(root.path()).expect("root URI"); + let outcome = load_environment_skills_from_root( + &file_system, + &root_uri, + /*restriction_product*/ None, + ) + .await; + + assert_eq!(outcome.warnings, Vec::::new()); + assert_eq!( + outcome.skills, + vec![ + EnvironmentSkillMetadata { + path_to_skills_md: PathUri::from_host_native_path(&inner_skill).unwrap(), + name: "inner:audit".to_string(), + description: "audit skill.".to_string(), + short_description: None, + dependencies: None, + policy: None, + }, + EnvironmentSkillMetadata { + path_to_skills_md: PathUri::from_host_native_path(&outer_skill).unwrap(), + name: "outer:deploy".to_string(), + description: "deploy skill.".to_string(), + short_description: None, + dependencies: None, + policy: None, + }, + EnvironmentSkillMetadata { + path_to_skills_md: PathUri::from_host_native_path(&standalone_skill).unwrap(), + name: "standalone".to_string(), + description: "standalone skill.".to_string(), + short_description: None, + dependencies: None, + policy: None, + }, + ] + ); + + let mut manifest_reads = file_system + .calls() + .read_files + .into_iter() + .filter(|path| path.basename().as_deref() == Some("plugin.json")) + .collect::>(); + manifest_reads.sort_by_key(ToString::to_string); + let mut expected_manifest_reads = [&outer_root, &inner_root] + .into_iter() + .map(|plugin_root| { + PathUri::from_host_native_path(plugin_root.join(".codex-plugin/plugin.json")).unwrap() + }) + .collect::>(); + expected_manifest_reads.sort_by_key(ToString::to_string); + assert_eq!(manifest_reads, expected_manifest_reads); +} + +#[tokio::test] +async fn reuses_walk_inventory_for_missing_skill_metadata() { + const SKILL_COUNT: usize = 66; + + let root = tempdir().expect("tempdir"); + let manifest_path = root.path().join(".codex-plugin/plugin.json"); + fs::create_dir_all(manifest_path.parent().expect("manifest parent")).expect("manifest dir"); + fs::write(&manifest_path, r#"{"name":"inventory"}"#).expect("manifest"); + + let mut skill_paths = Vec::new(); + for index in 0..SKILL_COUNT { + let name = format!("skill-{index}"); + let skill_path = root.path().join(&name).join("SKILL.md"); + fs::create_dir_all(skill_path.parent().expect("skill parent")).expect("skill dir"); + fs::write( + &skill_path, + format!("---\nname: {name}\ndescription: {name} skill.\n---\n"), + ) + .expect("skill"); + skill_paths.push(skill_path); + } + + let file_system = + RecordingFileSystem::new(LOCAL_FS.as_ref(), ManifestMetadataBehavior::Immediate); + let root_uri = PathUri::from_host_native_path(root.path()).expect("root URI"); + let outcome = load_environment_skills_from_root( + &file_system, + &root_uri, + /*restriction_product*/ None, + ) + .await; + + let mut expected_skills = skill_paths + .iter() + .enumerate() + .map(|(index, skill_path)| EnvironmentSkillMetadata { + path_to_skills_md: PathUri::from_host_native_path(skill_path).unwrap(), + name: format!("inventory:skill-{index}"), + description: format!("skill-{index} skill."), + short_description: None, + dependencies: None, + policy: None, + }) + .collect::>(); + expected_skills.sort_by(|left, right| { + left.name.cmp(&right.name).then_with(|| { + left.path_to_skills_md + .to_string() + .cmp(&right.path_to_skills_md.to_string()) + }) + }); + assert_eq!(outcome.skills, expected_skills); + assert_eq!(outcome.warnings, Vec::::new()); + + let mut expected_read_files = skill_paths + .iter() + .map(|path| PathUri::from_host_native_path(path).unwrap()) + .collect::>(); + let manifest_uri = PathUri::from_host_native_path(manifest_path).unwrap(); + expected_read_files.push(manifest_uri.clone()); + expected_read_files.sort_by_key(ToString::to_string); + assert_eq!( + file_system.calls(), + FileSystemCalls { + walks: 1, + read_files: expected_read_files, + metadata_files: vec![manifest_uri], + } + ); +} + +#[tokio::test] +async fn reads_skill_files_while_resolving_plugin_namespaces() { + let root = tempdir().expect("tempdir"); + let manifest_path = root.path().join(".codex-plugin/plugin.json"); + fs::create_dir_all(manifest_path.parent().expect("manifest parent")).expect("manifest dir"); + fs::write(&manifest_path, r#"{"name":"parallel"}"#).expect("manifest"); + let skill_path = root.path().join("demo/SKILL.md"); + fs::create_dir_all(skill_path.parent().expect("skill parent")).expect("skill dir"); + fs::write( + &skill_path, + "---\nname: demo\ndescription: demo skill.\n---\n", + ) + .expect("skill"); + + let file_system = RecordingFileSystem::new( + LOCAL_FS.as_ref(), + ManifestMetadataBehavior::WaitForSkillRead, + ); + let root_uri = PathUri::from_host_native_path(root.path()).expect("root URI"); + let outcome = tokio::time::timeout( + Duration::from_secs(5), + load_environment_skills_from_root( + &file_system, + &root_uri, + /*restriction_product*/ None, + ), + ) + .await + .expect("skill reads should start before namespace resolution finishes"); + + assert_eq!(outcome.warnings, Vec::::new()); + assert_eq!( + outcome.skills, + vec![EnvironmentSkillMetadata { + path_to_skills_md: PathUri::from_host_native_path(skill_path).unwrap(), + name: "parallel:demo".to_string(), + description: "demo skill.".to_string(), + short_description: None, + dependencies: None, + policy: None, + }] + ); +} diff --git a/codex-rs/ext/skills/src/loader/environment_tests.rs b/codex-rs/ext/skills/src/loader/environment_tests.rs new file mode 100644 index 0000000000..884b53aee5 --- /dev/null +++ b/codex-rs/ext/skills/src/loader/environment_tests.rs @@ -0,0 +1,151 @@ +use std::fs; + +use codex_core_skills::loader::load_environment_skills_from_root; +use codex_exec_server::CapabilityRootDiscoverRequest; +use codex_exec_server::CapabilityRootsDiscoverParams; +use codex_exec_server::LOCAL_FS; +use codex_exec_server::discover_capability_roots; +use codex_utils_path_uri::PathUri; +use pretty_assertions::assert_eq; +use tempfile::tempdir; + +use super::load_environment_skills_from_discovery; + +#[tokio::test] +async fn executor_bundle_parser_matches_the_existing_environment_loader() { + let root = tempdir().expect("tempdir"); + let plugin_manifest = root.path().join(".codex-plugin/plugin.json"); + let nested_manifest = root.path().join("nested/.claude-plugin/plugin.json"); + let deploy_skill = root.path().join("skills/deploy/SKILL.md"); + let deploy_metadata = root.path().join("skills/deploy/agents/openai.yaml"); + let audit_skill = root.path().join("nested/skills/audit/SKILL.md"); + for (path, contents) in [ + (&plugin_manifest, r#"{"name":"demo"}"#), + (&nested_manifest, r#"{"name":"nested"}"#), + ( + &deploy_skill, + "---\nname: deploy\ndescription: Deploy the service.\n---\n\nDeploy.\n", + ), + ( + &deploy_metadata, + "policy:\n allow_implicit_invocation: false\n", + ), + ( + &audit_skill, + "---\nname: audit\ndescription: Audit the service.\n---\n\nAudit.\n", + ), + ] { + fs::create_dir_all(path.parent().expect("test file parent")).expect("test directory"); + fs::write(path, contents).expect("test file"); + } + + let root_uri = PathUri::from_host_native_path(root.path()).expect("root URI"); + let existing = load_environment_skills_from_root( + LOCAL_FS.as_ref(), + &root_uri, + /*restriction_product*/ None, + ) + .await; + let response = discover_capability_roots( + LOCAL_FS.as_ref(), + CapabilityRootsDiscoverParams { + roots: vec![CapabilityRootDiscoverRequest { + id: "demo@1".to_string(), + path: root_uri, + sandbox: None, + }], + }, + ) + .await + .expect("capability discovery"); + let bundled = load_environment_skills_from_discovery( + response.roots.first().expect("discovered root"), + /*restriction_product*/ None, + ); + + assert_eq!(bundled.warnings, existing.warnings); + assert_eq!( + bundled + .skills + .iter() + .map(|skill| skill.metadata.clone()) + .collect::>(), + existing.skills + ); + assert_eq!( + bundled + .skills + .iter() + .map(|skill| skill.instructions.as_str()) + .collect::>(), + vec![ + "---\nname: deploy\ndescription: Deploy the service.\n---\n\nDeploy.\n", + "---\nname: audit\ndescription: Audit the service.\n---\n\nAudit.\n", + ] + ); +} + +#[tokio::test] +async fn executor_bundle_preserves_parent_namespace_and_manifest_precedence() { + let plugin = tempdir().expect("tempdir"); + for (relative_path, name) in [ + (".codex-plugin/plugin.json", "codex-name"), + (".claude-plugin/plugin.json", "claude-name"), + (".cursor-plugin/plugin.json", "cursor-name"), + ] { + let manifest = plugin.path().join(relative_path); + fs::create_dir_all(manifest.parent().expect("manifest parent")) + .expect("manifest directory"); + fs::write(&manifest, format!(r#"{{"name":"{name}"}}"#)).expect("manifest"); + } + let skills_root = plugin.path().join("skills"); + let skill_path = skills_root.join("search/SKILL.md"); + fs::create_dir_all(skill_path.parent().expect("skill parent")).expect("skill directory"); + fs::write( + &skill_path, + "---\nname: search\ndescription: Search the project.\n---\n\nSearch.\n", + ) + .expect("skill"); + + let root_uri = PathUri::from_host_native_path(&skills_root).expect("skills root URI"); + let existing = load_environment_skills_from_root( + LOCAL_FS.as_ref(), + &root_uri, + /*restriction_product*/ None, + ) + .await; + let response = discover_capability_roots( + LOCAL_FS.as_ref(), + CapabilityRootsDiscoverParams { + roots: vec![CapabilityRootDiscoverRequest { + id: "skills-only".to_string(), + path: root_uri, + sandbox: None, + }], + }, + ) + .await + .expect("capability discovery"); + let discovery = response.roots.first().expect("discovered root"); + let bundled = + load_environment_skills_from_discovery(discovery, /*restriction_product*/ None); + + assert_eq!(discovery.plugin, None); + assert_eq!(discovery.namespace_manifests.len(), 1); + assert!( + discovery.namespace_manifests[0] + .path + .to_string() + .ends_with("/.codex-plugin/plugin.json") + ); + assert_eq!(bundled.warnings, existing.warnings); + assert_eq!( + bundled + .skills + .iter() + .map(|skill| skill.metadata.clone()) + .collect::>(), + existing.skills + ); + assert_eq!(bundled.skills[0].metadata.name, "codex-name:search"); +} diff --git a/codex-rs/ext/skills/src/loader/metadata.rs b/codex-rs/ext/skills/src/loader/metadata.rs new file mode 100644 index 0000000000..e3537cfdd1 --- /dev/null +++ b/codex-rs/ext/skills/src/loader/metadata.rs @@ -0,0 +1,176 @@ +use std::path::PathBuf; + +use codex_protocol::protocol::Product; +use codex_skills::SkillDependencies; +use codex_skills::SkillParseError; +use codex_skills::SkillPolicy; +use codex_skills::SkillToolDependency; +use serde::Deserialize; + +use super::MAX_DEPENDENCY_COMMAND_LEN; +use super::MAX_DEPENDENCY_DESCRIPTION_LEN; +use super::MAX_DEPENDENCY_TRANSPORT_LEN; +use super::MAX_DEPENDENCY_TYPE_LEN; +use super::MAX_DEPENDENCY_URL_LEN; +use super::MAX_DEPENDENCY_VALUE_LEN; + +#[derive(Debug, Default, Deserialize)] +pub(super) struct SkillMetadataFile { + // Keep parsing the host-only interface fields so malformed interface metadata + // invalidates the file exactly as it does in the legacy loader. + #[serde(default, rename = "interface")] + _interface: Option, + #[serde(default)] + pub(super) dependencies: Option, + #[serde(default)] + pub(super) policy: Option, +} + +#[derive(Debug, Default, Deserialize)] +struct Interface { + #[serde(rename = "display_name")] + _display_name: Option, + #[serde(rename = "short_description")] + _short_description: Option, + #[serde(rename = "icon_small")] + _icon_small: Option, + #[serde(rename = "icon_large")] + _icon_large: Option, + #[serde(rename = "brand_color")] + _brand_color: Option, + #[serde(rename = "default_prompt")] + _default_prompt: Option, +} + +#[derive(Debug, Default, Deserialize)] +pub(super) struct Dependencies { + #[serde(default)] + tools: Vec, +} + +#[derive(Debug, Deserialize)] +pub(super) struct Policy { + #[serde(default)] + allow_implicit_invocation: Option, + #[serde(default)] + products: Vec, +} + +#[derive(Debug, Default, Deserialize)] +struct DependencyTool { + #[serde(rename = "type")] + kind: Option, + value: Option, + description: Option, + transport: Option, + command: Option, + url: Option, +} + +pub(super) fn resolve_dependencies( + dependencies: Option, +) -> Option { + let dependencies = dependencies?; + let tools = dependencies + .tools + .into_iter() + .filter_map(resolve_dependency_tool) + .collect::>(); + if tools.is_empty() { + None + } else { + Some(SkillDependencies { tools }) + } +} + +pub(super) fn resolve_policy(policy: Option) -> Option { + policy.map(|policy| SkillPolicy { + allow_implicit_invocation: policy.allow_implicit_invocation, + products: policy.products, + }) +} + +fn resolve_dependency_tool(tool: DependencyTool) -> Option { + let r#type = resolve_required_str( + tool.kind, + MAX_DEPENDENCY_TYPE_LEN, + "dependencies.tools.type", + )?; + let value = resolve_required_str( + tool.value, + MAX_DEPENDENCY_VALUE_LEN, + "dependencies.tools.value", + )?; + let description = resolve_str( + tool.description, + MAX_DEPENDENCY_DESCRIPTION_LEN, + "dependencies.tools.description", + ); + let transport = resolve_str( + tool.transport, + MAX_DEPENDENCY_TRANSPORT_LEN, + "dependencies.tools.transport", + ); + let command = resolve_str( + tool.command, + MAX_DEPENDENCY_COMMAND_LEN, + "dependencies.tools.command", + ); + let url = resolve_str(tool.url, MAX_DEPENDENCY_URL_LEN, "dependencies.tools.url"); + + Some(SkillToolDependency { + r#type, + value, + description, + transport, + command, + url, + }) +} + +pub(super) fn sanitize_single_line(raw: &str) -> String { + raw.split_whitespace().collect::>().join(" ") +} + +pub(super) fn validate_len( + value: &str, + max_len: usize, + field_name: &'static str, +) -> Result<(), SkillParseError> { + if value.is_empty() { + return Err(SkillParseError::MissingField(field_name)); + } + if value.chars().count() > max_len { + return Err(SkillParseError::InvalidField { + field: field_name, + reason: format!("exceeds maximum length of {max_len} characters"), + }); + } + Ok(()) +} + +fn resolve_str(value: Option, max_len: usize, field: &'static str) -> Option { + let value = value?; + let value = sanitize_single_line(&value); + if value.is_empty() { + tracing::warn!("ignoring {field}: value is empty"); + return None; + } + if value.chars().count() > max_len { + tracing::warn!("ignoring {field}: exceeds maximum length of {max_len} characters"); + return None; + } + Some(value) +} + +fn resolve_required_str( + value: Option, + max_len: usize, + field: &'static str, +) -> Option { + let Some(value) = value else { + tracing::warn!("ignoring {field}: value is missing"); + return None; + }; + resolve_str(Some(value), max_len, field) +} diff --git a/codex-rs/ext/skills/src/loader/mod.rs b/codex-rs/ext/skills/src/loader/mod.rs new file mode 100644 index 0000000000..f2e7ae7a1c --- /dev/null +++ b/codex-rs/ext/skills/src/loader/mod.rs @@ -0,0 +1,22 @@ +mod discovery; +mod environment; +mod metadata; +mod namespace; + +pub(crate) use environment::load_environment_skills_from_discovery; +pub(crate) use environment::load_environment_skills_from_root; + +pub(super) const SKILLS_FILENAME: &str = "SKILL.md"; +pub(super) const SKILLS_METADATA_DIR: &str = "agents"; +pub(super) const SKILLS_METADATA_FILENAME: &str = "openai.yaml"; +pub(super) const MAX_NAME_LEN: usize = 64; +pub(super) const MAX_QUALIFIED_NAME_LEN: usize = 128; +pub(super) const MAX_DESCRIPTION_LEN: usize = 1024; +pub(super) const MAX_DEPENDENCY_TYPE_LEN: usize = MAX_NAME_LEN; +pub(super) const MAX_DEPENDENCY_TRANSPORT_LEN: usize = MAX_NAME_LEN; +pub(super) const MAX_DEPENDENCY_VALUE_LEN: usize = MAX_DESCRIPTION_LEN; +pub(super) const MAX_DEPENDENCY_DESCRIPTION_LEN: usize = MAX_DESCRIPTION_LEN; +pub(super) const MAX_DEPENDENCY_COMMAND_LEN: usize = MAX_DESCRIPTION_LEN; +pub(super) const MAX_DEPENDENCY_URL_LEN: usize = MAX_DESCRIPTION_LEN; +pub(super) const MAX_SCAN_DEPTH: usize = 6; +pub(super) const MAX_SKILLS_DIRS_PER_ROOT: usize = 2000; diff --git a/codex-rs/ext/skills/src/loader/namespace.rs b/codex-rs/ext/skills/src/loader/namespace.rs new file mode 100644 index 0000000000..3027a235bf --- /dev/null +++ b/codex-rs/ext/skills/src/loader/namespace.rs @@ -0,0 +1,172 @@ +use std::collections::HashMap; +use std::collections::HashSet; + +use codex_exec_server::ExecutorFileSystem; +use codex_utils_path_uri::PathUri; +use codex_utils_plugins::plugin_namespace_for_root_uri; +use futures::StreamExt; + +use super::discovery::MAX_CONCURRENT_SKILL_LOADS; + +/// Resolves the namespace prefix applied to skill names during one skills scan. +/// +/// A plugin namespace is the plugin name from the nearest valid plugin manifest +/// above a skill path. For example, a skill named `search` beneath a plugin named +/// `sample` is exposed as `sample:search`. +/// +/// Resolving the namespace separately for every `SKILL.md` repeats the same +/// ancestor manifest probes for sibling skills. This resolver resolves relevant +/// roots once per scan, then selects the nearest matching root for each skill. +/// +/// The deepest matching canonical symlink root or nested plugin root wins, +/// followed by the namespace inherited from the scanned skills root. +pub(crate) struct SkillNamespaceResolver { + inherited_namespace: ResolvedSkillNamespace, + nested_namespaces: Vec<(PathUri, ResolvedSkillNamespace)>, +} + +impl SkillNamespaceResolver { + pub(crate) async fn discover( + fs: &dyn ExecutorFileSystem, + root: &PathUri, + skill_paths: &[PathUri], + plugin_roots: HashSet, + namespace_roots: HashSet, + ) -> Self { + // Only probe plugin roots above loaded skills; unused siblings cannot affect names. + let mut skill_ancestors = HashSet::new(); + for skill_path in skill_paths { + let mut ancestor = skill_path.parent(); + while let Some(path) = ancestor { + skill_ancestors.insert(path.clone()); + ancestor = path.parent(); + } + } + let plugin_roots = plugin_roots + .into_iter() + .filter(|plugin_root| skill_ancestors.contains(plugin_root)) + .collect::>(); + + // The scan root is already the fallback above if nothing else matches, exclude from the search. + let namespace_roots = namespace_roots + .into_iter() + .filter(|namespace_root| namespace_root != root) + .collect::>(); + let namespace_root_set = namespace_roots.iter().cloned().collect::>(); + let plugin_roots = plugin_roots + .into_iter() + .filter(|plugin_root| plugin_root != root && !namespace_root_set.contains(plugin_root)) + .collect::>(); + + let lookup_roots = std::iter::once(root.clone()) + .chain(namespace_roots.iter().cloned()) + .collect::>(); + let mut pending_lookups = lookup_roots + .iter() + .cloned() + .map(|lookup_root| (lookup_root.clone(), lookup_root)) + .collect::>(); + let mut direct_plugin_roots = plugin_roots.iter().cloned().collect::>(); + let mut namespaces_by_root = HashMap::new(); + let mut namespaces_by_lookup_root = HashMap::new(); + while !pending_lookups.is_empty() { + let probe_roots = pending_lookups + .iter() + .map(|(_, ancestor)| ancestor.clone()) + .chain(direct_plugin_roots.drain()) + .filter(|ancestor| !namespaces_by_root.contains_key(ancestor)) + .collect::>(); + namespaces_by_root.extend( + futures::stream::iter(probe_roots) + .map(|manifest_root| async move { + let namespace = plugin_namespace_for_root_uri(fs, &manifest_root).await; + (manifest_root, namespace) + }) + .buffered(MAX_CONCURRENT_SKILL_LOADS) + .collect::>() + .await, + ); + + let mut next_lookups = Vec::new(); + for (lookup_root, ancestor) in pending_lookups { + match namespaces_by_root.get(&ancestor) { + Some(Some(namespace)) => { + namespaces_by_lookup_root.insert(lookup_root, Some(namespace.clone())); + } + Some(None) => match ancestor.parent() { + Some(parent) => next_lookups.push((lookup_root, parent)), + None => { + namespaces_by_lookup_root.insert(lookup_root, None); + } + }, + None => unreachable!("pending namespace ancestor was not probed"), + } + } + pending_lookups = next_lookups; + } + + // Ordinary descendants fall back to the nearest valid manifest at or above the scan root. + let inherited_namespace = namespaces_by_lookup_root + .get(root) + .and_then(Option::as_ref) + .cloned() + .map(ResolvedSkillNamespace::Plugin) + .unwrap_or(ResolvedSkillNamespace::Plain); + let namespace_lookups = namespace_roots.into_iter().map(|namespace_root| { + let namespace = namespaces_by_lookup_root + .get(&namespace_root) + .and_then(Option::as_ref) + .cloned() + .map(ResolvedSkillNamespace::Plugin) + .unwrap_or(ResolvedSkillNamespace::Plain); + (namespace_root, namespace) + }); + // Invalid nested manifests are omitted, so the deepest remaining match wins. + let plugin_lookups = plugin_roots.into_iter().filter_map(|plugin_root| { + namespaces_by_root + .get(&plugin_root) + .and_then(Option::as_ref) + .cloned() + .map(|namespace| (plugin_root, ResolvedSkillNamespace::Plugin(namespace))) + }); + let nested_namespaces = namespace_lookups.chain(plugin_lookups).collect(); + + Self { + inherited_namespace, + nested_namespaces, + } + } + + pub(crate) fn for_skill(&self, root: &PathUri, path: &PathUri) -> &ResolvedSkillNamespace { + // Ancestor symlink targets cannot override skills still owned by the scan root. + let path_is_under_root = path.starts_with(root); + // The deepest matching path prefix is the nearest applicable namespace. + self.nested_namespaces + .iter() + .filter(|(namespace_root, _)| { + path.starts_with(namespace_root) + && (!path_is_under_root || !root.starts_with(namespace_root)) + }) + .max_by_key(|(namespace_root, _)| namespace_root.ancestors().count()) + .map(|(_, namespace)| namespace) + .unwrap_or(&self.inherited_namespace) + } +} + +/// The completed namespace resolution for a skill root. +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) enum ResolvedSkillNamespace { + /// No plugin namespace applies to matching skills. + Plain, + /// Qualify matching skill names with this plugin namespace. + Plugin(String), +} + +impl ResolvedSkillNamespace { + pub(crate) fn qualify(&self, base_name: &str) -> String { + match self { + Self::Plain => base_name.to_string(), + Self::Plugin(namespace) => format!("{namespace}:{base_name}"), + } + } +} diff --git a/codex-rs/ext/skills/src/provider/executor.rs b/codex-rs/ext/skills/src/provider/executor.rs index 3c9132d619..b095708d86 100644 --- a/codex-rs/ext/skills/src/provider/executor.rs +++ b/codex-rs/ext/skills/src/provider/executor.rs @@ -1,7 +1,5 @@ use std::sync::Arc; -use codex_core_skills::loader::load_environment_skills_from_discovery; -use codex_core_skills::loader::load_environment_skills_from_root; use codex_exec_server::EnvironmentManager; use codex_exec_server::FileSystemSandboxContext; use codex_protocol::capabilities::CapabilityRootLocation; @@ -20,6 +18,8 @@ use crate::catalog::SkillReadResult; use crate::catalog::SkillResourceId; use crate::catalog::SkillSearchResult; use crate::catalog::SkillSourceKind; +use crate::loader::load_environment_skills_from_discovery; +use crate::loader::load_environment_skills_from_root; use crate::provider::MAX_SKILL_RESOURCE_CONTENT_BYTES; use crate::provider::SkillListQuery; use crate::provider::SkillProvider; diff --git a/codex-rs/ext/skills/tests/executor_file_system_authority.rs b/codex-rs/ext/skills/tests/executor_file_system_authority.rs index 1fa0a20989..e3160b43ed 100644 --- a/codex-rs/ext/skills/tests/executor_file_system_authority.rs +++ b/codex-rs/ext/skills/tests/executor_file_system_authority.rs @@ -7,6 +7,7 @@ use std::sync::atomic::Ordering; use codex_core_skills::HostSkillsSnapshot; use codex_core_skills::loader::MAX_CONCURRENT_ROOT_SCANS; use codex_core_skills::loader::SkillRoot; +use codex_core_skills::loader::load_environment_skills_from_root as load_legacy_environment_skills_from_root; use codex_core_skills::loader::load_skills_from_roots; use codex_exec_server::CopyOptions; use codex_exec_server::CreateDirectoryOptions; @@ -24,6 +25,7 @@ use codex_protocol::capabilities::SelectedCapabilityRoot; use codex_protocol::models::PermissionProfile; use codex_protocol::permissions::FileSystemSandboxPolicy; use codex_protocol::permissions::NetworkSandboxPolicy; +use codex_protocol::protocol::Product; use codex_protocol::protocol::SkillScope; use codex_skills_extension::ExecutorSkillProvider; use codex_skills_extension::catalog::SkillAuthority; @@ -350,6 +352,413 @@ async fn selected_root_id_distinguishes_identical_executor_paths() { std::fs::remove_dir_all(test_root).expect("remove skill directory"); } +#[tokio::test] +async fn executor_discovery_routes_produce_equivalent_catalog_metadata() { + let id = NEXT_TEST_ROOT_ID.fetch_add(1, Ordering::Relaxed); + let test_root = std::env::temp_dir().join(format!( + "codex-executor-skill-parity-{}-{id}", + std::process::id() + )); + let plugin_manifest = test_root.join(".codex-plugin/plugin.json"); + let deploy_skill = test_root.join("skills/deploy/SKILL.md"); + let deploy_metadata = test_root.join("skills/deploy/agents/openai.yaml"); + let excluded_skill = test_root.join("skills/excluded/SKILL.md"); + let excluded_metadata = test_root.join("skills/excluded/agents/openai.yaml"); + let repaired_skill = test_root.join("skills/repaired/SKILL.md"); + let invalid_skill = test_root.join("skills/invalid/SKILL.md"); + let invalid_metadata_skill = test_root.join("skills/invalid-metadata/SKILL.md"); + let invalid_metadata = test_root.join("skills/invalid-metadata/agents/openai.yaml"); + for path in [ + &plugin_manifest, + &deploy_skill, + &deploy_metadata, + &excluded_skill, + &excluded_metadata, + &repaired_skill, + &invalid_skill, + &invalid_metadata_skill, + &invalid_metadata, + ] { + std::fs::create_dir_all(path.parent().expect("test file parent")) + .expect("create test directory"); + } + std::fs::write(&plugin_manifest, r#"{"name":"catalog"}"#).expect("write plugin manifest"); + std::fs::write( + &deploy_skill, + "---\nname: deploy\ndescription: Deploy the service.\nmetadata:\n short-description: Deploy safely.\n---\n\nDeploy.\n", + ) + .expect("write deploy skill"); + std::fs::write( + &deploy_metadata, + "dependencies:\n tools:\n - type: mcp\n value: deployer\n description: Deployment server.\npolicy:\n allow_implicit_invocation: false\n products:\n - codex\n", + ) + .expect("write deploy metadata"); + std::fs::write( + &excluded_skill, + "---\nname: excluded\ndescription: Excluded skill.\n---\n", + ) + .expect("write excluded skill"); + std::fs::write(&excluded_metadata, "policy:\n products:\n - chatgpt\n") + .expect("write excluded metadata"); + std::fs::write( + &repaired_skill, + "---\ndescription: Build for AWS: ECS\n---\n", + ) + .expect("write repaired skill"); + std::fs::write(&invalid_skill, "---\nname: invalid\n---\n").expect("write invalid skill"); + std::fs::write( + &invalid_metadata_skill, + "---\nname: invalid-metadata\ndescription: Invalid optional metadata.\n---\n", + ) + .expect("write invalid metadata skill"); + std::fs::write( + &invalid_metadata, + "interface: []\ndependencies:\n tools:\n - type: mcp\n value: must-be-ignored\n", + ) + .expect("write invalid metadata"); + + let manager = Arc::new(EnvironmentManager::default_for_tests()); + let provider = ExecutorSkillProvider::new_with_restriction_product( + Arc::clone(&manager), + Some(Product::Codex), + ); + let executor_roots = vec![SelectedCapabilityRoot { + id: "parity-root".to_string(), + location: CapabilityRootLocation::Environment { + environment_id: "local".to_string(), + path: PathUri::from_host_native_path(&test_root).expect("skill root URI"), + }, + }]; + let query = |executor_capability_discovery| SkillListQuery { + turn_id: "turn-1".to_string(), + executor_roots: executor_roots.clone(), + resolved_executor_roots: Vec::new(), + host_snapshot: None, + include_host_skills: false, + include_bundled_skills: true, + include_orchestrator_skills: false, + mcp_resources: None, + executor_capability_discovery, + }; + + let legacy_file_system = manager + .get_environment("local") + .expect("local environment") + .get_filesystem(); + let legacy = load_legacy_environment_skills_from_root( + legacy_file_system.as_ref(), + &PathUri::from_host_native_path(&test_root).expect("skill root URI"), + Some(Product::Codex), + ) + .await; + let direct = provider + .list(query(None)) + .await + .expect("list directly discovered executor skills"); + let discovery = ExecutorCapabilityDiscoveryCache::new(manager) + .snapshot(&executor_roots, &Default::default()) + .await; + let bundled = provider + .list(query(Some(discovery))) + .await + .expect("list bundled executor skills"); + + assert_eq!(bundled.warnings, direct.warnings); + let comparable_entries = |catalog: &codex_skills_extension::catalog::SkillCatalog| { + catalog + .entries + .iter() + .map(|entry| { + ( + entry.id.clone(), + entry.authority.clone(), + entry.name.clone(), + entry.description.clone(), + entry.short_description.clone(), + entry.display_path.clone(), + entry.dependencies.clone(), + entry.enabled, + entry.prompt_visible, + ) + }) + .collect::>() + }; + assert_eq!(comparable_entries(&bundled), comparable_entries(&direct)); + assert_eq!(legacy.warnings, direct.warnings); + assert_eq!(direct.warnings.len(), 1); + assert!(direct.warnings[0].contains("missing field `description`")); + let legacy_metadata = legacy + .skills + .iter() + .map(|skill| { + ( + skill.name.clone(), + skill.description.clone(), + skill.short_description.clone(), + skill.dependencies.clone(), + skill.allows_implicit_invocation(), + ) + }) + .collect::>(); + let catalog_metadata = direct + .entries + .iter() + .map(|entry| { + ( + entry.name.clone(), + entry.description.clone(), + entry.short_description.clone(), + entry.dependencies.clone(), + entry.prompt_visible, + ) + }) + .collect::>(); + assert_eq!(catalog_metadata, legacy_metadata); + let entry = direct + .entries + .iter() + .find(|entry| entry.name == "catalog:deploy") + .expect("deploy skill"); + assert_eq!( + entry + .dependencies + .as_ref() + .expect("skill dependencies") + .tools[0] + .value, + "deployer" + ); + assert!(!entry.prompt_visible); + let repaired = direct + .entries + .iter() + .find(|entry| entry.name == "catalog:repaired") + .expect("repaired skill"); + assert_eq!(repaired.description, "Build for AWS: ECS"); + let invalid_metadata = direct + .entries + .iter() + .find(|entry| entry.name == "catalog:invalid-metadata") + .expect("invalid metadata skill"); + assert_eq!(invalid_metadata.dependencies, None); + + std::fs::remove_dir_all(test_root).expect("remove skill directory"); +} + +#[tokio::test] +async fn pre_discovered_executor_catalog_snapshot() { + let test_root = create_local_skill_root("snapshot").expect("create local skill root"); + let manifest_dir = test_root.join(".codex-plugin"); + let metadata_dir = test_root.join("skill/agents"); + std::fs::create_dir_all(&manifest_dir).expect("create plugin manifest directory"); + std::fs::create_dir_all(&metadata_dir).expect("create skill metadata directory"); + std::fs::write( + manifest_dir.join("plugin.json"), + r#"{"name":"snapshot-plugin"}"#, + ) + .expect("write plugin manifest"); + std::fs::write( + metadata_dir.join("openai.yaml"), + "dependencies:\n tools:\n - type: mcp\n value: deployer\n description: Deployment server.\npolicy:\n allow_implicit_invocation: false\n", + ) + .expect("write skill metadata"); + + let manager = Arc::new(EnvironmentManager::default_for_tests()); + let provider = ExecutorSkillProvider::new_with_restriction_product( + Arc::clone(&manager), + /*restriction_product*/ None, + ); + let executor_roots = vec![SelectedCapabilityRoot { + id: "snapshot-root".to_string(), + location: CapabilityRootLocation::Environment { + environment_id: "local".to_string(), + path: PathUri::from_host_native_path(&test_root).expect("skill root URI"), + }, + }]; + let executor_capability_discovery = ExecutorCapabilityDiscoveryCache::new(manager) + .snapshot(&executor_roots, &Default::default()) + .await; + let catalog = provider + .list(SkillListQuery { + turn_id: "turn-1".to_string(), + executor_roots, + resolved_executor_roots: Vec::new(), + host_snapshot: None, + include_host_skills: false, + include_bundled_skills: true, + include_orchestrator_skills: false, + mcp_resources: None, + executor_capability_discovery: Some(executor_capability_discovery), + }) + .await + .expect("list pre-discovered executor skills"); + + let mut snapshot = serde_json::json!({ + "warnings": &catalog.warnings, + "entries": catalog + .entries + .iter() + .map(|entry| serde_json::json!({ + "authority": { + "kind": entry.authority.kind.to_string(), + "id": &entry.authority.id, + }, + "name": &entry.name, + "description": &entry.description, + "short_description": entry.short_description.as_deref(), + "dependencies": entry.dependencies.as_ref().map(|dependencies| { + dependencies.tools.iter().map(|tool| serde_json::json!({ + "type": &tool.r#type, + "value": &tool.value, + "description": tool.description.as_deref(), + "transport": tool.transport.as_deref(), + "command": tool.command.as_deref(), + "url": tool.url.as_deref(), + })).collect::>() + }), + "enabled": entry.enabled, + "prompt_visible": entry.prompt_visible, + })) + .collect::>(), + }); + snapshot.sort_all_objects(); + let snapshot_file = codex_utils_cargo_bin::find_resource!( + "tests/snapshots/executor_file_system_authority__pre_discovered_executor_catalog.snap" + ) + .expect("resolve catalog snapshot"); + let snapshot_dir = snapshot_file + .parent() + .unwrap_or_else(|| panic!("snapshot file has no parent: {}", snapshot_file.display())); + let mut settings = insta::Settings::clone_current(); + settings.set_snapshot_path(snapshot_dir); + settings.bind(|| { + insta::assert_snapshot!( + "pre_discovered_executor_catalog", + serde_json::to_string_pretty(&snapshot).expect("serialize catalog snapshot") + ); + }); + + std::fs::remove_dir_all(test_root).expect("remove skill directory"); +} + +#[tokio::test] +async fn direct_executor_discovery_preserves_hidden_nested_and_probed_metadata() { + let id = NEXT_TEST_ROOT_ID.fetch_add(1, Ordering::Relaxed); + let test_root = std::env::temp_dir().join(format!( + "codex-executor-skill-discovery-{}-{id}", + std::process::id() + )); + let outer_manifest = test_root.join(".codex-plugin/plugin.json"); + let hidden_skill = test_root.join(".hidden/deploy/SKILL.md"); + let hidden_metadata = test_root.join(".hidden/deploy/agents/openai.yaml"); + let inner_manifest = test_root.join("nested/.codex-plugin/plugin.json"); + let inner_skill = test_root.join("nested/skills/audit/SKILL.md"); + for (path, contents) in [ + (&outer_manifest, r#"{"name":"outer"}"#), + ( + &hidden_skill, + "---\nname: hidden\ndescription: Hidden skill.\n---\n", + ), + (&inner_manifest, r#"{"name":"inner"}"#), + ( + &inner_skill, + "---\nname: audit\ndescription: Audit skill.\n---\n", + ), + ] { + std::fs::create_dir_all(path.parent().expect("test file parent")) + .expect("create test directory"); + std::fs::write(path, contents).expect("write test file"); + } + std::fs::create_dir_all(hidden_metadata.parent().expect("metadata parent")) + .expect("create metadata directory"); + let metadata_contents = "policy:\n allow_implicit_invocation: false\n"; + #[cfg(unix)] + { + use std::os::unix::fs::symlink; + + let metadata_target = test_root.join("linked-openai.yaml"); + std::fs::write(&metadata_target, metadata_contents).expect("write metadata target"); + symlink(metadata_target, &hidden_metadata).expect("link metadata"); + } + #[cfg(not(unix))] + std::fs::write(&hidden_metadata, metadata_contents).expect("write metadata"); + + let root_uri = PathUri::from_host_native_path(&test_root).expect("skill root URI"); + let selected_root = SelectedCapabilityRoot { + id: "discovery-root".to_string(), + location: CapabilityRootLocation::Environment { + environment_id: "local".to_string(), + path: root_uri.clone(), + }, + }; + let manager = Arc::new(EnvironmentManager::default_for_tests()); + let legacy_file_system = manager + .get_environment("local") + .expect("local environment") + .get_filesystem(); + let legacy = load_legacy_environment_skills_from_root( + legacy_file_system.as_ref(), + &root_uri, + /*restriction_product*/ None, + ) + .await; + let provider = ExecutorSkillProvider::new_with_restriction_product( + manager, /*restriction_product*/ None, + ); + let direct = provider + .list(SkillListQuery { + turn_id: "turn-1".to_string(), + executor_roots: vec![selected_root], + resolved_executor_roots: Vec::new(), + host_snapshot: None, + include_host_skills: false, + include_bundled_skills: true, + include_orchestrator_skills: false, + mcp_resources: None, + executor_capability_discovery: None, + }) + .await + .expect("list directly discovered executor skills"); + + assert_eq!(direct.warnings, legacy.warnings); + let legacy_metadata = legacy + .skills + .iter() + .map(|skill| { + ( + skill.name.clone(), + skill.description.clone(), + skill.allows_implicit_invocation(), + ) + }) + .collect::>(); + let catalog_metadata = direct + .entries + .iter() + .map(|entry| { + ( + entry.name.clone(), + entry.description.clone(), + entry.prompt_visible, + ) + }) + .collect::>(); + assert_eq!(catalog_metadata, legacy_metadata); + assert_eq!( + catalog_metadata, + vec![ + ("inner:audit".to_string(), "Audit skill.".to_string(), true), + ( + "outer:hidden".to_string(), + "Hidden skill.".to_string(), + false, + ), + ] + ); + + std::fs::remove_dir_all(test_root).expect("remove skill directory"); +} + #[tokio::test] async fn high_level_discovery_reuses_materialized_skill_contents_for_reads() { let test_root = create_local_skill_root("materialized").expect("create local skill root"); diff --git a/codex-rs/ext/skills/tests/snapshots/executor_file_system_authority__pre_discovered_executor_catalog.snap b/codex-rs/ext/skills/tests/snapshots/executor_file_system_authority__pre_discovered_executor_catalog.snap new file mode 100644 index 0000000000..0b59ec0031 --- /dev/null +++ b/codex-rs/ext/skills/tests/snapshots/executor_file_system_authority__pre_discovered_executor_catalog.snap @@ -0,0 +1,30 @@ +--- +source: ext/skills/tests/executor_file_system_authority.rs +expression: "serde_json::to_string_pretty(&snapshot).expect(\"serialize catalog snapshot\")" +--- +{ + "entries": [ + { + "authority": { + "id": "snapshot-root", + "kind": "executor" + }, + "dependencies": [ + { + "command": null, + "description": "Deployment server.", + "transport": null, + "type": "mcp", + "url": null, + "value": "deployer" + } + ], + "description": "Synthetic executor skill.", + "enabled": true, + "name": "snapshot-plugin:synthetic", + "prompt_visible": false, + "short_description": null + } + ], + "warnings": [] +} diff --git a/codex-rs/otel/src/metrics/config.rs b/codex-rs/otel/src/metrics/config.rs index 6d9c18a87b..4228a0a71d 100644 --- a/codex-rs/otel/src/metrics/config.rs +++ b/codex-rs/otel/src/metrics/config.rs @@ -1,5 +1,7 @@ use crate::config::OtelExporter; use crate::metrics::Result; +use crate::metrics::names::API_CALL_COUNT_METRIC; +use crate::metrics::names::API_CALL_DURATION_METRIC; use crate::metrics::names::TOOL_CALL_COUNT_METRIC; use crate::metrics::names::TOOL_CALL_DURATION_METRIC; use crate::metrics::validation::validate_tag_key; @@ -8,7 +10,12 @@ use opentelemetry_sdk::metrics::InMemoryMetricExporter; use std::collections::BTreeMap; use std::time::Duration; -const RUNTIME_ONLY_METRICS: &[&str] = &[TOOL_CALL_COUNT_METRIC, TOOL_CALL_DURATION_METRIC]; +const RUNTIME_ONLY_METRICS: &[&str] = &[ + API_CALL_COUNT_METRIC, + API_CALL_DURATION_METRIC, + TOOL_CALL_COUNT_METRIC, + TOOL_CALL_DURATION_METRIC, +]; #[derive(Clone, Debug)] pub enum MetricsExporter { diff --git a/codex-rs/otel/src/provider.rs b/codex-rs/otel/src/provider.rs index 72a66d5651..448bf07ac0 100644 --- a/codex-rs/otel/src/provider.rs +++ b/codex-rs/otel/src/provider.rs @@ -462,6 +462,8 @@ mod shutdown_tests; #[cfg(test)] mod tests { use super::*; + use crate::metrics::API_CALL_COUNT_METRIC; + use crate::metrics::API_CALL_DURATION_METRIC; use crate::metrics::MetricsExporter; use crate::metrics::TOOL_CALL_COUNT_METRIC; use crate::metrics::TOOL_CALL_DURATION_METRIC; @@ -544,6 +546,8 @@ mod tests { config.exporter = MetricsExporter::InMemory(exporter.clone()); let metrics = MetricsClient::new(config)?; + metrics.counter(API_CALL_COUNT_METRIC, /*inc*/ 1, &[])?; + metrics.record_duration(API_CALL_DURATION_METRIC, Duration::from_millis(100), &[])?; metrics.counter(TOOL_CALL_COUNT_METRIC, /*inc*/ 1, &[])?; metrics.record_duration(TOOL_CALL_DURATION_METRIC, Duration::from_millis(25), &[])?; metrics.counter("codex.turns", /*inc*/ 1, &[])?; diff --git a/codex-rs/otel/tests/suite/otlp_http_loopback.rs b/codex-rs/otel/tests/suite/otlp_http_loopback.rs index b3351dabe8..01bd6c25db 100644 --- a/codex-rs/otel/tests/suite/otlp_http_loopback.rs +++ b/codex-rs/otel/tests/suite/otlp_http_loopback.rs @@ -186,6 +186,12 @@ fn otlp_http_exporter_sends_metrics_to_collector() -> Result<()> { ))?; metrics.counter("codex.turns", /*inc*/ 1, &[("source", "test")])?; + metrics.counter("codex.api_request", /*inc*/ 1, &[("status", "200")])?; + metrics.record_duration( + "codex.api_request.duration_ms", + Duration::from_millis(100), + &[("status", "200")], + )?; metrics.counter("codex.tool.call", /*inc*/ 1, &[("tool", "test")])?; metrics.record_duration( "codex.tool.call.duration_ms", @@ -227,6 +233,16 @@ fn otlp_http_exporter_sends_metrics_to_collector() -> Result<()> { "expected gauge not found; body prefix: {}", &body.chars().take(2000).collect::() ); + assert!( + body.contains("\"codex.api_request\""), + "expected API-request counter not found; body prefix: {}", + &body.chars().take(2000).collect::() + ); + assert!( + body.contains("\"codex.api_request.duration_ms\""), + "expected API-request duration not found; body prefix: {}", + &body.chars().take(2000).collect::() + ); assert!( body.contains("\"codex.tool.call\""), "expected tool-call counter not found; body prefix: {}", diff --git a/codex-rs/skills/Cargo.toml b/codex-rs/skills/Cargo.toml index 010e675186..f1c91c43a6 100644 --- a/codex-rs/skills/Cargo.toml +++ b/codex-rs/skills/Cargo.toml @@ -18,4 +18,9 @@ codex-protocol = { workspace = true } codex-utils-absolute-path = { workspace = true } codex-utils-path-uri = { workspace = true } include_dir = { workspace = true } +serde = { workspace = true, features = ["derive"] } +serde_yaml = { workspace = true } thiserror = { workspace = true } + +[dev-dependencies] +pretty_assertions = { workspace = true } diff --git a/codex-rs/skills/src/lib.rs b/codex-rs/skills/src/lib.rs index 76ce83b6c7..c6e88f01bc 100644 --- a/codex-rs/skills/src/lib.rs +++ b/codex-rs/skills/src/lib.rs @@ -1,4 +1,5 @@ mod model; +mod parser; pub use model::EnvironmentSkillMetadata; pub use model::SkillConfigRule; @@ -9,6 +10,9 @@ pub use model::SkillInterface; pub use model::SkillMetadata; pub use model::SkillPolicy; pub use model::SkillToolDependency; +pub use parser::ParsedSkillFrontmatter; +pub use parser::SkillParseError; +pub use parser::parse_skill_frontmatter_metadata; use codex_utils_absolute_path::AbsolutePathBuf; use include_dir::Dir; diff --git a/codex-rs/skills/src/parser.rs b/codex-rs/skills/src/parser.rs new file mode 100644 index 0000000000..aef295cd03 --- /dev/null +++ b/codex-rs/skills/src/parser.rs @@ -0,0 +1,225 @@ +use serde::Deserialize; +use thiserror::Error; + +const MAX_NAME_LEN: usize = 64; + +#[derive(Debug, Deserialize)] +struct SkillFrontmatter { + #[serde(default)] + name: Option, + #[serde(default)] + description: Option, + #[serde(default)] + metadata: SkillFrontmatterMetadata, +} + +#[derive(Debug, Default, Deserialize)] +struct SkillFrontmatterMetadata { + #[serde(default, rename = "short-description")] + short_description: Option, +} + +/// Validated metadata parsed from a `SKILL.md` frontmatter block. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ParsedSkillFrontmatter { + pub name: String, + pub description: String, + pub short_description: Option, +} + +/// Error produced while parsing or validating `SKILL.md` metadata. +#[derive(Debug, Error)] +pub enum SkillParseError { + #[error("missing YAML frontmatter delimited by ---")] + MissingFrontmatter, + #[error("invalid YAML: {0}")] + InvalidYaml(#[source] serde_yaml::Error), + #[error("missing field `{0}`")] + MissingField(&'static str), + #[error("invalid {field}: {reason}")] + InvalidField { field: &'static str, reason: String }, +} + +/// Parses and validates the metadata frontmatter from `SKILL.md` contents. +pub fn parse_skill_frontmatter_metadata( + contents: &str, + default_name: impl FnOnce() -> String, +) -> Result { + let frontmatter = extract_frontmatter(contents).ok_or(SkillParseError::MissingFrontmatter)?; + + let parsed: SkillFrontmatter = match serde_yaml::from_str(&frontmatter) { + Ok(parsed) => Ok(parsed), + Err(original_error) => match repair_frontmatter_scalar_fields(&frontmatter) { + // Some third-party skills use prose like `description: Build for AWS: ECS` + // or `argument-hint: `. Keep the repair line-oriented + // so unrelated invalid YAML still surfaces. + Some(repaired_frontmatter) => { + serde_yaml::from_str(&repaired_frontmatter).map_err(|_| original_error) + } + None => Err(original_error), + }, + } + .map_err(SkillParseError::InvalidYaml)?; + + let name = parsed + .name + .as_deref() + .map(sanitize_single_line) + .filter(|value| !value.is_empty()) + .unwrap_or_else(default_name); + let description = parsed + .description + .as_deref() + .map(sanitize_single_line) + .unwrap_or_default(); + let short_description = parsed + .metadata + .short_description + .as_deref() + .map(sanitize_single_line) + .filter(|value| !value.is_empty()); + + validate_len(&name, MAX_NAME_LEN, "name")?; + if description.is_empty() { + return Err(SkillParseError::MissingField("description")); + } + + Ok(ParsedSkillFrontmatter { + name, + description, + short_description, + }) +} + +fn sanitize_single_line(raw: &str) -> String { + raw.split_whitespace().collect::>().join(" ") +} + +fn repair_frontmatter_scalar_fields(frontmatter: &str) -> Option { + let mut changed = false; + let mut block_scalar_indent: Option = None; + let mut repaired_lines = Vec::new(); + for line in frontmatter.lines() { + let indent = line + .chars() + .take_while(|character| *character == ' ') + .count(); + if let Some(block_indent) = block_scalar_indent { + if line.trim().is_empty() || indent > block_indent { + repaired_lines.push(line.to_string()); + continue; + } + block_scalar_indent = None; + } + + let Some((key, value)) = line.split_once(':') else { + repaired_lines.push(line.to_string()); + continue; + }; + if key.trim().is_empty() || !value.chars().next().is_none_or(char::is_whitespace) { + repaired_lines.push(line.to_string()); + continue; + } + + let trimmed_start = value.trim_start(); + let leading_whitespace = &value[..value.len() - trimmed_start.len()]; + let mut scalar = trimmed_start; + let mut comment = ""; + for (index, character) in trimmed_start.char_indices() { + if character == '#' + && (index == 0 + || trimmed_start[..index] + .chars() + .next_back() + .is_some_and(char::is_whitespace)) + { + let comment_start = trimmed_start[..index].trim_end().len(); + scalar = &trimmed_start[..comment_start]; + comment = &trimmed_start[comment_start..]; + break; + } + } + + let scalar = scalar.trim_end(); + let Some(first_char) = scalar.chars().next() else { + repaired_lines.push(line.to_string()); + continue; + }; + if matches!(first_char, '|' | '>') { + block_scalar_indent = Some(indent); + repaired_lines.push(line.to_string()); + continue; + } + if matches!(first_char, '\'' | '"') { + repaired_lines.push(line.to_string()); + continue; + } + let mut has_colon_separator = false; + let mut chars = scalar.chars().peekable(); + while let Some(character) = chars.next() { + if character == ':' + && matches!(chars.peek(), Some(next_character) if next_character.is_whitespace()) + { + has_colon_separator = true; + break; + } + } + let invalid_flow_like_scalar = matches!(first_char, '[' | '{' | '@' | '`') + && serde_yaml::from_str::(scalar).is_err(); + if !has_colon_separator && !invalid_flow_like_scalar { + repaired_lines.push(line.to_string()); + continue; + } + + let quoted_scalar = format!("'{}'", scalar.replace('\'', "''")); + repaired_lines.push(format!( + "{key}:{leading_whitespace}{quoted_scalar}{comment}" + )); + changed = true; + } + changed.then(|| repaired_lines.join("\n")) +} + +fn validate_len( + value: &str, + max_len: usize, + field_name: &'static str, +) -> Result<(), SkillParseError> { + if value.is_empty() { + return Err(SkillParseError::MissingField(field_name)); + } + if value.chars().count() > max_len { + return Err(SkillParseError::InvalidField { + field: field_name, + reason: format!("exceeds maximum length of {max_len} characters"), + }); + } + Ok(()) +} + +fn extract_frontmatter(contents: &str) -> Option { + let mut lines = contents.lines(); + if !matches!(lines.next(), Some(line) if line.trim() == "---") { + return None; + } + + let mut frontmatter_lines = Vec::new(); + let mut found_closing = false; + for line in lines.by_ref() { + if line.trim() == "---" { + found_closing = true; + break; + } + frontmatter_lines.push(line); + } + + if frontmatter_lines.is_empty() || !found_closing { + return None; + } + + Some(frontmatter_lines.join("\n")) +} + +#[cfg(test)] +#[path = "parser_tests.rs"] +mod tests; diff --git a/codex-rs/skills/src/parser_tests.rs b/codex-rs/skills/src/parser_tests.rs new file mode 100644 index 0000000000..2bbad1e2c0 --- /dev/null +++ b/codex-rs/skills/src/parser_tests.rs @@ -0,0 +1,43 @@ +use pretty_assertions::assert_eq; + +use super::ParsedSkillFrontmatter; +use super::parse_skill_frontmatter_metadata; + +#[test] +fn parses_repairs_and_sanitizes_frontmatter() { + let parsed = parse_skill_frontmatter_metadata( + "---\nname: deploy service\ndescription: Build for AWS: ECS\nmetadata:\n short-description: Deploy safely\n---\n", + || "fallback".to_string(), + ) + .expect("valid frontmatter"); + + assert_eq!( + parsed, + ParsedSkillFrontmatter { + name: "deploy service".to_string(), + description: "Build for AWS: ECS".to_string(), + short_description: Some("Deploy safely".to_string()), + } + ); +} + +#[test] +fn uses_default_name_and_requires_description() { + let parsed = parse_skill_frontmatter_metadata("---\ndescription: Demo skill\n---\n", || { + "demo".to_string() + }) + .expect("valid frontmatter"); + assert_eq!( + parsed, + ParsedSkillFrontmatter { + name: "demo".to_string(), + description: "Demo skill".to_string(), + short_description: None, + } + ); + + let error = + parse_skill_frontmatter_metadata("---\nname: demo\n---\n", || "fallback".to_string()) + .expect_err("description should be required"); + assert_eq!(error.to_string(), "missing field `description`"); +} diff --git a/codex-rs/terminal-detection/src/lib.rs b/codex-rs/terminal-detection/src/lib.rs index c42733380d..5e9b197138 100644 --- a/codex-rs/terminal-detection/src/lib.rs +++ b/codex-rs/terminal-detection/src/lib.rs @@ -150,6 +150,7 @@ impl TerminalInfo { fn from_term(term: String, multiplexer: Option) -> Self { let name = match term.as_str() { "dumb" => TerminalName::Dumb, + "xterm-ghostty" => TerminalName::Ghostty, "wezterm" | "wezterm-mux" => TerminalName::WezTerm, _ => TerminalName::Unknown, }; @@ -293,7 +294,7 @@ pub fn terminal_info() -> TerminalInfo { /// - Otherwise, `TERM_PROGRAM` (plus `TERM_PROGRAM_VERSION`) drives the detected terminal name. /// This means `TERM_PROGRAM` can mask later probes (for example `WT_SESSION`). /// - Next, terminal-specific variables (WEZTERM, iTerm2, Apple Terminal, kitty, etc.) are checked. -/// - Finally, `TERM` is used as the capability fallback with `TerminalName::Unknown`. +/// - Finally, `TERM` is used as the capability fallback. /// /// tmux client term info is only consulted when a tmux multiplexer is detected, and it is /// derived from `tmux display-message` to surface the underlying terminal program instead of diff --git a/codex-rs/terminal-detection/src/terminal_tests.rs b/codex-rs/terminal-detection/src/terminal_tests.rs index 54f0a7a5a9..06f41090ab 100644 --- a/codex-rs/terminal-detection/src/terminal_tests.rs +++ b/codex-rs/terminal-detection/src/terminal_tests.rs @@ -239,6 +239,25 @@ fn detects_ghostty() { "Ghostty", "ghostty_term_program_user_agent" ); + + let env = FakeEnvironment::new().with_var("TERM", "xterm-ghostty"); + let terminal = detect_terminal_info_from_env(&env); + assert_eq!( + terminal, + terminal_info( + TerminalName::Ghostty, + /*term_program*/ None, + /*version*/ None, + Some("xterm-ghostty"), + /*multiplexer*/ None + ), + "ghostty_term_info" + ); + assert_eq!( + terminal.user_agent_token(), + "xterm-ghostty", + "ghostty_term_user_agent" + ); } #[test] diff --git a/codex-rs/tools/src/code_mode.rs b/codex-rs/tools/src/code_mode.rs index f9a88f83d0..8dc7cfc6b0 100644 --- a/codex-rs/tools/src/code_mode.rs +++ b/codex-rs/tools/src/code_mode.rs @@ -42,6 +42,20 @@ pub fn augment_tool_spec_for_code_mode(spec: ToolSpec) -> ToolSpec { tool.description = codex_code_mode::augment_tool_definition(definition).description; } + ResponsesApiNamespaceTool::Custom(tool) => { + let tool_name = + ToolName::namespaced(namespace.name.clone(), tool.name.clone()); + let definition = CodeModeToolDefinition { + name: code_mode_name_for_tool_name(&tool_name), + tool_name, + description: tool.description.clone(), + kind: CodeModeToolKind::Freeform, + input_schema: None, + output_schema: None, + }; + tool.description = + codex_code_mode::augment_tool_definition(definition).description; + } } } ToolSpec::Namespace(namespace) @@ -146,6 +160,17 @@ fn code_mode_tool_definitions_for_spec(spec: &ToolSpec) -> Vec { + let tool_name = ToolName::namespaced(namespace.name.clone(), tool.name.clone()); + CodeModeToolDefinition { + name: code_mode_name_for_tool_name(&tool_name), + tool_name, + description: tool.description.clone(), + kind: CodeModeToolKind::Freeform, + input_schema: None, + output_schema: None, + } + } }) .collect(), ToolSpec::ToolSearch { .. } | ToolSpec::WebSearch { .. } => Vec::new(), diff --git a/codex-rs/tools/src/code_mode_tests.rs b/codex-rs/tools/src/code_mode_tests.rs index c4c4c7ce26..bf5db21bfc 100644 --- a/codex-rs/tools/src/code_mode_tests.rs +++ b/codex-rs/tools/src/code_mode_tests.rs @@ -4,6 +4,8 @@ use crate::AdditionalProperties; use crate::FreeformTool; use crate::FreeformToolFormat; use crate::JsonSchema; +use crate::ResponsesApiNamespace; +use crate::ResponsesApiNamespaceTool; use crate::ResponsesApiTool; use crate::ToolName; use crate::ToolSpec; @@ -71,6 +73,7 @@ fn augment_tool_spec_for_code_mode_preserves_exec_tool_description() { augment_tool_spec_for_code_mode(ToolSpec::Freeform(FreeformTool { name: codex_code_mode::PUBLIC_TOOL_NAME.to_string(), description: "Run code".to_string(), + defer_loading: None, format: FreeformToolFormat { r#type: "grammar".to_string(), syntax: "lark".to_string(), @@ -80,6 +83,7 @@ fn augment_tool_spec_for_code_mode_preserves_exec_tool_description() { ToolSpec::Freeform(FreeformTool { name: codex_code_mode::PUBLIC_TOOL_NAME.to_string(), description: "Run code".to_string(), + defer_loading: None, format: FreeformToolFormat { r#type: "grammar".to_string(), syntax: "lark".to_string(), @@ -94,6 +98,7 @@ fn tool_spec_to_code_mode_tool_definition_returns_augmented_nested_tools() { let spec = ToolSpec::Freeform(FreeformTool { name: "apply_patch".to_string(), description: "Apply a patch".to_string(), + defer_loading: None, format: FreeformToolFormat { r#type: "grammar".to_string(), syntax: "lark".to_string(), @@ -120,6 +125,42 @@ declare const tools: { apply_patch(input: string): Promise; }; ); } +#[test] +fn tool_spec_to_code_mode_tool_definition_supports_namespaced_custom_tools() { + let spec = ToolSpec::Namespace(ResponsesApiNamespace { + name: "editor".to_string(), + description: "Editing tools".to_string(), + tools: vec![ResponsesApiNamespaceTool::Custom(FreeformTool { + name: "apply_patch".to_string(), + description: "Apply a patch".to_string(), + defer_loading: None, + format: FreeformToolFormat { + r#type: "grammar".to_string(), + syntax: "lark".to_string(), + definition: "start: \"patch\"".to_string(), + }, + })], + }); + + assert_eq!( + tool_spec_to_code_mode_tool_definition(&spec), + Some(codex_code_mode::ToolDefinition { + name: "editor__apply_patch".to_string(), + tool_name: ToolName::namespaced("editor", "apply_patch"), + description: r#"Apply a patch + +exec tool declaration: +```ts +declare const tools: { editor__apply_patch(input: string): Promise; }; +```"# + .to_string(), + kind: codex_code_mode::CodeModeToolKind::Freeform, + input_schema: None, + output_schema: None, + }) + ); +} + #[test] fn tool_spec_to_code_mode_tool_definition_skips_unsupported_variants() { assert_eq!( diff --git a/codex-rs/tools/src/responses_api.rs b/codex-rs/tools/src/responses_api.rs index 9911ad0eff..f7e4cc182b 100644 --- a/codex-rs/tools/src/responses_api.rs +++ b/codex-rs/tools/src/responses_api.rs @@ -12,6 +12,8 @@ use serde_json::Value; pub struct FreeformTool { pub name: String, pub description: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub defer_loading: Option, pub format: FreeformToolFormat, } @@ -61,9 +63,12 @@ pub fn default_namespace_description(namespace_name: &str) -> String { #[derive(Debug, Clone, Serialize, PartialEq)] #[serde(tag = "type")] +#[allow(clippy::large_enum_variant)] pub enum ResponsesApiNamespaceTool { #[serde(rename = "function")] Function(ResponsesApiTool), + #[serde(rename = "custom")] + Custom(FreeformTool), } pub fn dynamic_tool_to_responses_api_tool( diff --git a/codex-rs/tools/src/responses_api_tests.rs b/codex-rs/tools/src/responses_api_tests.rs index fc8dbff55d..f12d078d26 100644 --- a/codex-rs/tools/src/responses_api_tests.rs +++ b/codex-rs/tools/src/responses_api_tests.rs @@ -1,3 +1,4 @@ +use super::FreeformTool; use super::LoadableToolSpec; use super::ResponsesApiNamespace; use super::ResponsesApiNamespaceTool; @@ -13,6 +14,35 @@ use pretty_assertions::assert_eq; use serde_json::json; use std::collections::BTreeMap; +#[test] +fn freeform_tool_deferral_matches_function_tool_wire_shape() { + let mut expected_wire_shape = json!({ + "name": "apply_patch", + "description": "Apply a patch", + "format": { + "type": "grammar", + "syntax": "lark", + "definition": "start: \"patch\"", + }, + }); + + let mut tool: FreeformTool = serde_json::from_value(expected_wire_shape.clone()) + .expect("deserialize legacy freeform tool"); + + assert_eq!(tool.defer_loading, None); + assert_eq!( + serde_json::to_value(&tool).expect("serialize eager freeform tool"), + expected_wire_shape + ); + + tool.defer_loading = Some(true); + expected_wire_shape["defer_loading"] = json!(true); + assert_eq!( + serde_json::to_value(tool).expect("serialize deferred freeform tool"), + expected_wire_shape + ); +} + #[test] fn tool_definition_to_responses_api_tool_omits_false_defer_loading() { assert_eq!( diff --git a/codex-rs/tools/src/tool_search.rs b/codex-rs/tools/src/tool_search.rs index 2bde910199..81347b3e60 100644 --- a/codex-rs/tools/src/tool_search.rs +++ b/codex-rs/tools/src/tool_search.rs @@ -43,9 +43,15 @@ impl ToolSearchInfo { namespace.description = default_namespace_description(&namespace.name); } for tool in &mut namespace.tools { - let ResponsesApiNamespaceTool::Function(tool) = tool; - tool.defer_loading = Some(true); - tool.output_schema = None; + match tool { + ResponsesApiNamespaceTool::Function(tool) => { + tool.defer_loading = Some(true); + tool.output_schema = None; + } + ResponsesApiNamespaceTool::Custom(tool) => { + tool.defer_loading = Some(true); + } + } } LoadableToolSpec::Namespace(namespace) } @@ -73,8 +79,16 @@ fn default_tool_search_text(spec: &ToolSpec) -> String { push_search_part(&mut parts, namespace.name.clone()); push_search_part(&mut parts, namespace.description.clone()); for tool in &namespace.tools { - let ResponsesApiNamespaceTool::Function(tool) = tool; - append_function_search_text(tool, &mut parts); + match tool { + ResponsesApiNamespaceTool::Function(tool) => { + append_function_search_text(tool, &mut parts); + } + ResponsesApiNamespaceTool::Custom(tool) => { + push_search_part(&mut parts, tool.name.clone()); + push_search_part(&mut parts, tool.description.clone()); + push_search_part(&mut parts, tool.format.syntax.clone()); + } + } } } ToolSpec::ToolSearch { description, .. } => { diff --git a/codex-rs/tools/src/tool_search_tests.rs b/codex-rs/tools/src/tool_search_tests.rs index bf910c57f1..86c1db16ef 100644 --- a/codex-rs/tools/src/tool_search_tests.rs +++ b/codex-rs/tools/src/tool_search_tests.rs @@ -46,3 +46,63 @@ fn default_search_text_uses_model_visible_namespace_metadata_once() { "codex_app Manage Codex automations. automation_update automation update Create or update automations. Automation options. mode Update mode. schedule Schedule settings. timezone IANA timezone." ); } + +#[test] +fn mixed_namespaced_function_and_custom_tools_are_searchable() { + let function_tool = ResponsesApiTool { + name: "lookup_order".to_string(), + description: "Look up an order".to_string(), + strict: false, + defer_loading: None, + parameters: JsonSchema::object( + BTreeMap::new(), + /*required*/ None, + /*additional_properties*/ None, + ), + output_schema: Some(serde_json::json!({"type": "object"})), + }; + let custom_tool = crate::FreeformTool { + name: "apply_patch".to_string(), + description: "Apply a patch".to_string(), + defer_loading: None, + format: crate::FreeformToolFormat { + r#type: "grammar".to_string(), + syntax: "lark".to_string(), + definition: "start: \"patch\"".to_string(), + }, + }; + let spec = ToolSpec::Namespace(crate::ResponsesApiNamespace { + name: "editor".to_string(), + description: "Editing tools".to_string(), + tools: vec![ + ResponsesApiNamespaceTool::Function(function_tool.clone()), + ResponsesApiNamespaceTool::Custom(custom_tool.clone()), + ], + }); + + let search_info = ToolSearchInfo::from_tool_spec(spec, /*source_info*/ None) + .expect("mixed namespace should be searchable"); + + assert_eq!( + search_info.entry.search_text, + "editor Editing tools lookup_order lookup order Look up an order apply_patch Apply a patch lark" + ); + assert_eq!( + search_info.entry.output, + LoadableToolSpec::Namespace(crate::ResponsesApiNamespace { + name: "editor".to_string(), + description: "Editing tools".to_string(), + tools: vec![ + ResponsesApiNamespaceTool::Function(ResponsesApiTool { + defer_loading: Some(true), + output_schema: None, + ..function_tool + }), + ResponsesApiNamespaceTool::Custom(crate::FreeformTool { + defer_loading: Some(true), + ..custom_tool + }), + ], + }) + ); +} diff --git a/codex-rs/tools/src/tool_spec_tests.rs b/codex-rs/tools/src/tool_spec_tests.rs index b14d3f6960..9b59f780e5 100644 --- a/codex-rs/tools/src/tool_spec_tests.rs +++ b/codex-rs/tools/src/tool_spec_tests.rs @@ -74,6 +74,7 @@ fn tool_spec_name_covers_all_variants() { ToolSpec::Freeform(FreeformTool { name: "exec".to_string(), description: "Run a command".to_string(), + defer_loading: None, format: FreeformToolFormat { r#type: "grammar".to_string(), syntax: "lark".to_string(), @@ -173,21 +174,33 @@ fn namespace_tool_spec_serializes_expected_wire_shape() { serde_json::to_value(ToolSpec::Namespace(ResponsesApiNamespace { name: "mcp__demo__".to_string(), description: "Demo tools".to_string(), - tools: vec![ResponsesApiNamespaceTool::Function(ResponsesApiTool { - name: "lookup_order".to_string(), - description: "Look up an order".to_string(), - strict: false, - defer_loading: None, - parameters: JsonSchema::object( - BTreeMap::from([( - "order_id".to_string(), - JsonSchema::string(/*description*/ None), - )]), - /*required*/ None, - /*additional_properties*/ None, - ), - output_schema: None, - })], + tools: vec![ + ResponsesApiNamespaceTool::Function(ResponsesApiTool { + name: "lookup_order".to_string(), + description: "Look up an order".to_string(), + strict: false, + defer_loading: None, + parameters: JsonSchema::object( + BTreeMap::from([( + "order_id".to_string(), + JsonSchema::string(/*description*/ None), + )]), + /*required*/ None, + /*additional_properties*/ None, + ), + output_schema: None, + }), + ResponsesApiNamespaceTool::Custom(FreeformTool { + name: "apply_patch".to_string(), + description: "Apply a patch".to_string(), + defer_loading: None, + format: FreeformToolFormat { + r#type: "grammar".to_string(), + syntax: "lark".to_string(), + definition: "start: \"patch\"".to_string(), + }, + }), + ], })) .expect("serialize namespace tool"), json!({ @@ -207,6 +220,16 @@ fn namespace_tool_spec_serializes_expected_wire_shape() { }, }, }, + { + "type": "custom", + "name": "apply_patch", + "description": "Apply a patch", + "format": { + "type": "grammar", + "syntax": "lark", + "definition": "start: \"patch\"", + }, + }, ], }) ); diff --git a/codex-rs/tui/src/lib.rs b/codex-rs/tui/src/lib.rs index 67401985cd..490122d053 100644 --- a/codex-rs/tui/src/lib.rs +++ b/codex-rs/tui/src/lib.rs @@ -2222,7 +2222,7 @@ mod tests { Ok(()) } - async fn start_test_embedded_app_server( + pub(crate) async fn start_test_embedded_app_server( config: Config, ) -> color_eyre::Result { let state_db = diff --git a/codex-rs/tui/src/session_archive_commands.rs b/codex-rs/tui/src/session_archive_commands.rs index 1dea0ffe0f..ddf82f7461 100644 --- a/codex-rs/tui/src/session_archive_commands.rs +++ b/codex-rs/tui/src/session_archive_commands.rs @@ -5,6 +5,8 @@ use std::io::IsTerminal; use std::io::Write; +use std::path::Path; +use std::path::PathBuf; use std::sync::Arc; use crate::Cli; @@ -82,16 +84,25 @@ pub async fn run_session_archive_command( target: String, options: SessionArchiveCommandOptions, ) -> Result { - let mut app_server = start_app_server_for_archive_command(options).await?; - run_session_archive_action_with_app_server(&mut app_server, action, &target).await + let codex_home = find_codex_home().wrap_err("failed to find Codex home")?; + let mut app_server = + start_app_server_for_archive_command(options, codex_home.to_path_buf()).await?; + run_session_archive_action_with_app_server( + &mut app_server, + codex_home.as_path(), + action, + &target, + ) + .await } async fn run_session_archive_action_with_app_server( app_server: &mut AppServerSession, + codex_home: &Path, action: SessionArchiveAction, target: &str, ) -> Result { - let resolved = resolve_session_target(app_server, action, target).await?; + let resolved = resolve_session_target(app_server, codex_home, action, target).await?; let session_name = match action { SessionArchiveAction::Archive => { app_server.thread_archive(resolved.session_id).await?; @@ -120,6 +131,7 @@ async fn run_session_archive_action_with_app_server( async fn resolve_session_target( app_server: &mut AppServerSession, + codex_home: &Path, action: SessionArchiveAction, target: &str, ) -> Result { @@ -151,7 +163,9 @@ async fn resolve_session_target( SessionArchiveAction::Unarchive => ("archived", &[true]), }; for &archived in archived_values { - if let Some(thread) = lookup_session_by_exact_name(app_server, target, archived).await? { + if let Some(thread) = + lookup_session_by_exact_name(app_server, codex_home, target, archived).await? + { return session_target_from_app_server_thread(thread); } } @@ -162,50 +176,103 @@ async fn resolve_session_target( async fn lookup_session_by_exact_name( app_server: &mut AppServerSession, + codex_home: &Path, name: &str, archived: bool, ) -> Result> { - // Search is the fast path, but some stores attach renamed titles after applying the filter. - for search_term in [Some(name), None] { - let mut cursor = None; - loop { - let response = app_server - .thread_list(ThreadListParams { - cursor: cursor.clone(), - limit: Some(100), - sort_key: Some(ThreadSortKey::UpdatedAt), - sort_direction: None, - model_providers: None, - source_kinds: Some(super::resume_source_kinds( - /*include_non_interactive*/ false, - )), - archived: Some(archived), - section_id: None, - parent_thread_id: None, - ancestor_thread_id: None, - cwd: None, - use_state_db_only: false, - search_term: search_term.map(str::to_string), - }) - .await - .wrap_err("failed to list sessions while resolving session name")?; + let uses_remote_workspace = app_server.uses_remote_workspace(); + // Remote workspaces stay on their existing server-side path. Local workspaces trust SQLite + // names, then scan and repair only after a miss or an unusable rollout path. + let lookup_modes = if uses_remote_workspace { + &[SessionNameLookupMode::ScanAndRepair][..] + } else { + &[ + SessionNameLookupMode::StateDbOnly, + SessionNameLookupMode::ScanAndRepair, + ][..] + }; + for &lookup_mode in lookup_modes { + // Only the embedded server can safely use SQLite's recency cursor. Daemons may predate + // that sort key, and filesystem repair must paginate in the scanner's mtime order. + let sort_key = if lookup_mode == SessionNameLookupMode::StateDbOnly + && app_server.uses_embedded_app_server() + { + ThreadSortKey::RecencyAt + } else { + ThreadSortKey::UpdatedAt + }; + // Search is the fast path, but legacy stores attach renamed titles after filtering. + for search_term in [Some(name), None] { + let mut cursor = None; + loop { + let response = app_server + .thread_list(ThreadListParams { + cursor: cursor.clone(), + limit: Some(100), + sort_key: Some(sort_key), + sort_direction: None, + model_providers: None, + source_kinds: Some(super::resume_source_kinds( + /*include_non_interactive*/ false, + )), + archived: Some(archived), + section_id: None, + parent_thread_id: None, + ancestor_thread_id: None, + cwd: None, + use_state_db_only: lookup_mode == SessionNameLookupMode::StateDbOnly, + search_term: search_term.map(str::to_string), + }) + .await + .wrap_err("failed to list sessions while resolving session name")?; - if let Some(thread) = response - .data - .into_iter() - .find(|thread| thread.name.as_deref() == Some(name)) - { - return Ok(Some(thread)); + for thread in response + .data + .into_iter() + .filter(|thread| thread.name.as_deref() == Some(name)) + { + if !uses_remote_workspace { + // The action still requires a real rollout in the requested collection. + let thread_id = ThreadId::from_string(&thread.id).wrap_err_with(|| { + format!("app server returned invalid session id `{}`", thread.id) + })?; + let expected_root = codex_home.join(if archived { + codex_rollout::ARCHIVED_SESSIONS_SUBDIR + } else { + codex_rollout::SESSIONS_SUBDIR + }); + let valid_rollout = if let Some(path) = thread.path.as_deref() + && let Some(path) = codex_rollout::existing_rollout_path(path).await + && path.starts_with(expected_root) + && let Ok(session_meta) = + codex_rollout::read_session_meta_line(path.as_path()).await + { + session_meta.meta.id == thread_id + } else { + false + }; + if !valid_rollout { + continue; + } + } + return Ok(Some(thread)); + } + let Some(next_cursor) = response.next_cursor else { + break; + }; + cursor = Some(next_cursor); } - let Some(next_cursor) = response.next_cursor else { - break; - }; - cursor = Some(next_cursor); } } Ok(None) } +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum SessionNameLookupMode { + StateDbOnly, + ScanAndRepair, +} + fn session_target_from_app_server_thread(thread: AppServerThread) -> Result { let session_id = ThreadId::from_string(&thread.id) .wrap_err_with(|| format!("app server returned invalid session id `{}`", thread.id))?; @@ -246,6 +313,7 @@ fn confirm_session_delete(target: &ResolvedSessionTarget) -> Result { async fn start_app_server_for_archive_command( options: SessionArchiveCommandOptions, + codex_home: PathBuf, ) -> Result { let SessionArchiveCommandOptions { cli, @@ -259,8 +327,6 @@ async fn start_app_server_for_archive_command( let cli_kv_overrides = overrides_cli .parse_overrides() .map_err(|err| eyre!("failed to parse -c overrides: {err}"))?; - let codex_home = find_codex_home().wrap_err("failed to find Codex home")?; - let mut launch_loader_overrides = loader_overrides.clone(); if let Some(profile_v2) = cli.config_profile_v2.as_ref() { launch_loader_overrides.user_config_path = Some(resolve_profile_v2_config_path( @@ -413,3 +479,7 @@ async fn start_app_server_for_archive_command( .with_remote_cwd_override(remote_cwd_override), ) } + +#[cfg(test)] +#[path = "session_archive_commands_tests.rs"] +mod tests; diff --git a/codex-rs/tui/src/session_archive_commands_tests.rs b/codex-rs/tui/src/session_archive_commands_tests.rs new file mode 100644 index 0000000000..351247a651 --- /dev/null +++ b/codex-rs/tui/src/session_archive_commands_tests.rs @@ -0,0 +1,373 @@ +use std::path::PathBuf; +use std::sync::Arc; + +use chrono::DateTime; +use chrono::Utc; +use codex_protocol::ThreadId; +use codex_protocol::protocol::SessionMeta; +use codex_protocol::protocol::SessionMetaLine; +use codex_protocol::protocol::SessionSource; +use codex_state::ThreadMetadataBuilder; +use codex_utils_absolute_path::test_support::PathExt; +use pretty_assertions::assert_eq; +use tempfile::TempDir; + +use super::DeleteConfirmation; +use super::SessionArchiveAction; +use super::run_session_archive_action_with_app_server; +use crate::app_server_session::AppServerSession; +use crate::app_server_session::ThreadParamsMode; +use crate::legacy_core::config::Config; +use crate::legacy_core::config::ConfigBuilder; +use crate::tests::start_test_embedded_app_server; + +async fn build_config(temp_dir: &TempDir) -> std::io::Result { + ConfigBuilder::default() + .codex_home(temp_dir.path().to_path_buf()) + .build() + .await +} + +async fn state_runtime(config: &Config) -> std::io::Result> { + let runtime = codex_state::StateRuntime::init( + codex_state::SqliteConfig::new_for_testing(config.codex_home.as_path().abs()), + config.model_provider_id.clone(), + ) + .await + .map_err(std::io::Error::other)?; + runtime + .mark_backfill_complete(/*last_watermark*/ None) + .await + .map_err(std::io::Error::other)?; + Ok(runtime) +} + +async fn start_app_server(config: Config) -> color_eyre::Result { + Ok(AppServerSession::new( + codex_app_server_client::AppServerClient::InProcess( + start_test_embedded_app_server(config).await?, + ), + ThreadParamsMode::Embedded, + )) +} + +fn write_rollout( + config: &Config, + thread_id: ThreadId, + archived: bool, + timestamp: &str, + preview: &str, +) -> color_eyre::Result { + let subdir = if archived { + "archived_sessions" + } else { + "sessions/2025/02/01" + }; + let rollout_path = config + .codex_home + .join(subdir) + .join(format!("rollout-2025-02-01T10-00-00-{thread_id}.jsonl")); + std::fs::create_dir_all(rollout_path.parent().expect("rollout parent"))?; + let session_meta = SessionMetaLine { + meta: SessionMeta { + session_id: thread_id.into(), + id: thread_id, + timestamp: timestamp.to_string(), + cwd: config.codex_home.join("project").to_path_buf(), + originator: "codex".to_string(), + cli_version: "0.0.0".to_string(), + source: SessionSource::Cli, + model_provider: Some(config.model_provider_id.clone()), + ..Default::default() + }, + git: None, + }; + let lines = [ + serde_json::json!({ + "timestamp": timestamp, + "type": "session_meta", + "payload": session_meta, + }), + serde_json::json!({ + "timestamp": timestamp, + "type": "response_item", + "payload": { + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": preview}], + }, + }), + serde_json::json!({ + "timestamp": timestamp, + "type": "event_msg", + "payload": { + "type": "user_message", + "message": preview, + "kind": "plain", + }, + }), + ]; + std::fs::write( + &rollout_path, + lines + .iter() + .map(serde_json::Value::to_string) + .collect::>() + .join("\n") + + "\n", + )?; + Ok(rollout_path.to_path_buf()) +} + +fn thread_metadata( + config: &Config, + thread_id: ThreadId, + rollout_path: PathBuf, + name: &str, + archived: bool, +) -> codex_state::ThreadMetadata { + let created_at = DateTime::parse_from_rfc3339("2025-02-01T10:00:00Z") + .expect("timestamp should parse") + .with_timezone(&Utc); + let mut builder = ThreadMetadataBuilder::new( + thread_id, + rollout_path, + created_at, + serde_json::from_value(serde_json::json!("cli")) + .expect("cli session source should deserialize"), + ); + builder.cwd = config.codex_home.join("project").to_path_buf(); + let mut metadata = builder.build(config.model_provider_id.as_str()); + metadata.title = name.to_string(); + metadata.first_user_message = Some("preview text".to_string()); + metadata.preview = metadata.first_user_message.clone(); + metadata.archived_at = archived.then_some(created_at); + metadata +} + +#[tokio::test] +async fn archives_by_sqlite_name() -> color_eyre::Result<()> { + let temp_dir = TempDir::new()?; + let config = build_config(&temp_dir).await?; + let runtime = state_runtime(&config).await?; + let thread_id = ThreadId::new(); + let rollout_path = write_rollout( + &config, + thread_id, + /*archived*/ false, + "2025-02-01T10:00:00Z", + "preview", + )?; + runtime + .upsert_thread(&thread_metadata( + &config, + thread_id, + rollout_path, + "saved-session", + /*archived*/ false, + )) + .await + .map_err(std::io::Error::other)?; + + let mut app_server = start_app_server(config.clone()).await?; + let message = run_session_archive_action_with_app_server( + &mut app_server, + config.codex_home.as_path(), + SessionArchiveAction::Archive, + "saved-session", + ) + .await?; + app_server.shutdown().await?; + + assert_eq!( + ( + message, + runtime + .get_thread(thread_id) + .await + .map_err(std::io::Error::other)? + .is_some_and(|metadata| metadata.archived_at.is_some()), + ), + ( + format!("Archived session saved-session ({thread_id})."), + true, + ), + ); + Ok(()) +} + +#[tokio::test] +async fn unarchives_by_sqlite_name() -> color_eyre::Result<()> { + let temp_dir = TempDir::new()?; + let config = build_config(&temp_dir).await?; + let runtime = state_runtime(&config).await?; + let thread_id = ThreadId::new(); + let rollout_path = write_rollout( + &config, + thread_id, + /*archived*/ true, + "2025-02-01T10:00:00Z", + "preview", + )?; + runtime + .upsert_thread(&thread_metadata( + &config, + thread_id, + rollout_path, + "saved-session", + /*archived*/ true, + )) + .await + .map_err(std::io::Error::other)?; + + let mut app_server = start_app_server(config.clone()).await?; + let message = run_session_archive_action_with_app_server( + &mut app_server, + config.codex_home.as_path(), + SessionArchiveAction::Unarchive, + "saved-session", + ) + .await?; + app_server.shutdown().await?; + + assert_eq!( + ( + message, + runtime + .get_thread(thread_id) + .await + .map_err(std::io::Error::other)? + .is_some_and(|metadata| metadata.archived_at.is_some()), + ), + ( + format!("Unarchived session saved-session ({thread_id})."), + false, + ), + ); + Ok(()) +} + +#[tokio::test] +async fn deletes_valid_duplicate_after_stale_sqlite_hit() -> color_eyre::Result<()> { + let temp_dir = TempDir::new()?; + let config = build_config(&temp_dir).await?; + let runtime = state_runtime(&config).await?; + let stale_id = ThreadId::new(); + let stale_rollout_path = write_rollout( + &config, + stale_id, + /*archived*/ true, + "2025-02-01T10:00:00Z", + "stale preview", + )?; + runtime + .upsert_thread(&thread_metadata( + &config, + stale_id, + stale_rollout_path.clone(), + "saved-session", + /*archived*/ false, + )) + .await + .map_err(std::io::Error::other)?; + + let thread_id = ThreadId::new(); + let rollout_path = write_rollout( + &config, + thread_id, + /*archived*/ false, + "2025-02-01T10:00:00Z", + "preview", + )?; + codex_rollout::append_thread_name(config.codex_home.as_path(), thread_id, "saved-session") + .await?; + + let mut app_server = start_app_server(config.clone()).await?; + let message = run_session_archive_action_with_app_server( + &mut app_server, + config.codex_home.as_path(), + SessionArchiveAction::Delete(DeleteConfirmation::Skip), + "saved-session", + ) + .await?; + app_server.shutdown().await?; + + assert_eq!( + ( + message, + rollout_path.exists(), + stale_rollout_path.exists(), + runtime + .get_thread(thread_id) + .await + .map_err(std::io::Error::other)?, + runtime + .get_thread(stale_id) + .await + .map_err(std::io::Error::other)? + .is_some(), + ), + ( + format!("Deleted session saved-session ({thread_id})."), + false, + true, + None, + true, + ), + ); + Ok(()) +} + +#[tokio::test] +async fn trusts_sqlite_name_over_legacy_index_for_delete() -> color_eyre::Result<()> { + let temp_dir = TempDir::new()?; + let config = build_config(&temp_dir).await?; + let runtime = state_runtime(&config).await?; + let thread_id = ThreadId::new(); + let rollout_path = write_rollout( + &config, + thread_id, + /*archived*/ false, + "2025-02-01T10:00:00Z", + "preview", + )?; + runtime + .upsert_thread(&thread_metadata( + &config, + thread_id, + rollout_path.clone(), + "new-session", + /*archived*/ false, + )) + .await + .map_err(std::io::Error::other)?; + codex_rollout::append_thread_name(config.codex_home.as_path(), thread_id, "old-session") + .await?; + + let mut app_server = start_app_server(config.clone()).await?; + let message = run_session_archive_action_with_app_server( + &mut app_server, + config.codex_home.as_path(), + SessionArchiveAction::Delete(DeleteConfirmation::Skip), + "new-session", + ) + .await?; + app_server.shutdown().await?; + + assert_eq!( + ( + message, + rollout_path.exists(), + runtime + .get_thread(thread_id) + .await + .map_err(std::io::Error::other)?, + ), + ( + format!("Deleted session new-session ({thread_id})."), + false, + None, + ), + ); + Ok(()) +} diff --git a/codex-rs/tui/src/tui/keyboard_modes.rs b/codex-rs/tui/src/tui/keyboard_modes.rs index 28db590a3b..f6c08fcbb0 100644 --- a/codex-rs/tui/src/tui/keyboard_modes.rs +++ b/codex-rs/tui/src/tui/keyboard_modes.rs @@ -156,10 +156,12 @@ fn keyboard_enhancement_flags( let flags = KeyboardEnhancementFlags::DISAMBIGUATE_ESCAPE_CODES | KeyboardEnhancementFlags::REPORT_ALTERNATE_KEYS; - // iTerm can leak the release of an exit shortcut into the parent shell. + // iTerm and Ghostty can leak shortcut release events that the terminal consumes. // tmux's xterm key format also loses Shift-Enter when event types are // reported. Preserve repeat classification on transports that support it. - if terminal_name == TerminalName::Iterm2 || matches!(tmux_extended_keys_format, Some("xterm")) { + if matches!(terminal_name, TerminalName::Ghostty | TerminalName::Iterm2) + || matches!(tmux_extended_keys_format, Some("xterm")) + { flags } else { flags | KeyboardEnhancementFlags::REPORT_EVENT_TYPES @@ -331,6 +333,17 @@ mod tests { ); } + #[test] + fn keyboard_enhancement_suppresses_release_reporting_for_ghostty() { + assert_eq!( + ansi_for(PushKeyboardEnhancementFlags(keyboard_enhancement_flags( + TerminalName::Ghostty, + /*tmux_extended_keys_format*/ None + ))), + "\x1b[>5u" + ); + } + #[test] fn keyboard_enhancement_preserves_repeat_reporting_for_kitty() { assert_eq!( diff --git a/codex-rs/utils/audio/BUILD.bazel b/codex-rs/utils/audio/BUILD.bazel new file mode 100644 index 0000000000..eb2c15b509 --- /dev/null +++ b/codex-rs/utils/audio/BUILD.bazel @@ -0,0 +1,6 @@ +load("//:defs.bzl", "codex_rust_crate") + +codex_rust_crate( + name = "audio", + crate_name = "codex_utils_audio", +) diff --git a/codex-rs/utils/audio/Cargo.toml b/codex-rs/utils/audio/Cargo.toml new file mode 100644 index 0000000000..90540228a6 --- /dev/null +++ b/codex-rs/utils/audio/Cargo.toml @@ -0,0 +1,23 @@ +[package] +edition.workspace = true +license.workspace = true +name = "codex-utils-audio" +version.workspace = true + +[lints] +workspace = true + +[dependencies] +base64 = { workspace = true } +codex-protocol = { workspace = true } +codex-utils-cache = { workspace = true } +codex-utils-string = { workspace = true } +symphonia = { workspace = true } +thiserror = { workspace = true } +tracing = { workspace = true } + +[dev-dependencies] +pretty_assertions = { workspace = true } + +[lib] +doctest = false diff --git a/codex-rs/core/src/audio_preparation_tests.rs b/codex-rs/utils/audio/src/audio_preparation_tests.rs similarity index 100% rename from codex-rs/core/src/audio_preparation_tests.rs rename to codex-rs/utils/audio/src/audio_preparation_tests.rs diff --git a/codex-rs/core/src/audio_preparation.rs b/codex-rs/utils/audio/src/lib.rs similarity index 95% rename from codex-rs/core/src/audio_preparation.rs rename to codex-rs/utils/audio/src/lib.rs index 9f44420470..ab836b5631 100644 --- a/codex-rs/core/src/audio_preparation.rs +++ b/codex-rs/utils/audio/src/lib.rs @@ -1,3 +1,5 @@ +//! Audio preparation and duration-based token estimates for model inputs. + use base64::Engine; use base64::engine::general_purpose::STANDARD as BASE64_STANDARD; use codex_protocol::models::ContentItem; @@ -5,7 +7,7 @@ use codex_protocol::models::FunctionCallOutputContentItem; use codex_protocol::models::ResponseItem; use codex_utils_cache::BlockingLruCache; use codex_utils_cache::sha1_digest; -use codex_utils_output_truncation::approx_token_count; +use codex_utils_string::approx_token_count; use std::io::Cursor; use std::num::NonZeroUsize; use std::sync::LazyLock; @@ -58,7 +60,8 @@ impl AudioPreparationError { } } -pub(crate) fn prepare_response_items(items: &mut [ResponseItem]) { +/// Canonicalizes audio inputs and replaces unsupported inputs with text placeholders. +pub fn prepare_response_items(items: &mut [ResponseItem]) { for item in items { match item { ResponseItem::Message { content, .. } => prepare_message_content(content), @@ -141,7 +144,8 @@ fn canonical_audio_mime(mime: &str) -> Option<&'static str> { } } -pub(crate) fn estimate_audio_token_count(audio_url: &str) -> usize { +/// Estimates audio tokens from decoded duration, falling back to the data URL size. +pub fn estimate_audio_token_count(audio_url: &str) -> usize { let key = sha1_digest(audio_url.as_bytes()); AUDIO_TOKEN_ESTIMATE_CACHE.get_or_insert_with(key, || { let Some(duration_seconds) = audio_duration_seconds(audio_url) else { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 2d4c6f3497..9e4eeccb6b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -36,6 +36,9 @@ importers: sdk/typescript: devDependencies: + '@modelcontextprotocol/conformance': + specifier: github:modelcontextprotocol/conformance#49103de6ed70804e940637bf3e9e29e4a3f54e64 + version: https://codeload.github.com/modelcontextprotocol/conformance/tar.gz/49103de6ed70804e940637bf3e9e29e4a3f54e64 '@modelcontextprotocol/sdk': specifier: 1.26.0 version: 1.26.0(zod@3.25.76) @@ -74,7 +77,7 @@ importers: version: 10.9.2(@types/node@20.19.18)(typescript@5.9.2) tsup: specifier: ^8.5.0 - version: 8.5.0(postcss@8.5.6)(typescript@5.9.2)(yaml@2.8.1) + version: 8.5.0(postcss@8.5.6)(typescript@5.9.2)(yaml@2.9.0) typescript: specifier: ^5.9.2 version: 5.9.2 @@ -572,6 +575,11 @@ packages: '@jridgewell/trace-mapping@0.3.9': resolution: {integrity: sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==} + '@modelcontextprotocol/conformance@https://codeload.github.com/modelcontextprotocol/conformance/tar.gz/49103de6ed70804e940637bf3e9e29e4a3f54e64': + resolution: {tarball: https://codeload.github.com/modelcontextprotocol/conformance/tar.gz/49103de6ed70804e940637bf3e9e29e4a3f54e64} + version: 0.2.0-alpha.10 + hasBin: true + '@modelcontextprotocol/sdk@1.26.0': resolution: {integrity: sha512-Y5RmPncpiDtTXDbLKswIJzTqu2hyBKxTNsgKqKclDbhIgg1wgtf1fRuvxgTnRfcnxtvvgbIEcqUOzZrJ6iSReg==} engines: {node: '>=18'} @@ -594,6 +602,58 @@ packages: resolution: {integrity: sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==} engines: {node: '>= 8'} + '@octokit/auth-token@6.0.0': + resolution: {integrity: sha512-P4YJBPdPSpWTQ1NU4XYdvHvXJJDxM6YwpS0FZHRgP7YFkdVxsWcpWGy/NVqlAA7PcPCnMacXlRm1y2PFZRWL/w==} + engines: {node: '>= 20'} + + '@octokit/core@7.0.6': + resolution: {integrity: sha512-DhGl4xMVFGVIyMwswXeyzdL4uXD5OGILGX5N8Y+f6W7LhC1Ze2poSNrkF/fedpVDHEEZ+PHFW0vL14I+mm8K3Q==} + engines: {node: '>= 20'} + + '@octokit/endpoint@11.0.3': + resolution: {integrity: sha512-FWFlNxghg4HrXkD3ifYbS/IdL/mDHjh9QcsNyhQjN8dplUoZbejsdpmuqdA76nxj2xoWPs7p8uX2SNr9rYu0Ag==} + engines: {node: '>= 20'} + + '@octokit/graphql@9.0.3': + resolution: {integrity: sha512-grAEuupr/C1rALFnXTv6ZQhFuL1D8G5y8CN04RgrO4FIPMrtm+mcZzFG7dcBm+nq+1ppNixu+Jd78aeJOYxlGA==} + engines: {node: '>= 20'} + + '@octokit/openapi-types@27.0.0': + resolution: {integrity: sha512-whrdktVs1h6gtR+09+QsNk2+FO+49j6ga1c55YZudfEG+oKJVvJLQi3zkOm5JjiUXAagWK2tI2kTGKJ2Ys7MGA==} + + '@octokit/plugin-paginate-rest@14.0.0': + resolution: {integrity: sha512-fNVRE7ufJiAA3XUrha2omTA39M6IXIc6GIZLvlbsm8QOQCYvpq/LkMNGyFlB1d8hTDzsAXa3OKtybdMAYsV/fw==} + engines: {node: '>= 20'} + peerDependencies: + '@octokit/core': '>=6' + + '@octokit/plugin-request-log@6.0.0': + resolution: {integrity: sha512-UkOzeEN3W91/eBq9sPZNQ7sUBvYCqYbrrD8gTbBuGtHEuycE4/awMXcYvx6sVYo7LypPhmQwwpUe4Yyu4QZN5Q==} + engines: {node: '>= 20'} + peerDependencies: + '@octokit/core': '>=6' + + '@octokit/plugin-rest-endpoint-methods@17.0.0': + resolution: {integrity: sha512-B5yCyIlOJFPqUUeiD0cnBJwWJO8lkJs5d8+ze9QDP6SvfiXSz1BF+91+0MeI1d2yxgOhU/O+CvtiZ9jSkHhFAw==} + engines: {node: '>= 20'} + peerDependencies: + '@octokit/core': '>=6' + + '@octokit/request-error@7.1.0': + resolution: {integrity: sha512-KMQIfq5sOPpkQYajXHwnhjCC0slzCNScLHs9JafXc4RAJI+9f+jNDlBNaIMTvazOPLgb4BnlhGJOTbnN0wIjPw==} + engines: {node: '>= 20'} + + '@octokit/request@10.0.11': + resolution: {integrity: sha512-+s7HUxjfFqOMS9VlIwDffq0MikjSAK0gSpG73W+meAvVAvX4MBrHYTK5Bj3Uot55qFT4gzUtfzE4mGWY4Br8/Q==} + engines: {node: '>= 20'} + + '@octokit/rest@22.0.1': + resolution: {integrity: sha512-Jzbhzl3CEexhnivb1iQ0KJ7s5vvjMWcmRtq5aUsKmKDrRW6z3r84ngmiFKFvpZjpiU/9/S6ITPFRpn5s/3uQJw==} + engines: {node: '>= 20'} + + '@octokit/types@16.0.0': + resolution: {integrity: sha512-sKq+9r1Mm4efXW1FCk7hFSeJo4QKreL/tTbR0rz/qx/r1Oa2VV83LTA/H/MuCOX7uCIJmQVRKBcbmWoySjAnSg==} + '@pkgjs/parseargs@0.11.0': resolution: {integrity: sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==} engines: {node: '>=14'} @@ -893,6 +953,9 @@ packages: ajv@8.17.1: resolution: {integrity: sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==} + ajv@8.20.0: + resolution: {integrity: sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==} + ansi-escapes@4.3.2: resolution: {integrity: sha512-gKXj5ALrKWQLsYG9jlTRmR/xKluxHV+Z9QEwNIgCfM1/uwPMCuzVVnh5mwTd+OuBZcwSIMbqssNWRm1lE51QaQ==} engines: {node: '>=8'} @@ -969,6 +1032,9 @@ packages: resolution: {integrity: sha512-hY/u2lxLrbecMEWSB0IpGzGyDyeoMFQhCvZd2jGFSE5I17Fh01sYUBPCJtkWERw7zrac9+cIghxm/ytJa2X8iA==} hasBin: true + before-after-hook@4.0.0: + resolution: {integrity: sha512-q6tR3RPqIB1pMiTRMFcZwuG5T8vwp+vUvEG0vuI6B+Rikh5BfPp2fQ82c925FOs+b0lcFQ8CFrL+KbilfZFhOQ==} + body-parser@2.2.2: resolution: {integrity: sha512-oP5VkATKlNwcgvxi0vM0p/D3n2C3EReYVX+DNYs5TjZFn/oQt2j+4sVJtSMr18pdRr8wjTcBl6LoV+FUwzPmNA==} engines: {node: '>=18'} @@ -1073,6 +1139,10 @@ packages: color-name@1.1.4: resolution: {integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==} + commander@14.0.3: + resolution: {integrity: sha512-H+y0Jo/T1RZ9qPP4Eh1pkcQcLRglraJaSLoyOtHxu6AapkjWVCy2Sit1QQ4x3Dng8qDlSsZEet7g5Pq06MvTgw==} + engines: {node: '>=20'} + commander@4.1.1: resolution: {integrity: sha512-NOKm8xhkzAjzFx8B2v5OAHT+u5pRQc2UCa2Vq9jYL/31o2wi9mxBA7LIFs3sV5VSC49z6pEhfbMULvShKj26WA==} engines: {node: '>= 6'} @@ -1095,6 +1165,10 @@ packages: resolution: {integrity: sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==} engines: {node: '>= 0.6'} + content-type@2.0.0: + resolution: {integrity: sha512-j/O/d7GcZCyNl7/hwZAb606rzqkyvaDctLmckbxLzHvFBzTJHuGEdodATcP3yIRoDrLHkIATJuvzbFlp/ki2cQ==} + engines: {node: '>=18'} + convert-source-map@2.0.0: resolution: {integrity: sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==} @@ -1304,6 +1378,10 @@ packages: resolution: {integrity: sha512-Vo1ab+QXPzZ4tCa8SwIHJFaSzy4R6SHf7BY79rFBDf0idraZWAkYrDjDj8uWaSm3S2TK+hJ7/t1CEmZ7jXw+pg==} engines: {node: '>=18.0.0'} + eventsource-parser@3.1.0: + resolution: {integrity: sha512-kJezFj9YFAMLeORyi7aCLxLbD5/qWMQnoMVlVPyHIll7lgRJCc3JVln9Vgl9nwQi0YkMnhdGTMNn7CkRRAptMg==} + engines: {node: '>=18.0.0'} + eventsource@3.0.7: resolution: {integrity: sha512-CRT1WTyuQoD771GW56XEZFQ/ZoSfWid1alKGDYMmkt2yl8UXrVR4pspqWNEcqKvVIzg6PAltWjxcSSPrboA4iA==} engines: {node: '>=18.0.0'} @@ -1453,7 +1531,7 @@ packages: glob@7.2.3: resolution: {integrity: sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==} - deprecated: Glob versions prior to v9 are no longer supported + deprecated: Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me globals@14.0.0: resolution: {integrity: sha512-oahGvuMGQlPw/ivIYBjVSrWAfWLBeku5tpPE2fOPLi+WHffIWbuh2tCjhyQhTBPMf5E9jDEH4FOmTYgYwbKwtQ==} @@ -1775,6 +1853,9 @@ packages: json-stable-stringify-without-jsonify@1.0.1: resolution: {integrity: sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==} + json-with-bigint@3.5.10: + resolution: {integrity: sha512-Vcx+JVNEBts/xfcoCS69sKrOhOk/3TVlvlT+XzUOefVKnnrbYSCKpDCm10pohsJFtsJVYnwa/cXRZ4eElzaM6w==} + json5@2.2.3: resolution: {integrity: sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==} engines: {node: '>=6'} @@ -2093,10 +2174,6 @@ packages: pure-rand@6.1.0: resolution: {integrity: sha512-bVWawvoZoBYpp6yIoQtQXHZjmz35RSVHnUOTefl8Vcjr8snTPY1wnpSPMWekcFwbxI6gtmT7rSYPFvz71ldiOA==} - qs@6.14.0: - resolution: {integrity: sha512-YWWTjgABSKcvs/nWBi9PycY/JiPJqOD4JA6o9Sej2AtvSGarXxKC3OQSk4pAarbdQlKAh5D4FCQkJNkW+GAn3w==} - engines: {node: '>=0.6'} - qs@6.15.1: resolution: {integrity: sha512-6YHEFRL9mfgcAvql/XhwTvf5jKcOiiupt2FiJxHkiX1z4j7WL8J/jRHYLluORvc1XxB5rV20KoeK00gVJamspg==} engines: {node: '>=0.6'} @@ -2457,6 +2534,13 @@ packages: undici-types@6.21.0: resolution: {integrity: sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==} + undici@7.28.0: + resolution: {integrity: sha512-cRZYrTDwWznlnRiPjggAGxZXanty6M8RV1ff8Wm4LWXBp7/IG8v5DnOm74DtUBp9OONpK75YlPnIjQqX0dBDtA==} + engines: {node: '>=20.18.1'} + + universal-user-agent@7.0.3: + resolution: {integrity: sha512-TmnEAEAsBJVZM/AADELsK76llnwcf9vMKuPz8JflO1frO8Lchitr0fNaN9d+Ap0BjKtqWqd/J17qeDnXh8CL2A==} + unpipe@1.0.0: resolution: {integrity: sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==} engines: {node: '>= 0.8'} @@ -2524,8 +2608,8 @@ packages: yallist@3.1.1: resolution: {integrity: sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==} - yaml@2.8.1: - resolution: {integrity: sha512-lcYcMxX2PO9XMGvAJkJ3OsNMw+/7FKes7/hgerGUYWIoWu5j/+YQqcZr5JnPZWzOsEBgMbSbiSTn/dv/69Mkpw==} + yaml@2.9.0: + resolution: {integrity: sha512-2AvhNX3mb8zd6Zy7INTtSpl1F15HW6Wnqj0srWlkKLcpYl/gMIMJiyuGq2KeI2YFxUPjdlB+3Lc10seMLtL4cA==} engines: {node: '>= 14.6'} hasBin: true @@ -2558,6 +2642,9 @@ packages: zod@3.25.76: resolution: {integrity: sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==} + zod@4.4.3: + resolution: {integrity: sha512-ytENFjIJFl2UwYglde2jchW2Hwm4GJFLDiSXWdTrJQBIN9Fcyp7n4DhxJEiWNAJMV1/BqWfW/kkg71UDcHJyTQ==} + snapshots: '@babel/code-frame@7.27.1': @@ -3095,6 +3182,23 @@ snapshots: '@jridgewell/resolve-uri': 3.1.2 '@jridgewell/sourcemap-codec': 1.5.5 + '@modelcontextprotocol/conformance@https://codeload.github.com/modelcontextprotocol/conformance/tar.gz/49103de6ed70804e940637bf3e9e29e4a3f54e64': + dependencies: + '@modelcontextprotocol/sdk': 1.26.0(zod@4.4.3) + '@octokit/rest': 22.0.1 + ajv: 8.20.0 + ajv-formats: 3.0.1(ajv@8.20.0) + commander: 14.0.3 + eventsource-parser: 3.1.0 + express: 5.2.1 + jose: 6.1.3 + undici: 7.28.0 + yaml: 2.9.0 + zod: 4.4.3 + transitivePeerDependencies: + - '@cfworker/json-schema' + - supports-color + '@modelcontextprotocol/sdk@1.26.0(zod@3.25.76)': dependencies: '@hono/node-server': 1.19.13(hono@4.12.25) @@ -3117,6 +3221,28 @@ snapshots: transitivePeerDependencies: - supports-color + '@modelcontextprotocol/sdk@1.26.0(zod@4.4.3)': + dependencies: + '@hono/node-server': 1.19.13(hono@4.12.25) + ajv: 8.17.1 + ajv-formats: 3.0.1(ajv@8.17.1) + content-type: 1.0.5 + cors: 2.8.5 + cross-spawn: 7.0.6 + eventsource: 3.0.7 + eventsource-parser: 3.0.6 + express: 5.2.1 + express-rate-limit: 8.3.2(express@5.2.1) + hono: 4.12.25 + jose: 6.1.3 + json-schema-typed: 8.0.2 + pkce-challenge: 5.0.0 + raw-body: 3.0.1 + zod: 4.4.3 + zod-to-json-schema: 3.25.2(zod@4.4.3) + transitivePeerDependencies: + - supports-color + '@nodelib/fs.scandir@2.1.5': dependencies: '@nodelib/fs.stat': 2.0.5 @@ -3129,6 +3255,69 @@ snapshots: '@nodelib/fs.scandir': 2.1.5 fastq: 1.19.1 + '@octokit/auth-token@6.0.0': {} + + '@octokit/core@7.0.6': + dependencies: + '@octokit/auth-token': 6.0.0 + '@octokit/graphql': 9.0.3 + '@octokit/request': 10.0.11 + '@octokit/request-error': 7.1.0 + '@octokit/types': 16.0.0 + before-after-hook: 4.0.0 + universal-user-agent: 7.0.3 + + '@octokit/endpoint@11.0.3': + dependencies: + '@octokit/types': 16.0.0 + universal-user-agent: 7.0.3 + + '@octokit/graphql@9.0.3': + dependencies: + '@octokit/request': 10.0.11 + '@octokit/types': 16.0.0 + universal-user-agent: 7.0.3 + + '@octokit/openapi-types@27.0.0': {} + + '@octokit/plugin-paginate-rest@14.0.0(@octokit/core@7.0.6)': + dependencies: + '@octokit/core': 7.0.6 + '@octokit/types': 16.0.0 + + '@octokit/plugin-request-log@6.0.0(@octokit/core@7.0.6)': + dependencies: + '@octokit/core': 7.0.6 + + '@octokit/plugin-rest-endpoint-methods@17.0.0(@octokit/core@7.0.6)': + dependencies: + '@octokit/core': 7.0.6 + '@octokit/types': 16.0.0 + + '@octokit/request-error@7.1.0': + dependencies: + '@octokit/types': 16.0.0 + + '@octokit/request@10.0.11': + dependencies: + '@octokit/endpoint': 11.0.3 + '@octokit/request-error': 7.1.0 + '@octokit/types': 16.0.0 + content-type: 2.0.0 + json-with-bigint: 3.5.10 + universal-user-agent: 7.0.3 + + '@octokit/rest@22.0.1': + dependencies: + '@octokit/core': 7.0.6 + '@octokit/plugin-paginate-rest': 14.0.0(@octokit/core@7.0.6) + '@octokit/plugin-request-log': 6.0.0(@octokit/core@7.0.6) + '@octokit/plugin-rest-endpoint-methods': 17.0.0(@octokit/core@7.0.6) + + '@octokit/types@16.0.0': + dependencies: + '@octokit/openapi-types': 27.0.0 + '@pkgjs/parseargs@0.11.0': optional: true @@ -3393,6 +3582,10 @@ snapshots: optionalDependencies: ajv: 8.17.1 + ajv-formats@3.0.1(ajv@8.20.0): + optionalDependencies: + ajv: 8.20.0 + ajv@6.12.6: dependencies: fast-deep-equal: 3.1.3 @@ -3407,6 +3600,13 @@ snapshots: json-schema-traverse: 1.0.0 require-from-string: 2.0.2 + ajv@8.20.0: + dependencies: + fast-deep-equal: 3.1.3 + fast-uri: 3.1.1 + json-schema-traverse: 1.0.0 + require-from-string: 2.0.2 + ansi-escapes@4.3.2: dependencies: type-fest: 0.21.3 @@ -3499,6 +3699,8 @@ snapshots: baseline-browser-mapping@2.8.9: {} + before-after-hook@4.0.0: {} + body-parser@2.2.2: dependencies: bytes: 3.1.2 @@ -3602,6 +3804,8 @@ snapshots: color-name@1.1.4: {} + commander@14.0.3: {} + commander@4.1.1: {} concat-map@0.0.1: {} @@ -3616,6 +3820,8 @@ snapshots: content-type@1.0.5: {} + content-type@2.0.0: {} + convert-source-map@2.0.0: {} cookie-signature@1.2.2: {} @@ -3829,6 +4035,8 @@ snapshots: eventsource-parser@3.0.6: {} + eventsource-parser@3.1.0: {} + eventsource@3.0.7: dependencies: eventsource-parser: 3.0.6 @@ -3882,7 +4090,7 @@ snapshots: once: 1.4.0 parseurl: 1.3.3 proxy-addr: 2.0.7 - qs: 6.14.0 + qs: 6.15.1 range-parser: 1.2.1 router: 2.2.0 send: 1.2.0 @@ -4510,6 +4718,8 @@ snapshots: json-stable-stringify-without-jsonify@1.0.1: {} + json-with-bigint@3.5.10: {} + json5@2.2.3: {} keyv@4.5.4: @@ -4729,12 +4939,12 @@ snapshots: mlly: 1.8.0 pathe: 2.0.3 - postcss-load-config@6.0.1(postcss@8.5.6)(yaml@2.8.1): + postcss-load-config@6.0.1(postcss@8.5.6)(yaml@2.9.0): dependencies: lilconfig: 3.1.3 optionalDependencies: postcss: 8.5.6 - yaml: 2.8.1 + yaml: 2.9.0 postcss@8.5.6: dependencies: @@ -4769,10 +4979,6 @@ snapshots: pure-rand@6.1.0: {} - qs@6.14.0: - dependencies: - side-channel: 1.1.0 - qs@6.15.1: dependencies: side-channel: 1.1.0 @@ -5093,7 +5299,7 @@ snapshots: v8-compile-cache-lib: 3.0.1 yn: 3.1.1 - tsup@8.5.0(postcss@8.5.6)(typescript@5.9.2)(yaml@2.8.1): + tsup@8.5.0(postcss@8.5.6)(typescript@5.9.2)(yaml@2.9.0): dependencies: bundle-require: 5.1.0(esbuild@0.28.1) cac: 6.7.14 @@ -5104,7 +5310,7 @@ snapshots: fix-dts-default-cjs-exports: 1.0.1 joycon: 3.1.1 picocolors: 1.1.1 - postcss-load-config: 6.0.1(postcss@8.5.6)(yaml@2.8.1) + postcss-load-config: 6.0.1(postcss@8.5.6)(yaml@2.9.0) resolve-from: 5.0.0 rollup: 4.59.0 source-map: 0.8.0-beta.0 @@ -5157,6 +5363,10 @@ snapshots: undici-types@6.21.0: {} + undici@7.28.0: {} + + universal-user-agent@7.0.3: {} + unpipe@1.0.0: {} update-browserslist-db@1.1.3(browserslist@4.26.2): @@ -5222,8 +5432,7 @@ snapshots: yallist@3.1.1: {} - yaml@2.8.1: - optional: true + yaml@2.9.0: {} yargs-parser@21.1.1: {} @@ -5249,4 +5458,10 @@ snapshots: dependencies: zod: 3.25.76 + zod-to-json-schema@3.25.2(zod@4.4.3): + dependencies: + zod: 4.4.3 + zod@3.25.76: {} + + zod@4.4.3: {} diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 7f5e2d915e..a7c72d1f20 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -14,4 +14,6 @@ strictDepBuilds: true trustPolicy: no-downgrade trustPolicyIgnoreAfter: 10080 trustPolicyExclude: [] -allowBuilds: {} +allowBuilds: + # Build only the official MCP conformance CLI pinned by the TypeScript SDK. + "@modelcontextprotocol/conformance": true diff --git a/scripts/mcp_conformance/README.md b/scripts/mcp_conformance/README.md new file mode 100644 index 0000000000..d75aa1f3b8 --- /dev/null +++ b/scripts/mcp_conformance/README.md @@ -0,0 +1,121 @@ +# MCP client conformance + +This directory tests the actual Codex executable against the official Model +Context Protocol client conformance suite. It exercises the shipping legacy, +intermediate `2025-11-25`, and modern `2026-07-28` protocols, localhost HTTP, +stdio, OAuth, and additional transport and security regression fixtures. + +The official upstream suite is pinned to +`modelcontextprotocol/conformance@49103de6ed70804e940637bf3e9e29e4a3f54e64`. +Use Node.js 22 and Python 3.10 or later. + +## Run the conformance gate + +First install the frozen workspace dependencies and build Codex: + +```bash +pnpm install --frozen-lockfile +cargo build --locked --manifest-path codex-rs/Cargo.toml -p codex-cli --bin codex +``` + +From a published Codex checkout, run: + +```bash +python3 scripts/mcp_conformance/run_codex_compliance.py \ + codex-rs/target/debug/codex \ + --conformance-cli node_modules/@modelcontextprotocol/conformance/dist/index.js \ + --baseline-report scripts/mcp_conformance/regression-baseline-v1.json \ + --report /tmp/codex-mcp-conformance.json +``` + +The positional executable can also point to an already built Codex binary. +`--conformance-cli` selects the exact, lockfile-installed upstream JavaScript +runner instead of downloading a moving version during a test. + +## What the baseline means + +`regression-baseline-v1.json` is a compact, reviewed snapshot of the upstream +revision, required protocol versions, HTTP and stdio transports, enabled modern +feature, OAuth coverage, and individual passing and failing check identities. + +The gate exits successfully only when: + +- The upstream suite and modern feature match the committed baseline. +- The shipping legacy, intermediate, and modern protocols are actually tested. +- The required HTTP, stdio, and authentication scenarios are actually run. +- Every previously passing check still passes. +- No additional check fails. + +Existing known failures remain visible in the complete JSON report. In +particular, `success` describes complete upstream conformance and +`regressionGate.success` describes the no-new-regressions merge gate; the gate +does not relabel an existing failure as a pass. + +Create a compact baseline from a reviewed complete report without contacting +the upstream suite again: + +```bash +python3 scripts/mcp_conformance/run_codex_compliance.py \ + /absolute/path/to/codex \ + --baseline-report /absolute/path/to/full-conformance-report.json \ + --extract-baseline /tmp/mcp-conformance-regression-baseline-v1.json +``` + +Alternatively, add `--write-baseline /tmp/mcp-conformance-regression-baseline-v1.json` +to a complete conformance run. Review every baseline change; do not regenerate +it to conceal a regression. + +## Run the production reviewer regression gate + +The separate reviewer gate tests the real Codex app-server across all three +shipping, legacy, and modern protocol modes. It covers stdio and localhost +HTTP, exact-integer tool and elicitation schemas, bounded multi-round requests, +malformed discovery response IDs, repeated pagination cursors, SSE framing and +keepalives, and catalog boundaries. In a published Codex checkout, run: + +```bash +python3 scripts/mcp_conformance/review_regressions.py \ + /absolute/path/to/codex \ + --mode all \ + --baseline-report scripts/mcp_conformance/review-regression-baseline-v1.json \ + --report /tmp/codex-mcp-review-regressions.json +``` + +A complete, main-derived baseline records all 186 real check identities and all +21 required cases. Existing failures remain explicitly visible in the complete +report; `regressionGate.success: true` means there are no newly failing or +missing checks. Improvements are recorded under `fixedChecks`. The gate never +classifies an existing failure as a passing check. + +Extract a compact deterministic reviewer baseline from a reviewed complete +production report without rerunning the client: + +```bash +python3 scripts/mcp_conformance/review_regressions.py \ + /absolute/path/to/codex \ + --baseline-report /absolute/path/to/full-review-regressions.json \ + --extract-baseline /tmp/review-regression-baseline-v1.json +``` + +Review every baseline update. Do not regenerate a baseline to hide a regression. + +## Run the fixture self-tests + +```bash +env PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 \ + python3 -m pytest -q scripts/mcp_conformance +``` + +## Run the required SDK integration + +The existing required SDK workflow runs the complete Python fixture self-tests. +Its TypeScript job builds the actual Codex executable, sets `CODEX_EXEC_PATH`, +installs the pinned upstream conformance runner, and runs both the official +authenticated suite and the separate production reviewer regression matrix. +Neither gate can be skipped. To reproduce the focused integration locally: + +```bash +CODEX_EXEC_PATH=/absolute/path/to/codex \ + pnpm --filter @openai/codex-sdk test -- \ + --runInBand tests/mcpConformance.test.ts +``` diff --git a/scripts/mcp_conformance/codex_conformance_adapter.py b/scripts/mcp_conformance/codex_conformance_adapter.py new file mode 100644 index 0000000000..bdf2e1dfeb --- /dev/null +++ b/scripts/mcp_conformance/codex_conformance_adapter.py @@ -0,0 +1,976 @@ +#!/usr/bin/env python3 +"""Adapt one official MCP client scenario to Codex app-server requests.""" + +import ipaddress +import json +import os +import sys +import traceback +import urllib.error +import urllib.parse +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Mapping, Sequence +from urllib.request import HTTPRedirectHandler, ProxyHandler, Request, build_opener # noqa: TID251 + +_MODULE_DIR = Path(__file__).resolve().parent +if str(_MODULE_DIR) not in sys.path: + sys.path.insert(0, str(_MODULE_DIR)) + +from run_codex_compliance import ( # noqa: E402 - direct scripts must first add their sibling directory. + MODERN_VERSION, + TEST_SERVER_NAME, + AppServerClient, + AppServerError, + _call_tool, + _command_detail, + _isolated_environment, + _response_result, + _run_command, +) + + +@dataclass +class Step: + name: str + success: bool + detail: str + + +class AdapterFailure(RuntimeError): + pass + + +CIMD_CLIENT_METADATA_URL = "https://conformance-test.local/client-metadata.json" +PRE_REGISTERED_CLIENT_SECRET_ENV_VAR = "MCP_CONFORMANCE_CLIENT_SECRET" +AUTH_COMPLETION_METHOD = "mcpServer/oauthLogin/completed" +EXPECTED_AUTH_REJECTION_SCENARIOS = frozenset( + { + "auth/resource-mismatch", + "auth/iss-supported-missing", + "auth/iss-wrong-issuer", + "auth/iss-unexpected", + "auth/iss-normalized", + "auth/metadata-issuer-mismatch", + } +) + + +def _required_path(name: str) -> Path: + value = os.environ.get(name) + if not value: + raise AdapterFailure(f"{name} is required") + return Path(value).expanduser().resolve() + + +def _required_env(name: str) -> str: + value = os.environ.get(name) + if not value: + raise AdapterFailure(f"{name} is required") + return value + + +def _conformance_context() -> dict[str, object]: + raw = os.environ.get("MCP_CONFORMANCE_CONTEXT") + if not raw: + return {} + try: + decoded = json.loads(raw) + except json.JSONDecodeError as exc: + raise AdapterFailure(f"invalid MCP_CONFORMANCE_CONTEXT: {exc}") from exc + if not isinstance(decoded, dict): + raise AdapterFailure("MCP_CONFORMANCE_CONTEXT was not an object") + return decoded + + +def _result_or_raise( + response: Mapping[str, object], + operation: str, +) -> dict[str, object]: + result, detail = _response_result(response) + if result is None: + raise AdapterFailure(f"{operation}: {detail}") + return result + + +def _server_entry(inventory: Mapping[str, object]) -> dict[str, object]: + entries = inventory.get("data") + if not isinstance(entries, list): + raise AdapterFailure("mcpServerStatus/list returned no data array") + for entry in entries: + if isinstance(entry, dict) and entry.get("name") == TEST_SERVER_NAME: + return entry + raise AdapterFailure(f"{TEST_SERVER_NAME!r} was absent from MCP status") + + +def _thread_id(client: AppServerClient, workspace: Path) -> str: + result = _result_or_raise( + client.request( + "thread/start", + {"cwd": str(workspace), "ephemeral": True}, + ), + "thread/start", + ) + thread = result.get("thread") + if not isinstance(thread, dict) or not isinstance(thread.get("id"), str): + raise AdapterFailure("thread/start did not return a thread id") + return str(thread["id"]) + + +def _call( + client: AppServerClient, + *, + thread_id: str, + tool: str, + arguments: Mapping[str, object], +) -> None: + result, detail = _call_tool( + client, + thread_id=thread_id, + tool=tool, + arguments=arguments, + ) + if result is None: + raise AdapterFailure(f"tool {tool}: {detail}") + + +def _elicitation_content( + scenario: str, + _params: Mapping[str, object], +) -> Mapping[str, object]: + if scenario == "elicitation-sep1034-client-defaults": + # The official scenario deliberately supplies no values. Codex, as the + # MCP client under test, must materialize the JSON Schema defaults. + return {} + if scenario == "sep-2322-client-request-state": + return {"confirmed": True} + return {"confirmation": "confirmed"} + + +def _context_tool_calls() -> list[tuple[str, dict[str, object]]]: + context = _conformance_context() + if not context: + raise AdapterFailure( + "official scenario did not provide MCP_CONFORMANCE_CONTEXT" + ) + calls = context.get("toolCalls") + if not isinstance(calls, list): + raise AdapterFailure("official scenario context did not contain toolCalls") + result: list[tuple[str, dict[str, object]]] = [] + for call in calls: + if ( + not isinstance(call, dict) + or not isinstance(call.get("name"), str) + or not isinstance(call.get("arguments"), dict) + ): + raise AdapterFailure("official scenario contained an invalid tool call") + result.append((str(call["name"]), dict(call["arguments"]))) + return result + + +def _is_loopback_hostname(hostname: str | None) -> bool: + if hostname is None: + return False + if hostname.lower() == "localhost": + return True + try: + return ipaddress.ip_address(hostname).is_loopback + except ValueError: + return False + + +def _validated_callback_url(authorization_url: str, location: str) -> str: + authorization = urllib.parse.urlsplit(authorization_url) + query = urllib.parse.parse_qs(authorization.query, keep_blank_values=True) + redirect_values = query.get("redirect_uri") + state_values = query.get("state") + if not redirect_values or len(redirect_values) != 1: + raise AdapterFailure( + "authorization request did not contain exactly one redirect_uri" + ) + if not state_values or len(state_values) != 1: + raise AdapterFailure("authorization request did not contain exactly one state") + + redirect = urllib.parse.urlsplit(redirect_values[0]) + if ( + redirect.scheme != "http" + or not _is_loopback_hostname(redirect.hostname) + or redirect.username is not None + or redirect.password is not None + or redirect.port is None + or redirect.query + or redirect.fragment + ): + raise AdapterFailure("OAuth redirect_uri was not a safe loopback HTTP endpoint") + + callback_url = urllib.parse.urljoin(authorization_url, location) + callback = urllib.parse.urlsplit(callback_url) + expected_endpoint = ( + redirect.scheme, + redirect.hostname, + redirect.port, + redirect.path, + ) + actual_endpoint = ( + callback.scheme, + callback.hostname, + callback.port, + callback.path, + ) + if ( + actual_endpoint != expected_endpoint + or callback.username is not None + or callback.password is not None + or callback.fragment + ): + raise AdapterFailure( + "authorization server redirect did not target Codex's exact loopback callback" + ) + + callback_query = urllib.parse.parse_qs(callback.query, keep_blank_values=True) + callback_states = callback_query.get("state") + if callback_states != state_values: + raise AdapterFailure( + "authorization server redirect did not preserve OAuth state" + ) + for parameter in ("code", "error", "iss"): + values = callback_query.get(parameter) + if values is not None and (len(values) != 1 or not values[0]): + raise AdapterFailure( + f"authorization server redirect must contain exactly one nonempty {parameter}" + ) + if ("code" in callback_query) == ("error" in callback_query): + raise AdapterFailure( + "authorization server redirect must contain exactly one of code or error" + ) + return callback_url + + +class _NoRedirect(HTTPRedirectHandler): + def redirect_request( + self, + req: Request, + fp: object, + code: int, + msg: str, + headers: Mapping[str, str], + newurl: str, + ) -> Request | None: + del req, fp, code, msg, headers, newurl + return None + + +def _open_without_redirects(url: str, timeout_seconds: float) -> tuple[int, str | None]: + opener = build_opener( + ProxyHandler({}), + _NoRedirect(), + ) + request = Request( + url, + method="GET", + headers={"Accept": "text/html,application/xhtml+xml"}, + ) + try: + with opener.open(request, timeout=timeout_seconds) as response: + return response.status, response.headers.get("Location") + except urllib.error.HTTPError as exc: + # With redirects disabled urllib represents 3xx as HTTPError. Closing + # the body promptly prevents an authorization page from being retained. + try: + return exc.code, exc.headers.get("Location") + finally: + exc.close() + + +def _drive_headless_authorization( + authorization_url: str, + *, + timeout_seconds: float, +) -> None: + status, location = _open_without_redirects(authorization_url, timeout_seconds) + if status not in {301, 302, 303, 307, 308} or not location: + raise AdapterFailure( + "authorization endpoint did not issue the expected callback redirect" + ) + callback_url = _validated_callback_url(authorization_url, location) + callback_status, _ = _open_without_redirects(callback_url, timeout_seconds) + # Error callbacks may intentionally return a 4xx after notifying the OAuth + # waiter. The completion notification is the authoritative outcome. + if not 200 <= callback_status < 500: + raise AdapterFailure("Codex OAuth callback endpoint returned an invalid status") + + +def _oauth_client_id( + scenario: str, + context: Mapping[str, object], + *, + require_production_client_identity: bool = False, +) -> str | None: + if scenario == "auth/basic-cimd": + return None if require_production_client_identity else CIMD_CLIENT_METADATA_URL + if scenario == "auth/pre-registration": + client_id = context.get("client_id") + if not isinstance(client_id, str) or not client_id: + raise AdapterFailure("pre-registration context did not contain client_id") + return client_id + return None + + +def _write_auth_registration( + config_path: Path, + *, + server_url: str, + oauth_client_id: str | None, + oauth_client_secret_env_var: str | None = None, +) -> None: + existing = config_path.read_text(encoding="utf-8") if config_path.exists() else "" + block = f"\n[mcp_servers.{TEST_SERVER_NAME}]\nurl = {json.dumps(server_url, ensure_ascii=False)}\n" + if oauth_client_id is not None: + block += ( + f"\n[mcp_servers.{TEST_SERVER_NAME}.oauth]\n" + f"client_id = {json.dumps(oauth_client_id, ensure_ascii=False)}\n" + ) + if oauth_client_secret_env_var is not None: + block += ( + "client_secret_env_var = " + f"{json.dumps(oauth_client_secret_env_var, ensure_ascii=False)}\n" + ) + config_path.write_text(existing.rstrip() + "\n" + block, encoding="utf-8") + + +def _validate_oauth_secret_not_persisted(codex_home: Path, client_secret: str) -> None: + if not client_secret: + return + + candidates = [codex_home / "config.toml", *codex_home.rglob(".credentials.json")] + secret_bytes = client_secret.encode("utf-8") + for candidate in dict.fromkeys(candidates): + if not candidate.is_file(): + continue + if secret_bytes in candidate.read_bytes(): + raise AdapterFailure( + "environment-provided OAuth client secret was persisted in " + f"{candidate.relative_to(codex_home)}" + ) + + +def _oauth_login( + client: AppServerClient, + *, + scopes: Sequence[str] | None, + timeout_seconds: float, +) -> tuple[bool, str | None]: + event_index = len(client.events) + params: dict[str, object] = { + "name": TEST_SERVER_NAME, + "timeoutSecs": max(1, round(timeout_seconds)), + } + if scopes is not None: + params["scopes"] = list(scopes) + response = client.request("mcpServer/oauth/login", params) + result, detail = _response_result(response) + if result is None: + return False, detail + authorization_url = result.get("authorizationUrl") + if not isinstance(authorization_url, str): + raise AdapterFailure("mcpServer/oauth/login returned no authorization URL") + + _drive_headless_authorization( + authorization_url, + timeout_seconds=timeout_seconds, + ) + event = client.wait_for_notification( + AUTH_COMPLETION_METHOD, + predicate=lambda params: params.get("name") == TEST_SERVER_NAME, + after_event_index=event_index, + ) + params_value = event.get("params") + if not isinstance(params_value, dict): + raise AdapterFailure("OAuth completion notification did not contain params") + success = params_value.get("success") is True + error = params_value.get("error") + return success, str(error) if error is not None else None + + +def _reload_mcp(client: AppServerClient) -> None: + _result_or_raise( + client.request("config/mcpServer/reload", None), + "config/mcpServer/reload", + ) + + +def _auth_inventory(client: AppServerClient) -> dict[str, object]: + inventory = _result_or_raise( + client.request("mcpServerStatus/list", {"detail": "full"}), + "mcpServerStatus/list", + ) + _server_entry(inventory) + return inventory + + +def _auth_tool_call(client: AppServerClient, workspace: Path) -> None: + thread_id = _thread_id(client, workspace) + _call( + client, + thread_id=thread_id, + tool="test-tool", + arguments={}, + ) + + +def _login_reload_and_call( + client: AppServerClient, + *, + workspace: Path, + timeout_seconds: float, + scopes: Sequence[str] | None = None, +) -> None: + success, error = _oauth_login( + client, + scopes=scopes, + timeout_seconds=timeout_seconds, + ) + if not success: + raise AdapterFailure(f"OAuth login failed: {error or 'unknown error'}") + _reload_mcp(client) + _auth_inventory(client) + _auth_tool_call(client, workspace) + + +def _exercise_auth_scenario( + client: AppServerClient, + *, + scenario: str, + workspace: Path, + timeout_seconds: float, + require_automatic_auth: bool = False, +) -> str: + if scenario in EXPECTED_AUTH_REJECTION_SCENARIOS: + success, error = _oauth_login( + client, + scopes=None, + timeout_seconds=timeout_seconds, + ) + if success: + raise AdapterFailure( + "OAuth flow unexpectedly accepted authorization metadata that must be rejected" + ) + return f"rejected invalid authorization flow: {error or 'request rejected'}" + + if scenario == "auth/scope-step-up": + success, error = _oauth_login( + client, + scopes=None, + timeout_seconds=timeout_seconds, + ) + if not success: + raise AdapterFailure( + f"initial OAuth login failed: {error or 'unknown error'}" + ) + _reload_mcp(client) + _auth_inventory(client) + if require_automatic_auth: + _auth_tool_call(client, workspace) + return "Codex automatically recovered from the challenged OAuth scope" + try: + _auth_tool_call(client, workspace) + except AdapterFailure: + pass + else: + raise AdapterFailure( + "scope-step-up tool call did not request additional scope" + ) + + # The resource server challenges with only the missing scope. The Rust + # client must union it with the previously granted scope. + _login_reload_and_call( + client, + workspace=workspace, + timeout_seconds=timeout_seconds, + scopes=("mcp:write",), + ) + return "completed initial and scope-upgrade authorization flows" + + if scenario == "auth/scope-retry-limit": + if require_automatic_auth: + success, error = _oauth_login( + client, + scopes=None, + timeout_seconds=timeout_seconds, + ) + if not success: + raise AdapterFailure( + f"initial OAuth login failed: {error or 'unknown error'}" + ) + _reload_mcp(client) + _auth_inventory(client) + try: + _auth_tool_call(client, workspace) + except AdapterFailure: + return "observed Codex's production OAuth retry-limit behavior" + raise AdapterFailure( + "retry-limit scenario unexpectedly completed the tool call" + ) + for attempt in range(3): + success, error = _oauth_login( + client, + scopes=None if attempt == 0 else ("mcp:write",), + timeout_seconds=timeout_seconds, + ) + if not success: + raise AdapterFailure(f"OAuth retry failed: {error or 'unknown error'}") + _reload_mcp(client) + _auth_inventory(client) + try: + _auth_tool_call(client, workspace) + except AdapterFailure: + continue + raise AdapterFailure( + "retry-limit scenario unexpectedly completed the tool call" + ) + return "stopped after three unsuccessful authorization attempts" + + if scenario == "auth/authorization-server-migration": + success, error = _oauth_login( + client, + scopes=None, + timeout_seconds=timeout_seconds, + ) + if not success: + raise AdapterFailure( + f"initial OAuth login failed: {error or 'unknown error'}" + ) + _reload_mcp(client) + _auth_inventory(client) + if require_automatic_auth: + _auth_tool_call(client, workspace) + return ( + "Codex automatically registered with the migrated authorization server" + ) + try: + _auth_tool_call(client, workspace) + except AdapterFailure: + pass + else: + raise AdapterFailure("migration scenario did not require re-authorization") + _login_reload_and_call( + client, + workspace=workspace, + timeout_seconds=timeout_seconds, + ) + return ( + "re-authorized after the protected resource changed authorization servers" + ) + + _login_reload_and_call( + client, + workspace=workspace, + timeout_seconds=timeout_seconds, + ) + return "completed OAuth login, authenticated discovery, and tool call" + + +def _exercise_scenario( + client: AppServerClient, + *, + scenario: str, + workspace: Path, + inventory: Mapping[str, object], +) -> str: + thread_id = _thread_id(client, workspace) + + if scenario == "tools_call": + _call( + client, + thread_id=thread_id, + tool="add_numbers", + arguments={"a": 2, "b": 3}, + ) + return "called add_numbers" + + if scenario == "elicitation-sep1034-client-defaults": + _call( + client, + thread_id=thread_id, + tool="test_client_elicitation_defaults", + arguments={}, + ) + return "completed legacy elicitation with omitted optional fields" + + if scenario == "sse-retry": + _call( + client, + thread_id=thread_id, + tool="test_reconnection", + arguments={}, + ) + return "completed the SSE reconnection tool call" + + if scenario == "sep-2322-client-request-state": + for tool in ( + "test_mrtr_unrelated", + "test_mrtr_no_result_type", + "test_mrtr_echo_state", + "test_mrtr_no_state", + ): + _call(client, thread_id=thread_id, tool=tool, arguments={}) + return "completed all four MRTR flows" + + if scenario == "http-standard-headers": + _call( + client, + thread_id=thread_id, + tool="test_headers", + arguments={}, + ) + entry = _server_entry(inventory) + resources = entry.get("resources") + if not isinstance(resources, list) or not resources: + raise AdapterFailure("standard-header scenario exposed no resources") + first = resources[0] + if not isinstance(first, dict) or not isinstance(first.get("uri"), str): + raise AdapterFailure("standard-header scenario resource had no URI") + _result_or_raise( + client.request( + "mcpServer/resource/read", + { + "threadId": thread_id, + "server": TEST_SERVER_NAME, + "uri": first["uri"], + }, + ), + "mcpServer/resource/read", + ) + return "called a tool and read a resource" + + if scenario == "http-custom-headers": + for tool, arguments in _context_tool_calls(): + _call( + client, + thread_id=thread_id, + tool=tool, + arguments=arguments, + ) + return "called both custom-header tools with official values" + + if scenario == "http-invalid-tool-headers": + _call( + client, + thread_id=thread_id, + tool="valid_tool", + arguments={"region": "us-west1"}, + ) + return "called the valid tool after filtering malformed definitions" + + if scenario in { + "initialize", + "request-metadata", + "json-schema-ref-no-deref", + }: + # Discovery performed by mcpServerStatus/list is the behavior these + # scenarios observe. Starting a thread also exercises the initialized + # server through the same public app-server interface as the other + # scenarios. + return "completed MCP discovery" + + raise AdapterFailure(f"unsupported official scenario: {scenario}") + + +def run_adapter(server_url: str) -> dict[str, object]: + codex_binary = _required_path("CODEX_CONFORMANCE_BINARY") + codex_home = _required_path("CODEX_CONFORMANCE_HOME") + scenario = _required_env("MCP_CONFORMANCE_SCENARIO") + protocol_version = _required_env("MCP_CONFORMANCE_PROTOCOL_VERSION") + timeout_seconds = float(os.environ.get("CODEX_CONFORMANCE_TIMEOUT", "30")) + enable_modern_feature = ( + os.environ.get("CODEX_CONFORMANCE_ENABLE_MODERN_FEATURE", "1") != "0" + ) + require_automatic_auth = ( + os.environ.get("CODEX_CONFORMANCE_REQUIRE_AUTOMATIC_AUTH", "0") == "1" + ) + context = _conformance_context() + + codex_home.mkdir(parents=True, exist_ok=True) + workspace = codex_home / "workspace" + workspace.mkdir() + env = _isolated_environment(codex_home) + # Scenario context can contain ephemeral OAuth client secrets. The adapter + # consumes it directly and does not expose the full blob to Codex. + env.pop("MCP_CONFORMANCE_CONTEXT", None) + steps: list[Step] = [] + registered = False + error: str | None = None + + try: + if scenario.startswith("auth/"): + config_path = codex_home / "config.toml" + config_path.write_text( + 'mcp_oauth_credentials_store = "file"\n', + encoding="utf-8", + ) + steps.append( + Step( + "oauth_store_configuration", + True, + "configured the isolated file OAuth credential store", + ) + ) + + if protocol_version == MODERN_VERSION and enable_modern_feature: + feature = _run_command( + [ + str(codex_binary), + "features", + "enable", + "mcp_2026_07_28", + ], + env=env, + cwd=workspace, + timeout_seconds=timeout_seconds, + ) + feature_configured = feature.returncode == 0 + steps.append( + Step( + "modern_feature_configuration", + feature_configured, + "configured mcp_2026_07_28 before MCP startup" + if feature_configured + else _command_detail(feature), + ) + ) + if not feature_configured: + raise AdapterFailure("could not configure the modern MCP feature") + + oauth_client_id = _oauth_client_id( + scenario, + context, + require_production_client_identity=require_automatic_auth, + ) + oauth_client_secret = context.get("client_secret") + oauth_client_secret_env_var = None + if ( + scenario == "auth/pre-registration" + and isinstance(oauth_client_secret, str) + and oauth_client_secret + ): + env[PRE_REGISTERED_CLIENT_SECRET_ENV_VAR] = oauth_client_secret + oauth_client_secret_env_var = PRE_REGISTERED_CLIENT_SECRET_ENV_VAR + if scenario.startswith("auth/"): + _write_auth_registration( + codex_home / "config.toml", + server_url=server_url, + oauth_client_id=oauth_client_id, + oauth_client_secret_env_var=oauth_client_secret_env_var, + ) + registered = True + steps.append( + Step( + "mcp_registration", + True, + "wrote isolated registration without triggering CLI auto-login", + ) + ) + else: + add = _run_command( + [ + str(codex_binary), + "mcp", + "add", + TEST_SERVER_NAME, + "--url", + server_url, + ], + env=env, + cwd=workspace, + timeout_seconds=timeout_seconds, + ) + registered = add.returncode == 0 + steps.append(Step("mcp_add", registered, _command_detail(add))) + if not registered: + raise AdapterFailure("codex mcp add failed") + + get = _run_command( + [ + str(codex_binary), + "mcp", + "get", + TEST_SERVER_NAME, + "--json", + ], + env=env, + cwd=workspace, + timeout_seconds=timeout_seconds, + ) + registration_ok = False + if get.returncode == 0: + try: + decoded = json.loads(get.stdout) + transport = ( + decoded.get("transport") if isinstance(decoded, dict) else None + ) + registration_ok = ( + isinstance(transport, dict) + and transport.get("type") == "streamable_http" + and transport.get("url") == server_url + ) + except json.JSONDecodeError: + pass + steps.append( + Step( + "mcp_get", + registration_ok, + "registered official scenario URL" + if registration_ok + else _command_detail(get), + ) + ) + if not registration_ok: + raise AdapterFailure("Codex registration did not preserve the scenario URL") + + with AppServerClient( + codex_binary, + env=env, + cwd=workspace, + timeout_seconds=timeout_seconds, + elicitation_content=lambda params: _elicitation_content(scenario, params), + ) as client: + initialize = _result_or_raise( + client.request( + "initialize", + { + "clientInfo": { + "name": "official-mcp-conformance-adapter", + "title": "Official MCP conformance adapter", + "version": "1.0.0", + }, + "capabilities": { + "experimentalApi": True, + "requestAttestation": False, + "mcpServerOpenaiFormElicitation": True, + }, + }, + ), + "app-server initialize", + ) + steps.append(Step("app_server_initialize", bool(initialize), "initialized")) + client.notify("initialized") + + if protocol_version == MODERN_VERSION and enable_modern_feature: + feature = _result_or_raise( + client.request( + "experimentalFeature/enablement/set", + {"enablement": {"mcp_2026_07_28": True}}, + ), + "experimentalFeature/enablement/set", + ) + enabled = ( + isinstance(feature.get("enablement"), dict) + and feature["enablement"].get("mcp_2026_07_28") is True + ) + steps.append( + Step( + "modern_feature_enablement", + enabled, + "enabled mcp_2026_07_28" + if enabled + else f"unexpected response: {feature!r}", + ) + ) + if not enabled: + raise AdapterFailure("could not enable the modern MCP feature") + + if scenario.startswith("auth/"): + detail = _exercise_auth_scenario( + client, + scenario=scenario, + workspace=workspace, + timeout_seconds=timeout_seconds, + require_automatic_auth=require_automatic_auth, + ) + if scenario == "auth/pre-registration" and isinstance( + oauth_client_secret, + str, + ): + _validate_oauth_secret_not_persisted( + codex_home, oauth_client_secret + ) + steps.append( + Step( + "oauth_client_secret_not_persisted", + True, + "environment-provided confidential-client secret was not " + "written to configuration or file-backed credentials", + ) + ) + steps.append(Step("authentication", True, detail)) + else: + inventory = _result_or_raise( + client.request("mcpServerStatus/list", {"detail": "full"}), + "mcpServerStatus/list", + ) + _server_entry(inventory) + steps.append(Step("inventory", True, "official server discovered")) + detail = _exercise_scenario( + client, + scenario=scenario, + workspace=workspace, + inventory=inventory, + ) + steps.append(Step("scenario", True, detail)) + except (AdapterFailure, AppServerError, OSError, ValueError) as exc: + error = str(exc) + if not steps or steps[-1].success: + steps.append(Step("scenario", False, error)) + finally: + if registered: + remove = _run_command( + [str(codex_binary), "mcp", "remove", TEST_SERVER_NAME], + env=env, + cwd=workspace, + timeout_seconds=timeout_seconds, + ) + steps.append( + Step("mcp_remove", remove.returncode == 0, _command_detail(remove)) + ) + + success = bool(steps) and all(step.success for step in steps) + return { + "success": success, + "scenario": scenario, + "protocolVersion": protocol_version, + "automaticAuthRequired": require_automatic_auth, + "serverUrl": server_url, + "steps": [asdict(step) for step in steps], + "error": error, + } + + +def main(argv: Sequence[str] | None = None) -> int: + values = list(sys.argv[1:] if argv is None else argv) + report_path_value = os.environ.get("CODEX_CONFORMANCE_ADAPTER_REPORT") + report: dict[str, object] + try: + if len(values) != 1: + raise AdapterFailure("expected exactly one official scenario server URL") + report = run_adapter(values[0]) + except Exception as exc: # Preserve a diagnostic artifact for the parent. + report = { + "success": False, + "error": str(exc), + "traceback": traceback.format_exc(limit=20), + "steps": [], + } + + if report_path_value: + report_path = Path(report_path_value) + report_path.parent.mkdir(parents=True, exist_ok=True) + report_path.write_text( + json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + print(json.dumps(report, ensure_ascii=False, sort_keys=True)) + return 0 if report.get("success") is True else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/mcp_conformance/official_conformance.py b/scripts/mcp_conformance/official_conformance.py new file mode 100644 index 0000000000..a6cee00a17 --- /dev/null +++ b/scripts/mcp_conformance/official_conformance.py @@ -0,0 +1,518 @@ +"""Driver for the upstream MCP client conformance suite. + +The upstream suite owns the localhost HTTP server and the wire-level checks. +This module deliberately invokes one scenario at a time because the upstream +parallel suite path does not currently propagate the client adapter result. +""" + +import json +import os +import re +import shlex +import signal +import subprocess +import sys +import tempfile +import time +from dataclasses import dataclass, field +from pathlib import Path +from typing import Mapping, Sequence + +SHIPPING_LEGACY_VERSION = "2025-06-18" +LEGACY_VERSION = "2025-11-25" +MODERN_VERSION = "2026-07-28" + +# Reviewed on 2026-07-29. Pinning a commit instead of an npm prerelease keeps +# the scenario definitions and result semantics reproducible. +OFFICIAL_CONFORMANCE_GIT_REF = "49103de6ed70804e940637bf3e9e29e4a3f54e64" +OFFICIAL_CONFORMANCE_REPOSITORY = "modelcontextprotocol/conformance" + +OFFICIAL_NON_AUTH_SCENARIOS: Mapping[str, tuple[str, ...]] = { + SHIPPING_LEGACY_VERSION: ( + "initialize", + "tools_call", + ), + LEGACY_VERSION: ( + "initialize", + "tools_call", + "elicitation-sep1034-client-defaults", + "sse-retry", + ), + MODERN_VERSION: ( + "tools_call", + "request-metadata", + "sep-2322-client-request-state", + "http-standard-headers", + "http-custom-headers", + "http-invalid-tool-headers", + "json-schema-ref-no-deref", + ), +} + +OFFICIAL_AUTH_SCENARIOS: Mapping[str, tuple[str, ...]] = { + SHIPPING_LEGACY_VERSION: ( + "auth/token-endpoint-auth-basic", + "auth/token-endpoint-auth-post", + "auth/token-endpoint-auth-none", + ), + LEGACY_VERSION: ( + "auth/metadata-default", + "auth/metadata-var1", + "auth/metadata-var2", + "auth/metadata-var3", + "auth/basic-cimd", + "auth/scope-from-www-authenticate", + "auth/scope-from-scopes-supported", + "auth/scope-omitted-when-undefined", + "auth/scope-step-up", + "auth/scope-retry-limit", + "auth/token-endpoint-auth-basic", + "auth/token-endpoint-auth-post", + "auth/token-endpoint-auth-none", + "auth/pre-registration", + ), + MODERN_VERSION: ( + "auth/metadata-default", + "auth/metadata-var1", + "auth/metadata-var2", + "auth/metadata-var3", + "auth/basic-cimd", + "auth/scope-from-www-authenticate", + "auth/scope-from-scopes-supported", + "auth/scope-omitted-when-undefined", + "auth/scope-step-up", + "auth/scope-retry-limit", + "auth/token-endpoint-auth-basic", + "auth/token-endpoint-auth-post", + "auth/token-endpoint-auth-none", + "auth/pre-registration", + "auth/resource-mismatch", + "auth/offline-access-scope", + "auth/offline-access-not-supported", + "auth/authorization-server-migration", + "auth/iss-supported", + "auth/iss-not-advertised", + "auth/iss-supported-missing", + "auth/iss-wrong-issuer", + "auth/iss-unexpected", + "auth/iss-normalized", + "auth/metadata-issuer-mismatch", + ), +} + +# These scenarios cover separately negotiated protocol extensions rather than +# either dated release. Keep them visible for future adapters, but do not mix +# them into versioned release conformance percentages. +OFFICIAL_AUTH_EXTENSION_SCENARIOS: tuple[str, ...] = ( + "auth/client-credentials-jwt", + "auth/client-credentials-basic", + "auth/enterprise-managed-authorization", + "auth/dpop", + "auth/dpop-nonce", + "auth/wif-jwt-bearer", +) + +_SENSITIVE_JSON_FIELD = re.compile( + r'("(?:client_secret|private_key_pem|valid_jwt|wrong_audience_jwt|' + r'expired_jwt|idp_id_token)"\s*:\s*)"(?:\\.|[^"\\])*"' +) +_SENSITIVE_ESCAPED_JSON_FIELD = re.compile( + r'(\\"(?:client_secret|private_key_pem|valid_jwt|wrong_audience_jwt|' + r'expired_jwt|idp_id_token)\\"\s*:\s*)\\"(?:\\\\.|[^"\\])*\\"' +) +_SENSITIVE_URL_PARAMETER = re.compile( + r"([?&](?:code|state|code_challenge|code_verifier|access_token|" + r"""refresh_token|client_secret)=)[^&\s"'\\<>]+""", + re.IGNORECASE, +) +_BEARER_TOKEN = re.compile(r"(\bBearer\s+)[A-Za-z0-9._~+/=-]+", re.IGNORECASE) + + +def redact_sensitive_text(value: str) -> str: + value = _SENSITIVE_JSON_FIELD.sub(r'\1"[REDACTED]"', value) + value = _SENSITIVE_ESCAPED_JSON_FIELD.sub(r'\1\\"[REDACTED]\\"', value) + value = _SENSITIVE_URL_PARAMETER.sub(r"\1[REDACTED]", value) + return _BEARER_TOKEN.sub(r"\1[REDACTED]", value) + + +def _scrub_retained_artifacts(scenario_dir: Path) -> None: + for name in ("stdout.txt", "stderr.txt"): + for path in scenario_dir.rglob(name): + try: + original = path.read_text(encoding="utf-8") + redacted = redact_sensitive_text(original) + if redacted != original: + path.write_text(redacted, encoding="utf-8") + except OSError: + continue + for path in scenario_dir.rglob(".credentials.json"): + try: + path.unlink(missing_ok=True) + except OSError: + continue + + +@dataclass(frozen=True) +class OfficialCheck: + scenario: str + check_id: str + name: str + status: str + description: str + error_message: str | None = None + + +@dataclass +class OfficialScenarioResult: + scenario: str + success: bool + checks: list[OfficialCheck] = field(default_factory=list) + adapter_success: bool = False + adapter_detail: str = "" + runner_detail: str = "" + + +def default_conformance_command() -> list[str]: + return [ + "npx", + "--yes", + (f"github:{OFFICIAL_CONFORMANCE_REPOSITORY}#{OFFICIAL_CONFORMANCE_GIT_REF}"), + ] + + +def scenarios_for_mode( + mode: str, + requested: Sequence[str] | None = None, + *, + include_auth: bool = True, +) -> tuple[str, ...]: + non_auth = OFFICIAL_NON_AUTH_SCENARIOS.get(mode) + auth = OFFICIAL_AUTH_SCENARIOS.get(mode) + if non_auth is None or auth is None: + raise ValueError(f"unsupported MCP protocol version: {mode}") + available = non_auth + (auth if include_auth else ()) + if not requested: + return available + all_scenarios = { + scenario + for scenario_map in ( + OFFICIAL_NON_AUTH_SCENARIOS, + OFFICIAL_AUTH_SCENARIOS, + ) + for mode_scenarios in scenario_map.values() + for scenario in mode_scenarios + } + unknown = sorted(set(requested) - all_scenarios) + if unknown: + raise ValueError( + "scenarios are not part of the pinned versioned client suite: " + + ", ".join(unknown) + ) + disabled = sorted( + set(requested) + & { + scenario + for mode_scenarios in OFFICIAL_AUTH_SCENARIOS.values() + for scenario in mode_scenarios + } + if not include_auth + else () + ) + if disabled: + raise ValueError( + "authentication scenarios require auth coverage to be enabled: " + + ", ".join(disabled) + ) + requested_set = set(requested) + selected = tuple(scenario for scenario in available if scenario in requested_set) + if not selected: + raise ValueError( + f"requested scenarios are unavailable for MCP protocol version {mode}: " + + ", ".join(sorted(requested_set)) + ) + return selected + + +def _safe_scenario_name(scenario: str) -> str: + return re.sub(r"[^A-Za-z0-9_.-]+", "-", scenario) + + +def _make_adapter_launcher( + adapter_script: Path, + *, + require_automatic_auth: bool = False, +) -> Path: + # The official runner currently splits --command on spaces before spawning + # it. Put a one-token launcher in the system temp directory so arbitrary + # checkout paths (including paths with spaces) remain supported. + launcher_dir = Path(tempfile.mkdtemp(prefix="mcp-conformance-adapter-")) + automatic_auth = "1" if require_automatic_auth else "0" + if sys.platform == "win32": + launcher = launcher_dir / "client.cmd" + adapter_command = subprocess.list2cmdline([sys.executable, str(adapter_script)]) + launcher.write_text( + "@echo off\n" + f'set "CODEX_CONFORMANCE_REQUIRE_AUTOMATIC_AUTH={automatic_auth}"\n' + f"{adapter_command} %*\n", + encoding="utf-8", + ) + else: + launcher = launcher_dir / "client" + launcher.write_text( + "#!/bin/sh\n" + f"export CODEX_CONFORMANCE_REQUIRE_AUTOMATIC_AUTH={automatic_auth}\n" + f'exec {shlex.quote(sys.executable)} {shlex.quote(str(adapter_script))} "$@"\n', + encoding="utf-8", + ) + launcher.chmod(0o755) + return launcher + + +def _terminate_process_group( + process: subprocess.Popen[str], + *, + force: bool = False, +) -> None: + if sys.platform == "win32": + command = ["taskkill", "/T", "/PID", str(process.pid)] + if force: + command.insert(1, "/F") + try: + result = subprocess.run( + command, + check=False, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + if result.returncode == 0: + return + except OSError: + pass + try: + if force: + process.kill() + else: + process.terminate() + except (OSError, ProcessLookupError): + pass + return + + try: + os.killpg(process.pid, signal.SIGKILL if force else signal.SIGTERM) + except (OSError, ProcessLookupError): + pass + + +def _load_checks(result_dir: Path, scenario: str) -> list[OfficialCheck]: + check_files = sorted(result_dir.rglob("checks.json")) + if len(check_files) != 1: + return [] + try: + decoded = json.loads(check_files[0].read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return [] + if not isinstance(decoded, list): + return [] + + checks: list[OfficialCheck] = [] + for index, raw in enumerate(decoded, start=1): + if not isinstance(raw, dict): + continue + check_id = str(raw.get("id") or f"unnamed-{index}") + name = str(raw.get("name") or check_id) + status = str(raw.get("status") or "FAILURE").upper() + # INFO entries are the suite's HTTP trace, not conformance assertions. + # They remain available in the retained upstream checks.json artifact. + if status == "INFO": + continue + checks.append( + OfficialCheck( + scenario=scenario, + check_id=check_id, + name=name, + status=status, + description=str(raw.get("description") or name), + error_message=( + redact_sensitive_text(str(raw["errorMessage"])) + if raw.get("errorMessage") is not None + else None + ), + ) + ) + return checks + + +def _load_adapter_report(report_path: Path) -> tuple[bool, str]: + try: + decoded = json.loads(report_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + return False, f"Codex adapter did not write a valid report: {exc}" + if not isinstance(decoded, dict): + return False, "Codex adapter report was not an object" + success = decoded.get("success") is True + steps = decoded.get("steps") + failed_steps: list[str] = [] + if isinstance(steps, list): + for step in steps: + if isinstance(step, dict) and step.get("success") is not True: + failed_steps.append( + f"{step.get('name')}: {step.get('detail', 'failed')}" + ) + if success: + return True, "Codex adapter completed the scenario" + if failed_steps: + return False, redact_sensitive_text("; ".join(failed_steps))[-4_000:] + return False, redact_sensitive_text( + str(decoded.get("error") or "Codex adapter failed") + )[-4_000:] + + +def _run_scenario( + *, + conformance_command: Sequence[str], + adapter_launcher: Path, + codex_binary: Path, + mode: str, + scenario: str, + output_dir: Path, + timeout_seconds: float, + base_env: Mapping[str, str], + process_grace_seconds: float = 15, +) -> OfficialScenarioResult: + scenario_dir = output_dir / _safe_scenario_name(scenario) + scenario_dir.mkdir(parents=True, exist_ok=True) + adapter_report = scenario_dir / "codex-adapter.json" + adapter_home = scenario_dir / "codex-home" + adapter_home.mkdir() + + env = dict(base_env) + env.update( + { + "CODEX_CONFORMANCE_BINARY": str(codex_binary), + "CODEX_CONFORMANCE_HOME": str(adapter_home), + "CODEX_CONFORMANCE_ADAPTER_REPORT": str(adapter_report), + } + ) + command = [ + *conformance_command, + "client", + "--command", + str(adapter_launcher), + "--scenario", + scenario, + "--spec-version", + mode, + "--timeout", + str(round(timeout_seconds * 1_000)), + "--output-dir", + str(scenario_dir), + ] + process: subprocess.Popen[str] | None = None + try: + process = subprocess.Popen( + command, + env=env, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + start_new_session=sys.platform != "win32", + creationflags=( + getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0) + if sys.platform == "win32" + else 0 + ), + ) + try: + process_timeout = timeout_seconds + process_grace_seconds + stdout, stderr = process.communicate(timeout=process_timeout) + runner_detail = redact_sensitive_text((stderr or stdout).strip())[-8_000:] + returncode = process.returncode + except subprocess.TimeoutExpired: + _terminate_process_group(process) + try: + stdout, stderr = process.communicate(timeout=3) + except subprocess.TimeoutExpired: + _terminate_process_group(process, force=True) + stdout, stderr = process.communicate() + runner_detail = redact_sensitive_text( + (stderr or stdout) + + f"\nofficial scenario timed out after {process_timeout:g}s" + ).strip()[-8_000:] + returncode = 124 + except OSError as exc: + runner_detail = redact_sensitive_text(str(exc)) + returncode = 127 + + # A crashing upstream runner can exit before its still-running adapter has + # atomically written the diagnostic report. Give that report a short grace + # period so an upstream fixture crash is not mislabeled as an adapter + # failure, then terminate any orphaned descendants in the runner's process + # group. + if returncode != 0 and not adapter_report.is_file(): + report_deadline = time.monotonic() + min(process_grace_seconds, 10) + while time.monotonic() < report_deadline and not adapter_report.is_file(): + time.sleep(0.05) + if returncode != 0 and process is not None: + _terminate_process_group(process) + + _scrub_retained_artifacts(scenario_dir) + checks = _load_checks(scenario_dir, scenario) + adapter_success, adapter_detail = _load_adapter_report(adapter_report) + official_failure = any(check.status in {"FAILURE", "WARNING"} for check in checks) + success = ( + returncode == 0 and bool(checks) and adapter_success and not official_failure + ) + if not checks: + runner_detail = ( + "official runner did not produce exactly one checks.json; " + runner_detail + ).strip() + elif returncode != 0 and not official_failure and adapter_success: + runner_detail = ( + f"official runner exited with code {returncode}; " + runner_detail + ).strip() + return OfficialScenarioResult( + scenario=scenario, + success=success, + checks=checks, + adapter_success=adapter_success, + adapter_detail=adapter_detail, + runner_detail=runner_detail, + ) + + +def run_official_mode( + *, + conformance_command: Sequence[str], + adapter_script: Path, + codex_binary: Path, + mode: str, + scenarios: Sequence[str], + output_dir: Path, + timeout_seconds: float, + base_env: Mapping[str, str], +) -> list[OfficialScenarioResult]: + launcher = _make_adapter_launcher( + adapter_script, + require_automatic_auth=base_env.get("CODEX_CONFORMANCE_REQUIRE_AUTOMATIC_AUTH") + == "1", + ) + try: + return [ + _run_scenario( + conformance_command=conformance_command, + adapter_launcher=launcher, + codex_binary=codex_binary, + mode=mode, + scenario=scenario, + output_dir=output_dir, + timeout_seconds=timeout_seconds, + base_env=base_env, + ) + for scenario in scenarios + ] + finally: + try: + launcher.unlink() + launcher.parent.rmdir() + except OSError: + pass diff --git a/scripts/mcp_conformance/regression-baseline-v1.json b/scripts/mcp_conformance/regression-baseline-v1.json new file mode 100644 index 0000000000..08f01ed1fd --- /dev/null +++ b/scripts/mcp_conformance/regression-baseline-v1.json @@ -0,0 +1,4208 @@ +{ + "automaticAuthRequired": false, + "baselineKind": "mcp-conformance-regression-baseline-v1", + "checks": { + "failing": [ + { + "check_id": "harness/auth/metadata-var3/codex-adapter", + "mode": "2025-11-25", + "scenario": "auth/metadata-var3", + "source": "harness", + "transport": "official-http" + }, + { + "check_id": "harness/auth/pre-registration/codex-adapter", + "mode": "2025-11-25", + "scenario": "auth/pre-registration", + "source": "harness", + "transport": "official-http" + }, + { + "check_id": "harness/auth/scope-from-www-authenticate/codex-adapter", + "mode": "2025-11-25", + "scenario": "auth/scope-from-www-authenticate", + "source": "harness", + "transport": "official-http" + }, + { + "check_id": "harness/auth/scope-step-up/codex-adapter", + "mode": "2025-11-25", + "scenario": "auth/scope-step-up", + "source": "harness", + "transport": "official-http" + }, + { + "check_id": "authorization-request", + "mode": "2025-11-25", + "scenario": "auth/metadata-var3", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "authorization-server-metadata", + "mode": "2025-11-25", + "scenario": "auth/metadata-var3", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "client-registration", + "mode": "2025-11-25", + "scenario": "auth/metadata-var3", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "token-request", + "mode": "2025-11-25", + "scenario": "auth/metadata-var3", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pre-registration-auth", + "mode": "2025-11-25", + "scenario": "auth/pre-registration", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "scope-from-www-authenticate", + "mode": "2025-11-25", + "scenario": "auth/scope-from-www-authenticate", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "scope-step-up-initial", + "mode": "2025-11-25", + "scenario": "auth/scope-step-up", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "client-elicitation-sep1034-boolean-default", + "mode": "2025-11-25", + "scenario": "elicitation-sep1034-client-defaults", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "client-elicitation-sep1034-enum-default", + "mode": "2025-11-25", + "scenario": "elicitation-sep1034-client-defaults", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "client-elicitation-sep1034-integer-default", + "mode": "2025-11-25", + "scenario": "elicitation-sep1034-client-defaults", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "client-elicitation-sep1034-number-default", + "mode": "2025-11-25", + "scenario": "elicitation-sep1034-client-defaults", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "client-elicitation-sep1034-string-default", + "mode": "2025-11-25", + "scenario": "elicitation-sep1034-client-defaults", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "harness/auth/metadata-var3/codex-adapter", + "mode": "2026-07-28", + "scenario": "auth/metadata-var3", + "source": "harness", + "transport": "official-http" + }, + { + "check_id": "harness/auth/pre-registration/codex-adapter", + "mode": "2026-07-28", + "scenario": "auth/pre-registration", + "source": "harness", + "transport": "official-http" + }, + { + "check_id": "harness/auth/scope-from-www-authenticate/codex-adapter", + "mode": "2026-07-28", + "scenario": "auth/scope-from-www-authenticate", + "source": "harness", + "transport": "official-http" + }, + { + "check_id": "harness/auth/scope-step-up/codex-adapter", + "mode": "2026-07-28", + "scenario": "auth/scope-step-up", + "source": "harness", + "transport": "official-http" + }, + { + "check_id": "authorization-request", + "mode": "2026-07-28", + "scenario": "auth/metadata-var3", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "authorization-server-metadata", + "mode": "2026-07-28", + "scenario": "auth/metadata-var3", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "client-registration", + "mode": "2026-07-28", + "scenario": "auth/metadata-var3", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "token-request", + "mode": "2026-07-28", + "scenario": "auth/metadata-var3", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pre-registration-auth", + "mode": "2026-07-28", + "scenario": "auth/pre-registration", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "scope-from-www-authenticate", + "mode": "2026-07-28", + "scenario": "auth/scope-from-www-authenticate", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "scope-step-up-initial", + "mode": "2026-07-28", + "scenario": "auth/scope-step-up", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "sep-2350-scope-union-on-reauth", + "mode": "2026-07-28", + "scenario": "auth/scope-step-up", + "source": "official", + "transport": "official-http" + } + ], + "passing": [ + { + "check_id": "harness/auth/token-endpoint-auth-basic/codex-adapter", + "mode": "2025-06-18", + "scenario": "auth/token-endpoint-auth-basic", + "source": "harness", + "transport": "official-http" + }, + { + "check_id": "harness/auth/token-endpoint-auth-none/codex-adapter", + "mode": "2025-06-18", + "scenario": "auth/token-endpoint-auth-none", + "source": "harness", + "transport": "official-http" + }, + { + "check_id": "harness/auth/token-endpoint-auth-post/codex-adapter", + "mode": "2025-06-18", + "scenario": "auth/token-endpoint-auth-post", + "source": "harness", + "transport": "official-http" + }, + { + "check_id": "harness/initialize/codex-adapter", + "mode": "2025-06-18", + "scenario": "initialize", + "source": "harness", + "transport": "official-http" + }, + { + "check_id": "harness/tools_call/codex-adapter", + "mode": "2025-06-18", + "scenario": "tools_call", + "source": "harness", + "transport": "official-http" + }, + { + "check_id": "authorization-request", + "mode": "2025-06-18", + "scenario": "auth/token-endpoint-auth-basic", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "authorization-server-metadata", + "mode": "2025-06-18", + "scenario": "auth/token-endpoint-auth-basic", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "client-registration", + "mode": "2025-06-18", + "scenario": "auth/token-endpoint-auth-basic", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-code-challenge-sent", + "mode": "2025-06-18", + "scenario": "auth/token-endpoint-auth-basic", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-code-verifier-sent", + "mode": "2025-06-18", + "scenario": "auth/token-endpoint-auth-basic", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-s256-method-used", + "mode": "2025-06-18", + "scenario": "auth/token-endpoint-auth-basic", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-verifier-matches-challenge", + "mode": "2025-06-18", + "scenario": "auth/token-endpoint-auth-basic", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "prm-pathbased-requested", + "mode": "2025-06-18", + "scenario": "auth/token-endpoint-auth-basic", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "resource-parameter-consistency", + "mode": "2025-06-18", + "scenario": "auth/token-endpoint-auth-basic", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "resource-parameter-in-authorization", + "mode": "2025-06-18", + "scenario": "auth/token-endpoint-auth-basic", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "resource-parameter-in-token", + "mode": "2025-06-18", + "scenario": "auth/token-endpoint-auth-basic", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "resource-parameter-valid-uri", + "mode": "2025-06-18", + "scenario": "auth/token-endpoint-auth-basic", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "token-endpoint-auth-method", + "mode": "2025-06-18", + "scenario": "auth/token-endpoint-auth-basic", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "token-request", + "mode": "2025-06-18", + "scenario": "auth/token-endpoint-auth-basic", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "valid-bearer-token", + "mode": "2025-06-18", + "scenario": "auth/token-endpoint-auth-basic", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "authorization-request", + "mode": "2025-06-18", + "scenario": "auth/token-endpoint-auth-none", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "authorization-server-metadata", + "mode": "2025-06-18", + "scenario": "auth/token-endpoint-auth-none", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "client-registration", + "mode": "2025-06-18", + "scenario": "auth/token-endpoint-auth-none", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-code-challenge-sent", + "mode": "2025-06-18", + "scenario": "auth/token-endpoint-auth-none", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-code-verifier-sent", + "mode": "2025-06-18", + "scenario": "auth/token-endpoint-auth-none", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-s256-method-used", + "mode": "2025-06-18", + "scenario": "auth/token-endpoint-auth-none", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-verifier-matches-challenge", + "mode": "2025-06-18", + "scenario": "auth/token-endpoint-auth-none", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "prm-pathbased-requested", + "mode": "2025-06-18", + "scenario": "auth/token-endpoint-auth-none", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "resource-parameter-consistency", + "mode": "2025-06-18", + "scenario": "auth/token-endpoint-auth-none", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "resource-parameter-in-authorization", + "mode": "2025-06-18", + "scenario": "auth/token-endpoint-auth-none", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "resource-parameter-in-token", + "mode": "2025-06-18", + "scenario": "auth/token-endpoint-auth-none", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "resource-parameter-valid-uri", + "mode": "2025-06-18", + "scenario": "auth/token-endpoint-auth-none", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "token-endpoint-auth-method", + "mode": "2025-06-18", + "scenario": "auth/token-endpoint-auth-none", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "token-request", + "mode": "2025-06-18", + "scenario": "auth/token-endpoint-auth-none", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "valid-bearer-token", + "mode": "2025-06-18", + "scenario": "auth/token-endpoint-auth-none", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "authorization-request", + "mode": "2025-06-18", + "scenario": "auth/token-endpoint-auth-post", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "authorization-server-metadata", + "mode": "2025-06-18", + "scenario": "auth/token-endpoint-auth-post", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "client-registration", + "mode": "2025-06-18", + "scenario": "auth/token-endpoint-auth-post", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-code-challenge-sent", + "mode": "2025-06-18", + "scenario": "auth/token-endpoint-auth-post", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-code-verifier-sent", + "mode": "2025-06-18", + "scenario": "auth/token-endpoint-auth-post", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-s256-method-used", + "mode": "2025-06-18", + "scenario": "auth/token-endpoint-auth-post", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-verifier-matches-challenge", + "mode": "2025-06-18", + "scenario": "auth/token-endpoint-auth-post", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "prm-pathbased-requested", + "mode": "2025-06-18", + "scenario": "auth/token-endpoint-auth-post", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "resource-parameter-consistency", + "mode": "2025-06-18", + "scenario": "auth/token-endpoint-auth-post", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "resource-parameter-in-authorization", + "mode": "2025-06-18", + "scenario": "auth/token-endpoint-auth-post", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "resource-parameter-in-token", + "mode": "2025-06-18", + "scenario": "auth/token-endpoint-auth-post", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "resource-parameter-valid-uri", + "mode": "2025-06-18", + "scenario": "auth/token-endpoint-auth-post", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "token-endpoint-auth-method", + "mode": "2025-06-18", + "scenario": "auth/token-endpoint-auth-post", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "token-request", + "mode": "2025-06-18", + "scenario": "auth/token-endpoint-auth-post", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "valid-bearer-token", + "mode": "2025-06-18", + "scenario": "auth/token-endpoint-auth-post", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "mcp-client-initialization", + "mode": "2025-06-18", + "scenario": "initialize", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "tool-add-numbers", + "mode": "2025-06-18", + "scenario": "tools_call", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "wire-schema-valid", + "mode": "2025-06-18", + "scenario": "tools_call", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "app_server_initialize", + "mode": "2025-06-18", + "scenario": "", + "source": "supplemental", + "transport": "stdio" + }, + { + "check_id": "echo_tool", + "mode": "2025-06-18", + "scenario": "", + "source": "supplemental", + "transport": "stdio" + }, + { + "check_id": "ephemeral_thread", + "mode": "2025-06-18", + "scenario": "", + "source": "supplemental", + "transport": "stdio" + }, + { + "check_id": "inventory", + "mode": "2025-06-18", + "scenario": "", + "source": "supplemental", + "transport": "stdio" + }, + { + "check_id": "isolated_config_cleanup", + "mode": "2025-06-18", + "scenario": "", + "source": "supplemental", + "transport": "stdio" + }, + { + "check_id": "mcp_add", + "mode": "2025-06-18", + "scenario": "", + "source": "supplemental", + "transport": "stdio" + }, + { + "check_id": "mcp_get", + "mode": "2025-06-18", + "scenario": "", + "source": "supplemental", + "transport": "stdio" + }, + { + "check_id": "mcp_remove", + "mode": "2025-06-18", + "scenario": "", + "source": "supplemental", + "transport": "stdio" + }, + { + "check_id": "unicode_resource_read", + "mode": "2025-06-18", + "scenario": "", + "source": "supplemental", + "transport": "stdio" + }, + { + "check_id": "harness/auth/basic-cimd/codex-adapter", + "mode": "2025-11-25", + "scenario": "auth/basic-cimd", + "source": "harness", + "transport": "official-http" + }, + { + "check_id": "harness/auth/metadata-default/codex-adapter", + "mode": "2025-11-25", + "scenario": "auth/metadata-default", + "source": "harness", + "transport": "official-http" + }, + { + "check_id": "harness/auth/metadata-var1/codex-adapter", + "mode": "2025-11-25", + "scenario": "auth/metadata-var1", + "source": "harness", + "transport": "official-http" + }, + { + "check_id": "harness/auth/metadata-var2/codex-adapter", + "mode": "2025-11-25", + "scenario": "auth/metadata-var2", + "source": "harness", + "transport": "official-http" + }, + { + "check_id": "harness/auth/scope-from-scopes-supported/codex-adapter", + "mode": "2025-11-25", + "scenario": "auth/scope-from-scopes-supported", + "source": "harness", + "transport": "official-http" + }, + { + "check_id": "harness/auth/scope-omitted-when-undefined/codex-adapter", + "mode": "2025-11-25", + "scenario": "auth/scope-omitted-when-undefined", + "source": "harness", + "transport": "official-http" + }, + { + "check_id": "harness/auth/scope-retry-limit/codex-adapter", + "mode": "2025-11-25", + "scenario": "auth/scope-retry-limit", + "source": "harness", + "transport": "official-http" + }, + { + "check_id": "harness/auth/token-endpoint-auth-basic/codex-adapter", + "mode": "2025-11-25", + "scenario": "auth/token-endpoint-auth-basic", + "source": "harness", + "transport": "official-http" + }, + { + "check_id": "harness/auth/token-endpoint-auth-none/codex-adapter", + "mode": "2025-11-25", + "scenario": "auth/token-endpoint-auth-none", + "source": "harness", + "transport": "official-http" + }, + { + "check_id": "harness/auth/token-endpoint-auth-post/codex-adapter", + "mode": "2025-11-25", + "scenario": "auth/token-endpoint-auth-post", + "source": "harness", + "transport": "official-http" + }, + { + "check_id": "harness/elicitation-sep1034-client-defaults/codex-adapter", + "mode": "2025-11-25", + "scenario": "elicitation-sep1034-client-defaults", + "source": "harness", + "transport": "official-http" + }, + { + "check_id": "harness/initialize/codex-adapter", + "mode": "2025-11-25", + "scenario": "initialize", + "source": "harness", + "transport": "official-http" + }, + { + "check_id": "harness/sse-retry/codex-adapter", + "mode": "2025-11-25", + "scenario": "sse-retry", + "source": "harness", + "transport": "official-http" + }, + { + "check_id": "harness/tools_call/codex-adapter", + "mode": "2025-11-25", + "scenario": "tools_call", + "source": "harness", + "transport": "official-http" + }, + { + "check_id": "authorization-request", + "mode": "2025-11-25", + "scenario": "auth/basic-cimd", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "authorization-server-metadata", + "mode": "2025-11-25", + "scenario": "auth/basic-cimd", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "cimd-client-id-used", + "mode": "2025-11-25", + "scenario": "auth/basic-cimd", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-code-challenge-sent", + "mode": "2025-11-25", + "scenario": "auth/basic-cimd", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-code-verifier-sent", + "mode": "2025-11-25", + "scenario": "auth/basic-cimd", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-s256-method-used", + "mode": "2025-11-25", + "scenario": "auth/basic-cimd", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-verifier-matches-challenge", + "mode": "2025-11-25", + "scenario": "auth/basic-cimd", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "prm-pathbased-requested", + "mode": "2025-11-25", + "scenario": "auth/basic-cimd", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "token-request", + "mode": "2025-11-25", + "scenario": "auth/basic-cimd", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "valid-bearer-token", + "mode": "2025-11-25", + "scenario": "auth/basic-cimd", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "authorization-request", + "mode": "2025-11-25", + "scenario": "auth/metadata-default", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "authorization-server-metadata", + "mode": "2025-11-25", + "scenario": "auth/metadata-default", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "client-registration", + "mode": "2025-11-25", + "scenario": "auth/metadata-default", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-code-challenge-sent", + "mode": "2025-11-25", + "scenario": "auth/metadata-default", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-code-verifier-sent", + "mode": "2025-11-25", + "scenario": "auth/metadata-default", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-s256-method-used", + "mode": "2025-11-25", + "scenario": "auth/metadata-default", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-verifier-matches-challenge", + "mode": "2025-11-25", + "scenario": "auth/metadata-default", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "prm-pathbased-requested", + "mode": "2025-11-25", + "scenario": "auth/metadata-default", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "token-request", + "mode": "2025-11-25", + "scenario": "auth/metadata-default", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "valid-bearer-token", + "mode": "2025-11-25", + "scenario": "auth/metadata-default", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "authorization-request", + "mode": "2025-11-25", + "scenario": "auth/metadata-var1", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "authorization-server-metadata", + "mode": "2025-11-25", + "scenario": "auth/metadata-var1", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "client-registration", + "mode": "2025-11-25", + "scenario": "auth/metadata-var1", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-code-challenge-sent", + "mode": "2025-11-25", + "scenario": "auth/metadata-var1", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-code-verifier-sent", + "mode": "2025-11-25", + "scenario": "auth/metadata-var1", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-s256-method-used", + "mode": "2025-11-25", + "scenario": "auth/metadata-var1", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-verifier-matches-challenge", + "mode": "2025-11-25", + "scenario": "auth/metadata-var1", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "prm-pathbased-requested", + "mode": "2025-11-25", + "scenario": "auth/metadata-var1", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "token-request", + "mode": "2025-11-25", + "scenario": "auth/metadata-var1", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "valid-bearer-token", + "mode": "2025-11-25", + "scenario": "auth/metadata-var1", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "authorization-request", + "mode": "2025-11-25", + "scenario": "auth/metadata-var2", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "authorization-server-metadata", + "mode": "2025-11-25", + "scenario": "auth/metadata-var2", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "client-registration", + "mode": "2025-11-25", + "scenario": "auth/metadata-var2", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-code-challenge-sent", + "mode": "2025-11-25", + "scenario": "auth/metadata-var2", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-code-verifier-sent", + "mode": "2025-11-25", + "scenario": "auth/metadata-var2", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-s256-method-used", + "mode": "2025-11-25", + "scenario": "auth/metadata-var2", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-verifier-matches-challenge", + "mode": "2025-11-25", + "scenario": "auth/metadata-var2", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "prm-pathbased-requested", + "mode": "2025-11-25", + "scenario": "auth/metadata-var2", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "token-request", + "mode": "2025-11-25", + "scenario": "auth/metadata-var2", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "valid-bearer-token", + "mode": "2025-11-25", + "scenario": "auth/metadata-var2", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "authorization-request", + "mode": "2025-11-25", + "scenario": "auth/pre-registration", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "authorization-server-metadata", + "mode": "2025-11-25", + "scenario": "auth/pre-registration", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-code-challenge-sent", + "mode": "2025-11-25", + "scenario": "auth/pre-registration", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-code-verifier-sent", + "mode": "2025-11-25", + "scenario": "auth/pre-registration", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-s256-method-used", + "mode": "2025-11-25", + "scenario": "auth/pre-registration", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-verifier-matches-challenge", + "mode": "2025-11-25", + "scenario": "auth/pre-registration", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "prm-pathbased-requested", + "mode": "2025-11-25", + "scenario": "auth/pre-registration", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "token-request", + "mode": "2025-11-25", + "scenario": "auth/pre-registration", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "authorization-request", + "mode": "2025-11-25", + "scenario": "auth/scope-from-scopes-supported", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "authorization-server-metadata", + "mode": "2025-11-25", + "scenario": "auth/scope-from-scopes-supported", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "client-registration", + "mode": "2025-11-25", + "scenario": "auth/scope-from-scopes-supported", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-code-challenge-sent", + "mode": "2025-11-25", + "scenario": "auth/scope-from-scopes-supported", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-code-verifier-sent", + "mode": "2025-11-25", + "scenario": "auth/scope-from-scopes-supported", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-s256-method-used", + "mode": "2025-11-25", + "scenario": "auth/scope-from-scopes-supported", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-verifier-matches-challenge", + "mode": "2025-11-25", + "scenario": "auth/scope-from-scopes-supported", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "prm-pathbased-requested", + "mode": "2025-11-25", + "scenario": "auth/scope-from-scopes-supported", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "scope-from-scopes-supported", + "mode": "2025-11-25", + "scenario": "auth/scope-from-scopes-supported", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "token-request", + "mode": "2025-11-25", + "scenario": "auth/scope-from-scopes-supported", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "valid-bearer-token", + "mode": "2025-11-25", + "scenario": "auth/scope-from-scopes-supported", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "authorization-request", + "mode": "2025-11-25", + "scenario": "auth/scope-from-www-authenticate", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "authorization-server-metadata", + "mode": "2025-11-25", + "scenario": "auth/scope-from-www-authenticate", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "client-registration", + "mode": "2025-11-25", + "scenario": "auth/scope-from-www-authenticate", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-code-challenge-sent", + "mode": "2025-11-25", + "scenario": "auth/scope-from-www-authenticate", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-code-verifier-sent", + "mode": "2025-11-25", + "scenario": "auth/scope-from-www-authenticate", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-s256-method-used", + "mode": "2025-11-25", + "scenario": "auth/scope-from-www-authenticate", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-verifier-matches-challenge", + "mode": "2025-11-25", + "scenario": "auth/scope-from-www-authenticate", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "prm-pathbased-requested", + "mode": "2025-11-25", + "scenario": "auth/scope-from-www-authenticate", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "token-request", + "mode": "2025-11-25", + "scenario": "auth/scope-from-www-authenticate", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "valid-bearer-token", + "mode": "2025-11-25", + "scenario": "auth/scope-from-www-authenticate", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "authorization-request", + "mode": "2025-11-25", + "scenario": "auth/scope-omitted-when-undefined", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "authorization-server-metadata", + "mode": "2025-11-25", + "scenario": "auth/scope-omitted-when-undefined", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "client-registration", + "mode": "2025-11-25", + "scenario": "auth/scope-omitted-when-undefined", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-code-challenge-sent", + "mode": "2025-11-25", + "scenario": "auth/scope-omitted-when-undefined", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-code-verifier-sent", + "mode": "2025-11-25", + "scenario": "auth/scope-omitted-when-undefined", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-s256-method-used", + "mode": "2025-11-25", + "scenario": "auth/scope-omitted-when-undefined", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-verifier-matches-challenge", + "mode": "2025-11-25", + "scenario": "auth/scope-omitted-when-undefined", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "prm-pathbased-requested", + "mode": "2025-11-25", + "scenario": "auth/scope-omitted-when-undefined", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "scope-omitted-when-undefined", + "mode": "2025-11-25", + "scenario": "auth/scope-omitted-when-undefined", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "token-request", + "mode": "2025-11-25", + "scenario": "auth/scope-omitted-when-undefined", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "valid-bearer-token", + "mode": "2025-11-25", + "scenario": "auth/scope-omitted-when-undefined", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "authorization-request", + "mode": "2025-11-25", + "scenario": "auth/scope-retry-limit", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "authorization-server-metadata", + "mode": "2025-11-25", + "scenario": "auth/scope-retry-limit", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "client-registration", + "mode": "2025-11-25", + "scenario": "auth/scope-retry-limit", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-code-challenge-sent", + "mode": "2025-11-25", + "scenario": "auth/scope-retry-limit", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-code-verifier-sent", + "mode": "2025-11-25", + "scenario": "auth/scope-retry-limit", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-s256-method-used", + "mode": "2025-11-25", + "scenario": "auth/scope-retry-limit", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-verifier-matches-challenge", + "mode": "2025-11-25", + "scenario": "auth/scope-retry-limit", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "prm-pathbased-requested", + "mode": "2025-11-25", + "scenario": "auth/scope-retry-limit", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "scope-retry-limit", + "mode": "2025-11-25", + "scenario": "auth/scope-retry-limit", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "token-request", + "mode": "2025-11-25", + "scenario": "auth/scope-retry-limit", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "authorization-request", + "mode": "2025-11-25", + "scenario": "auth/scope-step-up", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "authorization-server-metadata", + "mode": "2025-11-25", + "scenario": "auth/scope-step-up", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "client-registration", + "mode": "2025-11-25", + "scenario": "auth/scope-step-up", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-code-challenge-sent", + "mode": "2025-11-25", + "scenario": "auth/scope-step-up", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-code-verifier-sent", + "mode": "2025-11-25", + "scenario": "auth/scope-step-up", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-s256-method-used", + "mode": "2025-11-25", + "scenario": "auth/scope-step-up", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-verifier-matches-challenge", + "mode": "2025-11-25", + "scenario": "auth/scope-step-up", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "prm-pathbased-requested", + "mode": "2025-11-25", + "scenario": "auth/scope-step-up", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "scope-step-up-escalation", + "mode": "2025-11-25", + "scenario": "auth/scope-step-up", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "token-request", + "mode": "2025-11-25", + "scenario": "auth/scope-step-up", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "valid-bearer-token", + "mode": "2025-11-25", + "scenario": "auth/scope-step-up", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "authorization-request", + "mode": "2025-11-25", + "scenario": "auth/token-endpoint-auth-basic", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "authorization-server-metadata", + "mode": "2025-11-25", + "scenario": "auth/token-endpoint-auth-basic", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "client-registration", + "mode": "2025-11-25", + "scenario": "auth/token-endpoint-auth-basic", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-code-challenge-sent", + "mode": "2025-11-25", + "scenario": "auth/token-endpoint-auth-basic", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-code-verifier-sent", + "mode": "2025-11-25", + "scenario": "auth/token-endpoint-auth-basic", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-s256-method-used", + "mode": "2025-11-25", + "scenario": "auth/token-endpoint-auth-basic", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-verifier-matches-challenge", + "mode": "2025-11-25", + "scenario": "auth/token-endpoint-auth-basic", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "prm-pathbased-requested", + "mode": "2025-11-25", + "scenario": "auth/token-endpoint-auth-basic", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "resource-parameter-consistency", + "mode": "2025-11-25", + "scenario": "auth/token-endpoint-auth-basic", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "resource-parameter-in-authorization", + "mode": "2025-11-25", + "scenario": "auth/token-endpoint-auth-basic", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "resource-parameter-in-token", + "mode": "2025-11-25", + "scenario": "auth/token-endpoint-auth-basic", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "resource-parameter-valid-uri", + "mode": "2025-11-25", + "scenario": "auth/token-endpoint-auth-basic", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "token-endpoint-auth-method", + "mode": "2025-11-25", + "scenario": "auth/token-endpoint-auth-basic", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "token-request", + "mode": "2025-11-25", + "scenario": "auth/token-endpoint-auth-basic", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "valid-bearer-token", + "mode": "2025-11-25", + "scenario": "auth/token-endpoint-auth-basic", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "authorization-request", + "mode": "2025-11-25", + "scenario": "auth/token-endpoint-auth-none", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "authorization-server-metadata", + "mode": "2025-11-25", + "scenario": "auth/token-endpoint-auth-none", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "client-registration", + "mode": "2025-11-25", + "scenario": "auth/token-endpoint-auth-none", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-code-challenge-sent", + "mode": "2025-11-25", + "scenario": "auth/token-endpoint-auth-none", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-code-verifier-sent", + "mode": "2025-11-25", + "scenario": "auth/token-endpoint-auth-none", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-s256-method-used", + "mode": "2025-11-25", + "scenario": "auth/token-endpoint-auth-none", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-verifier-matches-challenge", + "mode": "2025-11-25", + "scenario": "auth/token-endpoint-auth-none", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "prm-pathbased-requested", + "mode": "2025-11-25", + "scenario": "auth/token-endpoint-auth-none", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "resource-parameter-consistency", + "mode": "2025-11-25", + "scenario": "auth/token-endpoint-auth-none", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "resource-parameter-in-authorization", + "mode": "2025-11-25", + "scenario": "auth/token-endpoint-auth-none", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "resource-parameter-in-token", + "mode": "2025-11-25", + "scenario": "auth/token-endpoint-auth-none", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "resource-parameter-valid-uri", + "mode": "2025-11-25", + "scenario": "auth/token-endpoint-auth-none", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "token-endpoint-auth-method", + "mode": "2025-11-25", + "scenario": "auth/token-endpoint-auth-none", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "token-request", + "mode": "2025-11-25", + "scenario": "auth/token-endpoint-auth-none", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "valid-bearer-token", + "mode": "2025-11-25", + "scenario": "auth/token-endpoint-auth-none", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "authorization-request", + "mode": "2025-11-25", + "scenario": "auth/token-endpoint-auth-post", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "authorization-server-metadata", + "mode": "2025-11-25", + "scenario": "auth/token-endpoint-auth-post", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "client-registration", + "mode": "2025-11-25", + "scenario": "auth/token-endpoint-auth-post", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-code-challenge-sent", + "mode": "2025-11-25", + "scenario": "auth/token-endpoint-auth-post", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-code-verifier-sent", + "mode": "2025-11-25", + "scenario": "auth/token-endpoint-auth-post", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-s256-method-used", + "mode": "2025-11-25", + "scenario": "auth/token-endpoint-auth-post", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-verifier-matches-challenge", + "mode": "2025-11-25", + "scenario": "auth/token-endpoint-auth-post", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "prm-pathbased-requested", + "mode": "2025-11-25", + "scenario": "auth/token-endpoint-auth-post", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "resource-parameter-consistency", + "mode": "2025-11-25", + "scenario": "auth/token-endpoint-auth-post", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "resource-parameter-in-authorization", + "mode": "2025-11-25", + "scenario": "auth/token-endpoint-auth-post", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "resource-parameter-in-token", + "mode": "2025-11-25", + "scenario": "auth/token-endpoint-auth-post", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "resource-parameter-valid-uri", + "mode": "2025-11-25", + "scenario": "auth/token-endpoint-auth-post", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "token-endpoint-auth-method", + "mode": "2025-11-25", + "scenario": "auth/token-endpoint-auth-post", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "token-request", + "mode": "2025-11-25", + "scenario": "auth/token-endpoint-auth-post", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "valid-bearer-token", + "mode": "2025-11-25", + "scenario": "auth/token-endpoint-auth-post", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "mcp-client-initialization", + "mode": "2025-11-25", + "scenario": "initialize", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "client-sse-graceful-reconnect", + "mode": "2025-11-25", + "scenario": "sse-retry", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "client-sse-last-event-id", + "mode": "2025-11-25", + "scenario": "sse-retry", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "client-sse-retry-timing", + "mode": "2025-11-25", + "scenario": "sse-retry", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "tool-add-numbers", + "mode": "2025-11-25", + "scenario": "tools_call", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "wire-schema-valid", + "mode": "2025-11-25", + "scenario": "tools_call", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "app_server_initialize", + "mode": "2025-11-25", + "scenario": "", + "source": "supplemental", + "transport": "stdio" + }, + { + "check_id": "echo_tool", + "mode": "2025-11-25", + "scenario": "", + "source": "supplemental", + "transport": "stdio" + }, + { + "check_id": "ephemeral_thread", + "mode": "2025-11-25", + "scenario": "", + "source": "supplemental", + "transport": "stdio" + }, + { + "check_id": "inventory", + "mode": "2025-11-25", + "scenario": "", + "source": "supplemental", + "transport": "stdio" + }, + { + "check_id": "isolated_config_cleanup", + "mode": "2025-11-25", + "scenario": "", + "source": "supplemental", + "transport": "stdio" + }, + { + "check_id": "mcp_add", + "mode": "2025-11-25", + "scenario": "", + "source": "supplemental", + "transport": "stdio" + }, + { + "check_id": "mcp_get", + "mode": "2025-11-25", + "scenario": "", + "source": "supplemental", + "transport": "stdio" + }, + { + "check_id": "mcp_remove", + "mode": "2025-11-25", + "scenario": "", + "source": "supplemental", + "transport": "stdio" + }, + { + "check_id": "unicode_resource_read", + "mode": "2025-11-25", + "scenario": "", + "source": "supplemental", + "transport": "stdio" + }, + { + "check_id": "harness/auth/authorization-server-migration/codex-adapter", + "mode": "2026-07-28", + "scenario": "auth/authorization-server-migration", + "source": "harness", + "transport": "official-http" + }, + { + "check_id": "harness/auth/basic-cimd/codex-adapter", + "mode": "2026-07-28", + "scenario": "auth/basic-cimd", + "source": "harness", + "transport": "official-http" + }, + { + "check_id": "harness/auth/iss-normalized/codex-adapter", + "mode": "2026-07-28", + "scenario": "auth/iss-normalized", + "source": "harness", + "transport": "official-http" + }, + { + "check_id": "harness/auth/iss-not-advertised/codex-adapter", + "mode": "2026-07-28", + "scenario": "auth/iss-not-advertised", + "source": "harness", + "transport": "official-http" + }, + { + "check_id": "harness/auth/iss-supported/codex-adapter", + "mode": "2026-07-28", + "scenario": "auth/iss-supported", + "source": "harness", + "transport": "official-http" + }, + { + "check_id": "harness/auth/iss-supported-missing/codex-adapter", + "mode": "2026-07-28", + "scenario": "auth/iss-supported-missing", + "source": "harness", + "transport": "official-http" + }, + { + "check_id": "harness/auth/iss-unexpected/codex-adapter", + "mode": "2026-07-28", + "scenario": "auth/iss-unexpected", + "source": "harness", + "transport": "official-http" + }, + { + "check_id": "harness/auth/iss-wrong-issuer/codex-adapter", + "mode": "2026-07-28", + "scenario": "auth/iss-wrong-issuer", + "source": "harness", + "transport": "official-http" + }, + { + "check_id": "harness/auth/metadata-default/codex-adapter", + "mode": "2026-07-28", + "scenario": "auth/metadata-default", + "source": "harness", + "transport": "official-http" + }, + { + "check_id": "harness/auth/metadata-issuer-mismatch/codex-adapter", + "mode": "2026-07-28", + "scenario": "auth/metadata-issuer-mismatch", + "source": "harness", + "transport": "official-http" + }, + { + "check_id": "harness/auth/metadata-var1/codex-adapter", + "mode": "2026-07-28", + "scenario": "auth/metadata-var1", + "source": "harness", + "transport": "official-http" + }, + { + "check_id": "harness/auth/metadata-var2/codex-adapter", + "mode": "2026-07-28", + "scenario": "auth/metadata-var2", + "source": "harness", + "transport": "official-http" + }, + { + "check_id": "harness/auth/offline-access-not-supported/codex-adapter", + "mode": "2026-07-28", + "scenario": "auth/offline-access-not-supported", + "source": "harness", + "transport": "official-http" + }, + { + "check_id": "harness/auth/offline-access-scope/codex-adapter", + "mode": "2026-07-28", + "scenario": "auth/offline-access-scope", + "source": "harness", + "transport": "official-http" + }, + { + "check_id": "harness/auth/resource-mismatch/codex-adapter", + "mode": "2026-07-28", + "scenario": "auth/resource-mismatch", + "source": "harness", + "transport": "official-http" + }, + { + "check_id": "harness/auth/scope-from-scopes-supported/codex-adapter", + "mode": "2026-07-28", + "scenario": "auth/scope-from-scopes-supported", + "source": "harness", + "transport": "official-http" + }, + { + "check_id": "harness/auth/scope-omitted-when-undefined/codex-adapter", + "mode": "2026-07-28", + "scenario": "auth/scope-omitted-when-undefined", + "source": "harness", + "transport": "official-http" + }, + { + "check_id": "harness/auth/scope-retry-limit/codex-adapter", + "mode": "2026-07-28", + "scenario": "auth/scope-retry-limit", + "source": "harness", + "transport": "official-http" + }, + { + "check_id": "harness/auth/token-endpoint-auth-basic/codex-adapter", + "mode": "2026-07-28", + "scenario": "auth/token-endpoint-auth-basic", + "source": "harness", + "transport": "official-http" + }, + { + "check_id": "harness/auth/token-endpoint-auth-none/codex-adapter", + "mode": "2026-07-28", + "scenario": "auth/token-endpoint-auth-none", + "source": "harness", + "transport": "official-http" + }, + { + "check_id": "harness/auth/token-endpoint-auth-post/codex-adapter", + "mode": "2026-07-28", + "scenario": "auth/token-endpoint-auth-post", + "source": "harness", + "transport": "official-http" + }, + { + "check_id": "harness/http-custom-headers/codex-adapter", + "mode": "2026-07-28", + "scenario": "http-custom-headers", + "source": "harness", + "transport": "official-http" + }, + { + "check_id": "harness/http-invalid-tool-headers/codex-adapter", + "mode": "2026-07-28", + "scenario": "http-invalid-tool-headers", + "source": "harness", + "transport": "official-http" + }, + { + "check_id": "harness/http-standard-headers/codex-adapter", + "mode": "2026-07-28", + "scenario": "http-standard-headers", + "source": "harness", + "transport": "official-http" + }, + { + "check_id": "harness/json-schema-ref-no-deref/codex-adapter", + "mode": "2026-07-28", + "scenario": "json-schema-ref-no-deref", + "source": "harness", + "transport": "official-http" + }, + { + "check_id": "harness/request-metadata/codex-adapter", + "mode": "2026-07-28", + "scenario": "request-metadata", + "source": "harness", + "transport": "official-http" + }, + { + "check_id": "harness/sep-2322-client-request-state/codex-adapter", + "mode": "2026-07-28", + "scenario": "sep-2322-client-request-state", + "source": "harness", + "transport": "official-http" + }, + { + "check_id": "harness/tools_call/codex-adapter", + "mode": "2026-07-28", + "scenario": "tools_call", + "source": "harness", + "transport": "official-http" + }, + { + "check_id": "authorization-request", + "mode": "2026-07-28", + "scenario": "auth/authorization-server-migration", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "authorization-server-metadata", + "mode": "2026-07-28", + "scenario": "auth/authorization-server-migration", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "client-registration", + "mode": "2026-07-28", + "scenario": "auth/authorization-server-migration", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-code-challenge-sent", + "mode": "2026-07-28", + "scenario": "auth/authorization-server-migration", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-code-verifier-sent", + "mode": "2026-07-28", + "scenario": "auth/authorization-server-migration", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-s256-method-used", + "mode": "2026-07-28", + "scenario": "auth/authorization-server-migration", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-verifier-matches-challenge", + "mode": "2026-07-28", + "scenario": "auth/authorization-server-migration", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "prm-pathbased-requested", + "mode": "2026-07-28", + "scenario": "auth/authorization-server-migration", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "sep-2352-no-cross-as-credential-reuse", + "mode": "2026-07-28", + "scenario": "auth/authorization-server-migration", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "sep-2352-no-reuse-on-as-change", + "mode": "2026-07-28", + "scenario": "auth/authorization-server-migration", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "sep-2352-reregister-on-as-change", + "mode": "2026-07-28", + "scenario": "auth/authorization-server-migration", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "sep-837-application-type-present", + "mode": "2026-07-28", + "scenario": "auth/authorization-server-migration", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "token-request", + "mode": "2026-07-28", + "scenario": "auth/authorization-server-migration", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "valid-bearer-token", + "mode": "2026-07-28", + "scenario": "auth/authorization-server-migration", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "authorization-request", + "mode": "2026-07-28", + "scenario": "auth/basic-cimd", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "authorization-server-metadata", + "mode": "2026-07-28", + "scenario": "auth/basic-cimd", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "cimd-client-id-used", + "mode": "2026-07-28", + "scenario": "auth/basic-cimd", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-code-challenge-sent", + "mode": "2026-07-28", + "scenario": "auth/basic-cimd", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-code-verifier-sent", + "mode": "2026-07-28", + "scenario": "auth/basic-cimd", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-s256-method-used", + "mode": "2026-07-28", + "scenario": "auth/basic-cimd", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-verifier-matches-challenge", + "mode": "2026-07-28", + "scenario": "auth/basic-cimd", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "prm-pathbased-requested", + "mode": "2026-07-28", + "scenario": "auth/basic-cimd", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "token-request", + "mode": "2026-07-28", + "scenario": "auth/basic-cimd", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "valid-bearer-token", + "mode": "2026-07-28", + "scenario": "auth/basic-cimd", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "authorization-request", + "mode": "2026-07-28", + "scenario": "auth/iss-normalized", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "authorization-server-metadata", + "mode": "2026-07-28", + "scenario": "auth/iss-normalized", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "client-registration", + "mode": "2026-07-28", + "scenario": "auth/iss-normalized", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-code-challenge-sent", + "mode": "2026-07-28", + "scenario": "auth/iss-normalized", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-s256-method-used", + "mode": "2026-07-28", + "scenario": "auth/iss-normalized", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "prm-pathbased-requested", + "mode": "2026-07-28", + "scenario": "auth/iss-normalized", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "sep-2468-client-no-normalization", + "mode": "2026-07-28", + "scenario": "auth/iss-normalized", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "sep-837-application-type-present", + "mode": "2026-07-28", + "scenario": "auth/iss-normalized", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "authorization-request", + "mode": "2026-07-28", + "scenario": "auth/iss-not-advertised", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "authorization-server-metadata", + "mode": "2026-07-28", + "scenario": "auth/iss-not-advertised", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "client-registration", + "mode": "2026-07-28", + "scenario": "auth/iss-not-advertised", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-code-challenge-sent", + "mode": "2026-07-28", + "scenario": "auth/iss-not-advertised", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-code-verifier-sent", + "mode": "2026-07-28", + "scenario": "auth/iss-not-advertised", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-s256-method-used", + "mode": "2026-07-28", + "scenario": "auth/iss-not-advertised", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-verifier-matches-challenge", + "mode": "2026-07-28", + "scenario": "auth/iss-not-advertised", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "prm-pathbased-requested", + "mode": "2026-07-28", + "scenario": "auth/iss-not-advertised", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "sep-2468-client-proceed-no-iss", + "mode": "2026-07-28", + "scenario": "auth/iss-not-advertised", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "sep-837-application-type-present", + "mode": "2026-07-28", + "scenario": "auth/iss-not-advertised", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "token-request", + "mode": "2026-07-28", + "scenario": "auth/iss-not-advertised", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "valid-bearer-token", + "mode": "2026-07-28", + "scenario": "auth/iss-not-advertised", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "authorization-request", + "mode": "2026-07-28", + "scenario": "auth/iss-supported", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "authorization-server-metadata", + "mode": "2026-07-28", + "scenario": "auth/iss-supported", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "client-registration", + "mode": "2026-07-28", + "scenario": "auth/iss-supported", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-code-challenge-sent", + "mode": "2026-07-28", + "scenario": "auth/iss-supported", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-code-verifier-sent", + "mode": "2026-07-28", + "scenario": "auth/iss-supported", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-s256-method-used", + "mode": "2026-07-28", + "scenario": "auth/iss-supported", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-verifier-matches-challenge", + "mode": "2026-07-28", + "scenario": "auth/iss-supported", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "prm-pathbased-requested", + "mode": "2026-07-28", + "scenario": "auth/iss-supported", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "sep-2468-client-compare-iss-supported", + "mode": "2026-07-28", + "scenario": "auth/iss-supported", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "sep-837-application-type-present", + "mode": "2026-07-28", + "scenario": "auth/iss-supported", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "token-request", + "mode": "2026-07-28", + "scenario": "auth/iss-supported", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "valid-bearer-token", + "mode": "2026-07-28", + "scenario": "auth/iss-supported", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "authorization-request", + "mode": "2026-07-28", + "scenario": "auth/iss-supported-missing", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "authorization-server-metadata", + "mode": "2026-07-28", + "scenario": "auth/iss-supported-missing", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "client-registration", + "mode": "2026-07-28", + "scenario": "auth/iss-supported-missing", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-code-challenge-sent", + "mode": "2026-07-28", + "scenario": "auth/iss-supported-missing", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-s256-method-used", + "mode": "2026-07-28", + "scenario": "auth/iss-supported-missing", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "prm-pathbased-requested", + "mode": "2026-07-28", + "scenario": "auth/iss-supported-missing", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "sep-2468-client-reject-missing-iss", + "mode": "2026-07-28", + "scenario": "auth/iss-supported-missing", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "sep-837-application-type-present", + "mode": "2026-07-28", + "scenario": "auth/iss-supported-missing", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "authorization-request", + "mode": "2026-07-28", + "scenario": "auth/iss-unexpected", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "authorization-server-metadata", + "mode": "2026-07-28", + "scenario": "auth/iss-unexpected", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "client-registration", + "mode": "2026-07-28", + "scenario": "auth/iss-unexpected", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-code-challenge-sent", + "mode": "2026-07-28", + "scenario": "auth/iss-unexpected", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-s256-method-used", + "mode": "2026-07-28", + "scenario": "auth/iss-unexpected", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "prm-pathbased-requested", + "mode": "2026-07-28", + "scenario": "auth/iss-unexpected", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "sep-2468-client-compare-iss-unadvertised", + "mode": "2026-07-28", + "scenario": "auth/iss-unexpected", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "sep-837-application-type-present", + "mode": "2026-07-28", + "scenario": "auth/iss-unexpected", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "authorization-request", + "mode": "2026-07-28", + "scenario": "auth/iss-wrong-issuer", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "authorization-server-metadata", + "mode": "2026-07-28", + "scenario": "auth/iss-wrong-issuer", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "client-registration", + "mode": "2026-07-28", + "scenario": "auth/iss-wrong-issuer", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-code-challenge-sent", + "mode": "2026-07-28", + "scenario": "auth/iss-wrong-issuer", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-s256-method-used", + "mode": "2026-07-28", + "scenario": "auth/iss-wrong-issuer", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "prm-pathbased-requested", + "mode": "2026-07-28", + "scenario": "auth/iss-wrong-issuer", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "sep-2468-client-compare-iss-supported", + "mode": "2026-07-28", + "scenario": "auth/iss-wrong-issuer", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "sep-837-application-type-present", + "mode": "2026-07-28", + "scenario": "auth/iss-wrong-issuer", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "authorization-request", + "mode": "2026-07-28", + "scenario": "auth/metadata-default", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "authorization-server-metadata", + "mode": "2026-07-28", + "scenario": "auth/metadata-default", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "client-registration", + "mode": "2026-07-28", + "scenario": "auth/metadata-default", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-code-challenge-sent", + "mode": "2026-07-28", + "scenario": "auth/metadata-default", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-code-verifier-sent", + "mode": "2026-07-28", + "scenario": "auth/metadata-default", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-s256-method-used", + "mode": "2026-07-28", + "scenario": "auth/metadata-default", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-verifier-matches-challenge", + "mode": "2026-07-28", + "scenario": "auth/metadata-default", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "prm-pathbased-requested", + "mode": "2026-07-28", + "scenario": "auth/metadata-default", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "sep-837-application-type-present", + "mode": "2026-07-28", + "scenario": "auth/metadata-default", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "token-request", + "mode": "2026-07-28", + "scenario": "auth/metadata-default", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "valid-bearer-token", + "mode": "2026-07-28", + "scenario": "auth/metadata-default", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "authorization-server-metadata", + "mode": "2026-07-28", + "scenario": "auth/metadata-issuer-mismatch", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "prm-pathbased-requested", + "mode": "2026-07-28", + "scenario": "auth/metadata-issuer-mismatch", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "sep-2468-client-validate-metadata-issuer", + "mode": "2026-07-28", + "scenario": "auth/metadata-issuer-mismatch", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "authorization-request", + "mode": "2026-07-28", + "scenario": "auth/metadata-var1", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "authorization-server-metadata", + "mode": "2026-07-28", + "scenario": "auth/metadata-var1", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "client-registration", + "mode": "2026-07-28", + "scenario": "auth/metadata-var1", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-code-challenge-sent", + "mode": "2026-07-28", + "scenario": "auth/metadata-var1", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-code-verifier-sent", + "mode": "2026-07-28", + "scenario": "auth/metadata-var1", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-s256-method-used", + "mode": "2026-07-28", + "scenario": "auth/metadata-var1", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-verifier-matches-challenge", + "mode": "2026-07-28", + "scenario": "auth/metadata-var1", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "prm-pathbased-requested", + "mode": "2026-07-28", + "scenario": "auth/metadata-var1", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "sep-837-application-type-present", + "mode": "2026-07-28", + "scenario": "auth/metadata-var1", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "token-request", + "mode": "2026-07-28", + "scenario": "auth/metadata-var1", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "valid-bearer-token", + "mode": "2026-07-28", + "scenario": "auth/metadata-var1", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "authorization-request", + "mode": "2026-07-28", + "scenario": "auth/metadata-var2", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "authorization-server-metadata", + "mode": "2026-07-28", + "scenario": "auth/metadata-var2", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "client-registration", + "mode": "2026-07-28", + "scenario": "auth/metadata-var2", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-code-challenge-sent", + "mode": "2026-07-28", + "scenario": "auth/metadata-var2", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-code-verifier-sent", + "mode": "2026-07-28", + "scenario": "auth/metadata-var2", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-s256-method-used", + "mode": "2026-07-28", + "scenario": "auth/metadata-var2", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-verifier-matches-challenge", + "mode": "2026-07-28", + "scenario": "auth/metadata-var2", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "prm-pathbased-requested", + "mode": "2026-07-28", + "scenario": "auth/metadata-var2", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "sep-837-application-type-present", + "mode": "2026-07-28", + "scenario": "auth/metadata-var2", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "token-request", + "mode": "2026-07-28", + "scenario": "auth/metadata-var2", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "valid-bearer-token", + "mode": "2026-07-28", + "scenario": "auth/metadata-var2", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "authorization-request", + "mode": "2026-07-28", + "scenario": "auth/offline-access-not-supported", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "authorization-server-metadata", + "mode": "2026-07-28", + "scenario": "auth/offline-access-not-supported", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "client-registration", + "mode": "2026-07-28", + "scenario": "auth/offline-access-not-supported", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-code-challenge-sent", + "mode": "2026-07-28", + "scenario": "auth/offline-access-not-supported", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-code-verifier-sent", + "mode": "2026-07-28", + "scenario": "auth/offline-access-not-supported", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-s256-method-used", + "mode": "2026-07-28", + "scenario": "auth/offline-access-not-supported", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-verifier-matches-challenge", + "mode": "2026-07-28", + "scenario": "auth/offline-access-not-supported", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "prm-pathbased-requested", + "mode": "2026-07-28", + "scenario": "auth/offline-access-not-supported", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "sep-2207-offline-access-not-requested", + "mode": "2026-07-28", + "scenario": "auth/offline-access-not-supported", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "sep-837-application-type-present", + "mode": "2026-07-28", + "scenario": "auth/offline-access-not-supported", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "token-request", + "mode": "2026-07-28", + "scenario": "auth/offline-access-not-supported", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "valid-bearer-token", + "mode": "2026-07-28", + "scenario": "auth/offline-access-not-supported", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "authorization-request", + "mode": "2026-07-28", + "scenario": "auth/offline-access-scope", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "authorization-server-metadata", + "mode": "2026-07-28", + "scenario": "auth/offline-access-scope", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "client-registration", + "mode": "2026-07-28", + "scenario": "auth/offline-access-scope", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-code-challenge-sent", + "mode": "2026-07-28", + "scenario": "auth/offline-access-scope", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-code-verifier-sent", + "mode": "2026-07-28", + "scenario": "auth/offline-access-scope", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-s256-method-used", + "mode": "2026-07-28", + "scenario": "auth/offline-access-scope", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-verifier-matches-challenge", + "mode": "2026-07-28", + "scenario": "auth/offline-access-scope", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "prm-pathbased-requested", + "mode": "2026-07-28", + "scenario": "auth/offline-access-scope", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "sep-2207-client-metadata-grant-types", + "mode": "2026-07-28", + "scenario": "auth/offline-access-scope", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "sep-2207-offline-access-requested", + "mode": "2026-07-28", + "scenario": "auth/offline-access-scope", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "sep-837-application-type-present", + "mode": "2026-07-28", + "scenario": "auth/offline-access-scope", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "token-request", + "mode": "2026-07-28", + "scenario": "auth/offline-access-scope", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "valid-bearer-token", + "mode": "2026-07-28", + "scenario": "auth/offline-access-scope", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "authorization-request", + "mode": "2026-07-28", + "scenario": "auth/pre-registration", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "authorization-server-metadata", + "mode": "2026-07-28", + "scenario": "auth/pre-registration", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-code-challenge-sent", + "mode": "2026-07-28", + "scenario": "auth/pre-registration", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-code-verifier-sent", + "mode": "2026-07-28", + "scenario": "auth/pre-registration", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-s256-method-used", + "mode": "2026-07-28", + "scenario": "auth/pre-registration", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-verifier-matches-challenge", + "mode": "2026-07-28", + "scenario": "auth/pre-registration", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "prm-pathbased-requested", + "mode": "2026-07-28", + "scenario": "auth/pre-registration", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "token-request", + "mode": "2026-07-28", + "scenario": "auth/pre-registration", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "prm-pathbased-requested", + "mode": "2026-07-28", + "scenario": "auth/resource-mismatch", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "resource-mismatch-rejected", + "mode": "2026-07-28", + "scenario": "auth/resource-mismatch", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "authorization-request", + "mode": "2026-07-28", + "scenario": "auth/scope-from-scopes-supported", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "authorization-server-metadata", + "mode": "2026-07-28", + "scenario": "auth/scope-from-scopes-supported", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "client-registration", + "mode": "2026-07-28", + "scenario": "auth/scope-from-scopes-supported", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-code-challenge-sent", + "mode": "2026-07-28", + "scenario": "auth/scope-from-scopes-supported", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-code-verifier-sent", + "mode": "2026-07-28", + "scenario": "auth/scope-from-scopes-supported", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-s256-method-used", + "mode": "2026-07-28", + "scenario": "auth/scope-from-scopes-supported", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-verifier-matches-challenge", + "mode": "2026-07-28", + "scenario": "auth/scope-from-scopes-supported", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "prm-pathbased-requested", + "mode": "2026-07-28", + "scenario": "auth/scope-from-scopes-supported", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "scope-from-scopes-supported", + "mode": "2026-07-28", + "scenario": "auth/scope-from-scopes-supported", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "sep-837-application-type-present", + "mode": "2026-07-28", + "scenario": "auth/scope-from-scopes-supported", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "token-request", + "mode": "2026-07-28", + "scenario": "auth/scope-from-scopes-supported", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "valid-bearer-token", + "mode": "2026-07-28", + "scenario": "auth/scope-from-scopes-supported", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "authorization-request", + "mode": "2026-07-28", + "scenario": "auth/scope-from-www-authenticate", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "authorization-server-metadata", + "mode": "2026-07-28", + "scenario": "auth/scope-from-www-authenticate", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "client-registration", + "mode": "2026-07-28", + "scenario": "auth/scope-from-www-authenticate", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-code-challenge-sent", + "mode": "2026-07-28", + "scenario": "auth/scope-from-www-authenticate", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-code-verifier-sent", + "mode": "2026-07-28", + "scenario": "auth/scope-from-www-authenticate", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-s256-method-used", + "mode": "2026-07-28", + "scenario": "auth/scope-from-www-authenticate", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-verifier-matches-challenge", + "mode": "2026-07-28", + "scenario": "auth/scope-from-www-authenticate", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "prm-pathbased-requested", + "mode": "2026-07-28", + "scenario": "auth/scope-from-www-authenticate", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "sep-837-application-type-present", + "mode": "2026-07-28", + "scenario": "auth/scope-from-www-authenticate", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "token-request", + "mode": "2026-07-28", + "scenario": "auth/scope-from-www-authenticate", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "valid-bearer-token", + "mode": "2026-07-28", + "scenario": "auth/scope-from-www-authenticate", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "authorization-request", + "mode": "2026-07-28", + "scenario": "auth/scope-omitted-when-undefined", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "authorization-server-metadata", + "mode": "2026-07-28", + "scenario": "auth/scope-omitted-when-undefined", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "client-registration", + "mode": "2026-07-28", + "scenario": "auth/scope-omitted-when-undefined", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-code-challenge-sent", + "mode": "2026-07-28", + "scenario": "auth/scope-omitted-when-undefined", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-code-verifier-sent", + "mode": "2026-07-28", + "scenario": "auth/scope-omitted-when-undefined", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-s256-method-used", + "mode": "2026-07-28", + "scenario": "auth/scope-omitted-when-undefined", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-verifier-matches-challenge", + "mode": "2026-07-28", + "scenario": "auth/scope-omitted-when-undefined", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "prm-pathbased-requested", + "mode": "2026-07-28", + "scenario": "auth/scope-omitted-when-undefined", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "scope-omitted-when-undefined", + "mode": "2026-07-28", + "scenario": "auth/scope-omitted-when-undefined", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "sep-837-application-type-present", + "mode": "2026-07-28", + "scenario": "auth/scope-omitted-when-undefined", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "token-request", + "mode": "2026-07-28", + "scenario": "auth/scope-omitted-when-undefined", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "valid-bearer-token", + "mode": "2026-07-28", + "scenario": "auth/scope-omitted-when-undefined", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "authorization-request", + "mode": "2026-07-28", + "scenario": "auth/scope-retry-limit", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "authorization-server-metadata", + "mode": "2026-07-28", + "scenario": "auth/scope-retry-limit", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "client-registration", + "mode": "2026-07-28", + "scenario": "auth/scope-retry-limit", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-code-challenge-sent", + "mode": "2026-07-28", + "scenario": "auth/scope-retry-limit", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-code-verifier-sent", + "mode": "2026-07-28", + "scenario": "auth/scope-retry-limit", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-s256-method-used", + "mode": "2026-07-28", + "scenario": "auth/scope-retry-limit", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-verifier-matches-challenge", + "mode": "2026-07-28", + "scenario": "auth/scope-retry-limit", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "prm-pathbased-requested", + "mode": "2026-07-28", + "scenario": "auth/scope-retry-limit", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "scope-retry-limit", + "mode": "2026-07-28", + "scenario": "auth/scope-retry-limit", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "sep-837-application-type-present", + "mode": "2026-07-28", + "scenario": "auth/scope-retry-limit", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "token-request", + "mode": "2026-07-28", + "scenario": "auth/scope-retry-limit", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "authorization-request", + "mode": "2026-07-28", + "scenario": "auth/scope-step-up", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "authorization-server-metadata", + "mode": "2026-07-28", + "scenario": "auth/scope-step-up", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "client-registration", + "mode": "2026-07-28", + "scenario": "auth/scope-step-up", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-code-challenge-sent", + "mode": "2026-07-28", + "scenario": "auth/scope-step-up", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-code-verifier-sent", + "mode": "2026-07-28", + "scenario": "auth/scope-step-up", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-s256-method-used", + "mode": "2026-07-28", + "scenario": "auth/scope-step-up", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-verifier-matches-challenge", + "mode": "2026-07-28", + "scenario": "auth/scope-step-up", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "prm-pathbased-requested", + "mode": "2026-07-28", + "scenario": "auth/scope-step-up", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "scope-step-up-escalation", + "mode": "2026-07-28", + "scenario": "auth/scope-step-up", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "sep-837-application-type-present", + "mode": "2026-07-28", + "scenario": "auth/scope-step-up", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "token-request", + "mode": "2026-07-28", + "scenario": "auth/scope-step-up", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "valid-bearer-token", + "mode": "2026-07-28", + "scenario": "auth/scope-step-up", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "authorization-request", + "mode": "2026-07-28", + "scenario": "auth/token-endpoint-auth-basic", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "authorization-server-metadata", + "mode": "2026-07-28", + "scenario": "auth/token-endpoint-auth-basic", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "client-registration", + "mode": "2026-07-28", + "scenario": "auth/token-endpoint-auth-basic", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-code-challenge-sent", + "mode": "2026-07-28", + "scenario": "auth/token-endpoint-auth-basic", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-code-verifier-sent", + "mode": "2026-07-28", + "scenario": "auth/token-endpoint-auth-basic", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-s256-method-used", + "mode": "2026-07-28", + "scenario": "auth/token-endpoint-auth-basic", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-verifier-matches-challenge", + "mode": "2026-07-28", + "scenario": "auth/token-endpoint-auth-basic", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "prm-pathbased-requested", + "mode": "2026-07-28", + "scenario": "auth/token-endpoint-auth-basic", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "resource-parameter-consistency", + "mode": "2026-07-28", + "scenario": "auth/token-endpoint-auth-basic", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "resource-parameter-in-authorization", + "mode": "2026-07-28", + "scenario": "auth/token-endpoint-auth-basic", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "resource-parameter-in-token", + "mode": "2026-07-28", + "scenario": "auth/token-endpoint-auth-basic", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "resource-parameter-valid-uri", + "mode": "2026-07-28", + "scenario": "auth/token-endpoint-auth-basic", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "sep-837-application-type-present", + "mode": "2026-07-28", + "scenario": "auth/token-endpoint-auth-basic", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "token-endpoint-auth-method", + "mode": "2026-07-28", + "scenario": "auth/token-endpoint-auth-basic", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "token-request", + "mode": "2026-07-28", + "scenario": "auth/token-endpoint-auth-basic", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "valid-bearer-token", + "mode": "2026-07-28", + "scenario": "auth/token-endpoint-auth-basic", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "authorization-request", + "mode": "2026-07-28", + "scenario": "auth/token-endpoint-auth-none", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "authorization-server-metadata", + "mode": "2026-07-28", + "scenario": "auth/token-endpoint-auth-none", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "client-registration", + "mode": "2026-07-28", + "scenario": "auth/token-endpoint-auth-none", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-code-challenge-sent", + "mode": "2026-07-28", + "scenario": "auth/token-endpoint-auth-none", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-code-verifier-sent", + "mode": "2026-07-28", + "scenario": "auth/token-endpoint-auth-none", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-s256-method-used", + "mode": "2026-07-28", + "scenario": "auth/token-endpoint-auth-none", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-verifier-matches-challenge", + "mode": "2026-07-28", + "scenario": "auth/token-endpoint-auth-none", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "prm-pathbased-requested", + "mode": "2026-07-28", + "scenario": "auth/token-endpoint-auth-none", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "resource-parameter-consistency", + "mode": "2026-07-28", + "scenario": "auth/token-endpoint-auth-none", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "resource-parameter-in-authorization", + "mode": "2026-07-28", + "scenario": "auth/token-endpoint-auth-none", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "resource-parameter-in-token", + "mode": "2026-07-28", + "scenario": "auth/token-endpoint-auth-none", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "resource-parameter-valid-uri", + "mode": "2026-07-28", + "scenario": "auth/token-endpoint-auth-none", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "sep-837-application-type-present", + "mode": "2026-07-28", + "scenario": "auth/token-endpoint-auth-none", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "token-endpoint-auth-method", + "mode": "2026-07-28", + "scenario": "auth/token-endpoint-auth-none", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "token-request", + "mode": "2026-07-28", + "scenario": "auth/token-endpoint-auth-none", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "valid-bearer-token", + "mode": "2026-07-28", + "scenario": "auth/token-endpoint-auth-none", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "authorization-request", + "mode": "2026-07-28", + "scenario": "auth/token-endpoint-auth-post", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "authorization-server-metadata", + "mode": "2026-07-28", + "scenario": "auth/token-endpoint-auth-post", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "client-registration", + "mode": "2026-07-28", + "scenario": "auth/token-endpoint-auth-post", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-code-challenge-sent", + "mode": "2026-07-28", + "scenario": "auth/token-endpoint-auth-post", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-code-verifier-sent", + "mode": "2026-07-28", + "scenario": "auth/token-endpoint-auth-post", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-s256-method-used", + "mode": "2026-07-28", + "scenario": "auth/token-endpoint-auth-post", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "pkce-verifier-matches-challenge", + "mode": "2026-07-28", + "scenario": "auth/token-endpoint-auth-post", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "prm-pathbased-requested", + "mode": "2026-07-28", + "scenario": "auth/token-endpoint-auth-post", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "resource-parameter-consistency", + "mode": "2026-07-28", + "scenario": "auth/token-endpoint-auth-post", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "resource-parameter-in-authorization", + "mode": "2026-07-28", + "scenario": "auth/token-endpoint-auth-post", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "resource-parameter-in-token", + "mode": "2026-07-28", + "scenario": "auth/token-endpoint-auth-post", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "resource-parameter-valid-uri", + "mode": "2026-07-28", + "scenario": "auth/token-endpoint-auth-post", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "sep-837-application-type-present", + "mode": "2026-07-28", + "scenario": "auth/token-endpoint-auth-post", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "token-endpoint-auth-method", + "mode": "2026-07-28", + "scenario": "auth/token-endpoint-auth-post", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "token-request", + "mode": "2026-07-28", + "scenario": "auth/token-endpoint-auth-post", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "valid-bearer-token", + "mode": "2026-07-28", + "scenario": "auth/token-endpoint-auth-post", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "sep-2243-client-base64-unsafe", + "mode": "2026-07-28", + "scenario": "http-custom-headers", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "sep-2243-client-encode-values", + "mode": "2026-07-28", + "scenario": "http-custom-headers", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "sep-2243-client-mirrors-designated-params", + "mode": "2026-07-28", + "scenario": "http-custom-headers", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "sep-2243-client-omit-null", + "mode": "2026-07-28", + "scenario": "http-custom-headers", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "sep-2243-client-supports-custom-headers", + "mode": "2026-07-28", + "scenario": "http-custom-headers", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "sep-2243-client-reject-invalid-tool", + "mode": "2026-07-28", + "scenario": "http-invalid-tool-headers", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "sep-2243-x-mcp-header-charset", + "mode": "2026-07-28", + "scenario": "http-invalid-tool-headers", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "sep-2243-x-mcp-header-not-empty", + "mode": "2026-07-28", + "scenario": "http-invalid-tool-headers", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "sep-2243-x-mcp-header-primitive-only", + "mode": "2026-07-28", + "scenario": "http-invalid-tool-headers", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "sep-2243-x-mcp-header-unique", + "mode": "2026-07-28", + "scenario": "http-invalid-tool-headers", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "sep-2243-client-includes-standard-headers", + "mode": "2026-07-28", + "scenario": "http-standard-headers", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "sep-2106-no-network-ref-deref", + "mode": "2026-07-28", + "scenario": "json-schema-ref-no-deref", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "sep-2575-client-declares-elicitation-capability", + "mode": "2026-07-28", + "scenario": "request-metadata", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "sep-2575-client-populates-meta", + "mode": "2026-07-28", + "scenario": "request-metadata", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "sep-2575-client-retry-supported-version", + "mode": "2026-07-28", + "scenario": "request-metadata", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "sep-2575-client-sends-client-info", + "mode": "2026-07-28", + "scenario": "request-metadata", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "sep-2575-http-client-sends-version-header", + "mode": "2026-07-28", + "scenario": "request-metadata", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "sep-2575-http-version-header-matches-meta", + "mode": "2026-07-28", + "scenario": "request-metadata", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "sep-2322-client-jsonrpc-id-different", + "mode": "2026-07-28", + "scenario": "sep-2322-client-request-state", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "sep-2322-client-no-state-omitted", + "mode": "2026-07-28", + "scenario": "sep-2322-client-request-state", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "sep-2322-client-parallel-isolation", + "mode": "2026-07-28", + "scenario": "sep-2322-client-request-state", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "sep-2322-client-request-state-echoed", + "mode": "2026-07-28", + "scenario": "sep-2322-client-request-state", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "sep-2322-default-result-type-complete", + "mode": "2026-07-28", + "scenario": "sep-2322-client-request-state", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "tool-add-numbers", + "mode": "2026-07-28", + "scenario": "tools_call", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "wire-schema-valid", + "mode": "2026-07-28", + "scenario": "tools_call", + "source": "official", + "transport": "official-http" + }, + { + "check_id": "app_server_initialize", + "mode": "2026-07-28", + "scenario": "", + "source": "supplemental", + "transport": "stdio" + }, + { + "check_id": "echo_tool", + "mode": "2026-07-28", + "scenario": "", + "source": "supplemental", + "transport": "stdio" + }, + { + "check_id": "ephemeral_thread", + "mode": "2026-07-28", + "scenario": "", + "source": "supplemental", + "transport": "stdio" + }, + { + "check_id": "inventory", + "mode": "2026-07-28", + "scenario": "", + "source": "supplemental", + "transport": "stdio" + }, + { + "check_id": "isolated_config_cleanup", + "mode": "2026-07-28", + "scenario": "", + "source": "supplemental", + "transport": "stdio" + }, + { + "check_id": "mcp_add", + "mode": "2026-07-28", + "scenario": "", + "source": "supplemental", + "transport": "stdio" + }, + { + "check_id": "mcp_get", + "mode": "2026-07-28", + "scenario": "", + "source": "supplemental", + "transport": "stdio" + }, + { + "check_id": "mcp_remove", + "mode": "2026-07-28", + "scenario": "", + "source": "supplemental", + "transport": "stdio" + }, + { + "check_id": "modern_feature_enablement", + "mode": "2026-07-28", + "scenario": "", + "source": "supplemental", + "transport": "stdio" + }, + { + "check_id": "multi_round_trip_request", + "mode": "2026-07-28", + "scenario": "", + "source": "supplemental", + "transport": "stdio" + }, + { + "check_id": "per_request_metadata", + "mode": "2026-07-28", + "scenario": "", + "source": "supplemental", + "transport": "stdio" + }, + { + "check_id": "request_scoped_notifications", + "mode": "2026-07-28", + "scenario": "", + "source": "supplemental", + "transport": "stdio" + }, + { + "check_id": "unicode_resource_read", + "mode": "2026-07-28", + "scenario": "", + "source": "supplemental", + "transport": "stdio" + } + ] + }, + "modernFeatureEnablement": true, + "officialConformance": { + "authenticationIncluded": true, + "gitRef": "49103de6ed70804e940637bf3e9e29e4a3f54e64", + "repository": "modelcontextprotocol/conformance", + "scenarios": { + "2025-06-18": [ + "initialize", + "tools_call", + "auth/token-endpoint-auth-basic", + "auth/token-endpoint-auth-post", + "auth/token-endpoint-auth-none" + ], + "2025-11-25": [ + "initialize", + "tools_call", + "elicitation-sep1034-client-defaults", + "sse-retry", + "auth/metadata-default", + "auth/metadata-var1", + "auth/metadata-var2", + "auth/metadata-var3", + "auth/basic-cimd", + "auth/scope-from-www-authenticate", + "auth/scope-from-scopes-supported", + "auth/scope-omitted-when-undefined", + "auth/scope-step-up", + "auth/scope-retry-limit", + "auth/token-endpoint-auth-basic", + "auth/token-endpoint-auth-post", + "auth/token-endpoint-auth-none", + "auth/pre-registration" + ], + "2026-07-28": [ + "tools_call", + "request-metadata", + "sep-2322-client-request-state", + "http-standard-headers", + "http-custom-headers", + "http-invalid-tool-headers", + "json-schema-ref-no-deref", + "auth/metadata-default", + "auth/metadata-var1", + "auth/metadata-var2", + "auth/metadata-var3", + "auth/basic-cimd", + "auth/scope-from-www-authenticate", + "auth/scope-from-scopes-supported", + "auth/scope-omitted-when-undefined", + "auth/scope-step-up", + "auth/scope-retry-limit", + "auth/token-endpoint-auth-basic", + "auth/token-endpoint-auth-post", + "auth/token-endpoint-auth-none", + "auth/pre-registration", + "auth/resource-mismatch", + "auth/offline-access-scope", + "auth/offline-access-not-supported", + "auth/authorization-server-migration", + "auth/iss-supported", + "auth/iss-not-advertised", + "auth/iss-supported-missing", + "auth/iss-wrong-issuer", + "auth/iss-unexpected", + "auth/iss-normalized", + "auth/metadata-issuer-mismatch" + ] + } + }, + "requiredModes": [ + "2025-06-18", + "2025-11-25", + "2026-07-28" + ], + "schemaVersion": 4, + "transports": [ + "stdio", + "official-http" + ], + "versionCheck": { + "success": true + } +} diff --git a/scripts/mcp_conformance/review-regression-baseline-v1.json b/scripts/mcp_conformance/review-regression-baseline-v1.json new file mode 100644 index 0000000000..5688eafc04 --- /dev/null +++ b/scripts/mcp_conformance/review-regression-baseline-v1.json @@ -0,0 +1,1039 @@ +{ + "baselineKind": "mcp-review-regression-baseline-v1", + "checks": { + "failing": [ + { + "check_id": "review/catalog-over-limit/exact-limit-error", + "mode": "2025-06-18", + "transport": "review-http:catalog-over-limit" + }, + { + "check_id": "review/catalog-over-limit/server-info", + "mode": "2025-06-18", + "transport": "review-http:catalog-over-limit" + }, + { + "check_id": "review/catalog-over-limit/startup-failed", + "mode": "2025-06-18", + "transport": "review-http:catalog-over-limit" + }, + { + "check_id": "review/catalog-over-limit/tool-count", + "mode": "2025-06-18", + "transport": "review-http:catalog-over-limit" + }, + { + "check_id": "review/catalog-over-limit/exact-limit-error", + "mode": "2025-06-18", + "transport": "review-stdio:catalog-over-limit" + }, + { + "check_id": "review/catalog-over-limit/server-info", + "mode": "2025-06-18", + "transport": "review-stdio:catalog-over-limit" + }, + { + "check_id": "review/catalog-over-limit/startup-failed", + "mode": "2025-06-18", + "transport": "review-stdio:catalog-over-limit" + }, + { + "check_id": "review/catalog-over-limit/tool-count", + "mode": "2025-06-18", + "transport": "review-stdio:catalog-over-limit" + }, + { + "check_id": "review/catalog-over-limit/exact-limit-error", + "mode": "2025-11-25", + "transport": "review-http:catalog-over-limit" + }, + { + "check_id": "review/catalog-over-limit/server-info", + "mode": "2025-11-25", + "transport": "review-http:catalog-over-limit" + }, + { + "check_id": "review/catalog-over-limit/startup-failed", + "mode": "2025-11-25", + "transport": "review-http:catalog-over-limit" + }, + { + "check_id": "review/catalog-over-limit/tool-count", + "mode": "2025-11-25", + "transport": "review-http:catalog-over-limit" + }, + { + "check_id": "review/catalog-over-limit/exact-limit-error", + "mode": "2025-11-25", + "transport": "review-stdio:catalog-over-limit" + }, + { + "check_id": "review/catalog-over-limit/server-info", + "mode": "2025-11-25", + "transport": "review-stdio:catalog-over-limit" + }, + { + "check_id": "review/catalog-over-limit/startup-failed", + "mode": "2025-11-25", + "transport": "review-stdio:catalog-over-limit" + }, + { + "check_id": "review/catalog-over-limit/tool-count", + "mode": "2025-11-25", + "transport": "review-stdio:catalog-over-limit" + }, + { + "check_id": "review/catalog-over-limit/exact-limit-error", + "mode": "2026-07-28", + "transport": "review-http:catalog-over-limit" + }, + { + "check_id": "review/catalog-over-limit/server-info", + "mode": "2026-07-28", + "transport": "review-http:catalog-over-limit" + }, + { + "check_id": "review/catalog-over-limit/startup-failed", + "mode": "2026-07-28", + "transport": "review-http:catalog-over-limit" + }, + { + "check_id": "review/catalog-over-limit/tool-count", + "mode": "2026-07-28", + "transport": "review-http:catalog-over-limit" + }, + { + "check_id": "review/exact-large-integer-elicitation-round-trip", + "mode": "2026-07-28", + "transport": "review-http:review-regressions" + }, + { + "check_id": "review/exact-large-integer-elicitation-schema", + "mode": "2026-07-28", + "transport": "review-http:review-regressions" + }, + { + "check_id": "review/mrtr-input-requests-bounded-to-64", + "mode": "2026-07-28", + "transport": "review-http:review-regressions" + }, + { + "check_id": "review/catalog-over-limit/exact-limit-error", + "mode": "2026-07-28", + "transport": "review-stdio:catalog-over-limit" + }, + { + "check_id": "review/catalog-over-limit/server-info", + "mode": "2026-07-28", + "transport": "review-stdio:catalog-over-limit" + }, + { + "check_id": "review/catalog-over-limit/startup-failed", + "mode": "2026-07-28", + "transport": "review-stdio:catalog-over-limit" + }, + { + "check_id": "review/catalog-over-limit/tool-count", + "mode": "2026-07-28", + "transport": "review-stdio:catalog-over-limit" + }, + { + "check_id": "review/exact-large-integer-elicitation-round-trip", + "mode": "2026-07-28", + "transport": "review-stdio:review-regressions" + }, + { + "check_id": "review/exact-large-integer-elicitation-schema", + "mode": "2026-07-28", + "transport": "review-stdio:review-regressions" + }, + { + "check_id": "review/mrtr-input-requests-bounded-to-64", + "mode": "2026-07-28", + "transport": "review-stdio:review-regressions" + } + ], + "passing": [ + { + "check_id": "review/app-server-initialize", + "mode": "2025-06-18", + "transport": "review-http:catalog-max" + }, + { + "check_id": "review/catalog-at-limit/configured-server", + "mode": "2025-06-18", + "transport": "review-http:catalog-max" + }, + { + "check_id": "review/catalog-at-limit/server-info", + "mode": "2025-06-18", + "transport": "review-http:catalog-max" + }, + { + "check_id": "review/catalog-at-limit/startup-ready", + "mode": "2025-06-18", + "transport": "review-http:catalog-max" + }, + { + "check_id": "review/catalog-at-limit/tool-count", + "mode": "2025-06-18", + "transport": "review-http:catalog-max" + }, + { + "check_id": "review/ephemeral-thread", + "mode": "2025-06-18", + "transport": "review-http:catalog-max" + }, + { + "check_id": "review/isolated-registration-cleanup", + "mode": "2025-06-18", + "transport": "review-http:catalog-max" + }, + { + "check_id": "review/mcp-registration", + "mode": "2025-06-18", + "transport": "review-http:catalog-max" + }, + { + "check_id": "review/app-server-initialize", + "mode": "2025-06-18", + "transport": "review-http:catalog-over-limit" + }, + { + "check_id": "review/catalog-over-limit/configured-server", + "mode": "2025-06-18", + "transport": "review-http:catalog-over-limit" + }, + { + "check_id": "review/ephemeral-thread", + "mode": "2025-06-18", + "transport": "review-http:catalog-over-limit" + }, + { + "check_id": "review/isolated-registration-cleanup", + "mode": "2025-06-18", + "transport": "review-http:catalog-over-limit" + }, + { + "check_id": "review/mcp-registration", + "mode": "2025-06-18", + "transport": "review-http:catalog-over-limit" + }, + { + "check_id": "review/app-server-initialize", + "mode": "2025-06-18", + "transport": "review-stdio:catalog-max" + }, + { + "check_id": "review/catalog-at-limit/configured-server", + "mode": "2025-06-18", + "transport": "review-stdio:catalog-max" + }, + { + "check_id": "review/catalog-at-limit/server-info", + "mode": "2025-06-18", + "transport": "review-stdio:catalog-max" + }, + { + "check_id": "review/catalog-at-limit/startup-ready", + "mode": "2025-06-18", + "transport": "review-stdio:catalog-max" + }, + { + "check_id": "review/catalog-at-limit/tool-count", + "mode": "2025-06-18", + "transport": "review-stdio:catalog-max" + }, + { + "check_id": "review/ephemeral-thread", + "mode": "2025-06-18", + "transport": "review-stdio:catalog-max" + }, + { + "check_id": "review/isolated-registration-cleanup", + "mode": "2025-06-18", + "transport": "review-stdio:catalog-max" + }, + { + "check_id": "review/mcp-registration", + "mode": "2025-06-18", + "transport": "review-stdio:catalog-max" + }, + { + "check_id": "review/app-server-initialize", + "mode": "2025-06-18", + "transport": "review-stdio:catalog-over-limit" + }, + { + "check_id": "review/catalog-over-limit/configured-server", + "mode": "2025-06-18", + "transport": "review-stdio:catalog-over-limit" + }, + { + "check_id": "review/ephemeral-thread", + "mode": "2025-06-18", + "transport": "review-stdio:catalog-over-limit" + }, + { + "check_id": "review/isolated-registration-cleanup", + "mode": "2025-06-18", + "transport": "review-stdio:catalog-over-limit" + }, + { + "check_id": "review/mcp-registration", + "mode": "2025-06-18", + "transport": "review-stdio:catalog-over-limit" + }, + { + "check_id": "review/app-server-initialize", + "mode": "2025-06-18", + "transport": "review-stdio:review-regressions" + }, + { + "check_id": "review/ephemeral-thread", + "mode": "2025-06-18", + "transport": "review-stdio:review-regressions" + }, + { + "check_id": "review/exact-large-integer-tool-round-trip", + "mode": "2025-06-18", + "transport": "review-stdio:review-regressions" + }, + { + "check_id": "review/exact-large-integer-tool-schema", + "mode": "2025-06-18", + "transport": "review-stdio:review-regressions" + }, + { + "check_id": "review/isolated-registration-cleanup", + "mode": "2025-06-18", + "transport": "review-stdio:review-regressions" + }, + { + "check_id": "review/legacy-reserved-stdio-environment-preserved", + "mode": "2025-06-18", + "transport": "review-stdio:review-regressions" + }, + { + "check_id": "review/mcp-registration", + "mode": "2025-06-18", + "transport": "review-stdio:review-regressions" + }, + { + "check_id": "review/server-discovery", + "mode": "2025-06-18", + "transport": "review-stdio:review-regressions" + }, + { + "check_id": "review/app-server-initialize", + "mode": "2025-11-25", + "transport": "review-http:catalog-max" + }, + { + "check_id": "review/catalog-at-limit/configured-server", + "mode": "2025-11-25", + "transport": "review-http:catalog-max" + }, + { + "check_id": "review/catalog-at-limit/server-info", + "mode": "2025-11-25", + "transport": "review-http:catalog-max" + }, + { + "check_id": "review/catalog-at-limit/startup-ready", + "mode": "2025-11-25", + "transport": "review-http:catalog-max" + }, + { + "check_id": "review/catalog-at-limit/tool-count", + "mode": "2025-11-25", + "transport": "review-http:catalog-max" + }, + { + "check_id": "review/ephemeral-thread", + "mode": "2025-11-25", + "transport": "review-http:catalog-max" + }, + { + "check_id": "review/isolated-registration-cleanup", + "mode": "2025-11-25", + "transport": "review-http:catalog-max" + }, + { + "check_id": "review/mcp-registration", + "mode": "2025-11-25", + "transport": "review-http:catalog-max" + }, + { + "check_id": "review/app-server-initialize", + "mode": "2025-11-25", + "transport": "review-http:catalog-over-limit" + }, + { + "check_id": "review/catalog-over-limit/configured-server", + "mode": "2025-11-25", + "transport": "review-http:catalog-over-limit" + }, + { + "check_id": "review/ephemeral-thread", + "mode": "2025-11-25", + "transport": "review-http:catalog-over-limit" + }, + { + "check_id": "review/isolated-registration-cleanup", + "mode": "2025-11-25", + "transport": "review-http:catalog-over-limit" + }, + { + "check_id": "review/mcp-registration", + "mode": "2025-11-25", + "transport": "review-http:catalog-over-limit" + }, + { + "check_id": "review/app-server-initialize", + "mode": "2025-11-25", + "transport": "review-stdio:catalog-max" + }, + { + "check_id": "review/catalog-at-limit/configured-server", + "mode": "2025-11-25", + "transport": "review-stdio:catalog-max" + }, + { + "check_id": "review/catalog-at-limit/server-info", + "mode": "2025-11-25", + "transport": "review-stdio:catalog-max" + }, + { + "check_id": "review/catalog-at-limit/startup-ready", + "mode": "2025-11-25", + "transport": "review-stdio:catalog-max" + }, + { + "check_id": "review/catalog-at-limit/tool-count", + "mode": "2025-11-25", + "transport": "review-stdio:catalog-max" + }, + { + "check_id": "review/ephemeral-thread", + "mode": "2025-11-25", + "transport": "review-stdio:catalog-max" + }, + { + "check_id": "review/isolated-registration-cleanup", + "mode": "2025-11-25", + "transport": "review-stdio:catalog-max" + }, + { + "check_id": "review/mcp-registration", + "mode": "2025-11-25", + "transport": "review-stdio:catalog-max" + }, + { + "check_id": "review/app-server-initialize", + "mode": "2025-11-25", + "transport": "review-stdio:catalog-over-limit" + }, + { + "check_id": "review/catalog-over-limit/configured-server", + "mode": "2025-11-25", + "transport": "review-stdio:catalog-over-limit" + }, + { + "check_id": "review/ephemeral-thread", + "mode": "2025-11-25", + "transport": "review-stdio:catalog-over-limit" + }, + { + "check_id": "review/isolated-registration-cleanup", + "mode": "2025-11-25", + "transport": "review-stdio:catalog-over-limit" + }, + { + "check_id": "review/mcp-registration", + "mode": "2025-11-25", + "transport": "review-stdio:catalog-over-limit" + }, + { + "check_id": "review/app-server-initialize", + "mode": "2025-11-25", + "transport": "review-stdio:review-regressions" + }, + { + "check_id": "review/ephemeral-thread", + "mode": "2025-11-25", + "transport": "review-stdio:review-regressions" + }, + { + "check_id": "review/exact-large-integer-tool-round-trip", + "mode": "2025-11-25", + "transport": "review-stdio:review-regressions" + }, + { + "check_id": "review/exact-large-integer-tool-schema", + "mode": "2025-11-25", + "transport": "review-stdio:review-regressions" + }, + { + "check_id": "review/isolated-registration-cleanup", + "mode": "2025-11-25", + "transport": "review-stdio:review-regressions" + }, + { + "check_id": "review/legacy-reserved-stdio-environment-preserved", + "mode": "2025-11-25", + "transport": "review-stdio:review-regressions" + }, + { + "check_id": "review/mcp-registration", + "mode": "2025-11-25", + "transport": "review-stdio:review-regressions" + }, + { + "check_id": "review/server-discovery", + "mode": "2025-11-25", + "transport": "review-stdio:review-regressions" + }, + { + "check_id": "review/app-server-initialize", + "mode": "2026-07-28", + "transport": "review-http:catalog-max" + }, + { + "check_id": "review/catalog-at-limit/configured-server", + "mode": "2026-07-28", + "transport": "review-http:catalog-max" + }, + { + "check_id": "review/catalog-at-limit/server-info", + "mode": "2026-07-28", + "transport": "review-http:catalog-max" + }, + { + "check_id": "review/catalog-at-limit/startup-ready", + "mode": "2026-07-28", + "transport": "review-http:catalog-max" + }, + { + "check_id": "review/catalog-at-limit/tool-count", + "mode": "2026-07-28", + "transport": "review-http:catalog-max" + }, + { + "check_id": "review/ephemeral-thread", + "mode": "2026-07-28", + "transport": "review-http:catalog-max" + }, + { + "check_id": "review/isolated-registration-cleanup", + "mode": "2026-07-28", + "transport": "review-http:catalog-max" + }, + { + "check_id": "review/mcp-registration", + "mode": "2026-07-28", + "transport": "review-http:catalog-max" + }, + { + "check_id": "review/modern-feature-configuration", + "mode": "2026-07-28", + "transport": "review-http:catalog-max" + }, + { + "check_id": "review/modern-feature-enablement", + "mode": "2026-07-28", + "transport": "review-http:catalog-max" + }, + { + "check_id": "review/app-server-initialize", + "mode": "2026-07-28", + "transport": "review-http:catalog-over-limit" + }, + { + "check_id": "review/catalog-over-limit/configured-server", + "mode": "2026-07-28", + "transport": "review-http:catalog-over-limit" + }, + { + "check_id": "review/ephemeral-thread", + "mode": "2026-07-28", + "transport": "review-http:catalog-over-limit" + }, + { + "check_id": "review/isolated-registration-cleanup", + "mode": "2026-07-28", + "transport": "review-http:catalog-over-limit" + }, + { + "check_id": "review/mcp-registration", + "mode": "2026-07-28", + "transport": "review-http:catalog-over-limit" + }, + { + "check_id": "review/modern-feature-configuration", + "mode": "2026-07-28", + "transport": "review-http:catalog-over-limit" + }, + { + "check_id": "review/modern-feature-enablement", + "mode": "2026-07-28", + "transport": "review-http:catalog-over-limit" + }, + { + "check_id": "review/app-server-initialize", + "mode": "2026-07-28", + "transport": "review-http:discovery-mismatched-id" + }, + { + "check_id": "review/isolated-registration-cleanup", + "mode": "2026-07-28", + "transport": "review-http:discovery-mismatched-id" + }, + { + "check_id": "review/mcp-registration", + "mode": "2026-07-28", + "transport": "review-http:discovery-mismatched-id" + }, + { + "check_id": "review/modern-feature-configuration", + "mode": "2026-07-28", + "transport": "review-http:discovery-mismatched-id" + }, + { + "check_id": "review/modern-feature-enablement", + "mode": "2026-07-28", + "transport": "review-http:discovery-mismatched-id" + }, + { + "check_id": "review/reject-mismatched-discovery-response-id", + "mode": "2026-07-28", + "transport": "review-http:discovery-mismatched-id" + }, + { + "check_id": "review/app-server-initialize", + "mode": "2026-07-28", + "transport": "review-http:discovery-null-id" + }, + { + "check_id": "review/isolated-registration-cleanup", + "mode": "2026-07-28", + "transport": "review-http:discovery-null-id" + }, + { + "check_id": "review/mcp-registration", + "mode": "2026-07-28", + "transport": "review-http:discovery-null-id" + }, + { + "check_id": "review/modern-feature-configuration", + "mode": "2026-07-28", + "transport": "review-http:discovery-null-id" + }, + { + "check_id": "review/modern-feature-enablement", + "mode": "2026-07-28", + "transport": "review-http:discovery-null-id" + }, + { + "check_id": "review/reject-null-discovery-response-id", + "mode": "2026-07-28", + "transport": "review-http:discovery-null-id" + }, + { + "check_id": "review/app-server-initialize", + "mode": "2026-07-28", + "transport": "review-http:repeated-cursor" + }, + { + "check_id": "review/isolated-registration-cleanup", + "mode": "2026-07-28", + "transport": "review-http:repeated-cursor" + }, + { + "check_id": "review/mcp-registration", + "mode": "2026-07-28", + "transport": "review-http:repeated-cursor" + }, + { + "check_id": "review/modern-feature-configuration", + "mode": "2026-07-28", + "transport": "review-http:repeated-cursor" + }, + { + "check_id": "review/modern-feature-enablement", + "mode": "2026-07-28", + "transport": "review-http:repeated-cursor" + }, + { + "check_id": "review/repeated-pagination-cursor-bounded", + "mode": "2026-07-28", + "transport": "review-http:repeated-cursor" + }, + { + "check_id": "review/app-server-initialize", + "mode": "2026-07-28", + "transport": "review-http:review-regressions" + }, + { + "check_id": "review/ephemeral-thread", + "mode": "2026-07-28", + "transport": "review-http:review-regressions" + }, + { + "check_id": "review/exact-large-integer-tool-round-trip", + "mode": "2026-07-28", + "transport": "review-http:review-regressions" + }, + { + "check_id": "review/exact-large-integer-tool-schema", + "mode": "2026-07-28", + "transport": "review-http:review-regressions" + }, + { + "check_id": "review/isolated-registration-cleanup", + "mode": "2026-07-28", + "transport": "review-http:review-regressions" + }, + { + "check_id": "review/mcp-registration", + "mode": "2026-07-28", + "transport": "review-http:review-regressions" + }, + { + "check_id": "review/modern-feature-configuration", + "mode": "2026-07-28", + "transport": "review-http:review-regressions" + }, + { + "check_id": "review/modern-feature-enablement", + "mode": "2026-07-28", + "transport": "review-http:review-regressions" + }, + { + "check_id": "review/modern-paginated-tools-page-two", + "mode": "2026-07-28", + "transport": "review-http:review-regressions" + }, + { + "check_id": "review/server-discovery", + "mode": "2026-07-28", + "transport": "review-http:review-regressions" + }, + { + "check_id": "review/app-server-initialize", + "mode": "2026-07-28", + "transport": "review-http:sse-comment-flood" + }, + { + "check_id": "review/ephemeral-thread", + "mode": "2026-07-28", + "transport": "review-http:sse-comment-flood" + }, + { + "check_id": "review/isolated-registration-cleanup", + "mode": "2026-07-28", + "transport": "review-http:sse-comment-flood" + }, + { + "check_id": "review/mcp-registration", + "mode": "2026-07-28", + "transport": "review-http:sse-comment-flood" + }, + { + "check_id": "review/modern-feature-configuration", + "mode": "2026-07-28", + "transport": "review-http:sse-comment-flood" + }, + { + "check_id": "review/modern-feature-enablement", + "mode": "2026-07-28", + "transport": "review-http:sse-comment-flood" + }, + { + "check_id": "review/sse-comment-flood-excluded-from-event-size-limit", + "mode": "2026-07-28", + "transport": "review-http:sse-comment-flood" + }, + { + "check_id": "review/sse-cr-discovery", + "mode": "2026-07-28", + "transport": "review-http:sse-comment-flood" + }, + { + "check_id": "review/app-server-initialize", + "mode": "2026-07-28", + "transport": "review-http:sse-cr-comments" + }, + { + "check_id": "review/ephemeral-thread", + "mode": "2026-07-28", + "transport": "review-http:sse-cr-comments" + }, + { + "check_id": "review/isolated-registration-cleanup", + "mode": "2026-07-28", + "transport": "review-http:sse-cr-comments" + }, + { + "check_id": "review/mcp-registration", + "mode": "2026-07-28", + "transport": "review-http:sse-cr-comments" + }, + { + "check_id": "review/modern-feature-configuration", + "mode": "2026-07-28", + "transport": "review-http:sse-cr-comments" + }, + { + "check_id": "review/modern-feature-enablement", + "mode": "2026-07-28", + "transport": "review-http:sse-cr-comments" + }, + { + "check_id": "review/sse-carriage-return-and-comment-framing", + "mode": "2026-07-28", + "transport": "review-http:sse-cr-comments" + }, + { + "check_id": "review/sse-cr-discovery", + "mode": "2026-07-28", + "transport": "review-http:sse-cr-comments" + }, + { + "check_id": "review/app-server-initialize", + "mode": "2026-07-28", + "transport": "review-stdio:catalog-max" + }, + { + "check_id": "review/catalog-at-limit/configured-server", + "mode": "2026-07-28", + "transport": "review-stdio:catalog-max" + }, + { + "check_id": "review/catalog-at-limit/server-info", + "mode": "2026-07-28", + "transport": "review-stdio:catalog-max" + }, + { + "check_id": "review/catalog-at-limit/startup-ready", + "mode": "2026-07-28", + "transport": "review-stdio:catalog-max" + }, + { + "check_id": "review/catalog-at-limit/tool-count", + "mode": "2026-07-28", + "transport": "review-stdio:catalog-max" + }, + { + "check_id": "review/ephemeral-thread", + "mode": "2026-07-28", + "transport": "review-stdio:catalog-max" + }, + { + "check_id": "review/isolated-registration-cleanup", + "mode": "2026-07-28", + "transport": "review-stdio:catalog-max" + }, + { + "check_id": "review/mcp-registration", + "mode": "2026-07-28", + "transport": "review-stdio:catalog-max" + }, + { + "check_id": "review/modern-feature-configuration", + "mode": "2026-07-28", + "transport": "review-stdio:catalog-max" + }, + { + "check_id": "review/modern-feature-enablement", + "mode": "2026-07-28", + "transport": "review-stdio:catalog-max" + }, + { + "check_id": "review/app-server-initialize", + "mode": "2026-07-28", + "transport": "review-stdio:catalog-over-limit" + }, + { + "check_id": "review/catalog-over-limit/configured-server", + "mode": "2026-07-28", + "transport": "review-stdio:catalog-over-limit" + }, + { + "check_id": "review/ephemeral-thread", + "mode": "2026-07-28", + "transport": "review-stdio:catalog-over-limit" + }, + { + "check_id": "review/isolated-registration-cleanup", + "mode": "2026-07-28", + "transport": "review-stdio:catalog-over-limit" + }, + { + "check_id": "review/mcp-registration", + "mode": "2026-07-28", + "transport": "review-stdio:catalog-over-limit" + }, + { + "check_id": "review/modern-feature-configuration", + "mode": "2026-07-28", + "transport": "review-stdio:catalog-over-limit" + }, + { + "check_id": "review/modern-feature-enablement", + "mode": "2026-07-28", + "transport": "review-stdio:catalog-over-limit" + }, + { + "check_id": "review/app-server-initialize", + "mode": "2026-07-28", + "transport": "review-stdio:review-regressions" + }, + { + "check_id": "review/ephemeral-thread", + "mode": "2026-07-28", + "transport": "review-stdio:review-regressions" + }, + { + "check_id": "review/exact-large-integer-tool-round-trip", + "mode": "2026-07-28", + "transport": "review-stdio:review-regressions" + }, + { + "check_id": "review/exact-large-integer-tool-schema", + "mode": "2026-07-28", + "transport": "review-stdio:review-regressions" + }, + { + "check_id": "review/isolated-registration-cleanup", + "mode": "2026-07-28", + "transport": "review-stdio:review-regressions" + }, + { + "check_id": "review/mcp-registration", + "mode": "2026-07-28", + "transport": "review-stdio:review-regressions" + }, + { + "check_id": "review/modern-feature-configuration", + "mode": "2026-07-28", + "transport": "review-stdio:review-regressions" + }, + { + "check_id": "review/modern-feature-enablement", + "mode": "2026-07-28", + "transport": "review-stdio:review-regressions" + }, + { + "check_id": "review/modern-paginated-tools-page-two", + "mode": "2026-07-28", + "transport": "review-stdio:review-regressions" + }, + { + "check_id": "review/server-discovery", + "mode": "2026-07-28", + "transport": "review-stdio:review-regressions" + } + ] + }, + "requiredCases": [ + { + "mode": "2025-06-18", + "transport": "review-http:catalog-max" + }, + { + "mode": "2025-06-18", + "transport": "review-http:catalog-over-limit" + }, + { + "mode": "2025-06-18", + "transport": "review-stdio:catalog-max" + }, + { + "mode": "2025-06-18", + "transport": "review-stdio:catalog-over-limit" + }, + { + "mode": "2025-06-18", + "transport": "review-stdio:review-regressions" + }, + { + "mode": "2025-11-25", + "transport": "review-http:catalog-max" + }, + { + "mode": "2025-11-25", + "transport": "review-http:catalog-over-limit" + }, + { + "mode": "2025-11-25", + "transport": "review-stdio:catalog-max" + }, + { + "mode": "2025-11-25", + "transport": "review-stdio:catalog-over-limit" + }, + { + "mode": "2025-11-25", + "transport": "review-stdio:review-regressions" + }, + { + "mode": "2026-07-28", + "transport": "review-http:catalog-max" + }, + { + "mode": "2026-07-28", + "transport": "review-http:catalog-over-limit" + }, + { + "mode": "2026-07-28", + "transport": "review-http:discovery-mismatched-id" + }, + { + "mode": "2026-07-28", + "transport": "review-http:discovery-null-id" + }, + { + "mode": "2026-07-28", + "transport": "review-http:repeated-cursor" + }, + { + "mode": "2026-07-28", + "transport": "review-http:review-regressions" + }, + { + "mode": "2026-07-28", + "transport": "review-http:sse-comment-flood" + }, + { + "mode": "2026-07-28", + "transport": "review-http:sse-cr-comments" + }, + { + "mode": "2026-07-28", + "transport": "review-stdio:catalog-max" + }, + { + "mode": "2026-07-28", + "transport": "review-stdio:catalog-over-limit" + }, + { + "mode": "2026-07-28", + "transport": "review-stdio:review-regressions" + } + ], + "requiredModes": [ + "2025-06-18", + "2025-11-25", + "2026-07-28" + ], + "reviewer": "codex-mcp-regression-suite", + "schemaVersion": 1, + "summary": { + "casesPassed": 13, + "casesTotal": 21, + "failed": 30, + "passed": 156, + "total": 186 + } +} diff --git a/scripts/mcp_conformance/review_regressions.py b/scripts/mcp_conformance/review_regressions.py new file mode 100644 index 0000000000..7a9d059e45 --- /dev/null +++ b/scripts/mcp_conformance/review_regressions.py @@ -0,0 +1,1294 @@ +#!/usr/bin/env python3 +"""Run reviewer-derived MCP regressions against a real Codex app-server.""" + +import argparse +import json +import os +import shutil +import sys +import tempfile +import threading +import time +from contextlib import contextmanager, nullcontext +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Iterator, Mapping, Sequence + +_MODULE_DIR = Path(__file__).resolve().parent +if str(_MODULE_DIR) not in sys.path: + sys.path.insert(0, str(_MODULE_DIR)) + +from run_codex_compliance import ( # noqa: E402 - direct scripts must first add their sibling directory. + LEGACY_VERSION, + MISMATCHED_DISCOVERY_ID_PROFILE, + MODERN_VERSION, + NULL_DISCOVERY_ID_PROFILE, + REPEATED_CURSOR_PROFILE, + REVIEW_EXACT_INTEGER, + REVIEW_MRTR_INPUT_REQUEST_COUNT, + REVIEW_PROFILE, + SHIPPING_LEGACY_VERSION, + SSE_COMMENT_FLOOD_PROFILE, + SSE_CR_COMMENTS_PROFILE, + TEST_SERVER_NAME, + AppServerClient, + AppServerError, + CaseResult, + ProtocolServer, + _call_tool, + _command_detail, + _isolated_environment, + _response_result, + _run_command, + make_http_server, +) +from server import ( # noqa: E402 - direct scripts must first add their sibling directory. + CATALOG_MAX_PROFILE, + CATALOG_OVER_LIMIT_PROFILE, + MAX_CATALOG_ITEMS, +) + +REVIEW_MODES = (SHIPPING_LEGACY_VERSION, LEGACY_VERSION, MODERN_VERSION) +REVIEW_REPORT_SCHEMA_VERSION = 1 +REVIEW_BASELINE_KIND = "mcp-review-regression-baseline-v1" +REVIEWER = "codex-mcp-regression-suite" +LEGACY_ENVIRONMENT_SENTINEL = "review-legacy-protocol-environment" +MRTR_REQUEST_LIMIT = 64 +CATALOG_BOUNDARY_PROFILES = (CATALOG_MAX_PROFILE, CATALOG_OVER_LIMIT_PROFILE) +CATALOG_BOUNDARY_TRANSPORTS = ("stdio", "http") +CATALOG_LIMIT_ERROR = ( + f"tools/list exceeded the catalog limit of {MAX_CATALOG_ITEMS} items" +) + + +@dataclass(frozen=True, order=True) +class _ReviewCheckIdentity: + mode: str + transport: str + check_id: str + + +def _required_review_cases() -> set[tuple[str, str]]: + cases = {(mode, f"review-stdio:{REVIEW_PROFILE}") for mode in REVIEW_MODES} + cases.update( + (MODERN_VERSION, f"review-http:{profile}") + for profile in ( + REVIEW_PROFILE, + REPEATED_CURSOR_PROFILE, + MISMATCHED_DISCOVERY_ID_PROFILE, + NULL_DISCOVERY_ID_PROFILE, + SSE_CR_COMMENTS_PROFILE, + SSE_COMMENT_FLOOD_PROFILE, + ) + ) + cases.update( + (mode, f"review-{transport}:{profile}") + for mode in REVIEW_MODES + for transport in CATALOG_BOUNDARY_TRANSPORTS + for profile in CATALOG_BOUNDARY_PROFILES + ) + return cases + + +@contextmanager +def _running_review_http_fixture(mode: str, profile: str) -> Iterator[str]: + server = make_http_server( + ProtocolServer(mode, profile=profile), + "127.0.0.1", + 0, + log_requests=False, + ) + thread = threading.Thread( + target=server.serve_forever, + name=f"mcp-review-{profile}-{mode}", + daemon=True, + ) + thread.start() + try: + _, port = server.server_address + yield f"http://127.0.0.1:{port}/mcp" + finally: + server.shutdown() + server.server_close() + thread.join(timeout=3) + + +def _review_registration_command( + codex_binary: Path, + server_script: Path, + *, + transport: str, + mode: str, + profile: str, + http_url: str | None, +) -> list[str]: + command = [str(codex_binary), "mcp", "add", TEST_SERVER_NAME] + if transport == "stdio": + protocol_environment = ( + MODERN_VERSION if mode == MODERN_VERSION else LEGACY_ENVIRONMENT_SENTINEL + ) + command.extend( + [ + "--env", + f"CODEX_MCP_PROTOCOL_VERSION={protocol_environment}", + "--", + sys.executable, + str(server_script), + "--mode", + mode, + "--transport", + "stdio", + "--profile", + profile, + ] + ) + else: + if http_url is None: + raise ValueError("review HTTP transport requires a fixture URL") + command.extend(["--url", http_url]) + return command + + +def _review_inventory_entry(result: Mapping[str, object]) -> dict[str, object] | None: + entries = result.get("data") + if not isinstance(entries, list): + return None + for entry in entries: + if isinstance(entry, dict) and entry.get("name") == TEST_SERVER_NAME: + return entry + return None + + +def _tool_input_schema( + entry: Mapping[str, object], + tool_name: str, +) -> Mapping[str, object] | None: + tools = entry.get("tools") + if not isinstance(tools, dict): + return None + tool = tools.get(tool_name) + if not isinstance(tool, dict): + return None + schema = tool.get("inputSchema") + if not isinstance(schema, dict): + schema = tool.get("input_schema") + return schema if isinstance(schema, dict) else None + + +def _exact_integer_property(schema: Mapping[str, object] | None) -> bool: + if schema is None: + return False + properties = schema.get("properties") + if not isinstance(properties, dict): + return False + value = properties.get("value") + return ( + isinstance(value, dict) + and value.get("type") == "integer" + and isinstance(value.get("minimum"), int) + and not isinstance(value.get("minimum"), bool) + and value.get("minimum") == REVIEW_EXACT_INTEGER + and isinstance(value.get("default"), int) + and not isinstance(value.get("default"), bool) + and value.get("default") == REVIEW_EXACT_INTEGER + ) + + +def _elicitation_schema(request: Mapping[str, object]) -> Mapping[str, object] | None: + for key in ("requestedSchema", "requested_schema", "schema"): + schema = request.get(key) + if isinstance(schema, dict): + return schema + for key in ("request", "elicitation", "params"): + nested = request.get(key) + if isinstance(nested, dict): + found = _elicitation_schema(nested) + if found is not None: + return found + return None + + +def _review_elicitation_content(params: Mapping[str, object]) -> Mapping[str, object]: + schema = _elicitation_schema(params) + if _exact_integer_property(schema): + return {"value": REVIEW_EXACT_INTEGER} + return {"value": "review-confirmed", "confirmation": "confirmed"} + + +def _mrtr_budget_is_bounded( + request_count: int, + *, + elapsed_seconds: float, + timeout_seconds: float, +) -> bool: + return ( + 0 <= request_count <= MRTR_REQUEST_LIMIT + and elapsed_seconds <= timeout_seconds + 1 + ) + + +def _initialize_review_client( + case: CaseResult, + client: AppServerClient, + *, + mode: str, +) -> bool: + response, detail = _response_result( + client.request( + "initialize", + { + "clientInfo": { + "name": "mcp-review-regression-runner", + "title": "MCP review regression runner", + "version": "1.0.0", + }, + "capabilities": { + "experimentalApi": True, + "requestAttestation": False, + "mcpServerOpenaiFormElicitation": True, + }, + }, + ) + ) + if not case.check("review/app-server-initialize", response is not None, detail): + return False + client.notify("initialized") + if mode != MODERN_VERSION: + return True + + feature, feature_detail = _response_result( + client.request( + "experimentalFeature/enablement/set", + {"enablement": {"mcp_2026_07_28": True}}, + ) + ) + enabled = ( + feature is not None + and isinstance(feature.get("enablement"), dict) + and feature["enablement"].get("mcp_2026_07_28") is True + ) + return case.check( + "review/modern-feature-enablement", + enabled, + "enabled the production modern MCP feature" if enabled else feature_detail, + ) + + +def _review_thread_id( + case: CaseResult, + client: AppServerClient, + workspace: Path, +) -> str | None: + response, detail = _response_result( + client.request("thread/start", {"cwd": str(workspace), "ephemeral": True}) + ) + thread = response.get("thread") if response is not None else None + thread_id = thread.get("id") if isinstance(thread, dict) else None + if not case.check( + "review/ephemeral-thread", + isinstance(thread_id, str), + "created isolated review thread" if isinstance(thread_id, str) else detail, + ): + return None + return str(thread_id) + + +def _run_normal_review_checks( + case: CaseResult, + client: AppServerClient, + *, + mode: str, + workspace: Path, +) -> None: + inventory, inventory_detail = _response_result( + client.request("mcpServerStatus/list", {"detail": "full"}) + ) + entry = _review_inventory_entry(inventory) if inventory is not None else None + if not case.check( + "review/server-discovery", + entry is not None, + "discovered the adversarial review fixture" + if entry is not None + else inventory_detail, + ): + return + assert entry is not None + + tools = entry.get("tools") + if not isinstance(tools, dict): + tools = {} + if mode == MODERN_VERSION: + required_page_two = { + "review_integer_elicitation", + "review_large_integer", + "review_mrtr_cap", + "review_protocol_env", + } + missing = sorted(required_page_two - set(tools)) + case.check( + "review/modern-paginated-tools-page-two", + not missing, + "all protected second-page tools reached the app-server inventory" + if not missing + else "second-page tools missing: " + ", ".join(missing), + ) + + schema = _tool_input_schema(entry, "review_large_integer") + exact_schema = _exact_integer_property(schema) + case.check( + "review/exact-large-integer-tool-schema", + exact_schema, + "preserved integer minimum and default 9007199254740993" + if exact_schema + else "the app-server rounded or omitted the 2^53+1 tool schema", + ) + + thread_id = _review_thread_id(case, client, workspace) + if thread_id is None: + return + + integer, integer_detail = _call_tool( + client, + thread_id=thread_id, + tool="review_large_integer", + arguments={"value": REVIEW_EXACT_INTEGER}, + ) + content = integer.get("structuredContent") if integer is not None else None + exact_value = ( + isinstance(content, dict) + and isinstance(content.get("value"), int) + and not isinstance(content.get("value"), bool) + and content.get("value") == REVIEW_EXACT_INTEGER + ) + case.check( + "review/exact-large-integer-tool-round-trip", + exact_value, + "round-tripped 9007199254740993 without floating-point conversion" + if exact_value + else integer_detail, + ) + + if mode != MODERN_VERSION: + environment, environment_detail = _call_tool( + client, + thread_id=thread_id, + tool="review_protocol_env", + arguments={}, + ) + observed = ( + environment.get("structuredContent") if environment is not None else None + ) + preserved = ( + isinstance(observed, dict) + and observed.get("value") == LEGACY_ENVIRONMENT_SENTINEL + ) + case.check( + "review/legacy-reserved-stdio-environment-preserved", + preserved, + "forwarded the explicitly configured legacy protocol environment unchanged" + if preserved + else environment_detail, + ) + return + + before = len(client.elicitation_requests) + elicitation, elicitation_detail = _call_tool( + client, + thread_id=thread_id, + tool="review_integer_elicitation", + arguments={}, + ) + requests = client.elicitation_requests[before:] + exact_elicitation = any( + _exact_integer_property(_elicitation_schema(request)) for request in requests + ) + case.check( + "review/exact-large-integer-elicitation-schema", + exact_elicitation, + "app-server preserved 9007199254740993 in the actual elicitation schema" + if exact_elicitation + else "the production elicitation schema rounded or omitted the 2^53+1 integer", + ) + elicitation_content = ( + elicitation.get("structuredContent") if elicitation is not None else None + ) + completed = ( + isinstance(elicitation_content, dict) + and elicitation_content.get("value") == REVIEW_EXACT_INTEGER + ) + case.check( + "review/exact-large-integer-elicitation-round-trip", + completed, + "completed elicitation using the exact 2^53+1 integer" + if completed + else elicitation_detail, + ) + + mrtr_before = len(client.elicitation_requests) + started = time.monotonic() + mrtr_result = None + try: + mrtr_result, mrtr_detail = _call_tool( + client, + thread_id=thread_id, + tool="review_mrtr_cap", + arguments={}, + ) + except AppServerError as exc: + mrtr_detail = str(exc) + elapsed = time.monotonic() - started + observed = len(client.elicitation_requests) - mrtr_before + bounded = mrtr_result is None and _mrtr_budget_is_bounded( + observed, + elapsed_seconds=elapsed, + timeout_seconds=client.timeout_seconds, + ) + case.check( + "review/mrtr-input-requests-bounded-to-64", + bounded, + ( + f"rejected {REVIEW_MRTR_INPUT_REQUEST_COUNT} simultaneous input requests; " + f"observed {observed} elicitations in {elapsed:.2f}s" + if bounded + else ( + f"observed {observed} elicitation requests for a " + f"{REVIEW_MRTR_INPUT_REQUEST_COUNT}-request response in " + f"{elapsed:.2f}s; {mrtr_detail}" + ) + ), + ) + + +def _run_malformed_discovery_check( + case: CaseResult, + client: AppServerClient, + *, + profile: str, +) -> None: + name = ( + "review/reject-null-discovery-response-id" + if profile == NULL_DISCOVERY_ID_PROFILE + else "review/reject-mismatched-discovery-response-id" + ) + try: + inventory, detail = _response_result( + client.request("mcpServerStatus/list", {"detail": "full"}) + ) + except AppServerError as exc: + case.check(name, True, f"rejected malformed discovery response: {exc}") + return + entry = _review_inventory_entry(inventory) if inventory is not None else None + tools = entry.get("tools") if entry is not None else None + rejected = ( + inventory is None or entry is None or not isinstance(tools, dict) or not tools + ) + case.check( + name, + rejected, + "rejected the uncorrelated discovery response without silently downgrading" + if rejected + else "accepted an uncorrelated discovery response and exposed server tools", + ) + + +def _run_repeated_cursor_check(case: CaseResult, client: AppServerClient) -> None: + started = time.monotonic() + try: + inventory, _ = _response_result( + client.request("mcpServerStatus/list", {"detail": "full"}) + ) + entry = _review_inventory_entry(inventory) if inventory is not None else None + tools = entry.get("tools") if entry is not None else None + rejected = ( + inventory is None + or entry is None + or not isinstance(tools, dict) + or not tools + ) + detail = ( + "rejected the repeated pagination cursor within the startup deadline" + if rejected + else "accepted or exposed a catalog with a repeated pagination cursor" + ) + except AppServerError as exc: + rejected = True + detail = f"terminated repeated-cursor pagination: {exc}" + elapsed = time.monotonic() - started + case.check( + "review/repeated-pagination-cursor-bounded", + rejected and elapsed <= client.timeout_seconds + 2, + f"{detail}; elapsed={elapsed:.2f}s", + ) + + +def _run_catalog_boundary_checks( + case: CaseResult, + client: AppServerClient, + *, + profile: str, + workspace: Path, +) -> None: + if profile not in CATALOG_BOUNDARY_PROFILES: + raise ValueError(f"unknown catalog boundary profile: {profile}") + + at_limit = profile == CATALOG_MAX_PROFILE + check_prefix = ( + "review/catalog-at-limit" if at_limit else "review/catalog-over-limit" + ) + expected_status = "ready" if at_limit else "failed" + event_index = len(client.events) + if _review_thread_id(case, client, workspace) is None: + return + + try: + event = client.wait_for_notification( + "mcpServer/startupStatus/updated", + predicate=lambda params: ( + params.get("name") == TEST_SERVER_NAME + and params.get("status") in {"ready", "failed"} + ), + after_event_index=event_index, + ) + except AppServerError as exc: + case.check( + f"{check_prefix}/startup-{expected_status}", + False, + f"did not observe the required {expected_status} startup: {exc}", + ) + return + + params = event.get("params") + case.check( + f"{check_prefix}/startup-{expected_status}", + isinstance(params, dict) and params.get("status") == expected_status, + f"observed the expected {expected_status} MCP server startup" + if isinstance(params, dict) and params.get("status") == expected_status + else ( + "observed MCP server startup " + f"{params.get('status')!r}; expected {expected_status!r}" + if isinstance(params, dict) + else "MCP server startup notification did not contain parameters" + ), + ) + if not isinstance(params, dict): + return + + if not at_limit: + error = params.get("error") + case.check( + f"{check_prefix}/exact-limit-error", + isinstance(error, str) and CATALOG_LIMIT_ERROR in error, + error + if isinstance(error, str) + else f"startup failure did not report {CATALOG_LIMIT_ERROR!r}", + ) + + inventory, detail = _response_result( + client.request("mcpServerStatus/list", {"detail": "full"}) + ) + entry = _review_inventory_entry(inventory) if inventory is not None else None + if not case.check( + f"{check_prefix}/configured-server", + entry is not None, + "retained the configured catalog-boundary server" + if entry is not None + else detail, + ): + return + assert entry is not None + + server_info = entry.get("serverInfo") + expected_server_info = ( + isinstance(server_info, dict) if at_limit else server_info is None + ) + case.check( + f"{check_prefix}/server-info", + expected_server_info, + "exposed initialized server information" + if at_limit and expected_server_info + else "left rejected server information null" + if not at_limit and expected_server_info + else "catalog server information did not match its startup status", + ) + + tools = entry.get("tools") + actual_count = len(tools) if isinstance(tools, dict) else None + expected_count = MAX_CATALOG_ITEMS if at_limit else 0 + case.check( + f"{check_prefix}/tool-count", + isinstance(tools, dict) and actual_count == expected_count, + f"discovered exactly {expected_count} tools" + if actual_count == expected_count + else f"expected {expected_count} discovered tools, observed {actual_count}", + ) + + +def _run_sse_check( + case: CaseResult, + client: AppServerClient, + workspace: Path, + *, + profile: str, +) -> None: + inventory, detail = _response_result( + client.request("mcpServerStatus/list", {"detail": "full"}) + ) + entry = _review_inventory_entry(inventory) if inventory is not None else None + if not case.check( + "review/sse-cr-discovery", + entry is not None, + "discovered the CR-only SSE fixture" if entry is not None else detail, + ): + return + thread_id = _review_thread_id(case, client, workspace) + if thread_id is None: + return + result, detail = _call_tool( + client, + thread_id=thread_id, + tool="progress", + arguments={}, + meta={"progressToken": "review-cr-only-sse"}, + ) + content = result.get("content") if result is not None else None + accepted = isinstance(content, list) and bool(content) + flood = profile == SSE_COMMENT_FLOOD_PROFILE + case.check( + "review/sse-comment-flood-excluded-from-event-size-limit" + if flood + else "review/sse-carriage-return-and-comment-framing", + accepted, + "accepted more than 8 MiB of valid SSE comment keepalives" + if accepted and flood + else "accepted valid CR-only SSE lines and ignored comment keepalives" + if accepted + else detail, + ) + + +def _run_review_case( + codex_binary: Path, + server_script: Path, + *, + mode: str, + profile: str, + transport: str, + case_home: Path, + timeout_seconds: float, +) -> CaseResult: + started = time.monotonic() + case = CaseResult(transport=f"review-{transport}:{profile}", mode=mode) + case_home.mkdir(parents=True, exist_ok=True) + workspace = case_home / "workspace" + workspace.mkdir(exist_ok=True) + env = _isolated_environment(case_home) + registered = False + fixture = ( + _running_review_http_fixture(mode, profile) + if transport == "http" + else nullcontext(None) + ) + + try: + if mode == MODERN_VERSION: + feature = _run_command( + [str(codex_binary), "features", "enable", "mcp_2026_07_28"], + env=env, + cwd=workspace, + timeout_seconds=timeout_seconds, + ) + if not case.check( + "review/modern-feature-configuration", + feature.returncode == 0, + _command_detail(feature), + ): + return case + + with fixture as http_url: + registration = _run_command( + _review_registration_command( + codex_binary, + server_script, + transport=transport, + mode=mode, + profile=profile, + http_url=http_url, + ), + env=env, + cwd=workspace, + timeout_seconds=timeout_seconds, + ) + registered = case.check( + "review/mcp-registration", + registration.returncode == 0, + _command_detail(registration), + ) + if not registered: + return case + + with AppServerClient( + codex_binary, + env=env, + cwd=workspace, + timeout_seconds=timeout_seconds, + elicitation_content=_review_elicitation_content, + ) as client: + if not _initialize_review_client(case, client, mode=mode): + return case + if profile in ( + MISMATCHED_DISCOVERY_ID_PROFILE, + NULL_DISCOVERY_ID_PROFILE, + ): + _run_malformed_discovery_check(case, client, profile=profile) + elif profile == REPEATED_CURSOR_PROFILE: + _run_repeated_cursor_check(case, client) + elif profile in CATALOG_BOUNDARY_PROFILES: + _run_catalog_boundary_checks( + case, client, profile=profile, workspace=workspace + ) + elif profile in (SSE_CR_COMMENTS_PROFILE, SSE_COMMENT_FLOOD_PROFILE): + _run_sse_check(case, client, workspace, profile=profile) + else: + _run_normal_review_checks( + case, client, mode=mode, workspace=workspace + ) + except (AppServerError, OSError, ValueError) as exc: + case.check("review/client-runtime", False, str(exc)) + finally: + if registered: + removal = _run_command( + [str(codex_binary), "mcp", "remove", TEST_SERVER_NAME], + env=env, + cwd=workspace, + timeout_seconds=timeout_seconds, + ) + case.check( + "review/isolated-registration-cleanup", + removal.returncode == 0, + _command_detail(removal), + ) + case.finish(started) + return case + + +def run_review_regressions( + codex_binary: Path, + *, + modes: Sequence[str] = REVIEW_MODES, + server_script: Path | None = None, + timeout_seconds: float = 8, + artifact_parent: Path | None = None, + keep_artifacts: bool = False, +) -> dict[str, object]: + server_script = server_script or _MODULE_DIR / "server.py" + root = Path(tempfile.mkdtemp(prefix="codex-mcp-review-", dir=artifact_parent)) + cases: list[CaseResult] = [] + try: + for mode in modes: + cases.append( + _run_review_case( + codex_binary, + server_script, + mode=mode, + profile=REVIEW_PROFILE, + transport="stdio", + case_home=root / f"review-stdio-{mode}", + timeout_seconds=timeout_seconds, + ) + ) + if mode != MODERN_VERSION: + continue + for profile in ( + REVIEW_PROFILE, + REPEATED_CURSOR_PROFILE, + MISMATCHED_DISCOVERY_ID_PROFILE, + NULL_DISCOVERY_ID_PROFILE, + SSE_CR_COMMENTS_PROFILE, + SSE_COMMENT_FLOOD_PROFILE, + ): + cases.append( + _run_review_case( + codex_binary, + server_script, + mode=mode, + profile=profile, + transport="http", + case_home=root / f"review-http-{mode}-{profile}", + timeout_seconds=timeout_seconds, + ) + ) + + for mode in modes: + for transport in CATALOG_BOUNDARY_TRANSPORTS: + for profile in CATALOG_BOUNDARY_PROFILES: + cases.append( + _run_review_case( + codex_binary, + server_script, + mode=mode, + profile=profile, + transport=transport, + case_home=root / f"review-{transport}-{mode}-{profile}", + timeout_seconds=timeout_seconds, + ) + ) + + checks = [check for case in cases for check in case.checks] + failures = [check for check in checks if not check.success] + return { + "schemaVersion": REVIEW_REPORT_SCHEMA_VERSION, + "success": bool(cases) and all(case.success for case in cases), + "codexBinary": str(codex_binary), + "reviewer": REVIEWER, + "modes": list(modes), + "summary": { + "passed": len(checks) - len(failures), + "failed": len(failures), + "total": len(checks), + "casesPassed": sum(case.success for case in cases), + "casesTotal": len(cases), + }, + "cases": [asdict(case) for case in cases], + "artifacts": str(root) if keep_artifacts else None, + } + finally: + if not keep_artifacts: + shutil.rmtree(root, ignore_errors=True) + + +def _required_review_checks( + report: Mapping[str, object], + *, + label: str, + errors: list[str], +) -> dict[_ReviewCheckIdentity, bool]: + if report.get("schemaVersion") != REVIEW_REPORT_SCHEMA_VERSION: + errors.append( + f"{label} does not use reviewer report schema " + f"{REVIEW_REPORT_SCHEMA_VERSION}" + ) + if report.get("reviewer") != REVIEWER: + errors.append(f"{label} does not identify the production reviewer probes") + + compact = report.get("baselineKind") is not None + modes = report.get("requiredModes" if compact else "modes") + if modes != list(REVIEW_MODES): + errors.append(f"{label} does not run all three required MCP protocol modes") + + expected_cases = _required_review_cases() + observed_cases: set[tuple[str, str]] = set() + identities: dict[_ReviewCheckIdentity, bool] = {} + + if compact: + if report.get("baselineKind") != REVIEW_BASELINE_KIND: + errors.append(f"{label} uses an unsupported reviewer baseline format") + + raw_cases = report.get("requiredCases") + if not isinstance(raw_cases, list): + errors.append(f"{label} does not contain required reviewer cases") + raw_cases = [] + for case in raw_cases: + if not isinstance(case, dict): + errors.append(f"{label} contains a malformed reviewer case") + continue + mode = case.get("mode") + transport = case.get("transport") + if not isinstance(mode, str) or not isinstance(transport, str): + errors.append(f"{label} contains a malformed reviewer case") + continue + identity = (mode, transport) + if identity in observed_cases: + errors.append(f"{label} contains a duplicate reviewer case") + continue + observed_cases.add(identity) + + raw_checks = report.get("checks") + if not isinstance(raw_checks, dict): + errors.append(f"{label} does not contain reviewer check identities") + raw_checks = {} + for bucket, success in (("passing", True), ("failing", False)): + records = raw_checks.get(bucket) + if not isinstance(records, list): + errors.append(f"{label} does not contain {bucket} reviewer checks") + continue + for record in records: + if not isinstance(record, dict): + errors.append( + f"{label} contains a malformed {bucket} reviewer check" + ) + continue + mode = record.get("mode") + transport = record.get("transport") + check_id = record.get("check_id") + if ( + not isinstance(mode, str) + or not isinstance(transport, str) + or not isinstance(check_id, str) + or not check_id.startswith("review/") + or (mode, transport) not in expected_cases + ): + errors.append( + f"{label} contains a malformed {bucket} reviewer check" + ) + continue + identity = _ReviewCheckIdentity(mode, transport, check_id) + if identity in identities: + errors.append(f"{label} contains a duplicate reviewer check") + continue + identities[identity] = success + else: + raw_cases = report.get("cases") + if not isinstance(raw_cases, list): + errors.append(f"{label} does not contain production reviewer cases") + raw_cases = [] + for case in raw_cases: + if not isinstance(case, dict): + errors.append(f"{label} contains a malformed reviewer case") + continue + mode = case.get("mode") + transport = case.get("transport") + if ( + not isinstance(mode, str) + or not isinstance(transport, str) + or (mode, transport) not in expected_cases + ): + errors.append(f"{label} contains an unexpected reviewer case") + continue + case_identity = (mode, transport) + if case_identity in observed_cases: + errors.append(f"{label} contains a duplicate reviewer case") + continue + observed_cases.add(case_identity) + + checks = case.get("checks") + if not isinstance(checks, list) or not checks: + errors.append( + f"{label} has no reviewer checks for {transport} / {mode}" + ) + continue + case_success = True + for check in checks: + if not isinstance(check, dict): + errors.append(f"{label} contains a malformed reviewer check") + continue + check_id = check.get("name") + success = check.get("success") + if ( + not isinstance(check_id, str) + or not check_id.startswith("review/") + or not isinstance(success, bool) + ): + errors.append(f"{label} contains a malformed reviewer check") + continue + identity = _ReviewCheckIdentity(mode, transport, check_id) + if identity in identities: + errors.append(f"{label} contains a duplicate reviewer check") + continue + identities[identity] = success + case_success = case_success and success + if case.get("success") is not case_success: + errors.append( + f"{label} reports inconsistent reviewer case success " + f"for {transport} / {mode}" + ) + + missing_cases = sorted(expected_cases - observed_cases) + unexpected_cases = sorted(observed_cases - expected_cases) + for mode, transport in missing_cases: + errors.append(f"{label} is missing required reviewer case {transport} / {mode}") + for mode, transport in unexpected_cases: + errors.append(f"{label} contains unexpected reviewer case {transport} / {mode}") + + summary = report.get("summary") + passing = sum(identities.values()) + failed = len(identities) - passing + cases_passed = sum( + all( + success + for identity, success in identities.items() + if (identity.mode, identity.transport) == case + ) + and any((identity.mode, identity.transport) == case for identity in identities) + for case in observed_cases + ) + expected_summary = { + "passed": passing, + "failed": failed, + "total": len(identities), + "casesPassed": cases_passed, + "casesTotal": len(observed_cases), + } + if not isinstance(summary, dict) or any( + summary.get(name) != value for name, value in expected_summary.items() + ): + errors.append(f"{label} reports inconsistent reviewer check totals") + if not compact and report.get("success") is not ( + bool(observed_cases) and failed == 0 + ): + errors.append(f"{label} reports inconsistent overall reviewer success") + return identities + + +def _compact_review_regression_baseline( + report: Mapping[str, object], +) -> dict[str, object]: + errors: list[str] = [] + checks = _required_review_checks(report, label="baseline", errors=errors) + if errors: + raise ValueError("; ".join(errors)) + + summary = report.get("summary") + assert isinstance(summary, dict) + return { + "baselineKind": REVIEW_BASELINE_KIND, + "schemaVersion": REVIEW_REPORT_SCHEMA_VERSION, + "reviewer": REVIEWER, + "requiredModes": list(REVIEW_MODES), + "requiredCases": [ + {"mode": mode, "transport": transport} + for mode, transport in sorted(_required_review_cases()) + ], + "summary": { + key: summary[key] + for key in ("passed", "failed", "total", "casesPassed", "casesTotal") + }, + "checks": { + "passing": [ + asdict(identity) + for identity, success in sorted(checks.items()) + if success + ], + "failing": [ + asdict(identity) + for identity, success in sorted(checks.items()) + if not success + ], + }, + } + + +def _evaluate_review_regression_gate( + report: Mapping[str, object], + baseline_report: Mapping[str, object], + *, + baseline_path: Path | None = None, +) -> dict[str, object]: + errors: list[str] = [] + baseline_checks = _required_review_checks( + baseline_report, label="baseline", errors=errors + ) + candidate_checks = _required_review_checks(report, label="candidate", errors=errors) + known_failures: list[dict[str, object]] = [] + new_failures: list[dict[str, object]] = [] + missing_checks: list[dict[str, object]] = [] + fixed_checks: list[dict[str, object]] = [] + + for identity, success in sorted(candidate_checks.items()): + if success: + continue + if baseline_checks.get(identity) is False: + known_failures.append(asdict(identity)) + else: + new_failures.append(asdict(identity)) + + for identity, baseline_success in sorted(baseline_checks.items()): + candidate_success = candidate_checks.get(identity) + if candidate_success is None: + missing_checks.append(asdict(identity)) + elif not baseline_success and candidate_success: + fixed_checks.append(asdict(identity)) + + return { + "success": not errors and not new_failures and not missing_checks, + "requiredModes": list(REVIEW_MODES), + "baselineReport": str(baseline_path) if baseline_path is not None else None, + "configurationErrors": errors, + "knownFailures": known_failures, + "newFailures": new_failures, + "missingChecks": missing_checks, + "fixedChecks": fixed_checks, + } + + +def _write_review_json(path: Path, value: Mapping[str, object]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + json.dumps(value, ensure_ascii=False, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + + +def _parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Run reviewer-derived MCP client regression tests against Codex." + ) + parser.add_argument("codex_binary", type=Path) + parser.add_argument("--mode", choices=("all", *REVIEW_MODES), default="all") + parser.add_argument("--timeout", type=float, default=8) + parser.add_argument("--server-script", type=Path, default=_MODULE_DIR / "server.py") + parser.add_argument("--report", type=Path) + parser.add_argument( + "--baseline-report", + type=Path, + help="Compare every production reviewer check with a reviewed baseline.", + ) + parser.add_argument( + "--write-baseline", + type=Path, + help="Write a compact reviewer baseline from the completed full matrix.", + ) + parser.add_argument( + "--extract-baseline", + type=Path, + help="Extract a compact reviewer baseline from --baseline-report and exit.", + ) + parser.add_argument("--artifact-parent", type=Path) + parser.add_argument("--keep-artifacts", action="store_true") + parser.add_argument("--json", action="store_true") + return parser.parse_args(argv) + + +def _print_report(report: Mapping[str, object]) -> None: + print( + "Codex MCP reviewer regressions: " + + ("PASS" if report.get("success") is True else "FAIL") + ) + print(f"Binary: {report.get('codexBinary')}") + cases = report.get("cases") + if isinstance(cases, list): + for case in cases: + if not isinstance(case, dict): + continue + print(f"\n{case.get('transport')} / {case.get('mode')}") + for check in case.get("checks", []): + if isinstance(check, dict): + status = "PASS" if check.get("success") else "FAIL" + print(f" {status} {check.get('name')}: {check.get('detail')}") + summary = report.get("summary") + if isinstance(summary, dict): + print( + f"\nSummary: {summary.get('passed')}/{summary.get('total')} checks; " + f"{summary.get('casesPassed')}/{summary.get('casesTotal')} cases" + ) + gate = report.get("regressionGate") + if isinstance(gate, dict): + status = "PASS" if gate.get("success") is True else "FAIL" + known = gate.get("knownFailures") + new = gate.get("newFailures") + missing = gate.get("missingChecks") + fixed = gate.get("fixedChecks") + print( + f"Regression gate: {status}; " + f"{len(known) if isinstance(known, list) else 0} known, " + f"{len(new) if isinstance(new, list) else 0} new, " + f"{len(missing) if isinstance(missing, list) else 0} missing, " + f"{len(fixed) if isinstance(fixed, list) else 0} fixed" + ) + errors = gate.get("configurationErrors") + if isinstance(errors, list): + for error in errors: + print(f" Configuration error: {error}") + + +def main(argv: Sequence[str] | None = None) -> int: + args = _parse_args(argv) + codex_binary = args.codex_binary.expanduser().resolve() + server_script = args.server_script.expanduser().resolve() + if not codex_binary.is_file() or not os.access(codex_binary, os.X_OK): + print(f"error: Codex binary is not executable: {codex_binary}", file=sys.stderr) + return 2 + if not server_script.is_file(): + print(f"error: review fixture does not exist: {server_script}", file=sys.stderr) + return 2 + if args.timeout <= 0: + print("error: --timeout must be positive", file=sys.stderr) + return 2 + if args.extract_baseline is not None and args.baseline_report is None: + print("error: --extract-baseline requires --baseline-report", file=sys.stderr) + return 2 + if args.extract_baseline is not None and args.write_baseline is not None: + print( + "error: --extract-baseline cannot be combined with --write-baseline", + file=sys.stderr, + ) + return 2 + + baseline_report: dict[str, object] | None = None + baseline_path: Path | None = None + if args.baseline_report is not None: + baseline_path = args.baseline_report.expanduser().resolve() + try: + loaded_baseline = json.loads(baseline_path.read_text(encoding="utf-8")) + except (OSError, UnicodeError, json.JSONDecodeError) as exc: + print( + f"error: cannot read reviewer baseline {baseline_path}: {exc}", + file=sys.stderr, + ) + return 2 + if not isinstance(loaded_baseline, dict): + print( + f"error: reviewer baseline must be a JSON object: {baseline_path}", + file=sys.stderr, + ) + return 2 + baseline_report = loaded_baseline + + if args.extract_baseline is not None: + assert baseline_report is not None + output_path = args.extract_baseline.expanduser().resolve() + try: + _write_review_json( + output_path, _compact_review_regression_baseline(baseline_report) + ) + except (OSError, ValueError) as exc: + print(f"error: cannot extract reviewer baseline: {exc}", file=sys.stderr) + return 2 + print(f"Reviewer regression baseline: {output_path}") + return 0 + + parent = None + if args.artifact_parent is not None: + parent = args.artifact_parent.expanduser().resolve() + parent.mkdir(parents=True, exist_ok=True) + + modes = REVIEW_MODES if args.mode == "all" else (args.mode,) + report = run_review_regressions( + codex_binary, + modes=modes, + server_script=server_script, + timeout_seconds=args.timeout, + artifact_parent=parent, + keep_artifacts=args.keep_artifacts, + ) + if baseline_report is not None: + report["regressionGate"] = _evaluate_review_regression_gate( + report, + baseline_report, + baseline_path=baseline_path, + ) + if args.write_baseline is not None: + output_path = args.write_baseline.expanduser().resolve() + try: + _write_review_json(output_path, _compact_review_regression_baseline(report)) + except (OSError, ValueError) as exc: + print(f"error: cannot write reviewer baseline: {exc}", file=sys.stderr) + return 2 + if args.report is not None: + path = args.report.expanduser().resolve() + _write_review_json(path, report) + if args.json: + print(json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True)) + else: + _print_report(report) + if args.report is not None: + print(f"JSON report: {args.report.expanduser().resolve()}") + if args.write_baseline is not None: + print(f"Reviewer regression baseline: {args.write_baseline.resolve()}") + if baseline_report is not None: + gate = report.get("regressionGate") + return 0 if isinstance(gate, dict) and gate.get("success") is True else 1 + return 0 if report.get("success") is True else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/mcp_conformance/run_codex_compliance.py b/scripts/mcp_conformance/run_codex_compliance.py new file mode 100644 index 0000000000..bfb31cc6c5 --- /dev/null +++ b/scripts/mcp_conformance/run_codex_compliance.py @@ -0,0 +1,2201 @@ +#!/usr/bin/env python3 +"""Black-box MCP compliance runner for a supplied Codex binary.""" + +import argparse +import importlib.util +import json +import os +import queue +import shutil +import subprocess +import sys +import tempfile +import threading +import time +from collections import deque +from contextlib import contextmanager, nullcontext +from dataclasses import asdict, dataclass, field +from datetime import datetime, timezone +from pathlib import Path +from typing import Callable, Iterator, Mapping, Sequence + +_MODULE_DIR = Path(__file__).resolve().parent +if str(_MODULE_DIR) not in sys.path: + # Some execution environments set PYTHONSAFEPATH. Add only this + # trusted package directory so direct `python path/to/script.py` usage + # continues to work alongside the Bazel entry point. + sys.path.insert(0, str(_MODULE_DIR)) + +from official_conformance import ( # noqa: E402 - direct scripts must first add their sibling directory. + OFFICIAL_CONFORMANCE_GIT_REF, + OFFICIAL_CONFORMANCE_REPOSITORY, + OfficialScenarioResult, + default_conformance_command, + run_official_mode, + scenarios_for_mode, +) + +_FIXTURE_MODULE_PATH = Path(__file__).resolve().with_name("server.py") +_FIXTURE_SPEC = importlib.util.spec_from_file_location( + "_mcp_spec_test_fixture_server", + _FIXTURE_MODULE_PATH, +) +if _FIXTURE_SPEC is None or _FIXTURE_SPEC.loader is None: + raise RuntimeError(f"could not load MCP fixture module: {_FIXTURE_MODULE_PATH}") +_FIXTURE_MODULE = importlib.util.module_from_spec(_FIXTURE_SPEC) +sys.modules[_FIXTURE_SPEC.name] = _FIXTURE_MODULE +_FIXTURE_SPEC.loader.exec_module(_FIXTURE_MODULE) + +LEGACY_VERSION = _FIXTURE_MODULE.LEGACY_VERSION +SHIPPING_LEGACY_VERSION = _FIXTURE_MODULE.SHIPPING_LEGACY_VERSION +MODERN_VERSION = _FIXTURE_MODULE.MODERN_VERSION +MISMATCHED_DISCOVERY_ID_PROFILE = _FIXTURE_MODULE.MISMATCHED_DISCOVERY_ID_PROFILE +NULL_DISCOVERY_ID_PROFILE = _FIXTURE_MODULE.NULL_DISCOVERY_ID_PROFILE +REPEATED_CURSOR_PROFILE = _FIXTURE_MODULE.REPEATED_CURSOR_PROFILE +REVIEW_EXACT_INTEGER = _FIXTURE_MODULE.REVIEW_EXACT_INTEGER +REVIEW_MRTR_INPUT_REQUEST_COUNT = _FIXTURE_MODULE.REVIEW_MRTR_INPUT_REQUEST_COUNT +REVIEW_PROFILE = _FIXTURE_MODULE.REVIEW_PROFILE +RESOURCE_URIS = _FIXTURE_MODULE.RESOURCE_URIS +SERVER_NAME = _FIXTURE_MODULE.SERVER_NAME +SERVER_VERSION = _FIXTURE_MODULE.SERVER_VERSION +SSE_COMMENT_FLOOD_PROFILE = _FIXTURE_MODULE.SSE_COMMENT_FLOOD_PROFILE +SSE_CR_COMMENTS_PROFILE = _FIXTURE_MODULE.SSE_CR_COMMENTS_PROFILE +ProtocolServer = _FIXTURE_MODULE.ProtocolServer +make_http_server = _FIXTURE_MODULE.make_http_server + +REPORT_SCHEMA_VERSION = 4 +COMPACT_REGRESSION_BASELINE_KIND = "mcp-conformance-regression-baseline-v1" +REQUIRED_REGRESSION_MODES = ( + SHIPPING_LEGACY_VERSION, + LEGACY_VERSION, + MODERN_VERSION, +) +TEST_SERVER_NAME = "mcp_spec_fixture" +TEST_META_KEY = "com.openai/mcp-spec-test" +CHECK_ORDER = ( + "mcp_add", + "mcp_get", + "app_server_initialize", + "modern_feature_enablement", + "inventory", + "ephemeral_thread", + "echo_tool", + "unicode_resource_read", + "per_request_metadata", + "request_scoped_notifications", + "multi_round_trip_request", + "http_header_mirroring", + "app_server_protocol", + "case_runtime", + "mcp_remove", + "isolated_config_cleanup", +) + + +@dataclass +class CommandResult: + returncode: int + stdout: str + stderr: str + + +@dataclass +class CheckResult: + name: str + success: bool + detail: str + status: str | None = None + source: str = "supplemental" + scenario: str | None = None + check_id: str | None = None + category: str | None = None + + def __post_init__(self) -> None: + if self.status is None: + self.status = "PASS" if self.success else "FAIL" + + +@dataclass +class CaseResult: + transport: str + mode: str + success: bool = False + duration_seconds: float = 0.0 + checks: list[CheckResult] = field(default_factory=list) + diagnostics: str | None = None + + def check(self, name: str, success: bool, detail: str) -> bool: + self.checks.append( + CheckResult( + name=name, + success=success, + detail=detail, + status="PASS" if success else "FAIL", + ) + ) + return success + + def finish(self, started_at: float) -> None: + self.duration_seconds = round(time.monotonic() - started_at, 3) + self.success = bool(self.checks) and all(check.success for check in self.checks) + + +@dataclass(frozen=True, order=True) +class _RegressionCheckIdentity: + mode: str + transport: str + source: str + scenario: str + check_id: str + + +class AppServerError(RuntimeError): + pass + + +class AppServerClient: + """Minimal JSONL client for the Codex app-server protocol.""" + + def __init__( + self, + codex_binary: Path, + *, + env: Mapping[str, str], + cwd: Path, + timeout_seconds: float, + elicitation_content: ( + Callable[[Mapping[str, object]], Mapping[str, object]] | None + ) = None, + ) -> None: + self.timeout_seconds = timeout_seconds + self._elicitation_content = elicitation_content + self._next_id = 1 + self._messages: queue.Queue[dict[str, object] | None] = queue.Queue() + self._write_lock = threading.Lock() + self._stderr: deque[str] = deque(maxlen=200) + self._parse_errors: deque[str] = deque(maxlen=20) + self.events: list[dict[str, object]] = [] + self.elicitation_requests: list[dict[str, object]] = [] + self.process = subprocess.Popen( + [str(codex_binary), "app-server"], + cwd=cwd, + env=dict(env), + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + bufsize=1, + ) + self._stdout_thread = threading.Thread( + target=self._read_stdout, + name="codex-app-server-stdout", + daemon=True, + ) + self._stderr_thread = threading.Thread( + target=self._read_stderr, + name="codex-app-server-stderr", + daemon=True, + ) + self._stdout_thread.start() + self._stderr_thread.start() + + def __enter__(self) -> "AppServerClient": + return self + + def __exit__(self, *_: object) -> None: + self.close() + + def _read_stdout(self) -> None: + assert self.process.stdout is not None + for line in self.process.stdout: + stripped = line.strip() + if not stripped: + continue + try: + message = json.loads(stripped) + except json.JSONDecodeError: + self._parse_errors.append(stripped[:500]) + continue + if isinstance(message, dict): + self._messages.put(message) + self._messages.put(None) + + def _read_stderr(self) -> None: + assert self.process.stderr is not None + for line in self.process.stderr: + self._stderr.append(line.rstrip()) + + def _send(self, message: Mapping[str, object]) -> None: + if self.process.poll() is not None: + raise AppServerError( + f"Codex app-server exited with code {self.process.returncode}" + ) + assert self.process.stdin is not None + encoded = json.dumps(message, ensure_ascii=False, separators=(",", ":")) + with self._write_lock: + self.process.stdin.write(encoded + "\n") + self.process.stdin.flush() + + def notify(self, method: str) -> None: + self._send({"method": method}) + + def request( + self, + method: str, + params: Mapping[str, object] | None, + ) -> dict[str, object]: + request_id = self._next_id + self._next_id += 1 + message: dict[str, object] = {"id": request_id, "method": method} + if params is not None: + message["params"] = dict(params) + self._send(message) + + deadline = time.monotonic() + self.timeout_seconds + while True: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise AppServerError( + f"timed out waiting for app-server response to {method}" + ) + try: + message = self._messages.get(timeout=remaining) + except queue.Empty as exc: + raise AppServerError( + f"timed out waiting for app-server response to {method}" + ) from exc + if message is None: + raise AppServerError( + f"Codex app-server closed stdout while handling {method}" + ) + if message.get("id") == request_id and "method" not in message: + return message + if "id" in message and isinstance(message.get("method"), str): + self._handle_server_request(message) + else: + self.events.append(message) + + def wait_for_notification( + self, + method: str, + *, + predicate: Callable[[Mapping[str, object]], bool] | None = None, + after_event_index: int = 0, + ) -> dict[str, object]: + for event in self.events[after_event_index:]: + if event.get("method") != method: + continue + params = event.get("params") + if isinstance(params, dict) and (predicate is None or predicate(params)): + return event + + deadline = time.monotonic() + self.timeout_seconds + while True: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise AppServerError( + f"timed out waiting for app-server notification {method}" + ) + try: + message = self._messages.get(timeout=remaining) + except queue.Empty as exc: + raise AppServerError( + f"timed out waiting for app-server notification {method}" + ) from exc + if message is None: + raise AppServerError( + f"Codex app-server closed stdout while waiting for {method}" + ) + if "id" in message and isinstance(message.get("method"), str): + self._handle_server_request(message) + continue + self.events.append(message) + if message.get("method") != method: + continue + params = message.get("params") + if isinstance(params, dict) and (predicate is None or predicate(params)): + return message + + def _handle_server_request(self, message: dict[str, object]) -> None: + self.events.append(message) + request_id = message.get("id") + method = message.get("method") + if method == "mcpServer/elicitation/request": + params = message.get("params") + if isinstance(params, dict): + self.elicitation_requests.append(params) + else: + params = {} + content = ( + dict(self._elicitation_content(params)) + if self._elicitation_content is not None + else {"confirmation": "confirmed"} + ) + self._send( + { + "id": request_id, + "result": { + "action": "accept", + "content": content, + "_meta": None, + }, + } + ) + return + self._send( + { + "id": request_id, + "error": { + "code": -32601, + "message": f"unsupported test client request: {method}", + }, + } + ) + + def diagnostic_text(self) -> str: + parts: list[str] = [] + if self._parse_errors: + parts.append("non-JSON stdout: " + " | ".join(self._parse_errors)) + startup_failures = [ + event + for event in self.events + if event.get("method") == "mcpServer/startupStatus/updated" + and isinstance(event.get("params"), dict) + and event["params"].get("status") not in ("starting", "ready") + ] + if startup_failures: + parts.append( + "startup events: " + + json.dumps(startup_failures[-3:], ensure_ascii=False) + ) + if self._stderr: + parts.append("stderr:\n" + "\n".join(self._stderr)) + return "\n".join(parts)[-8_000:] + + def close(self) -> None: + if self.process.poll() is None: + self.process.terminate() + try: + self.process.wait(timeout=3) + except subprocess.TimeoutExpired: + self.process.kill() + self.process.wait(timeout=3) + for stream in (self.process.stdin, self.process.stdout, self.process.stderr): + if stream is not None: + stream.close() + self._stdout_thread.join(timeout=1) + self._stderr_thread.join(timeout=1) + + +def _run_command( + command: Sequence[str], + *, + env: Mapping[str, str], + cwd: Path, + timeout_seconds: float, +) -> CommandResult: + try: + completed = subprocess.run( + list(command), + cwd=cwd, + env=dict(env), + text=True, + capture_output=True, + timeout=timeout_seconds, + check=False, + ) + return CommandResult( + completed.returncode, + completed.stdout.strip(), + completed.stderr.strip(), + ) + except subprocess.TimeoutExpired as exc: + stdout = ( + exc.stdout.decode() if isinstance(exc.stdout, bytes) else exc.stdout or "" + ) + stderr = ( + exc.stderr.decode() if isinstance(exc.stderr, bytes) else exc.stderr or "" + ) + return CommandResult( + 124, + stdout.strip(), + (stderr + f"\ncommand timed out after {timeout_seconds:g}s").strip(), + ) + + +def _response_result( + response: Mapping[str, object], +) -> tuple[dict[str, object] | None, str]: + error = response.get("error") + if isinstance(error, dict): + return None, json.dumps(error, ensure_ascii=False, sort_keys=True) + result = response.get("result") + if not isinstance(result, dict): + return None, "response did not contain an object result" + return result, "ok" + + +def _command_detail(result: CommandResult) -> str: + detail = result.stdout or result.stderr or f"exit code {result.returncode}" + return detail[-2_000:] + + +def _result_detail( + result: Mapping[str, object] | None, + fallback: str, +) -> str: + if result is None: + return fallback + return ( + "unexpected result: " + + json.dumps(result, ensure_ascii=False, sort_keys=True, default=str) + )[-2_000:] + + +def _isolated_environment(codex_home: Path) -> dict[str, str]: + env = dict(os.environ) + env["CODEX_HOME"] = str(codex_home) + # Direct app-server MCP calls do not need a model or credentials. + for name in ("CODEX_API_KEY", "CODEX_ACCESS_TOKEN", "OPENAI_API_KEY"): + env.pop(name, None) + return env + + +@contextmanager +def _running_http_fixture(mode: str) -> Iterator[str]: + httpd = make_http_server( + ProtocolServer(mode), + "127.0.0.1", + 0, + log_requests=False, + ) + thread = threading.Thread( + target=httpd.serve_forever, + name=f"mcp-fixture-{mode}", + daemon=True, + ) + thread.start() + try: + _, port = httpd.server_address + yield f"http://127.0.0.1:{port}/mcp" + finally: + httpd.shutdown() + httpd.server_close() + thread.join(timeout=3) + + +def _registration_command( + codex_binary: Path, + server_script: Path, + *, + transport: str, + mode: str, + http_url: str | None, +) -> list[str]: + command = [str(codex_binary), "mcp", "add", TEST_SERVER_NAME] + if transport == "stdio": + if mode == MODERN_VERSION: + command.extend(["--env", f"CODEX_MCP_PROTOCOL_VERSION={MODERN_VERSION}"]) + return [ + *command, + "--", + sys.executable, + str(server_script), + "--mode", + mode, + "--transport", + "stdio", + ] + assert http_url is not None + return [*command, "--url", http_url] + + +def _validate_registration( + config: Mapping[str, object], + *, + transport: str, + mode: str, + http_url: str | None, +) -> tuple[bool, str]: + if config.get("name") != TEST_SERVER_NAME or config.get("enabled") is not True: + return False, "registered server name or enabled state was incorrect" + value = config.get("transport") + if not isinstance(value, dict): + return False, "registered server did not contain a transport object" + if transport == "stdio": + args = value.get("args") + if value.get("type") != "stdio" or not isinstance(args, list): + return False, f"unexpected stdio transport: {value!r}" + if mode not in args or "stdio" not in args: + return False, f"stdio registration omitted mode or transport: {args!r}" + env = value.get("env") + modern_opt_in = ( + isinstance(env, dict) + and env.get("CODEX_MCP_PROTOCOL_VERSION") == MODERN_VERSION + ) + if mode == MODERN_VERSION and not modern_opt_in: + return False, "modern stdio registration omitted its protocol opt-in" + if mode in (SHIPPING_LEGACY_VERSION, LEGACY_VERSION) and modern_opt_in: + return ( + False, + "legacy stdio registration unexpectedly enabled the modern protocol", + ) + elif value.get("type") != "streamable_http" or value.get("url") != http_url: + return False, f"unexpected HTTP transport: {value!r}" + return True, f"registered {value.get('type')} transport" + + +def _validate_inventory( + result: Mapping[str, object], + *, + mode: str, +) -> tuple[bool, str]: + entries = result.get("data") + if not isinstance(entries, list): + return False, "mcpServerStatus/list result did not contain data" + entry = next( + ( + item + for item in entries + if isinstance(item, dict) and item.get("name") == TEST_SERVER_NAME + ), + None, + ) + if not isinstance(entry, dict): + return False, f"{TEST_SERVER_NAME!r} was absent from MCP status" + + missing: list[str] = [] + server_info = entry.get("serverInfo") + if ( + not isinstance(server_info, dict) + or server_info.get("name") != SERVER_NAME + or server_info.get("version") != SERVER_VERSION + ): + missing.append("server identity") + tools = entry.get("tools") + expected_tools = {"echo", "client_metadata", "progress"} + if mode == MODERN_VERSION: + expected_tools.add("request_input") + if not isinstance(tools, dict): + missing.append("tool inventory") + else: + absent_tools = sorted(expected_tools - set(tools)) + if absent_tools: + missing.append("tools " + ", ".join(absent_tools)) + resources = entry.get("resources") + resource_uris = ( + { + item.get("uri") + for item in resources + if isinstance(item, dict) and isinstance(item.get("uri"), str) + } + if isinstance(resources, list) + else set() + ) + absent_resources = sorted(set(RESOURCE_URIS) - resource_uris) + if absent_resources: + missing.append("paginated resources " + ", ".join(absent_resources)) + if missing: + return False, "missing " + "; ".join(missing) + return True, "identity, tools, and both paginated resources discovered" + + +def _validate_echo( + result: Mapping[str, object], + sentinel: str, +) -> tuple[bool, str]: + structured = result.get("structuredContent") + if isinstance(structured, dict) and structured.get("text") == sentinel: + return True, "echo returned the exact sentinel" + return False, f"unexpected echo result: {dict(result)!r}" + + +def _ratio(passed: int, total: int) -> dict[str, int | float]: + percentage = round((passed / total) * 100, 1) if total else 0.0 + return { + "passed": passed, + "total": total, + "percentage": percentage, + } + + +def _check_ratio(checks: Sequence[CheckResult]) -> dict[str, int | float]: + applicable = [check for check in checks if check.status != "SKIP"] + return _ratio( + sum(check.success for check in applicable), + len(applicable), + ) + + +def _scenario_ratio( + checks: Sequence[CheckResult], + *, + category: str, +) -> dict[str, int | float]: + by_scenario: dict[str, list[CheckResult]] = {} + for check in checks: + if ( + check.scenario is None + or check.category != category + or check.source not in {"official", "harness"} + ): + continue + by_scenario.setdefault(check.scenario, []).append(check) + return _ratio( + sum( + all(check.success for check in scenario_checks) + for scenario_checks in by_scenario.values() + ), + len(by_scenario), + ) + + +def _summarize_modes(cases: Sequence[CaseResult]) -> list[dict[str, object]]: + summaries: list[dict[str, object]] = [] + modes = dict.fromkeys(case.mode for case in cases) + for mode in modes: + mode_cases = [case for case in cases if case.mode == mode] + checks = [check for case in mode_cases for check in case.checks] + official_checks = [check for check in checks if check.source == "official"] + official_non_auth_checks = [ + check for check in official_checks if check.category == "non-auth" + ] + official_auth_checks = [ + check for check in official_checks if check.category == "auth" + ] + harness_checks = [check for check in checks if check.source == "harness"] + supplemental_checks = [ + check for check in checks if check.source == "supplemental" + ] + summaries.append( + { + "mode": mode, + "checks": _check_ratio(checks), + "officialChecks": _check_ratio(official_checks), + "officialNonAuthChecks": _check_ratio(official_non_auth_checks), + "officialAuthChecks": _check_ratio(official_auth_checks), + "officialNonAuthScenarios": _scenario_ratio( + checks, + category="non-auth", + ), + "officialAuthScenarios": _scenario_ratio( + checks, + category="auth", + ), + "harnessChecks": _check_ratio(harness_checks), + "supplementalChecks": _check_ratio(supplemental_checks), + "transportCases": _ratio( + sum(case.success for case in mode_cases), + len(mode_cases), + ), + } + ) + return summaries + + +def _build_test_matrix(cases: Sequence[CaseResult]) -> dict[str, object]: + columns = [ + { + "key": f"{case.transport}:{case.mode}", + "transport": case.transport, + "mode": case.mode, + } + for case in cases + ] + checks_by_case = { + f"{case.transport}:{case.mode}": {check.name: check for check in case.checks} + for case in cases + } + observed = {check.name for case in cases for check in case.checks} + ordered_names = [name for name in CHECK_ORDER if name in observed] + ordered_names.extend(sorted(observed - set(ordered_names))) + rows: list[dict[str, object]] = [] + for name in ordered_names: + results: dict[str, str] = {} + for column in columns: + key = str(column["key"]) + check = checks_by_case[key].get(name) + results[key] = "N/A" if check is None else check.status + rows.append({"test": name, "results": results}) + return {"columns": columns, "rows": rows} + + +def _call_tool( + client: AppServerClient, + *, + thread_id: str, + tool: str, + arguments: Mapping[str, object], + meta: Mapping[str, object] | None = None, +) -> tuple[dict[str, object] | None, str]: + params: dict[str, object] = { + "threadId": thread_id, + "server": TEST_SERVER_NAME, + "tool": tool, + "arguments": dict(arguments), + } + if meta is not None: + params["_meta"] = dict(meta) + return _response_result(client.request("mcpServer/tool/call", params)) + + +def _exercise_app_server( + case: CaseResult, + client: AppServerClient, + *, + transport: str, + mode: str, + workspace: Path, + enable_modern_feature: bool, +) -> None: + initialize, initialize_detail = _response_result( + client.request( + "initialize", + { + "clientInfo": { + "name": "mcp-spec-compliance-runner", + "title": "MCP spec compliance runner", + "version": "1.0.0", + }, + "capabilities": { + "experimentalApi": True, + "requestAttestation": False, + "mcpServerOpenaiFormElicitation": True, + }, + }, + ) + ) + if not case.check( + "app_server_initialize", + initialize is not None, + initialize_detail, + ): + return + client.notify("initialized") + + if mode == MODERN_VERSION and enable_modern_feature: + feature_result, feature_detail = _response_result( + client.request( + "experimentalFeature/enablement/set", + {"enablement": {"mcp_2026_07_28": True}}, + ) + ) + enabled = ( + feature_result is not None + and isinstance(feature_result.get("enablement"), dict) + and feature_result["enablement"].get("mcp_2026_07_28") is True + ) + if not case.check( + "modern_feature_enablement", + enabled, + ( + "runtime feature mcp_2026_07_28 enabled" + if enabled + else _result_detail(feature_result, feature_detail) + ), + ): + return + + inventory, inventory_detail = _response_result( + client.request("mcpServerStatus/list", {}) + ) + if inventory is None: + case.check("inventory", False, inventory_detail) + else: + valid, detail = _validate_inventory(inventory, mode=mode) + case.check("inventory", valid, detail) + + thread_result, thread_detail = _response_result( + client.request( + "thread/start", + {"cwd": str(workspace), "ephemeral": True}, + ) + ) + thread_id: str | None = None + if thread_result is not None: + thread_value = thread_result.get("thread") + if isinstance(thread_value, dict) and isinstance(thread_value.get("id"), str): + thread_id = str(thread_value["id"]) + if not case.check( + "ephemeral_thread", + thread_id is not None, + "ephemeral thread created" if thread_id is not None else thread_detail, + ): + return + assert thread_id is not None + + sentinel = f"codex-mcp-{transport}-{mode}" + echo, echo_detail = _call_tool( + client, + thread_id=thread_id, + tool="echo", + arguments={"text": sentinel}, + meta={TEST_META_KEY: sentinel}, + ) + if echo is None: + case.check("echo_tool", False, echo_detail) + else: + valid, detail = _validate_echo(echo, sentinel) + case.check("echo_tool", valid, detail) + + resource, resource_detail = _response_result( + client.request( + "mcpServer/resource/read", + { + "threadId": thread_id, + "server": TEST_SERVER_NAME, + "uri": RESOURCE_URIS[1], + }, + ) + ) + expected_resource_text = f"fixture contents for {RESOURCE_URIS[1]}" + contents = resource.get("contents") if resource is not None else None + resource_ok = ( + isinstance(contents, list) + and len(contents) == 1 + and isinstance(contents[0], dict) + and contents[0].get("text") == expected_resource_text + ) + case.check( + "unicode_resource_read", + resource_ok, + ( + "read the Unicode resource" + if resource_ok + else _result_detail(resource, resource_detail) + ), + ) + + if mode != MODERN_VERSION: + return + + metadata, metadata_detail = _call_tool( + client, + thread_id=thread_id, + tool="client_metadata", + arguments={}, + meta={TEST_META_KEY: sentinel}, + ) + structured = metadata.get("structuredContent") if metadata is not None else None + metadata_ok = ( + isinstance(structured, dict) + and structured.get("io.modelcontextprotocol/protocolVersion") == MODERN_VERSION + and isinstance( + structured.get("io.modelcontextprotocol/clientCapabilities"), + dict, + ) + and structured.get(TEST_META_KEY) == sentinel + ) + case.check( + "per_request_metadata", + metadata_ok, + ( + "required and caller metadata were preserved" + if metadata_ok + else _result_detail(metadata, metadata_detail) + ), + ) + + progress, progress_detail = _call_tool( + client, + thread_id=thread_id, + tool="progress", + arguments={}, + meta={ + "progressToken": "fixture-progress", + "io.modelcontextprotocol/logLevel": "info", + }, + ) + progress_content = progress.get("content") if progress is not None else None + progress_ok = isinstance(progress_content, list) and bool(progress_content) + case.check( + "request_scoped_notifications", + progress_ok, + ( + "progress and log notifications were consumed before the result" + if progress_ok + else _result_detail(progress, progress_detail) + ), + ) + + elicitations_before = len(client.elicitation_requests) + mrtr, mrtr_detail = _call_tool( + client, + thread_id=thread_id, + tool="request_input", + arguments={}, + ) + mrtr_structured = mrtr.get("structuredContent") if mrtr is not None else None + new_elicitations = client.elicitation_requests[elicitations_before:] + mrtr_ok = ( + isinstance(mrtr_structured, dict) + and mrtr_structured.get("confirmation") == "confirmed" + and any( + item.get("serverName") == TEST_SERVER_NAME and item.get("mode") == "form" + for item in new_elicitations + ) + ) + case.check( + "multi_round_trip_request", + mrtr_ok, + ( + "elicitation was surfaced, answered, retried, and completed" + if mrtr_ok + else _result_detail(mrtr, mrtr_detail) + ), + ) + + if transport == "http": + arguments = { + "region": "us-west1", + "attempt": 3, + "enabled": True, + "greeting": "Hello, 世界", + } + mirrored, mirrored_detail = _call_tool( + client, + thread_id=thread_id, + tool="header_echo", + arguments=arguments, + ) + mirrored_structured = ( + mirrored.get("structuredContent") if mirrored is not None else None + ) + mirrored_ok = mirrored_structured == arguments + case.check( + "http_header_mirroring", + mirrored_ok, + ( + "method, name, scalar, and Base64 headers matched" + if mirrored_ok + else _result_detail(mirrored, mirrored_detail) + ), + ) + + +def _run_case( + codex_binary: Path, + server_script: Path, + *, + transport: str, + mode: str, + case_home: Path, + timeout_seconds: float, + enable_modern_feature: bool, +) -> CaseResult: + started_at = time.monotonic() + case = CaseResult(transport=transport, mode=mode) + case_home.mkdir(parents=True) + workspace = case_home / "workspace" + workspace.mkdir() + env = _isolated_environment(case_home) + client: AppServerClient | None = None + registered = False + + http_context = ( + _running_http_fixture(mode) if transport == "http" else nullcontext(None) + ) + try: + with http_context as http_url: + add = _run_command( + _registration_command( + codex_binary, + server_script, + transport=transport, + mode=mode, + http_url=http_url, + ), + env=env, + cwd=workspace, + timeout_seconds=timeout_seconds, + ) + registered = case.check( + "mcp_add", + add.returncode == 0, + _command_detail(add), + ) + if registered: + get = _run_command( + [ + str(codex_binary), + "mcp", + "get", + TEST_SERVER_NAME, + "--json", + ], + env=env, + cwd=workspace, + timeout_seconds=timeout_seconds, + ) + config: dict[str, object] | None = None + if get.returncode == 0: + try: + decoded = json.loads(get.stdout) + if isinstance(decoded, dict): + config = decoded + except json.JSONDecodeError: + pass + if config is None: + case.check("mcp_get", False, _command_detail(get)) + else: + valid, detail = _validate_registration( + config, + transport=transport, + mode=mode, + http_url=http_url, + ) + case.check("mcp_get", valid, detail) + + client = AppServerClient( + codex_binary, + env=env, + cwd=workspace, + timeout_seconds=timeout_seconds, + ) + try: + _exercise_app_server( + case, + client, + transport=transport, + mode=mode, + workspace=workspace, + enable_modern_feature=enable_modern_feature, + ) + except (AppServerError, OSError) as exc: + case.check("app_server_protocol", False, str(exc)) + finally: + client.close() + diagnostic_text = client.diagnostic_text() + if diagnostic_text and any( + not check.success for check in case.checks + ): + case.diagnostics = diagnostic_text + except OSError as exc: + case.check("case_runtime", False, str(exc)) + finally: + if client is not None and client.process.poll() is None: + client.close() + if registered: + remove = _run_command( + [str(codex_binary), "mcp", "remove", TEST_SERVER_NAME], + env=env, + cwd=workspace, + timeout_seconds=timeout_seconds, + ) + case.check( + "mcp_remove", + remove.returncode == 0, + _command_detail(remove), + ) + listed = _run_command( + [str(codex_binary), "mcp", "list", "--json"], + env=env, + cwd=workspace, + timeout_seconds=timeout_seconds, + ) + empty = False + if listed.returncode == 0: + try: + empty = json.loads(listed.stdout) == [] + except json.JSONDecodeError: + pass + case.check( + "isolated_config_cleanup", + empty, + "no MCP servers remained" if empty else _command_detail(listed), + ) + case.finish(started_at) + return case + + +def _official_check_result( + result: OfficialScenarioResult, + *, + index: int, + duplicate: bool, +) -> CheckResult: + check = result.checks[index] + if check.status == "SUCCESS": + status = "PASS" + success = True + elif check.status in {"SKIPPED", "INFO"}: + status = "SKIP" + success = True + else: + status = "FAIL" + success = False + suffix = f"{check.name}#{index + 1}" if duplicate else check.name + name = f"official/{result.scenario}/{suffix}" + detail = check.error_message or check.description + return CheckResult( + name=name, + success=success, + detail=detail, + status=status, + source="official", + scenario=result.scenario, + check_id=check.check_id, + category="auth" if result.scenario.startswith("auth/") else "non-auth", + ) + + +def _run_official_case( + codex_binary: Path, + adapter_script: Path, + *, + conformance_command: Sequence[str], + mode: str, + scenarios: Sequence[str], + case_home: Path, + timeout_seconds: float, + enable_modern_feature: bool, + require_automatic_auth: bool = False, +) -> CaseResult: + started_at = time.monotonic() + case = CaseResult(transport="official-http", mode=mode) + case_home.mkdir(parents=True) + env = _isolated_environment(case_home) + env["CODEX_CONFORMANCE_TIMEOUT"] = str(timeout_seconds) + env["CODEX_CONFORMANCE_ENABLE_MODERN_FEATURE"] = ( + "1" if enable_modern_feature else "0" + ) + env["CODEX_CONFORMANCE_REQUIRE_AUTOMATIC_AUTH"] = ( + "1" if require_automatic_auth else "0" + ) + results = run_official_mode( + conformance_command=conformance_command, + adapter_script=adapter_script, + codex_binary=codex_binary, + mode=mode, + scenarios=scenarios, + output_dir=case_home / "official-results", + timeout_seconds=timeout_seconds, + base_env=env, + ) + + diagnostics: list[str] = [] + for result in results: + counts: dict[str, int] = {} + for check in result.checks: + counts[check.name] = counts.get(check.name, 0) + 1 + for index, check in enumerate(result.checks): + case.checks.append( + _official_check_result( + result, + index=index, + duplicate=counts[check.name] > 1, + ) + ) + + case.checks.append( + CheckResult( + name=f"harness/{result.scenario}/codex-adapter", + success=result.adapter_success, + detail=result.adapter_detail, + status="PASS" if result.adapter_success else "FAIL", + source="harness", + scenario=result.scenario, + category=( + "auth" if result.scenario.startswith("auth/") else "non-auth" + ), + ) + ) + if ( + result.adapter_success + and not result.success + and not any( + check.status in {"FAILURE", "WARNING"} for check in result.checks + ) + ): + case.checks.append( + CheckResult( + name=f"harness/{result.scenario}/official-runner", + success=False, + detail=result.runner_detail or "official runner failed", + status="FAIL", + source="harness", + scenario=result.scenario, + category=( + "auth" if result.scenario.startswith("auth/") else "non-auth" + ), + ) + ) + if not result.success and result.runner_detail: + diagnostics.append(f"{result.scenario}:\n{result.runner_detail}") + + if not results: + case.checks.append( + CheckResult( + name="harness/official-scenario-selection", + success=False, + detail=f"no official scenarios selected for {mode}", + status="FAIL", + source="harness", + ) + ) + if diagnostics: + case.diagnostics = "\n\n".join(diagnostics)[-16_000:] + case.finish(started_at) + return case + + +def run_compliance( + codex_binary: Path, + *, + server_script: Path, + adapter_script: Path, + conformance_command: Sequence[str], + official_scenarios: Sequence[str] | None, + modes: Sequence[str], + transports: Sequence[str], + timeout_seconds: float, + artifact_parent: Path | None, + keep_artifacts: bool, + enable_modern_feature: bool = True, + include_auth: bool = True, + require_automatic_auth: bool = False, +) -> tuple[dict[str, object], Path | None]: + started_at = datetime.now(timezone.utc) + run_root = Path( + tempfile.mkdtemp( + prefix="codex-mcp-compliance-", + dir=artifact_parent, + ) + ) + version_env = _isolated_environment(run_root) + version = _run_command( + [str(codex_binary), "--version"], + env=version_env, + cwd=run_root, + timeout_seconds=timeout_seconds, + ) + + cases: list[CaseResult] = [] + selected_scenarios: dict[str, tuple[str, ...]] = { + mode: scenarios_for_mode( + mode, + official_scenarios, + include_auth=include_auth, + ) + for mode in modes + } + + # Transport order is intentional: supplemental local stdio first, then the + # official suite's loopback HTTP server. + for transport in transports: + for mode in modes: + if transport == "stdio": + case_home = run_root / f"stdio-{mode}" + cases.append( + _run_case( + codex_binary, + server_script, + transport="stdio", + mode=mode, + case_home=case_home, + timeout_seconds=timeout_seconds, + enable_modern_feature=enable_modern_feature, + ) + ) + else: + if not selected_scenarios[mode]: + continue + case_home = run_root / f"official-http-{mode}" + cases.append( + _run_official_case( + codex_binary, + adapter_script, + conformance_command=conformance_command, + mode=mode, + scenarios=selected_scenarios[mode], + case_home=case_home, + timeout_seconds=timeout_seconds, + enable_modern_feature=enable_modern_feature, + require_automatic_auth=require_automatic_auth, + ) + ) + + passed = sum(case.success for case in cases) + finished_at = datetime.now(timezone.utc) + report: dict[str, object] = { + "schemaVersion": REPORT_SCHEMA_VERSION, + "success": version.returncode == 0 and passed == len(cases), + "startedAt": started_at.isoformat(), + "finishedAt": finished_at.isoformat(), + "codexBinary": str(codex_binary), + "codexVersion": version.stdout or None, + "modernFeatureEnablement": enable_modern_feature, + "automaticAuthRequired": require_automatic_auth, + "officialConformance": { + "repository": OFFICIAL_CONFORMANCE_REPOSITORY, + "gitRef": OFFICIAL_CONFORMANCE_GIT_REF, + "scope": ( + "all versioned client scenarios" + if include_auth + else "versioned non-auth client scenarios" + ), + "authenticationIncluded": include_auth, + "scenarios": selected_scenarios, + }, + "versionCheck": { + "success": version.returncode == 0, + "detail": _command_detail(version), + }, + "summary": { + "passed": passed, + "failed": len(cases) - passed, + "total": len(cases), + }, + "modeSummaries": _summarize_modes(cases), + "testMatrix": _build_test_matrix(cases), + "cases": [asdict(case) for case in cases], + "artifacts": str(run_root) if keep_artifacts else None, + } + + retained: Path | None = run_root if keep_artifacts else None + if not keep_artifacts: + shutil.rmtree(run_root, ignore_errors=True) + return report, retained + + +def _regression_check_identity( + mode: str, + transport: str, + check: Mapping[str, object], +) -> _RegressionCheckIdentity | None: + source = check.get("source") + scenario = check.get("scenario") + name = check.get("name") + check_id = check.get("check_id") + if source not in {"official", "harness", "supplemental"}: + return None + if not isinstance(name, str) or not name: + return None + if scenario is not None and (not isinstance(scenario, str) or not scenario): + return None + if check_id is not None and (not isinstance(check_id, str) or not check_id): + return None + + # The upstream runner numbers repeated request observations by arrival + # order. Neither that number nor the number of passing retries is a stable + # protocol assertion, so prefer the upstream check ID when it exists. + stable_name = name + stem, separator, suffix = name.rpartition("#") + if separator and stem and suffix.isdecimal(): + stable_name = stem + return _RegressionCheckIdentity( + mode=mode, + transport=transport, + source=source, + scenario=scenario if isinstance(scenario, str) else "", + check_id=check_id if isinstance(check_id, str) else stable_name, + ) + + +def _required_regression_checks( + report: Mapping[str, object], + *, + label: str, + errors: list[str], +) -> dict[_RegressionCheckIdentity, set[str]]: + if report.get("schemaVersion") != REPORT_SCHEMA_VERSION: + errors.append(f"{label} does not use report schema {REPORT_SCHEMA_VERSION}") + + version_check = report.get("versionCheck") + if not isinstance(version_check, dict) or version_check.get("success") is not True: + errors.append(f"{label} does not contain a successful Codex version check") + if report.get("modernFeatureEnablement") is not True: + errors.append(f"{label} did not enable the modern MCP feature") + + official = report.get("officialConformance") + selected: Mapping[str, object] = {} + if not isinstance(official, dict): + errors.append(f"{label} does not contain official conformance metadata") + else: + if official.get("repository") != OFFICIAL_CONFORMANCE_REPOSITORY: + errors.append(f"{label} uses a different upstream conformance repository") + if official.get("gitRef") != OFFICIAL_CONFORMANCE_GIT_REF: + errors.append( + f"{label} uses a different pinned upstream conformance revision" + ) + if official.get("authenticationIncluded") is not True: + errors.append(f"{label} does not include the required OAuth scenarios") + scenarios = official.get("scenarios") + if isinstance(scenarios, dict): + selected = scenarios + else: + errors.append(f"{label} does not contain a versioned scenario catalog") + + baseline_kind = report.get("baselineKind") + if baseline_kind is not None: + if baseline_kind != COMPACT_REGRESSION_BASELINE_KIND: + errors.append(f"{label} uses an unsupported compact baseline format") + return {} + if report.get("requiredModes") != list(REQUIRED_REGRESSION_MODES): + errors.append( + f"{label} does not require the shipping, intermediate, and modern MCP versions" + ) + transports = report.get("transports") + if ( + not isinstance(transports, list) + or len(transports) != 2 + or set(transports) != {"stdio", "official-http"} + ): + errors.append(f"{label} does not require both stdio and official HTTP") + + raw_identities = report.get("checks") + if not isinstance(raw_identities, dict): + errors.append(f"{label} does not contain compact baseline check identities") + return {} + + identities: dict[_RegressionCheckIdentity, set[str]] = {} + for bucket, status in (("passing", "PASS"), ("failing", "FAIL")): + records = raw_identities.get(bucket) + if not isinstance(records, list): + errors.append(f"{label} does not contain {bucket} baseline identities") + continue + seen: set[_RegressionCheckIdentity] = set() + for record in records: + if not isinstance(record, dict): + errors.append( + f"{label} contains a malformed {bucket} baseline identity" + ) + continue + mode = record.get("mode") + transport = record.get("transport") + source = record.get("source") + scenario = record.get("scenario") + check_id = record.get("check_id") + if ( + mode not in REQUIRED_REGRESSION_MODES + or transport not in {"stdio", "official-http"} + or source not in {"official", "harness", "supplemental"} + or not isinstance(scenario, str) + or not isinstance(check_id, str) + or not check_id + ): + errors.append( + f"{label} contains a malformed {bucket} baseline identity" + ) + continue + identity = _RegressionCheckIdentity( + mode, transport, source, scenario, check_id + ) + if identity in seen: + errors.append( + f"{label} contains a duplicate {bucket} baseline identity" + ) + continue + seen.add(identity) + identities.setdefault(identity, set()).add(status) + + for mode in REQUIRED_REGRESSION_MODES: + expected_scenarios = scenarios_for_mode(mode, include_auth=True) + actual_scenarios = selected.get(mode) + if ( + not isinstance(actual_scenarios, (list, tuple)) + or tuple(actual_scenarios) != expected_scenarios + ): + errors.append( + f"{label} does not run the complete authenticated {mode} scenario catalog" + ) + for transport in ("stdio", "official-http"): + if not any( + identity.mode == mode and identity.transport == transport + for identity in identities + ): + errors.append( + f"{label} is missing the required {transport} case for {mode}" + ) + observed = { + identity.scenario + for identity in identities + if identity.mode == mode + and identity.transport == "official-http" + and identity.source in {"official", "harness"} + } + missing = sorted(set(expected_scenarios) - observed) + unexpected = sorted(observed - set(expected_scenarios)) + if missing: + errors.append( + f"{label} did not observe {mode} scenarios: {', '.join(missing)}" + ) + if unexpected: + errors.append( + f"{label} observed unexpected {mode} scenarios: {', '.join(unexpected)}" + ) + return identities + + case_list = report.get("cases") + cases: dict[tuple[str, str], Mapping[str, object]] = {} + if not isinstance(case_list, list): + errors.append(f"{label} does not contain transport cases") + case_list = [] + for case in case_list: + if not isinstance(case, dict): + errors.append(f"{label} contains a malformed transport case") + continue + mode = case.get("mode") + transport = case.get("transport") + if mode not in REQUIRED_REGRESSION_MODES: + continue + if not isinstance(mode, str) or not isinstance(transport, str): + errors.append(f"{label} contains a malformed required transport case") + continue + key = (mode, transport) + if key in cases: + errors.append(f"{label} contains a duplicate {transport} case for {mode}") + continue + cases[key] = case + + identities: dict[_RegressionCheckIdentity, set[str]] = {} + for mode in REQUIRED_REGRESSION_MODES: + expected_scenarios = scenarios_for_mode(mode, include_auth=True) + actual_scenarios = selected.get(mode) + if ( + not isinstance(actual_scenarios, (list, tuple)) + or tuple(actual_scenarios) != expected_scenarios + ): + errors.append( + f"{label} does not run the complete authenticated {mode} scenario catalog" + ) + + for transport in ("stdio", "official-http"): + case = cases.get((mode, transport)) + if case is None: + errors.append( + f"{label} is missing the required {transport} case for {mode}" + ) + continue + raw_checks = case.get("checks") + if not isinstance(raw_checks, list) or not raw_checks: + errors.append(f"{label} has no {transport} checks for {mode}") + continue + + observed_scenarios: set[str] = set() + for check in raw_checks: + if not isinstance(check, dict): + errors.append( + f"{label} contains a malformed {transport} check for {mode}" + ) + continue + identity = _regression_check_identity(mode, transport, check) + status = check.get("status") + success = check.get("success") + if ( + identity is None + or status not in {"PASS", "FAIL", "SKIP"} + or not isinstance(success, bool) + or success != (status != "FAIL") + ): + errors.append( + f"{label} contains a malformed {transport} check for {mode}" + ) + continue + identities.setdefault(identity, set()).add(status) + if transport == "official-http" and identity.source in { + "official", + "harness", + }: + observed_scenarios.add(identity.scenario) + + if transport == "official-http": + expected = set(expected_scenarios) + missing = sorted(expected - observed_scenarios) + unexpected = sorted(observed_scenarios - expected) + if missing: + errors.append( + f"{label} did not observe {mode} scenarios: {', '.join(missing)}" + ) + if unexpected: + errors.append( + f"{label} observed unexpected {mode} scenarios: {', '.join(unexpected)}" + ) + + return identities + + +def _compact_regression_baseline(report: Mapping[str, object]) -> dict[str, object]: + errors: list[str] = [] + identities = _required_regression_checks(report, label="baseline", errors=errors) + automatic_auth = report.get("automaticAuthRequired") + if not isinstance(automatic_auth, bool): + errors.append("baseline does not record its production OAuth policy") + if any( + identity.mode == SHIPPING_LEGACY_VERSION and "FAIL" in statuses + for identity, statuses in identities.items() + ): + errors.append("baseline contains a shipping MCP regression") + if errors: + raise ValueError("; ".join(errors)) + + official = report.get("officialConformance") + assert isinstance(official, dict) + scenarios = official.get("scenarios") + assert isinstance(scenarios, dict) + return { + "baselineKind": COMPACT_REGRESSION_BASELINE_KIND, + "schemaVersion": REPORT_SCHEMA_VERSION, + "requiredModes": list(REQUIRED_REGRESSION_MODES), + "transports": ["stdio", "official-http"], + "versionCheck": {"success": True}, + "modernFeatureEnablement": True, + "automaticAuthRequired": automatic_auth, + "officialConformance": { + "repository": OFFICIAL_CONFORMANCE_REPOSITORY, + "gitRef": OFFICIAL_CONFORMANCE_GIT_REF, + "authenticationIncluded": True, + "scenarios": {mode: scenarios[mode] for mode in REQUIRED_REGRESSION_MODES}, + }, + "checks": { + "passing": [ + asdict(identity) + for identity, statuses in sorted(identities.items()) + if "PASS" in statuses + ], + "failing": [ + asdict(identity) + for identity, statuses in sorted(identities.items()) + if "FAIL" in statuses + ], + }, + } + + +def _write_json_file(path: Path, value: Mapping[str, object]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + json.dumps(value, ensure_ascii=False, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + + +def _write_compact_regression_baseline( + report: Mapping[str, object], + path: Path, +) -> None: + _write_json_file(path, _compact_regression_baseline(report)) + + +def _evaluate_regression_gate( + report: Mapping[str, object], + baseline_report: Mapping[str, object], + *, + baseline_path: Path | None = None, +) -> dict[str, object]: + errors: list[str] = [] + baseline_checks = _required_regression_checks( + baseline_report, label="baseline", errors=errors + ) + candidate_checks = _required_regression_checks( + report, label="candidate", errors=errors + ) + + baseline_auth = baseline_report.get("automaticAuthRequired") + candidate_auth = report.get("automaticAuthRequired") + if not isinstance(baseline_auth, bool) or not isinstance(candidate_auth, bool): + errors.append( + "baseline and candidate must record their production OAuth policy" + ) + elif baseline_auth != candidate_auth: + errors.append("baseline and candidate use different production OAuth policies") + + new_failures: list[dict[str, object]] = [] + missing_checks: list[dict[str, object]] = [] + known_failures: list[dict[str, object]] = [] + fixed_checks: list[dict[str, object]] = [] + + for identity, statuses in sorted(candidate_checks.items()): + baseline_statuses = baseline_checks.get(identity, set()) + if "FAIL" not in statuses: + continue + if identity.mode == SHIPPING_LEGACY_VERSION: + errors.append(f"shipping MCP check failed: {identity.check_id}") + if "FAIL" in baseline_statuses: + known_failures.append(asdict(identity)) + else: + new_failures.append(asdict(identity)) + + for identity, baseline_statuses in sorted(baseline_checks.items()): + candidate_statuses = candidate_checks.get(identity, set()) + if identity.mode == SHIPPING_LEGACY_VERSION and "FAIL" in baseline_statuses: + errors.append( + f"baseline shipping MCP check is not passing: {identity.check_id}" + ) + if "PASS" in baseline_statuses and "PASS" not in candidate_statuses: + missing_checks.append(asdict(identity)) + elif ( + "FAIL" in baseline_statuses + and "FAIL" not in candidate_statuses + and "PASS" not in candidate_statuses + ): + missing_checks.append(asdict(identity)) + elif "FAIL" in baseline_statuses and "FAIL" not in candidate_statuses: + fixed_checks.append(asdict(identity)) + + return { + "success": not errors and not new_failures and not missing_checks, + "requiredModes": list(REQUIRED_REGRESSION_MODES), + "baselineReport": str(baseline_path) if baseline_path is not None else None, + "configurationErrors": errors, + "newFailures": new_failures, + "missingChecks": missing_checks, + "knownFailures": known_failures, + "fixedChecks": fixed_checks, + } + + +def _format_ratio(value: Mapping[str, object]) -> str: + passed = value.get("passed", 0) + total = value.get("total", 0) + if total == 0: + return "0/0 (N/A)" + percentage = value.get("percentage", 0) + return f"{passed}/{total} ({percentage:g}%)" + + +def _print_table(headers: Sequence[str], rows: Sequence[Sequence[str]]) -> None: + widths = [ + max(len(headers[index]), *(len(row[index]) for row in rows)) + for index in range(len(headers)) + ] + + def print_row(row: Sequence[str], separator: str = " | ") -> None: + print( + separator.join( + value.ljust(widths[index]) for index, value in enumerate(row) + ) + ) + + print_row(headers) + print("-+-".join("-" * width for width in widths)) + for row in rows: + print_row(row) + + +def _print_human_report(report: Mapping[str, object]) -> None: + success = report.get("success") is True + print(f"Codex MCP compliance: {'PASS' if success else 'FAIL'}") + print(f"Binary: {report.get('codexBinary')}") + print(f"Version: {report.get('codexVersion') or 'unknown'}") + failure_details: list[tuple[str, str, str]] = [] + cases = report.get("cases") + if isinstance(cases, list): + for case in cases: + if not isinstance(case, dict): + continue + label = f"{case.get('transport')} / {case.get('mode')}" + print(f" {'PASS' if case.get('success') else 'FAIL'} {label}") + checks = case.get("checks") + if not isinstance(checks, list): + continue + for check in checks: + if isinstance(check, dict) and check.get("success") is not True: + failure_details.append( + ( + label, + str(check.get("name")), + str(check.get("detail")), + ) + ) + + mode_summaries = report.get("modeSummaries") + if isinstance(mode_summaries, list) and mode_summaries: + print("\nPass rates by protocol mode") + summary_rows: list[list[str]] = [] + for item in mode_summaries: + if not isinstance(item, dict): + continue + checks = item.get("checks") + official_checks = item.get("officialChecks") + official_non_auth_checks = item.get("officialNonAuthChecks") + official_auth_checks = item.get("officialAuthChecks") + official_non_auth_scenarios = item.get("officialNonAuthScenarios") + official_auth_scenarios = item.get("officialAuthScenarios") + harness_checks = item.get("harnessChecks") + supplemental_checks = item.get("supplementalChecks") + transport_cases = item.get("transportCases") + if not all( + isinstance(value, dict) + for value in ( + checks, + official_checks, + official_non_auth_checks, + official_auth_checks, + official_non_auth_scenarios, + official_auth_scenarios, + harness_checks, + supplemental_checks, + transport_cases, + ) + ): + continue + summary_rows.append( + [ + str(item.get("mode")), + _format_ratio(official_non_auth_scenarios), + _format_ratio(official_auth_scenarios), + _format_ratio(official_checks), + _format_ratio(harness_checks), + _format_ratio(supplemental_checks), + _format_ratio(checks), + _format_ratio(transport_cases), + ] + ) + _print_table( + [ + "Mode", + "Non-auth scenarios", + "Auth scenarios", + "Official assertions", + "Harness", + "Supplemental", + "All checks", + "Transport cases", + ], + summary_rows, + ) + + test_matrix = report.get("testMatrix") + if isinstance(test_matrix, dict): + columns = test_matrix.get("columns") + rows = test_matrix.get("rows") + if isinstance(columns, list) and isinstance(rows, list) and columns: + matrix_headers = ["Test"] + column_keys: list[str] = [] + for column in columns: + if not isinstance(column, dict): + continue + key = str(column.get("key")) + column_keys.append(key) + matrix_headers.append(f"{column.get('transport')} {column.get('mode')}") + matrix_rows: list[list[str]] = [] + for row in rows: + if not isinstance(row, dict) or not isinstance( + row.get("results"), dict + ): + continue + results = row["results"] + matrix_rows.append( + [ + str(row.get("test")), + *(str(results.get(key, "N/A")) for key in column_keys), + ] + ) + print("\nPer-test results") + _print_table(matrix_headers, matrix_rows) + + if failure_details: + print("\nFailure details") + for label, name, detail in failure_details: + print(f" {label} / {name}: {detail}") + + summary = report.get("summary") + if isinstance(summary, dict): + print(f"Summary: {summary.get('passed')}/{summary.get('total')} cases passed") + gate = report.get("regressionGate") + if isinstance(gate, dict): + passed = gate.get("success") is True + known = gate.get("knownFailures") + known_count = len(known) if isinstance(known, list) else 0 + print( + f"Regression gate: {'PASS' if passed else 'FAIL'} " + f"({known_count} acknowledged baseline failures)" + ) + for category in ("configurationErrors", "newFailures", "missingChecks"): + values = gate.get(category) + if isinstance(values, list): + for value in values: + print(f" {category}: {value}") + artifacts = report.get("artifacts") + if isinstance(artifacts, str): + print(f"Artifacts: {artifacts}") + + +def _parse_args(argv: Sequence[str] | None) -> argparse.Namespace: + parser = argparse.ArgumentParser( + description=( + "Run the full versioned official MCP client conformance suite against a " + "supplied Codex binary, plus the supplemental local stdio suite." + ) + ) + parser.add_argument("codex_binary", type=Path) + parser.add_argument( + "--mode", + choices=("all", SHIPPING_LEGACY_VERSION, LEGACY_VERSION, MODERN_VERSION), + default="all", + help="Protocol era to test (default: both).", + ) + parser.add_argument( + "--transport", + choices=("all", "stdio", "http"), + default="all", + help=( + "Transport to test: supplemental stdio, official localhost HTTP, " + "or both (default: both)." + ), + ) + parser.add_argument( + "--server-script", + type=Path, + default=Path(__file__).resolve().with_name("server.py"), + help="Path to the MCP fixture server.py.", + ) + parser.add_argument( + "--adapter-script", + type=Path, + default=Path(__file__).resolve().with_name("codex_conformance_adapter.py"), + help="Path to the Codex adapter used by the official client suite.", + ) + parser.add_argument( + "--conformance-cli", + type=Path, + help=( + "Use an existing official conformance executable. By default the " + "runner bootstraps the pinned upstream Git commit with npx." + ), + ) + parser.add_argument( + "--official-scenario", + action="append", + help=( + "Run only this official versioned scenario. Repeat to select more " + "(default: every applicable scenario for the selected mode)." + ), + ) + parser.add_argument( + "--auth", + action=argparse.BooleanOptionalAction, + default=True, + help=( + "Include official OAuth authorization scenarios in HTTP coverage " + "(default: enabled). Use --no-auth for a faster non-auth slice." + ), + ) + parser.add_argument( + "--require-automatic-auth", + action="store_true", + help=( + "Require Codex to recover from OAuth scope escalation and " + "authorization-server migration without harness-injected re-login; " + "also exercise production client-metadata selection." + ), + ) + parser.add_argument( + "--timeout", + type=float, + default=45.0, + help="Per-command and per-request timeout in seconds.", + ) + parser.add_argument( + "--report", + type=Path, + help="Also write the complete JSON report to this path.", + ) + parser.add_argument( + "--baseline-report", + type=Path, + help=( + "Require the complete shipping, intermediate, and modern authenticated " + "HTTP and stdio matrices to preserve every passing check from this report. " + "Known baseline failures remain visible and new failures fail the gate." + ), + ) + parser.add_argument( + "--write-baseline", + type=Path, + help="Write a compact, deterministic regression baseline from the completed run.", + ) + parser.add_argument( + "--extract-baseline", + type=Path, + help=( + "Write a compact, deterministic baseline from --baseline-report " + "and exit without rerunning conformance." + ), + ) + parser.add_argument( + "--json", + action="store_true", + help="Print the report as JSON instead of the human summary.", + ) + parser.add_argument( + "--artifact-parent", + type=Path, + help="Parent directory for isolated temporary Codex homes.", + ) + parser.add_argument( + "--keep-artifacts", + action="store_true", + help="Keep isolated Codex homes and include their path in the report.", + ) + parser.add_argument( + "--enable-modern-feature", + action=argparse.BooleanOptionalAction, + default=True, + help=( + "Configure mcp_2026_07_28 before MCP startup and verify it through " + "the app-server runtime setter for modern cases (default: enabled)." + ), + ) + return parser.parse_args(argv) + + +def main(argv: Sequence[str] | None = None) -> int: + args = _parse_args(argv) + codex_binary = args.codex_binary.expanduser().resolve() + server_script = args.server_script.expanduser().resolve() + adapter_script = args.adapter_script.expanduser().resolve() + modes = ( + (SHIPPING_LEGACY_VERSION, LEGACY_VERSION, MODERN_VERSION) + if args.mode == "all" + else (args.mode,) + ) + transports = ("stdio", "http") if args.transport == "all" else (args.transport,) + if not codex_binary.is_file() or not os.access(codex_binary, os.X_OK): + print(f"error: Codex binary is not executable: {codex_binary}", file=sys.stderr) + return 2 + if "stdio" in transports and not server_script.is_file(): + print(f"error: fixture server does not exist: {server_script}", file=sys.stderr) + return 2 + if "http" in transports and not adapter_script.is_file(): + print( + f"error: official conformance adapter does not exist: {adapter_script}", + file=sys.stderr, + ) + return 2 + if args.timeout <= 0: + print("error: --timeout must be positive", file=sys.stderr) + return 2 + if args.extract_baseline is not None and args.baseline_report is None: + print("error: --extract-baseline requires --baseline-report", file=sys.stderr) + return 2 + if args.extract_baseline is not None and args.write_baseline is not None: + print( + "error: --extract-baseline cannot be combined with --write-baseline", + file=sys.stderr, + ) + return 2 + + baseline_report: dict[str, object] | None = None + baseline_path: Path | None = None + if args.baseline_report is not None: + baseline_path = args.baseline_report.expanduser().resolve() + try: + loaded_baseline = json.loads(baseline_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + print( + f"error: cannot read baseline report {baseline_path}: {exc}", + file=sys.stderr, + ) + return 2 + if not isinstance(loaded_baseline, dict): + print( + f"error: baseline report must be a JSON object: {baseline_path}", + file=sys.stderr, + ) + return 2 + baseline_report = loaded_baseline + + if args.extract_baseline is not None: + assert baseline_report is not None + output_path = args.extract_baseline.expanduser().resolve() + try: + _write_compact_regression_baseline(baseline_report, output_path) + except (OSError, ValueError) as exc: + print( + f"error: cannot extract regression baseline {output_path}: {exc}", + file=sys.stderr, + ) + return 2 + print(f"Regression baseline: {output_path}") + return 0 + + conformance_command: list[str] + if args.conformance_cli is None: + conformance_command = default_conformance_command() + else: + conformance_cli = args.conformance_cli.expanduser().resolve() + if not conformance_cli.is_file(): + print( + f"error: official conformance CLI does not exist: {conformance_cli}", + file=sys.stderr, + ) + return 2 + if os.access(conformance_cli, os.X_OK): + conformance_command = [str(conformance_cli)] + elif conformance_cli.suffix in {".js", ".mjs"} and shutil.which("node"): + conformance_command = [str(shutil.which("node")), str(conformance_cli)] + else: + print( + "error: --conformance-cli must be executable (or a JavaScript " + f"file runnable with node): {conformance_cli}", + file=sys.stderr, + ) + return 2 + + try: + for mode in modes: + scenarios_for_mode( + mode, + args.official_scenario, + include_auth=args.auth, + ) + except ValueError as exc: + print(f"error: {exc}", file=sys.stderr) + return 2 + + artifact_parent: Path | None = None + if args.artifact_parent is not None: + artifact_parent = args.artifact_parent.expanduser().resolve() + artifact_parent.mkdir(parents=True, exist_ok=True) + + report, _ = run_compliance( + codex_binary, + server_script=server_script, + adapter_script=adapter_script, + conformance_command=conformance_command, + official_scenarios=args.official_scenario, + modes=modes, + transports=transports, + timeout_seconds=args.timeout, + artifact_parent=artifact_parent, + keep_artifacts=args.keep_artifacts, + enable_modern_feature=args.enable_modern_feature, + include_auth=args.auth, + require_automatic_auth=args.require_automatic_auth, + ) + if baseline_report is not None: + report["regressionGate"] = _evaluate_regression_gate( + report, + baseline_report, + baseline_path=baseline_path, + ) + if args.write_baseline is not None: + output_path = args.write_baseline.expanduser().resolve() + try: + _write_compact_regression_baseline(report, output_path) + except (OSError, ValueError) as exc: + print( + f"error: cannot write regression baseline {output_path}: {exc}", + file=sys.stderr, + ) + return 2 + + if args.report is not None: + report_path = args.report.expanduser().resolve() + _write_json_file(report_path, report) + if args.json: + print(json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True)) + else: + _print_human_report(report) + if args.report is not None: + print(f"JSON report: {args.report.expanduser().resolve()}") + if baseline_report is not None: + gate = report.get("regressionGate") + return 0 if isinstance(gate, dict) and gate.get("success") is True else 1 + return 0 if report["success"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/mcp_conformance/server.py b/scripts/mcp_conformance/server.py new file mode 100644 index 0000000000..294fcb4c48 --- /dev/null +++ b/scripts/mcp_conformance/server.py @@ -0,0 +1,1744 @@ +#!/usr/bin/env python3 +"""MCP test server for shipping and draft legacy protocols and MCP 2026-07-28.""" + +import argparse +import base64 +import binascii +import json +import os +import sys +import threading +import uuid +from dataclasses import dataclass, field +from http import HTTPStatus +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import IO, Mapping, Sequence +from urllib.parse import urlsplit + +SHIPPING_LEGACY_VERSION = "2025-06-18" +LEGACY_VERSION = "2025-11-25" +MODERN_VERSION = "2026-07-28" +SERVER_NAME = "openai-mcp-spec-test-server" +SERVER_VERSION = "0.1.0" + +DEFAULT_PROFILE = "default" +REVIEW_PROFILE = "review-regressions" +REPEATED_CURSOR_PROFILE = "repeated-cursor" +MISMATCHED_DISCOVERY_ID_PROFILE = "discovery-mismatched-id" +NULL_DISCOVERY_ID_PROFILE = "discovery-null-id" +SSE_CR_COMMENTS_PROFILE = "sse-cr-comments" +SSE_COMMENT_FLOOD_PROFILE = "sse-comment-flood" +CATALOG_MAX_PROFILE = "catalog-max" +CATALOG_OVER_LIMIT_PROFILE = "catalog-over-limit" +FIXTURE_PROFILES = ( + DEFAULT_PROFILE, + REVIEW_PROFILE, + REPEATED_CURSOR_PROFILE, + MISMATCHED_DISCOVERY_ID_PROFILE, + NULL_DISCOVERY_ID_PROFILE, + SSE_CR_COMMENTS_PROFILE, + SSE_COMMENT_FLOOD_PROFILE, + CATALOG_MAX_PROFILE, + CATALOG_OVER_LIMIT_PROFILE, +) +MAX_CATALOG_ITEMS = 1_024 +REVIEW_EXACT_INTEGER = 9_007_199_254_740_993 +REVIEW_MAX_INTEGER = 9_223_372_036_854_775_807 +REVIEW_MRTR_INPUT_REQUEST_COUNT = 65 +REVIEW_PAGE_CURSOR = "review-page-2" +REVIEW_SSE_EVENT_LIMIT_BYTES = 8 * 1_024 * 1_024 +REVIEW_SSE_COMMENT_LINE_BYTES = 2_048 +REVIEW_SSE_COMMENT_LINE_COUNT = ( + REVIEW_SSE_EVENT_LIMIT_BYTES // REVIEW_SSE_COMMENT_LINE_BYTES + 1 +) + +PARSE_ERROR = -32700 +INVALID_REQUEST = -32600 +METHOD_NOT_FOUND = -32601 +INVALID_PARAMS = -32602 +INTERNAL_ERROR = -32603 +HEADER_MISMATCH = -32020 +MISSING_REQUIRED_CLIENT_CAPABILITY = -32021 +UNSUPPORTED_PROTOCOL_VERSION = -32022 + +SERVER_INFO_META = { + "io.modelcontextprotocol/serverInfo": { + "name": SERVER_NAME, + "version": SERVER_VERSION, + } +} + +RESOURCE_URIS = ( + "test://fixture/readme", + "test://fixture/こんにちは", +) + + +@dataclass +class ConnectionState: + initialize_seen: bool = False + initialized: bool = False + + +@dataclass +class ResponsePlan: + response: dict[str, object] | None + notifications: list[dict[str, object]] = field(default_factory=list) + http_status: int = HTTPStatus.OK + force_sse: bool = False + + +class HeaderValidationError(ValueError): + pass + + +def _error_response( + request_id: object, + code: int, + message: str, + *, + data: dict[str, object] | None = None, +) -> dict[str, object]: + error: dict[str, object] = {"code": code, "message": message} + if data is not None: + error["data"] = data + return {"jsonrpc": "2.0", "id": request_id, "error": error} + + +def _response(request_id: object, result: dict[str, object]) -> dict[str, object]: + return {"jsonrpc": "2.0", "id": request_id, "result": result} + + +def _modern_result( + fields: Mapping[str, object] | None = None, + *, + cache_scope: str | None = None, + ttl_ms: int | None = None, + meta: Mapping[str, object] | None = None, +) -> dict[str, object]: + result: dict[str, object] = { + "resultType": "complete", + "_meta": dict(meta or SERVER_INFO_META), + } + if fields is not None: + result.update(fields) + if cache_scope is not None: + result["cacheScope"] = cache_scope + if ttl_ms is not None: + result["ttlMs"] = ttl_ms + return result + + +def _tool_definitions( + modern: bool, + *, + profile: str = DEFAULT_PROFILE, +) -> list[dict[str, object]]: + definitions: list[dict[str, object]] = [ + { + "name": "client_metadata", + "description": "Return the per-request MCP client metadata observed by the server.", + "inputSchema": {"type": "object", "additionalProperties": False}, + "outputSchema": {"type": "object"}, + "annotations": {"readOnlyHint": True}, + }, + { + "name": "echo", + "description": "Echo text through a normal MCP tool result.", + "inputSchema": { + "type": "object", + "properties": {"text": {"type": "string"}}, + "required": ["text"], + "additionalProperties": False, + }, + "outputSchema": { + "type": "object", + "properties": {"text": {"type": "string"}}, + "required": ["text"], + "additionalProperties": False, + }, + "annotations": {"readOnlyHint": True}, + }, + { + "name": "fail", + "description": "Return a tool execution error without using a JSON-RPC error.", + "inputSchema": {"type": "object", "additionalProperties": False}, + "annotations": {"readOnlyHint": True}, + }, + { + "name": "header_echo", + "description": "Echo values that modern HTTP clients must mirror into headers.", + "inputSchema": { + "type": "object", + "properties": { + "region": {"type": "string", "x-mcp-header": "Region"}, + "attempt": { + "type": "integer", + "minimum": -9_007_199_254_740_991, + "maximum": 9_007_199_254_740_991, + "x-mcp-header": "Attempt", + }, + "enabled": {"type": "boolean", "x-mcp-header": "Enabled"}, + "greeting": {"type": "string", "x-mcp-header": "Greeting"}, + }, + "required": ["region", "attempt", "enabled", "greeting"], + "additionalProperties": False, + }, + "outputSchema": {"type": "object"}, + "annotations": {"readOnlyHint": True}, + }, + { + "name": "progress", + "description": "Emit request-scoped progress and opt-in log notifications.", + "inputSchema": {"type": "object", "additionalProperties": False}, + "annotations": {"readOnlyHint": True}, + }, + ] + if modern: + # This tool first returns InputRequiredResult, not CallToolResult. Keep + # output-schema coverage on echo so the MRTR fixture tests one concern. + definitions.append( + { + "name": "request_input", + "description": "Exercise the 2026-07-28 multi round-trip request pattern.", + "inputSchema": {"type": "object", "additionalProperties": False}, + "annotations": {"readOnlyHint": True}, + } + ) + if profile in (REVIEW_PROFILE, REPEATED_CURSOR_PROFILE): + definitions.extend( + [ + { + "name": "review_large_integer", + "description": "Echo an integer without losing JSON-schema precision.", + "inputSchema": { + "type": "object", + "properties": { + "value": { + "type": "integer", + "minimum": REVIEW_EXACT_INTEGER, + "maximum": REVIEW_MAX_INTEGER, + "default": REVIEW_EXACT_INTEGER, + } + }, + "required": ["value"], + "additionalProperties": False, + }, + "outputSchema": { + "type": "object", + "properties": {"value": {"type": "integer"}}, + "required": ["value"], + "additionalProperties": False, + }, + "annotations": {"readOnlyHint": True}, + }, + { + "name": "review_protocol_env", + "description": "Report the MCP protocol-version environment variable.", + "inputSchema": {"type": "object", "additionalProperties": False}, + "outputSchema": { + "type": "object", + "properties": {"value": {"type": ["string", "null"]}}, + "required": ["value"], + "additionalProperties": False, + }, + "annotations": {"readOnlyHint": True}, + }, + ] + ) + if modern: + definitions.extend( + [ + { + "name": "review_integer_elicitation", + "description": "Request an integer form without losing schema precision.", + "inputSchema": { + "type": "object", + "additionalProperties": False, + }, + "annotations": {"readOnlyHint": True}, + }, + { + "name": "review_mrtr_cap", + "description": ( + "Return more simultaneous input requests than the client cap." + ), + "inputSchema": { + "type": "object", + "additionalProperties": False, + }, + "annotations": {"readOnlyHint": True}, + }, + ] + ) + if profile in (CATALOG_MAX_PROFILE, CATALOG_OVER_LIMIT_PROFILE): + catalog_size = MAX_CATALOG_ITEMS + if profile == CATALOG_OVER_LIMIT_PROFILE: + catalog_size += 1 + definitions.extend( + { + "name": f"catalog_boundary_{index:04d}", + "description": "Exercise the MCP catalog item boundary.", + "inputSchema": { + "type": "object", + "additionalProperties": False, + }, + "annotations": {"readOnlyHint": True}, + } + for index in range(catalog_size - len(definitions)) + ) + return sorted(definitions, key=lambda tool: str(tool["name"])) + + +class ProtocolServer: + def __init__(self, mode: str, *, profile: str = DEFAULT_PROFILE) -> None: + if mode not in (SHIPPING_LEGACY_VERSION, LEGACY_VERSION, MODERN_VERSION): + raise ValueError(f"unsupported mode: {mode}") + if profile not in FIXTURE_PROFILES: + raise ValueError(f"unsupported profile: {profile}") + self.mode = mode + self.profile = profile + + @property + def modern(self) -> bool: + return self.mode == MODERN_VERSION + + def handle( + self, + message: object, + state: ConnectionState, + ) -> ResponsePlan: + if not isinstance(message, dict): + return ResponsePlan( + _error_response(None, INVALID_REQUEST, "Request must be a JSON object"), + http_status=HTTPStatus.BAD_REQUEST, + ) + + if message.get("jsonrpc") != "2.0" or not isinstance( + message.get("method"), str + ): + return ResponsePlan( + _error_response( + message.get("id"), + INVALID_REQUEST, + "Invalid JSON-RPC 2.0 request", + ), + http_status=HTTPStatus.BAD_REQUEST, + ) + + method = str(message["method"]) + is_notification = "id" not in message + request_id = message.get("id") + params = message.get("params", {}) + if not isinstance(params, dict): + return ResponsePlan( + None + if is_notification + else _error_response( + request_id, INVALID_PARAMS, "params must be an object" + ), + http_status=HTTPStatus.BAD_REQUEST, + ) + + if is_notification: + return self._handle_notification(method, params, state) + + if self.modern: + return self._handle_modern(request_id, method, params) + return self._handle_legacy(request_id, method, params, state) + + def _handle_notification( + self, + method: str, + params: dict[str, object], + state: ConnectionState, + ) -> ResponsePlan: + if not self.modern and method == "notifications/initialized": + if state.initialize_seen: + state.initialized = True + return ResponsePlan(None, http_status=HTTPStatus.ACCEPTED) + if method == "notifications/cancelled": + return ResponsePlan(None, http_status=HTTPStatus.ACCEPTED) + return ResponsePlan(None, http_status=HTTPStatus.ACCEPTED) + + def _handle_modern( + self, + request_id: object, + method: str, + params: dict[str, object], + ) -> ResponsePlan: + if method == "initialize": + return ResponsePlan( + _error_response( + request_id, + METHOD_NOT_FOUND, + f"initialize is not supported; this server speaks MCP {MODERN_VERSION}", + data={"supported": [MODERN_VERSION]}, + ), + http_status=HTTPStatus.NOT_FOUND, + ) + + meta = params.get("_meta") + if not isinstance(meta, dict): + return ResponsePlan( + _error_response( + request_id, + INVALID_PARAMS, + "params._meta is required on every 2026-07-28 request", + ), + http_status=HTTPStatus.BAD_REQUEST, + ) + + version = meta.get("io.modelcontextprotocol/protocolVersion") + capabilities = meta.get("io.modelcontextprotocol/clientCapabilities") + if not isinstance(version, str) or not isinstance(capabilities, dict): + return ResponsePlan( + _error_response( + request_id, + INVALID_PARAMS, + "_meta must include protocolVersion and clientCapabilities", + ), + http_status=HTTPStatus.BAD_REQUEST, + ) + if version != MODERN_VERSION: + return ResponsePlan( + _error_response( + request_id, + UNSUPPORTED_PROTOCOL_VERSION, + "Unsupported protocol version", + data={"supported": [MODERN_VERSION], "requested": version}, + ), + http_status=HTTPStatus.BAD_REQUEST, + ) + + if method == "server/discover": + response_id = request_id + if self.profile == MISMATCHED_DISCOVERY_ID_PROFILE: + response_id = f"review-mismatched-{request_id}" + elif self.profile == NULL_DISCOVERY_ID_PROFILE: + response_id = None + return ResponsePlan( + _response( + response_id, + _modern_result( + { + "supportedVersions": [MODERN_VERSION], + "capabilities": self._modern_capabilities(), + "instructions": ( + "Use echo for a basic call, header_echo for HTTP " + "header mirroring, request_input for MRTR, and " + "progress for request-scoped notifications." + ), + }, + cache_scope="public", + ttl_ms=60_000, + ), + ) + ) + if method == "tools/list": + return self._list_tools(request_id, params) + if method == "tools/call": + return self._call_tool(request_id, params, meta) + if method == "resources/list": + return self._list_resources(request_id, params) + if method == "resources/templates/list": + return ResponsePlan( + _response( + request_id, + _modern_result( + { + "resourceTemplates": [ + { + "name": "fixture-by-name", + "uriTemplate": "test://fixture/{name}", + "description": "Read a named fixture resource.", + "mimeType": "text/plain", + } + ] + }, + cache_scope="public", + ttl_ms=5_000, + ), + ) + ) + if method == "resources/read": + return self._read_resource(request_id, params) + if method == "prompts/list": + return ResponsePlan( + _response( + request_id, + _modern_result( + {"prompts": self._prompts()}, + cache_scope="public", + ttl_ms=5_000, + ), + ) + ) + if method == "prompts/get": + return self._get_prompt(request_id, params) + if method == "subscriptions/listen": + return self._listen(request_id, params) + + return ResponsePlan( + _error_response(request_id, METHOD_NOT_FOUND, f"Unknown method: {method}"), + http_status=HTTPStatus.NOT_FOUND, + ) + + def _handle_legacy( + self, + request_id: object, + method: str, + params: dict[str, object], + state: ConnectionState, + ) -> ResponsePlan: + if method == "initialize": + requested_version = params.get("protocolVersion") + if not isinstance(requested_version, str): + return ResponsePlan( + _error_response( + request_id, + INVALID_PARAMS, + "initialize requires protocolVersion", + ) + ) + state.initialize_seen = True + return ResponsePlan( + _response( + request_id, + { + "protocolVersion": self.mode, + "capabilities": self._legacy_capabilities(), + "serverInfo": { + "name": SERVER_NAME, + "version": SERVER_VERSION, + }, + "instructions": f"Legacy {self.mode} compatibility fixture.", + }, + ) + ) + + if method == "server/discover": + return ResponsePlan( + _error_response( + request_id, + METHOD_NOT_FOUND, + "Unknown method: server/discover", + ), + http_status=HTTPStatus.NOT_FOUND, + ) + if not state.initialized: + return ResponsePlan( + _error_response( + request_id, + INVALID_REQUEST, + "Server has not received notifications/initialized", + ) + ) + if method == "ping": + return ResponsePlan(_response(request_id, {})) + if method == "logging/setLevel": + return ResponsePlan(_response(request_id, {})) + if method == "tools/list": + return self._list_tools(request_id, params) + if method == "tools/call": + return self._call_tool(request_id, params, {}) + if method == "resources/list": + return self._list_resources(request_id, params) + if method == "resources/templates/list": + return ResponsePlan( + _response( + request_id, + { + "resourceTemplates": [ + { + "name": "fixture-by-name", + "uriTemplate": "test://fixture/{name}", + "description": "Read a named fixture resource.", + "mimeType": "text/plain", + } + ] + }, + ) + ) + if method == "resources/read": + return self._read_resource(request_id, params) + if method == "prompts/list": + return ResponsePlan(_response(request_id, {"prompts": self._prompts()})) + if method == "prompts/get": + return self._get_prompt(request_id, params) + return ResponsePlan( + _error_response(request_id, METHOD_NOT_FOUND, f"Unknown method: {method}"), + http_status=HTTPStatus.NOT_FOUND, + ) + + def _list_tools( + self, + request_id: object, + params: dict[str, object], + ) -> ResponsePlan: + definitions = _tool_definitions(self.modern, profile=self.profile) + fields: dict[str, object] + if self.modern and self.profile in (REVIEW_PROFILE, REPEATED_CURSOR_PROFILE): + cursor = params.get("cursor") + if cursor is None: + fields = { + "tools": definitions[:2], + "nextCursor": REVIEW_PAGE_CURSOR, + } + elif cursor == REVIEW_PAGE_CURSOR: + fields = {"tools": definitions[2:]} + if self.profile == REPEATED_CURSOR_PROFILE: + fields["nextCursor"] = REVIEW_PAGE_CURSOR + else: + return ResponsePlan( + _error_response(request_id, INVALID_PARAMS, "Invalid tool cursor") + ) + else: + fields = {"tools": definitions} + + result = ( + _modern_result(fields, cache_scope="public", ttl_ms=1_000) + if self.modern + else fields + ) + return ResponsePlan(_response(request_id, result)) + + @staticmethod + def _modern_capabilities() -> dict[str, object]: + return { + "tools": {"listChanged": True}, + "resources": {"subscribe": True, "listChanged": True}, + "prompts": {"listChanged": True}, + "logging": {}, + "extensions": {"com.openai/mcp-spec-test": {}}, + } + + @staticmethod + def _legacy_capabilities() -> dict[str, object]: + return { + "tools": {"listChanged": True}, + "resources": {"subscribe": True, "listChanged": True}, + "prompts": {"listChanged": True}, + "logging": {}, + } + + def _call_tool( + self, + request_id: object, + params: dict[str, object], + request_meta: dict[str, object], + ) -> ResponsePlan: + name = params.get("name") + arguments = params.get("arguments", {}) + if not isinstance(name, str) or not isinstance(arguments, dict): + return ResponsePlan( + _error_response( + request_id, + INVALID_PARAMS, + "tools/call requires a string name and object arguments", + ) + ) + known = { + str(tool["name"]) + for tool in _tool_definitions(self.modern, profile=self.profile) + } + if name not in known: + return ResponsePlan( + _error_response(request_id, INVALID_PARAMS, f"Unknown tool: {name}") + ) + + if name == "request_input": + return self._request_input(request_id, params, request_meta) + if name == "review_integer_elicitation": + return self._review_integer_elicitation(request_id, params, request_meta) + if name == "review_mrtr_cap": + return self._review_mrtr_cap(request_id, request_meta) + + notifications: list[dict[str, object]] = [] + if name == "progress": + progress_token = request_meta.get("progressToken") + if isinstance(progress_token, (str, int)) and not isinstance( + progress_token, bool + ): + notifications.append( + { + "jsonrpc": "2.0", + "method": "notifications/progress", + "params": { + "progressToken": progress_token, + "progress": 1, + "total": 1, + "message": "fixture complete", + }, + } + ) + if isinstance(request_meta.get("io.modelcontextprotocol/logLevel"), str): + notifications.append( + { + "jsonrpc": "2.0", + "method": "notifications/message", + "params": { + "level": "info", + "logger": SERVER_NAME, + "data": "request-scoped fixture log", + }, + } + ) + + if name == "review_large_integer": + value = arguments.get("value") + if ( + not isinstance(value, int) + or isinstance(value, bool) + or not REVIEW_EXACT_INTEGER <= value <= REVIEW_MAX_INTEGER + ): + return ResponsePlan( + _error_response( + request_id, + INVALID_PARAMS, + "review_large_integer requires an exactly representable JSON integer", + ) + ) + structured = {"value": value} + fields: dict[str, object] = { + "content": [{"type": "text", "text": json.dumps(structured)}], + "structuredContent": structured, + } + elif name == "review_protocol_env": + structured = {"value": os.environ.get("CODEX_MCP_PROTOCOL_VERSION")} + fields = { + "content": [{"type": "text", "text": json.dumps(structured)}], + "structuredContent": structured, + } + elif name == "echo": + text = arguments.get("text") + if not isinstance(text, str): + return ResponsePlan( + _error_response( + request_id, + INVALID_PARAMS, + "echo requires a string text argument", + ) + ) + fields = { + "content": [{"type": "text", "text": text}], + "structuredContent": {"text": text}, + } + elif name == "client_metadata": + fields = { + "content": [ + { + "type": "text", + "text": json.dumps(request_meta, sort_keys=True), + } + ], + "structuredContent": request_meta, + } + elif name == "header_echo": + required = { + "region": str, + "attempt": int, + "enabled": bool, + "greeting": str, + } + if any( + key not in arguments + or not isinstance(arguments[key], expected_type) + or (expected_type is int and isinstance(arguments[key], bool)) + for key, expected_type in required.items() + ) or not ( + -9_007_199_254_740_991 + <= int(arguments["attempt"]) + <= 9_007_199_254_740_991 + ): + return ResponsePlan( + _error_response( + request_id, + INVALID_PARAMS, + "header_echo arguments have invalid types", + ) + ) + fields = { + "content": [ + { + "type": "text", + "text": json.dumps( + arguments, ensure_ascii=False, sort_keys=True + ), + } + ], + "structuredContent": arguments, + } + elif name == "fail": + fields = { + "content": [ + { + "type": "text", + "text": "Intentional tool execution failure", + } + ], + "isError": True, + } + else: + fields = { + "content": [{"type": "text", "text": "progress complete"}], + } + + result = _modern_result(fields) if self.modern else fields + return ResponsePlan( + _response(request_id, result), + notifications=notifications, + force_sse=bool(notifications), + ) + + def _review_integer_elicitation( + self, + request_id: object, + params: dict[str, object], + request_meta: dict[str, object], + ) -> ResponsePlan: + capabilities = request_meta.get( + "io.modelcontextprotocol/clientCapabilities", {} + ) + if not isinstance(capabilities, dict) or not isinstance( + capabilities.get("elicitation"), dict + ): + return ResponsePlan( + _error_response( + request_id, + MISSING_REQUIRED_CLIENT_CAPABILITY, + "review_integer_elicitation requires elicitation support", + data={"requiredCapabilities": {"elicitation": {"form": {}}}}, + ), + http_status=HTTPStatus.BAD_REQUEST, + ) + + input_responses = params.get("inputResponses") + request_state = params.get("requestState") + expected_state = "opaque:review_integer_elicitation:v1" + if input_responses is None and request_state is None: + return ResponsePlan( + _response( + request_id, + { + "resultType": "input_required", + "_meta": dict(SERVER_INFO_META), + "inputRequests": { + "large_integer": { + "method": "elicitation/create", + "params": { + "mode": "form", + "message": "Return the exact large integer.", + "requestedSchema": { + "type": "object", + "properties": { + "value": { + "type": "integer", + "minimum": REVIEW_EXACT_INTEGER, + "maximum": REVIEW_MAX_INTEGER, + "default": REVIEW_EXACT_INTEGER, + } + }, + "required": ["value"], + }, + }, + } + }, + "requestState": expected_state, + }, + ) + ) + + if request_state != expected_state or not isinstance(input_responses, dict): + return ResponsePlan( + _error_response( + request_id, + INVALID_PARAMS, + "integer elicitation retry must echo requestState and inputResponses", + ) + ) + + response = input_responses.get("large_integer") + if not isinstance(response, dict) or response.get("action") != "accept": + return ResponsePlan( + _error_response( + request_id, INVALID_PARAMS, "integer input must be accepted" + ) + ) + + content = response.get("content") + value = content.get("value") if isinstance(content, dict) else None + if ( + not isinstance(value, int) + or isinstance(value, bool) + or not REVIEW_EXACT_INTEGER <= value <= REVIEW_MAX_INTEGER + ): + return ResponsePlan( + _error_response( + request_id, + INVALID_PARAMS, + "integer elicitation requires the exact integer form response", + ) + ) + + return ResponsePlan( + _response( + request_id, + _modern_result( + { + "content": [{"type": "text", "text": str(value)}], + "structuredContent": {"value": value}, + } + ), + ) + ) + + def _review_mrtr_cap( + self, + request_id: object, + request_meta: dict[str, object], + ) -> ResponsePlan: + capabilities = request_meta.get( + "io.modelcontextprotocol/clientCapabilities", {} + ) + if not isinstance(capabilities, dict) or not isinstance( + capabilities.get("elicitation"), dict + ): + return ResponsePlan( + _error_response( + request_id, + MISSING_REQUIRED_CLIENT_CAPABILITY, + "review_mrtr_cap requires elicitation support", + data={"requiredCapabilities": {"elicitation": {"form": {}}}}, + ), + http_status=HTTPStatus.BAD_REQUEST, + ) + + return ResponsePlan( + _response( + request_id, + { + "resultType": "input_required", + "_meta": dict(SERVER_INFO_META), + "inputRequests": { + f"review-input-{index:02d}": { + "method": "elicitation/create", + "params": { + "mode": "form", + "message": f"Review MRTR input {index}.", + "requestedSchema": { + "type": "object", + "properties": {"value": {"type": "string"}}, + "required": ["value"], + }, + }, + } + for index in range(REVIEW_MRTR_INPUT_REQUEST_COUNT) + }, + "requestState": "opaque:review_mrtr_cap:v1", + }, + ) + ) + + def _request_input( + self, + request_id: object, + params: dict[str, object], + request_meta: dict[str, object], + ) -> ResponsePlan: + capabilities = request_meta.get( + "io.modelcontextprotocol/clientCapabilities", {} + ) + if not isinstance(capabilities, dict) or not isinstance( + capabilities.get("elicitation"), dict + ): + return ResponsePlan( + _error_response( + request_id, + MISSING_REQUIRED_CLIENT_CAPABILITY, + "request_input requires elicitation support", + data={"requiredCapabilities": {"elicitation": {"form": {}}}}, + ), + http_status=HTTPStatus.BAD_REQUEST, + ) + + input_responses = params.get("inputResponses") + request_state = params.get("requestState") + expected_state = "opaque:request_input:v1" + if input_responses is None and request_state is None: + return ResponsePlan( + _response( + request_id, + { + "resultType": "input_required", + "_meta": dict(SERVER_INFO_META), + "inputRequests": { + "confirmation": { + "method": "elicitation/create", + "params": { + "mode": "form", + "message": "Confirm the MCP 2026-07-28 MRTR test.", + "requestedSchema": { + "type": "object", + "properties": { + "confirmation": { + "type": "string", + "default": "confirmed", + } + }, + "required": ["confirmation"], + }, + }, + } + }, + "requestState": expected_state, + }, + ) + ) + + if request_state != expected_state or not isinstance(input_responses, dict): + return ResponsePlan( + _error_response( + request_id, + INVALID_PARAMS, + "MRTR retry must echo requestState and provide inputResponses", + ) + ) + confirmation = input_responses.get("confirmation") + if not isinstance(confirmation, dict) or confirmation.get("action") != "accept": + return ResponsePlan( + _error_response( + request_id, + INVALID_PARAMS, + "confirmation input must be accepted", + ) + ) + content = confirmation.get("content") + if not isinstance(content, dict) or not isinstance( + content.get("confirmation"), str + ): + return ResponsePlan( + _error_response( + request_id, + INVALID_PARAMS, + "confirmation content is required", + ) + ) + value = str(content["confirmation"]) + return ResponsePlan( + _response( + request_id, + _modern_result( + { + "content": [{"type": "text", "text": value}], + "structuredContent": {"confirmation": value}, + } + ), + ) + ) + + def _list_resources( + self, + request_id: object, + params: dict[str, object], + ) -> ResponsePlan: + cursor = params.get("cursor") + if cursor is None: + resources = [self._resource(RESOURCE_URIS[0])] + next_cursor: str | None = "page-2" + elif cursor == "page-2": + resources = [self._resource(RESOURCE_URIS[1])] + next_cursor = None + else: + return ResponsePlan( + _error_response(request_id, INVALID_PARAMS, "Invalid resource cursor") + ) + + fields: dict[str, object] = {"resources": resources} + if next_cursor is not None: + fields["nextCursor"] = next_cursor + result = ( + _modern_result(fields, cache_scope="public", ttl_ms=2_000) + if self.modern + else fields + ) + return ResponsePlan(_response(request_id, result)) + + @staticmethod + def _resource(uri: str) -> dict[str, object]: + return { + "name": uri.rsplit("/", 1)[-1], + "title": f"Fixture resource: {uri.rsplit('/', 1)[-1]}", + "uri": uri, + "description": "A deterministic text resource from the MCP spec fixture.", + "mimeType": "text/plain", + } + + def _read_resource( + self, + request_id: object, + params: dict[str, object], + ) -> ResponsePlan: + uri = params.get("uri") + if uri not in RESOURCE_URIS: + return ResponsePlan( + _error_response(request_id, INVALID_PARAMS, "Resource not found") + ) + fields: dict[str, object] = { + "contents": [ + { + "uri": uri, + "mimeType": "text/plain", + "text": f"fixture contents for {uri}", + } + ] + } + result = ( + _modern_result(fields, cache_scope="private", ttl_ms=500) + if self.modern + else fields + ) + return ResponsePlan(_response(request_id, result)) + + @staticmethod + def _prompts() -> list[dict[str, object]]: + return [ + { + "name": "fixture.greeting", + "title": "Fixture greeting", + "description": "Create a deterministic greeting.", + "arguments": [ + { + "name": "name", + "description": "Name to greet.", + "required": True, + } + ], + } + ] + + def _get_prompt( + self, + request_id: object, + params: dict[str, object], + ) -> ResponsePlan: + if params.get("name") != "fixture.greeting": + return ResponsePlan( + _error_response(request_id, INVALID_PARAMS, "Unknown prompt") + ) + arguments = params.get("arguments", {}) + if not isinstance(arguments, dict) or not isinstance( + arguments.get("name"), str + ): + return ResponsePlan( + _error_response( + request_id, + INVALID_PARAMS, + "fixture.greeting requires the name argument", + ) + ) + fields: dict[str, object] = { + "description": "A deterministic fixture greeting.", + "messages": [ + { + "role": "user", + "content": { + "type": "text", + "text": f"Hello, {arguments['name']}!", + }, + } + ], + } + result = _modern_result(fields) if self.modern else fields + return ResponsePlan(_response(request_id, result)) + + def _listen( + self, + request_id: object, + params: dict[str, object], + ) -> ResponsePlan: + requested = params.get("notifications") + if not isinstance(requested, dict): + return ResponsePlan( + _error_response( + request_id, + INVALID_PARAMS, + "subscriptions/listen requires a notifications filter", + ) + ) + + acknowledged: dict[str, object] = {} + if requested.get("toolsListChanged") is True: + acknowledged["toolsListChanged"] = True + if requested.get("promptsListChanged") is True: + acknowledged["promptsListChanged"] = True + if requested.get("resourcesListChanged") is True: + acknowledged["resourcesListChanged"] = True + subscriptions = requested.get("resourceSubscriptions") + valid_subscriptions = ( + [uri for uri in subscriptions if uri in RESOURCE_URIS] + if isinstance(subscriptions, list) + and all(isinstance(uri, str) for uri in subscriptions) + else [] + ) + if valid_subscriptions: + acknowledged["resourceSubscriptions"] = valid_subscriptions + + subscription_meta = { + "io.modelcontextprotocol/subscriptionId": request_id, + } + notifications: list[dict[str, object]] = [ + { + "jsonrpc": "2.0", + "method": "notifications/subscriptions/acknowledged", + "params": { + "_meta": subscription_meta, + "notifications": acknowledged, + }, + } + ] + notification_methods = ( + ("toolsListChanged", "notifications/tools/list_changed"), + ("promptsListChanged", "notifications/prompts/list_changed"), + ("resourcesListChanged", "notifications/resources/list_changed"), + ) + for filter_name, method in notification_methods: + if acknowledged.get(filter_name) is True: + notifications.append( + { + "jsonrpc": "2.0", + "method": method, + "params": {"_meta": subscription_meta}, + } + ) + for uri in valid_subscriptions: + notifications.append( + { + "jsonrpc": "2.0", + "method": "notifications/resources/updated", + "params": {"_meta": subscription_meta, "uri": uri}, + } + ) + + closing_meta = { + **SERVER_INFO_META, + "io.modelcontextprotocol/subscriptionId": request_id, + } + return ResponsePlan( + _response( + request_id, + _modern_result(meta=closing_meta), + ), + notifications=notifications, + force_sse=True, + ) + + +def _decode_mirrored_header(value: str) -> str: + if value.startswith("=?base64?") and value.endswith("?="): + encoded = value[len("=?base64?") : -len("?=")] + try: + return base64.b64decode(encoded, validate=True).decode("utf-8") + except (binascii.Error, UnicodeDecodeError) as exc: + raise HeaderValidationError("invalid Base64 mirrored header") from exc + if value.startswith("=?base64?") or value.endswith("?="): + raise HeaderValidationError("malformed Base64 sentinel") + if value != value.strip() or any( + char != "\t" and (ord(char) < 0x20 or ord(char) > 0x7E) for char in value + ): + raise HeaderValidationError("unsafe plain mirrored header") + return value + + +def validate_modern_http_headers( + message: dict[str, object], + headers: Mapping[str, str], +) -> None: + normalized = {name.lower(): value for name, value in headers.items()} + accept = { + value.strip().split(";", 1)[0].lower() + for value in normalized.get("accept", "").split(",") + } + if not {"application/json", "text/event-stream"}.issubset(accept): + raise HeaderValidationError( + "Accept must include application/json and text/event-stream" + ) + + method = message.get("method") + params = message.get("params") + if not isinstance(method, str) or not isinstance(params, dict): + return + meta = params.get("_meta") + + header_version = normalized.get("mcp-protocol-version") + if header_version is None: + raise HeaderValidationError("missing MCP-Protocol-Version header") + if isinstance(meta, dict): + body_version = meta.get("io.modelcontextprotocol/protocolVersion") + if isinstance(body_version, str) and body_version != header_version: + raise HeaderValidationError( + "MCP-Protocol-Version header does not match request _meta" + ) + + header_method = normalized.get("mcp-method") + if header_method is None: + raise HeaderValidationError("missing Mcp-Method header") + if header_method != method: + raise HeaderValidationError("Mcp-Method header does not match request method") + + name_source: object | None = None + if method in ("tools/call", "prompts/get"): + name_source = params.get("name") + elif method == "resources/read": + name_source = params.get("uri") + if name_source is not None: + header_name = normalized.get("mcp-name") + if header_name is None: + raise HeaderValidationError("missing Mcp-Name header") + if ( + not isinstance(name_source, str) + or _decode_mirrored_header(header_name) != name_source + ): + raise HeaderValidationError("Mcp-Name header does not match request body") + + if method != "tools/call" or params.get("name") != "header_echo": + return + arguments = params.get("arguments", {}) + if not isinstance(arguments, dict): + return + mirrored = { + "region": "region", + "attempt": "attempt", + "enabled": "enabled", + "greeting": "greeting", + } + for argument_name, header_suffix in mirrored.items(): + header_key = f"mcp-param-{header_suffix}" + argument = arguments.get(argument_name) + header_value = normalized.get(header_key) + if argument is None: + if header_value is not None: + raise HeaderValidationError( + f"Mcp-Param-{header_suffix} must be omitted" + ) + continue + if header_value is None: + raise HeaderValidationError(f"missing Mcp-Param-{header_suffix} header") + decoded = _decode_mirrored_header(header_value) + if isinstance(argument, bool): + expected = "true" if argument else "false" + elif isinstance(argument, int): + expected = str(argument) + elif isinstance(argument, str): + expected = argument + else: + raise HeaderValidationError( + f"unsupported mirrored argument type for {argument_name}" + ) + if decoded != expected: + raise HeaderValidationError( + f"Mcp-Param-{header_suffix} header does not match request body" + ) + + +def _encode_sse_messages( + messages: Sequence[Mapping[str, object]], + *, + profile: str, +) -> bytes: + if profile == SSE_COMMENT_FLOOD_PROFILE: + comment_prefix = b": reviewer keepalive " + comment = ( + comment_prefix + + b"x" * (REVIEW_SSE_COMMENT_LINE_BYTES - len(comment_prefix) - 1) + + b"\r" + ) + prefix = comment * REVIEW_SSE_COMMENT_LINE_COUNT + separator = "\r\r" + elif profile == SSE_CR_COMMENTS_PROFILE: + prefix = b": reviewer keepalive\r" + separator = "\r\r" + else: + prefix = b"" + separator = "\n\n" + + return prefix + b"".join( + ( + f"data: {json.dumps(message, ensure_ascii=False, separators=(',', ':'))}" + f"{separator}" + ).encode() + for message in messages + ) + + +class FixtureHTTPServer(ThreadingHTTPServer): + daemon_threads = True + + def __init__( + self, + address: tuple[str, int], + fixture: ProtocolServer, + *, + endpoint: str, + allowed_origins: Sequence[str], + log_requests: bool, + ) -> None: + self.fixture = fixture + self.endpoint = endpoint + self.allowed_origins = set(allowed_origins) + self.log_requests = log_requests + self.sessions: dict[str, ConnectionState] = {} + self.sessions_lock = threading.Lock() + super().__init__(address, FixtureRequestHandler) + + +class FixtureRequestHandler(BaseHTTPRequestHandler): + server: FixtureHTTPServer + + def do_GET(self) -> None: + if self.path == "/healthz": + body = json.dumps( + {"status": "ok", "mode": self.server.fixture.mode} + ).encode() + self._send_bytes(HTTPStatus.OK, "application/json", body) + return + self._send_bytes( + HTTPStatus.METHOD_NOT_ALLOWED, + "application/json", + b"", + extra_headers={"Allow": "POST"}, + ) + + def do_DELETE(self) -> None: + if self.path != self.server.endpoint or self.server.fixture.modern: + self._send_bytes( + HTTPStatus.METHOD_NOT_ALLOWED, + "application/json", + b"", + extra_headers={"Allow": "POST"}, + ) + return + session_id = self.headers.get("Mcp-Session-Id") + with self.server.sessions_lock: + existed = ( + session_id is not None + and self.server.sessions.pop(session_id, None) is not None + ) + self._send_bytes( + HTTPStatus.OK if existed else HTTPStatus.NOT_FOUND, + "application/json", + b"", + ) + + def do_POST(self) -> None: + if self.path != self.server.endpoint: + self._send_json_error( + None, + HTTPStatus.NOT_FOUND, + METHOD_NOT_FOUND, + f"Unknown MCP endpoint: {self.path}", + ) + return + if not self._origin_is_allowed(): + self._send_json_error( + None, + HTTPStatus.FORBIDDEN, + INVALID_REQUEST, + "Origin is not allowed", + ) + return + if self.headers.get_content_type() != "application/json": + self._send_json_error( + None, + HTTPStatus.UNSUPPORTED_MEDIA_TYPE, + INVALID_REQUEST, + "Content-Type must be application/json", + ) + return + try: + content_length = int(self.headers.get("Content-Length", "")) + except ValueError: + content_length = -1 + if content_length < 0 or content_length > 1_048_576: + self._send_json_error( + None, + HTTPStatus.BAD_REQUEST, + INVALID_REQUEST, + "Invalid Content-Length", + ) + return + try: + message = json.loads(self.rfile.read(content_length)) + except (json.JSONDecodeError, UnicodeDecodeError): + self._send_json_error( + None, + HTTPStatus.BAD_REQUEST, + PARSE_ERROR, + "Parse error", + ) + return + if not isinstance(message, dict): + self._send_json_error( + None, + HTTPStatus.BAD_REQUEST, + INVALID_REQUEST, + "Request must be a JSON object", + ) + return + + request_id = message.get("id") + if self.server.fixture.modern: + if "id" in message: + try: + validate_modern_http_headers(message, self.headers) + except HeaderValidationError as exc: + self._send_json_error( + request_id, + HTTPStatus.BAD_REQUEST, + HEADER_MISMATCH, + f"Header mismatch: {exc}", + ) + return + state = ConnectionState() + response_headers: dict[str, str] = {} + else: + state, response_headers = self._legacy_state(message) + if state is None: + return + + plan = self.server.fixture.handle(message, state) + if plan.response is None: + self._send_bytes( + plan.http_status, + "application/json", + b"", + extra_headers=response_headers, + ) + return + if plan.force_sse: + messages = [*plan.notifications, plan.response] + body = _encode_sse_messages(messages, profile=self.server.fixture.profile) + self._send_bytes( + plan.http_status, + "text/event-stream", + body, + extra_headers={ + **response_headers, + "Cache-Control": "no-cache", + "X-Accel-Buffering": "no", + }, + ) + return + body = json.dumps( + plan.response, ensure_ascii=False, separators=(",", ":") + ).encode() + self._send_bytes( + plan.http_status, + "application/json", + body, + extra_headers=response_headers, + ) + + def _legacy_state( + self, + message: dict[str, object], + ) -> tuple[ConnectionState | None, dict[str, str]]: + method = message.get("method") + if method == "initialize": + state = ConnectionState() + session_id = uuid.uuid4().hex + with self.server.sessions_lock: + self.server.sessions[session_id] = state + return state, {"Mcp-Session-Id": session_id} + if method == "server/discover": + return ConnectionState(), {} + session_id = self.headers.get("Mcp-Session-Id") + with self.server.sessions_lock: + state = ( + self.server.sessions.get(session_id) if session_id is not None else None + ) + if state is None: + self._send_json_error( + message.get("id"), + HTTPStatus.BAD_REQUEST, + INVALID_REQUEST, + "Missing or unknown Mcp-Session-Id", + ) + return None, {} + return state, {"Mcp-Session-Id": session_id} + + def _origin_is_allowed(self) -> bool: + origin = self.headers.get("Origin") + if origin is None: + return True + if origin in self.server.allowed_origins: + return True + parsed = urlsplit(origin) + return parsed.scheme in ("http", "https") and parsed.hostname in ( + "localhost", + "127.0.0.1", + "::1", + ) + + def _send_json_error( + self, + request_id: object, + status: int, + code: int, + message: str, + ) -> None: + body = json.dumps( + _error_response(request_id, code, message), separators=(",", ":") + ).encode() + self._send_bytes(status, "application/json", body) + + def _send_bytes( + self, + status: int, + content_type: str, + body: bytes, + *, + extra_headers: Mapping[str, str] | None = None, + ) -> None: + self.send_response(status) + self.send_header("Content-Type", content_type) + self.send_header("Content-Length", str(len(body))) + if extra_headers is not None: + for name, value in extra_headers.items(): + self.send_header(name, value) + self.end_headers() + if body: + self.wfile.write(body) + + def log_message(self, format: str, *args: object) -> None: + if not self.server.log_requests: + return + print( + f"{self.address_string()} - {format % args}", + file=sys.stderr, + flush=True, + ) + + +def make_http_server( + fixture: ProtocolServer, + host: str, + port: int, + *, + endpoint: str = "/mcp", + allowed_origins: Sequence[str] = (), + log_requests: bool = True, +) -> FixtureHTTPServer: + return FixtureHTTPServer( + (host, port), + fixture, + endpoint=endpoint, + allowed_origins=allowed_origins, + log_requests=log_requests, + ) + + +def run_stdio( + fixture: ProtocolServer, + stdin: IO[str], + stdout: IO[str], +) -> None: + state = ConnectionState() + for line in stdin: + try: + message = json.loads(line) + except json.JSONDecodeError: + stdout.write( + json.dumps( + _error_response(None, PARSE_ERROR, "Parse error"), + separators=(",", ":"), + ) + + "\n" + ) + stdout.flush() + continue + plan = fixture.handle(message, state) + for notification in plan.notifications: + stdout.write( + json.dumps(notification, ensure_ascii=False, separators=(",", ":")) + + "\n" + ) + if plan.response is not None: + stdout.write( + json.dumps(plan.response, ensure_ascii=False, separators=(",", ":")) + + "\n" + ) + stdout.flush() + + +def _parse_args(argv: Sequence[str] | None) -> argparse.Namespace: + parser = argparse.ArgumentParser( + description=( + "Run a deterministic MCP server for testing shipping 2025-06-18, " + "2025-11-25 compatibility, or 2026-07-28 draft compliance." + ) + ) + parser.add_argument( + "--mode", + choices=(SHIPPING_LEGACY_VERSION, LEGACY_VERSION, MODERN_VERSION), + default=MODERN_VERSION, + ) + parser.add_argument( + "--transport", + choices=("stdio", "http"), + default="stdio", + ) + parser.add_argument( + "--profile", + choices=FIXTURE_PROFILES, + default=DEFAULT_PROFILE, + help="Select an optional deterministic reviewer-regression fixture profile.", + ) + parser.add_argument("--host", default="127.0.0.1") + parser.add_argument("--port", type=int, default=8765) + parser.add_argument("--endpoint", default="/mcp") + parser.add_argument( + "--allowed-origin", + action="append", + default=[], + help="Additional exact Origin value accepted by the HTTP transport.", + ) + return parser.parse_args(argv) + + +def main(argv: Sequence[str] | None = None) -> int: + args = _parse_args(argv) + fixture = ProtocolServer(args.mode, profile=args.profile) + if args.transport == "stdio": + print( + f"{SERVER_NAME} starting stdio mode={args.mode}", + file=sys.stderr, + flush=True, + ) + run_stdio(fixture, sys.stdin, sys.stdout) + return 0 + + server = make_http_server( + fixture, + args.host, + args.port, + endpoint=args.endpoint, + allowed_origins=args.allowed_origin, + ) + print( + f"{SERVER_NAME} listening on http://{args.host}:{args.port}{args.endpoint} " + f"mode={args.mode}", + file=sys.stderr, + flush=True, + ) + try: + server.serve_forever() + except KeyboardInterrupt: + pass + finally: + server.server_close() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/mcp_conformance/test_codex_compliance.py b/scripts/mcp_conformance/test_codex_compliance.py new file mode 100644 index 0000000000..e97374d515 --- /dev/null +++ b/scripts/mcp_conformance/test_codex_compliance.py @@ -0,0 +1,1202 @@ +import os +import sys +import json +from copy import deepcopy +from pathlib import Path + +import pytest +from run_codex_compliance import ( + COMPACT_REGRESSION_BASELINE_KIND, + LEGACY_VERSION, + MODERN_VERSION, + OFFICIAL_CONFORMANCE_GIT_REF, + OFFICIAL_CONFORMANCE_REPOSITORY, + REPORT_SCHEMA_VERSION, + REQUIRED_REGRESSION_MODES, + RESOURCE_URIS, + SERVER_NAME, + SERVER_VERSION, + SHIPPING_LEGACY_VERSION, + TEST_SERVER_NAME, + CaseResult, + CheckResult, + OfficialScenarioResult, + _build_test_matrix, + _compact_regression_baseline, + _evaluate_regression_gate, + _isolated_environment, + _parse_args, + _registration_command, + _run_official_case, + _scenario_ratio, + _summarize_modes, + _validate_inventory, + _validate_registration, + _write_compact_regression_baseline, + _write_json_file, + main, + scenarios_for_mode, +) + + +def test_registration_commands_select_stdio_then_http_shapes() -> None: + codex = Path("/opt/codex") + server = Path("/src/server.py") + + stdio = _registration_command( + codex, + server, + transport="stdio", + mode=MODERN_VERSION, + http_url=None, + ) + http = _registration_command( + codex, + server, + transport="http", + mode=MODERN_VERSION, + http_url="http://127.0.0.1:8765/mcp", + ) + + assert stdio == [ + "/opt/codex", + "mcp", + "add", + TEST_SERVER_NAME, + "--env", + f"CODEX_MCP_PROTOCOL_VERSION={MODERN_VERSION}", + "--", + sys.executable, + "/src/server.py", + "--mode", + MODERN_VERSION, + "--transport", + "stdio", + ] + assert http == [ + "/opt/codex", + "mcp", + "add", + TEST_SERVER_NAME, + "--url", + "http://127.0.0.1:8765/mcp", + ] + + +def test_registration_validation_checks_transport_and_mode() -> None: + valid, _ = _validate_registration( + { + "name": TEST_SERVER_NAME, + "enabled": True, + "transport": { + "type": "stdio", + "command": sys.executable, + "args": [ + "/src/server.py", + "--mode", + LEGACY_VERSION, + "--transport", + "stdio", + ], + }, + }, + transport="stdio", + mode=LEGACY_VERSION, + http_url=None, + ) + assert valid + + valid, _ = _validate_registration( + { + "name": TEST_SERVER_NAME, + "enabled": True, + "transport": { + "type": "stdio", + "command": sys.executable, + "args": [ + "/src/server.py", + "--mode", + MODERN_VERSION, + "--transport", + "stdio", + ], + "env": {"CODEX_MCP_PROTOCOL_VERSION": MODERN_VERSION}, + }, + }, + transport="stdio", + mode=MODERN_VERSION, + http_url=None, + ) + assert valid + + valid, detail = _validate_registration( + { + "name": TEST_SERVER_NAME, + "enabled": True, + "transport": { + "type": "stdio", + "command": sys.executable, + "args": [ + "/src/server.py", + "--mode", + MODERN_VERSION, + "--transport", + "stdio", + ], + }, + }, + transport="stdio", + mode=MODERN_VERSION, + http_url=None, + ) + assert not valid + assert "protocol opt-in" in detail + + valid, detail = _validate_registration( + { + "name": TEST_SERVER_NAME, + "enabled": True, + "transport": { + "type": "streamable_http", + "url": "http://127.0.0.1:2/mcp", + }, + }, + transport="http", + mode=MODERN_VERSION, + http_url="http://127.0.0.1:1/mcp", + ) + assert not valid + assert "unexpected HTTP transport" in detail + + +def test_inventory_validation_requires_modern_tool_and_all_pages() -> None: + inventory = { + "data": [ + { + "name": TEST_SERVER_NAME, + "serverInfo": { + "name": SERVER_NAME, + "version": SERVER_VERSION, + }, + "tools": { + name: {"name": name} + for name in ( + "echo", + "client_metadata", + "progress", + "request_input", + ) + }, + "resources": [{"uri": uri} for uri in RESOURCE_URIS], + } + ] + } + + valid, _ = _validate_inventory(inventory, mode=MODERN_VERSION) + assert valid + + inventory["data"][0]["tools"].pop("request_input") + inventory["data"][0]["resources"].pop() + valid, detail = _validate_inventory(inventory, mode=MODERN_VERSION) + assert not valid + assert "request_input" in detail + assert RESOURCE_URIS[1] in detail + + +def test_isolated_environment_drops_model_credentials( + tmp_path: Path, + monkeypatch, +) -> None: + monkeypatch.setenv("CODEX_API_KEY", "secret") + monkeypatch.setenv("CODEX_ACCESS_TOKEN", "secret") + monkeypatch.setenv("OPENAI_API_KEY", "secret") + + env = _isolated_environment(tmp_path) + + assert env["CODEX_HOME"] == str(tmp_path) + assert "CODEX_API_KEY" not in env + assert "CODEX_ACCESS_TOKEN" not in env + assert "OPENAI_API_KEY" not in env + assert env.get("PATH") == os.environ.get("PATH") + + +def test_mode_summaries_separate_official_and_supplemental_percentages() -> None: + legacy = CaseResult( + transport="stdio", + mode=LEGACY_VERSION, + success=True, + checks=[ + CheckResult("mcp_add", True, "ok"), + CheckResult("inventory", True, "ok"), + CheckResult("echo_tool", True, "ok"), + ], + ) + modern = CaseResult( + transport="http", + mode=MODERN_VERSION, + success=False, + checks=[ + CheckResult("mcp_add", True, "ok"), + CheckResult("modern_feature_enablement", True, "ok"), + CheckResult("inventory", False, "failed"), + CheckResult("http_header_mirroring", False, "failed"), + CheckResult( + "official/example/pass", + True, + "ok", + source="official", + category="non-auth", + scenario="example-pass", + ), + CheckResult( + "official/example/skipped", + True, + "not exercised", + status="SKIP", + source="official", + category="auth", + scenario="example-skipped", + ), + ], + ) + + summaries = _summarize_modes([legacy, modern]) + + assert summaries == [ + { + "mode": LEGACY_VERSION, + "checks": {"passed": 3, "total": 3, "percentage": 100.0}, + "officialChecks": {"passed": 0, "total": 0, "percentage": 0.0}, + "officialNonAuthChecks": { + "passed": 0, + "total": 0, + "percentage": 0.0, + }, + "officialAuthChecks": { + "passed": 0, + "total": 0, + "percentage": 0.0, + }, + "officialNonAuthScenarios": { + "passed": 0, + "total": 0, + "percentage": 0.0, + }, + "officialAuthScenarios": { + "passed": 0, + "total": 0, + "percentage": 0.0, + }, + "harnessChecks": {"passed": 0, "total": 0, "percentage": 0.0}, + "supplementalChecks": { + "passed": 3, + "total": 3, + "percentage": 100.0, + }, + "transportCases": {"passed": 1, "total": 1, "percentage": 100.0}, + }, + { + "mode": MODERN_VERSION, + "checks": {"passed": 3, "total": 5, "percentage": 60.0}, + "officialChecks": { + "passed": 1, + "total": 1, + "percentage": 100.0, + }, + "officialNonAuthChecks": { + "passed": 1, + "total": 1, + "percentage": 100.0, + }, + "officialAuthChecks": { + "passed": 0, + "total": 0, + "percentage": 0.0, + }, + "officialNonAuthScenarios": { + "passed": 1, + "total": 1, + "percentage": 100.0, + }, + "officialAuthScenarios": { + "passed": 1, + "total": 1, + "percentage": 100.0, + }, + "harnessChecks": {"passed": 0, "total": 0, "percentage": 0.0}, + "supplementalChecks": { + "passed": 2, + "total": 4, + "percentage": 50.0, + }, + "transportCases": {"passed": 0, "total": 1, "percentage": 0.0}, + }, + ] + + +def test_matrix_reports_pass_fail_and_not_applicable() -> None: + legacy = CaseResult( + transport="stdio", + mode=LEGACY_VERSION, + checks=[ + CheckResult("inventory", True, "ok"), + CheckResult("echo_tool", True, "ok"), + ], + ) + modern = CaseResult( + transport="http", + mode=MODERN_VERSION, + checks=[ + CheckResult("inventory", False, "failed"), + CheckResult("http_header_mirroring", False, "failed"), + ], + ) + + matrix = _build_test_matrix([legacy, modern]) + + assert matrix["columns"] == [ + { + "key": f"stdio:{LEGACY_VERSION}", + "transport": "stdio", + "mode": LEGACY_VERSION, + }, + { + "key": f"http:{MODERN_VERSION}", + "transport": "http", + "mode": MODERN_VERSION, + }, + ] + assert matrix["rows"] == [ + { + "test": "inventory", + "results": { + f"stdio:{LEGACY_VERSION}": "PASS", + f"http:{MODERN_VERSION}": "FAIL", + }, + }, + { + "test": "echo_tool", + "results": { + f"stdio:{LEGACY_VERSION}": "PASS", + f"http:{MODERN_VERSION}": "N/A", + }, + }, + { + "test": "http_header_mirroring", + "results": { + f"stdio:{LEGACY_VERSION}": "N/A", + f"http:{MODERN_VERSION}": "FAIL", + }, + }, + ] + + +def test_scenario_ratio_includes_adapter_failures() -> None: + checks = [ + CheckResult( + "official/auth/example/assertion", + True, + "ok", + source="official", + scenario="auth/example", + category="auth", + ), + CheckResult( + "harness/auth/example/codex-adapter", + False, + "adapter failed", + source="harness", + scenario="auth/example", + category="auth", + ), + CheckResult( + "official/auth/other/assertion", + True, + "ok", + source="official", + scenario="auth/other", + category="auth", + ), + ] + + assert _scenario_ratio(checks, category="auth") == { + "passed": 1, + "total": 2, + "percentage": 50.0, + } + + +@pytest.mark.parametrize("mode", [LEGACY_VERSION, MODERN_VERSION]) +@pytest.mark.parametrize( + ("adapter_success", "expected_status"), + [(False, "FAIL"), (True, "PASS")], +) +def test_official_case_preserves_adapter_check_identity_on_failure_and_success( + mode: str, + adapter_success: bool, + expected_status: str, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + scenario = "auth/pre-registration" + result = OfficialScenarioResult( + scenario=scenario, + success=adapter_success, + adapter_success=adapter_success, + adapter_detail="adapter succeeded" if adapter_success else "adapter failed", + ) + monkeypatch.setattr( + "run_codex_compliance.run_official_mode", + lambda **_kwargs: [result], + ) + + case = _run_official_case( + Path("/opt/codex"), + Path("/src/codex_conformance_adapter.py"), + conformance_command=["conformance"], + mode=mode, + scenarios=[scenario], + case_home=tmp_path / mode, + timeout_seconds=1.0, + enable_modern_feature=True, + ) + + assert case.checks == [ + CheckResult( + name=f"harness/{scenario}/codex-adapter", + success=adapter_success, + detail=result.adapter_detail, + status=expected_status, + source="harness", + scenario=scenario, + category="auth", + ) + ] + assert case.success is adapter_success + + +@pytest.mark.parametrize("mode", [LEGACY_VERSION, MODERN_VERSION]) +def test_official_case_preserves_independent_runner_failure_with_successful_adapter( + mode: str, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + scenario = "auth/pre-registration" + result = OfficialScenarioResult( + scenario=scenario, + success=False, + adapter_success=True, + adapter_detail="adapter succeeded", + runner_detail="official runner failed", + ) + monkeypatch.setattr( + "run_codex_compliance.run_official_mode", + lambda **_kwargs: [result], + ) + + case = _run_official_case( + Path("/opt/codex"), + Path("/src/codex_conformance_adapter.py"), + conformance_command=["conformance"], + mode=mode, + scenarios=[scenario], + case_home=tmp_path / mode, + timeout_seconds=1.0, + enable_modern_feature=True, + ) + + assert case.checks == [ + CheckResult( + name=f"harness/{scenario}/codex-adapter", + success=True, + detail="adapter succeeded", + status="PASS", + source="harness", + scenario=scenario, + category="auth", + ), + CheckResult( + name=f"harness/{scenario}/official-runner", + success=False, + detail="official runner failed", + status="FAIL", + source="harness", + scenario=scenario, + category="auth", + ), + ] + assert case.success is False + assert case.diagnostics == f"{scenario}:\nofficial runner failed" + + +def _regression_report() -> dict[str, object]: + cases: list[dict[str, object]] = [] + known_modern = { + "auth/metadata-var3": "authorization-server-metadata", + "auth/scope-from-www-authenticate": "scope-from-www-authenticate", + "auth/scope-step-up": "scope-step-up-initial", + "auth/pre-registration": "pre-registration-auth", + } + scenarios = { + mode: list(scenarios_for_mode(mode, include_auth=True)) + for mode in REQUIRED_REGRESSION_MODES + } + + for mode in REQUIRED_REGRESSION_MODES: + cases.append( + { + "mode": mode, + "transport": "stdio", + "success": True, + "checks": [ + { + "name": "mcp_add", + "success": True, + "status": "PASS", + "source": "supplemental", + "scenario": None, + "check_id": None, + } + ], + } + ) + official_checks: list[dict[str, object]] = [] + for scenario in scenarios[mode]: + if mode == MODERN_VERSION and scenario == "request-metadata": + official_checks.append( + { + "name": "harness/request-metadata/official-runner", + "success": False, + "status": "FAIL", + "source": "harness", + "scenario": scenario, + "check_id": None, + } + ) + continue + + known = mode == MODERN_VERSION and scenario in known_modern + official_checks.append( + { + "name": f"official/{scenario}/assertion", + "success": not known, + "status": "FAIL" if known else "PASS", + "source": "official", + "scenario": scenario, + "check_id": ( + known_modern[scenario] if known else f"assertion-{scenario}" + ), + } + ) + if known: + official_checks.append( + { + "name": f"harness/{scenario}/codex-adapter", + "success": False, + "status": "FAIL", + "source": "harness", + "scenario": scenario, + "check_id": None, + } + ) + cases.append( + { + "mode": mode, + "transport": "official-http", + "success": mode == SHIPPING_LEGACY_VERSION, + "checks": official_checks, + } + ) + + return { + "schemaVersion": REPORT_SCHEMA_VERSION, + "success": False, + "versionCheck": {"success": True}, + "modernFeatureEnablement": True, + "automaticAuthRequired": False, + "officialConformance": { + "repository": OFFICIAL_CONFORMANCE_REPOSITORY, + "gitRef": OFFICIAL_CONFORMANCE_GIT_REF, + "authenticationIncluded": True, + "scenarios": scenarios, + }, + "cases": cases, + } + + +def _regression_case( + report: dict[str, object], + mode: str, + transport: str, +) -> dict[str, object]: + cases = report["cases"] + assert isinstance(cases, list) + return next( + case + for case in cases + if isinstance(case, dict) + and case["mode"] == mode + and case["transport"] == transport + ) + + +def _regression_scenario_check( + report: dict[str, object], + *, + mode: str = MODERN_VERSION, + scenario: str, + source: str = "official", +) -> dict[str, object]: + checks = _regression_case(report, mode, "official-http")["checks"] + assert isinstance(checks, list) + return next( + check + for check in checks + if isinstance(check, dict) + and check["scenario"] == scenario + and check["source"] == source + ) + + +def test_regression_gate_preserves_truthful_known_modern_failures() -> None: + baseline = _regression_report() + candidate = deepcopy(baseline) + + gate = _evaluate_regression_gate(candidate, baseline) + + assert candidate["success"] is False + assert gate["success"] is True + assert gate["requiredModes"] == [ + SHIPPING_LEGACY_VERSION, + LEGACY_VERSION, + MODERN_VERSION, + ] + assert len(gate["knownFailures"]) == 9 + assert gate["newFailures"] == [] + assert gate["missingChecks"] == [] + assert gate["configurationErrors"] == [] + + +def test_regression_gate_rejects_a_new_modern_failure() -> None: + baseline = _regression_report() + candidate = deepcopy(baseline) + check = _regression_scenario_check(candidate, scenario="tools_call") + check["success"] = False + check["status"] = "FAIL" + + gate = _evaluate_regression_gate(candidate, baseline) + + assert gate["success"] is False + assert any(item["scenario"] == "tools_call" for item in gate["newFailures"]) + + +def test_regression_gate_rejects_a_new_intermediate_oauth_failure() -> None: + baseline = _regression_report() + candidate = deepcopy(baseline) + check = _regression_scenario_check( + candidate, + mode=LEGACY_VERSION, + scenario="auth/metadata-default", + ) + check["success"] = False + check["status"] = "FAIL" + + gate = _evaluate_regression_gate(candidate, baseline) + + assert gate["success"] is False + assert any( + item["mode"] == LEGACY_VERSION and item["scenario"] == "auth/metadata-default" + for item in gate["newFailures"] + ) + + +def test_regression_gate_rejects_new_failures_inside_a_known_failing_scenario() -> None: + baseline = _regression_report() + candidate = deepcopy(baseline) + checks = _regression_case(candidate, MODERN_VERSION, "official-http")["checks"] + assert isinstance(checks, list) + checks.append( + { + "name": "official/auth/metadata-var3/new assertion", + "success": False, + "status": "FAIL", + "source": "official", + "scenario": "auth/metadata-var3", + "check_id": "previously-passing-metadata-assertion", + } + ) + + gate = _evaluate_regression_gate(candidate, baseline) + + assert gate["success"] is False + assert any( + item["check_id"] == "previously-passing-metadata-assertion" + for item in gate["newFailures"] + ) + + +def test_regression_gate_accepts_fixed_known_failures() -> None: + baseline = _regression_report() + candidate = deepcopy(baseline) + check = _regression_scenario_check(candidate, scenario="auth/metadata-var3") + check["success"] = True + check["status"] = "PASS" + + gate = _evaluate_regression_gate(candidate, baseline) + + assert gate["success"] is True + assert any( + item["check_id"] == "authorization-server-metadata" + for item in gate["fixedChecks"] + ) + + +def test_regression_gate_accepts_fixed_oauth_official_and_adapter_checks() -> None: + baseline = _regression_report() + candidate = deepcopy(baseline) + scenario = "auth/pre-registration" + for source in ("official", "harness"): + check = _regression_scenario_check( + candidate, + scenario=scenario, + source=source, + ) + check["success"] = True + check["status"] = "PASS" + + gate = _evaluate_regression_gate(candidate, baseline) + + assert gate["success"] is True + assert gate["configurationErrors"] == [] + assert gate["newFailures"] == [] + assert gate["missingChecks"] == [] + assert gate["fixedChecks"] == [ + { + "mode": MODERN_VERSION, + "transport": "official-http", + "source": "harness", + "scenario": scenario, + "check_id": f"harness/{scenario}/codex-adapter", + }, + { + "mode": MODERN_VERSION, + "transport": "official-http", + "source": "official", + "scenario": scenario, + "check_id": "pre-registration-auth", + }, + ] + + +def test_regression_gate_rejects_a_missing_fixed_oauth_adapter_check() -> None: + baseline = _regression_report() + candidate = deepcopy(baseline) + scenario = "auth/pre-registration" + check = _regression_scenario_check(candidate, scenario=scenario) + check["success"] = True + check["status"] = "PASS" + checks = _regression_case(candidate, MODERN_VERSION, "official-http")["checks"] + assert isinstance(checks, list) + checks[:] = [ + item + for item in checks + if item.get("scenario") != scenario or item.get("source") != "harness" + ] + + gate = _evaluate_regression_gate(candidate, baseline) + + assert gate["success"] is False + assert gate["configurationErrors"] == [] + assert gate["newFailures"] == [] + assert gate["missingChecks"] == [ + { + "mode": MODERN_VERSION, + "transport": "official-http", + "source": "harness", + "scenario": scenario, + "check_id": f"harness/{scenario}/codex-adapter", + } + ] + assert gate["fixedChecks"] == [ + { + "mode": MODERN_VERSION, + "transport": "official-http", + "source": "official", + "scenario": scenario, + "check_id": "pre-registration-auth", + } + ] + + +def test_regression_gate_rejects_a_skipped_previous_failure() -> None: + baseline = _regression_report() + candidate = deepcopy(baseline) + check = _regression_scenario_check(candidate, scenario="auth/metadata-var3") + check["success"] = True + check["status"] = "SKIP" + + gate = _evaluate_regression_gate(candidate, baseline) + + assert gate["success"] is False + assert any( + item["check_id"] == "authorization-server-metadata" + for item in gate["missingChecks"] + ) + + +def test_regression_gate_rejects_a_missing_previously_passing_check() -> None: + baseline = _regression_report() + candidate = deepcopy(baseline) + checks = _regression_case(candidate, MODERN_VERSION, "official-http")["checks"] + assert isinstance(checks, list) + checks[:] = [check for check in checks if check.get("scenario") != "tools_call"] + + gate = _evaluate_regression_gate(candidate, baseline) + + assert gate["success"] is False + assert any(item["scenario"] == "tools_call" for item in gate["missingChecks"]) + assert any("did not observe" in item for item in gate["configurationErrors"]) + + +@pytest.mark.parametrize("mode", REQUIRED_REGRESSION_MODES) +@pytest.mark.parametrize("transport", ["stdio", "official-http"]) +def test_regression_gate_requires_every_version_and_both_transports( + mode: str, + transport: str, +) -> None: + baseline = _regression_report() + candidate = deepcopy(baseline) + cases = candidate["cases"] + assert isinstance(cases, list) + cases[:] = [ + case + for case in cases + if case.get("mode") != mode or case.get("transport") != transport + ] + + gate = _evaluate_regression_gate(candidate, baseline) + + assert gate["success"] is False + assert any( + f"missing the required {transport} case for {mode}" in item + for item in gate["configurationErrors"] + ) + + +def test_regression_gate_requires_the_complete_modern_authenticated_catalog() -> None: + baseline = _regression_report() + candidate = deepcopy(baseline) + official = candidate["officialConformance"] + assert isinstance(official, dict) + scenarios = official["scenarios"] + assert isinstance(scenarios, dict) + scenarios[MODERN_VERSION] = [ + scenario + for scenario in scenarios[MODERN_VERSION] + if scenario != "auth/pre-registration" + ] + + gate = _evaluate_regression_gate(candidate, baseline) + + assert gate["success"] is False + assert any("complete authenticated" in item for item in gate["configurationErrors"]) + + +def test_regression_gate_requires_modern_feature_enablement() -> None: + baseline = _regression_report() + candidate = deepcopy(baseline) + candidate["modernFeatureEnablement"] = False + + gate = _evaluate_regression_gate(candidate, baseline) + + assert gate["success"] is False + assert any("modern MCP feature" in item for item in gate["configurationErrors"]) + + +def test_regression_gate_requires_the_same_pinned_upstream_suite() -> None: + baseline = _regression_report() + candidate = deepcopy(baseline) + official = candidate["officialConformance"] + assert isinstance(official, dict) + official["gitRef"] = "0" * 40 + + gate = _evaluate_regression_gate(candidate, baseline) + + assert gate["success"] is False + assert any("pinned upstream" in item for item in gate["configurationErrors"]) + + +def test_regression_gate_requires_oauth_scenarios() -> None: + baseline = _regression_report() + candidate = deepcopy(baseline) + official = candidate["officialConformance"] + assert isinstance(official, dict) + official["authenticationIncluded"] = False + + gate = _evaluate_regression_gate(candidate, baseline) + + assert gate["success"] is False + assert any("required OAuth" in item for item in gate["configurationErrors"]) + + +def test_regression_gate_rejects_different_production_oauth_policies() -> None: + baseline = _regression_report() + candidate = deepcopy(baseline) + candidate["automaticAuthRequired"] = True + + gate = _evaluate_regression_gate(candidate, baseline) + + assert gate["success"] is False + assert any( + "different production OAuth" in item for item in gate["configurationErrors"] + ) + + +def test_regression_gate_rejects_unsupported_report_schema() -> None: + baseline = _regression_report() + baseline["schemaVersion"] = REPORT_SCHEMA_VERSION + 1 + + gate = _evaluate_regression_gate(_regression_report(), baseline) + + assert gate["success"] is False + assert any("report schema" in item for item in gate["configurationErrors"]) + + +def test_regression_gate_rejects_any_shipping_failure() -> None: + baseline = _regression_report() + candidate = deepcopy(baseline) + check = _regression_scenario_check( + candidate, + mode=SHIPPING_LEGACY_VERSION, + scenario="initialize", + ) + check["success"] = False + check["status"] = "FAIL" + + gate = _evaluate_regression_gate(candidate, baseline) + + assert gate["success"] is False + assert any( + "shipping MCP check failed" in item for item in gate["configurationErrors"] + ) + + +def test_regression_gate_ignores_retry_observation_renumbering() -> None: + baseline = _regression_report() + check = _regression_scenario_check(baseline, scenario="tools_call") + check["name"] = "official/tools_call/authorization metadata#4" + candidate = deepcopy(baseline) + candidate_check = _regression_scenario_check(candidate, scenario="tools_call") + candidate_check["name"] = "official/tools_call/authorization metadata#57" + checks = _regression_case(candidate, MODERN_VERSION, "official-http")["checks"] + assert isinstance(checks, list) + repeated = deepcopy(candidate_check) + repeated["name"] = "official/tools_call/authorization metadata#58" + checks.append(repeated) + + gate = _evaluate_regression_gate(candidate, baseline) + + assert gate["success"] is True + assert gate["newFailures"] == [] + assert gate["missingChecks"] == [] + + +def test_regression_cli_accepts_a_baseline_report() -> None: + args = _parse_args( + ["/opt/codex", "--baseline-report", "/tmp/codex-mcp-baseline.json"] + ) + + assert args.baseline_report == Path("/tmp/codex-mcp-baseline.json") + assert args.mode == "all" + assert args.auth is True + assert args.enable_modern_feature is True + + +def test_regression_gate_accepts_a_compact_baseline() -> None: + report = _regression_report() + + compact = _compact_regression_baseline(report) + gate = _evaluate_regression_gate(deepcopy(report), compact) + + assert compact["baselineKind"] == COMPACT_REGRESSION_BASELINE_KIND + assert compact["requiredModes"] == [ + SHIPPING_LEGACY_VERSION, + LEGACY_VERSION, + MODERN_VERSION, + ] + assert compact["transports"] == ["stdio", "official-http"] + assert "cases" not in compact + assert gate["success"] is True + assert len(gate["knownFailures"]) == 9 + + +def test_regression_gate_accepts_the_real_in_memory_tuple_scenario_catalog() -> None: + baseline = _compact_regression_baseline(_regression_report()) + candidate = _regression_report() + official = candidate["officialConformance"] + assert isinstance(official, dict) + scenarios = official["scenarios"] + assert isinstance(scenarios, dict) + official["scenarios"] = { + mode: tuple(selected) for mode, selected in scenarios.items() + } + + gate = _evaluate_regression_gate(candidate, baseline) + + assert gate["success"] is True + assert gate["configurationErrors"] == [] + + +def test_regression_gate_rejects_a_new_failure_against_a_compact_baseline() -> None: + baseline = _compact_regression_baseline(_regression_report()) + candidate = _regression_report() + check = _regression_scenario_check(candidate, scenario="tools_call") + check["success"] = False + check["status"] = "FAIL" + + gate = _evaluate_regression_gate(candidate, baseline) + + assert gate["success"] is False + assert any(item["scenario"] == "tools_call" for item in gate["newFailures"]) + + +def test_regression_gate_rejects_malformed_compact_identities() -> None: + compact = _compact_regression_baseline(_regression_report()) + checks = compact["checks"] + assert isinstance(checks, dict) + passing = checks["passing"] + assert isinstance(passing, list) + passing[0]["check_id"] = "" + + gate = _evaluate_regression_gate(_regression_report(), compact) + + assert gate["success"] is False + assert any( + "malformed passing baseline identity" in e for e in gate["configurationErrors"] + ) + + +def test_regression_gate_rejects_an_unknown_compact_baseline_format() -> None: + compact = _compact_regression_baseline(_regression_report()) + compact["baselineKind"] = "unrecognized-baseline" + + gate = _evaluate_regression_gate(_regression_report(), compact) + + assert gate["success"] is False + assert any("unsupported compact baseline" in e for e in gate["configurationErrors"]) + + +def test_json_artifacts_are_sorted_human_readable_and_newline_terminated( + tmp_path: Path, +) -> None: + path = tmp_path / "nested" / "report.json" + payload = {"z": {"z": "café", "a": [2, 1]}, "a": True} + + _write_json_file(path, payload) + + assert path.read_text(encoding="utf-8") == ( + "{\n" + ' "a": true,\n' + ' "z": {\n' + ' "a": [\n' + " 2,\n" + " 1\n" + " ],\n" + ' "z": "café"\n' + " }\n" + "}\n" + ) + assert json.loads(path.read_text(encoding="utf-8")) == payload + + +def test_committed_regression_baseline_is_sorted_human_readable_json() -> None: + baseline = Path(__file__).with_name("regression-baseline-v1.json") + content = baseline.read_text(encoding="utf-8") + + assert content == ( + json.dumps(json.loads(content), ensure_ascii=False, indent=2, sort_keys=True) + + "\n" + ) + + +def test_compact_regression_baselines_are_deterministic(tmp_path: Path) -> None: + first = tmp_path / "first.json" + second = tmp_path / "second.json" + + _write_compact_regression_baseline(_regression_report(), first) + _write_compact_regression_baseline(_regression_report(), second) + + assert first.read_bytes() == second.read_bytes() + content = first.read_text(encoding="utf-8") + assert content == ( + json.dumps(json.loads(content), ensure_ascii=False, indent=2, sort_keys=True) + + "\n" + ) + assert json.loads(content)["baselineKind"] == (COMPACT_REGRESSION_BASELINE_KIND) + + +def test_regression_cli_extracts_compact_baseline_without_starting_conformance( + tmp_path: Path, +) -> None: + codex = tmp_path / "codex" + codex.write_text("#!/bin/sh\nexit 99\n", encoding="utf-8") + codex.chmod(0o755) + source = tmp_path / "full-report.json" + source.write_text(json.dumps(_regression_report()), encoding="utf-8") + extracted = tmp_path / "compact.json" + + assert ( + main( + [ + str(codex), + "--baseline-report", + str(source), + "--extract-baseline", + str(extracted), + ] + ) + == 0 + ) + assert json.loads(extracted.read_text(encoding="utf-8"))["baselineKind"] == ( + COMPACT_REGRESSION_BASELINE_KIND + ) + + +def test_regression_cli_rejects_extraction_without_a_source_report( + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + codex = tmp_path / "codex" + codex.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") + codex.chmod(0o755) + + assert main([str(codex), "--extract-baseline", str(tmp_path / "compact.json")]) == 2 + assert "requires --baseline-report" in capsys.readouterr().err + + +@pytest.mark.parametrize("payload", ["{", "[]", "null"]) +def test_regression_cli_rejects_malformed_baselines( + payload: str, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + codex = tmp_path / "codex" + codex.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") + codex.chmod(0o755) + baseline = tmp_path / "baseline.json" + baseline.write_text(payload, encoding="utf-8") + + assert main([str(codex), "--baseline-report", str(baseline)]) == 2 + assert "baseline report" in capsys.readouterr().err + + +def test_regression_cli_rejects_missing_baseline_before_starting_conformance( + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + codex = tmp_path / "codex" + codex.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") + codex.chmod(0o755) + + assert main([str(codex), "--baseline-report", str(tmp_path / "missing.json")]) == 2 + assert "cannot read baseline report" in capsys.readouterr().err diff --git a/scripts/mcp_conformance/test_official_conformance.py b/scripts/mcp_conformance/test_official_conformance.py new file mode 100644 index 0000000000..be8a9650ab --- /dev/null +++ b/scripts/mcp_conformance/test_official_conformance.py @@ -0,0 +1,685 @@ +import json +import subprocess +import sys +import urllib.parse +from pathlib import Path + +import codex_conformance_adapter +import official_conformance +import pytest +import run_codex_compliance +from codex_conformance_adapter import ( + CIMD_CLIENT_METADATA_URL, + AdapterFailure, + _exercise_auth_scenario, + _oauth_client_id, + _validate_oauth_secret_not_persisted, + _validated_callback_url, + _write_auth_registration, +) +from official_conformance import ( + LEGACY_VERSION, + MODERN_VERSION, + OFFICIAL_CONFORMANCE_GIT_REF, + SHIPPING_LEGACY_VERSION, + _make_adapter_launcher, + _run_scenario, + _scrub_retained_artifacts, + _terminate_process_group, + default_conformance_command, + redact_sensitive_text, + run_official_mode, + scenarios_for_mode, +) + + +def _authorization_url(redirect_uri: str, state: str = "test-state") -> str: + return "http://127.0.0.1:8765/authorize?" + urllib.parse.urlencode( + { + "client_id": "test-client", + "redirect_uri": redirect_uri, + "state": state, + } + ) + + +def test_auth_adapter_only_accepts_exact_loopback_callback() -> None: + redirect_uri = "http://127.0.0.1:32123/callback" + authorization_url = _authorization_url(redirect_uri) + callback_url = redirect_uri + "?code=test-code&state=test-state" + + assert _validated_callback_url(authorization_url, callback_url) == callback_url + + with pytest.raises(AdapterFailure, match="exact loopback"): + _validated_callback_url( + authorization_url, + "http://127.0.0.1:32124/callback?code=test-code&state=test-state", + ) + with pytest.raises(AdapterFailure, match="preserve OAuth state"): + _validated_callback_url( + authorization_url, + redirect_uri + "?code=test-code&state=wrong-state", + ) + with pytest.raises(AdapterFailure, match="safe loopback"): + _validated_callback_url( + _authorization_url("https://example.com/callback"), + "https://example.com/callback?code=test-code&state=test-state", + ) + + +@pytest.mark.parametrize( + ("query", "message"), + [ + ("code=first&code=second&state=test-state", "exactly one nonempty code"), + ("code=&state=test-state", "exactly one nonempty code"), + ("error=first&error=second&state=test-state", "exactly one nonempty error"), + ("error=&state=test-state", "exactly one nonempty error"), + ("code=test-code&error=access_denied&state=test-state", "exactly one of"), + ("code=test-code&state=test-state&state=test-state", "preserve OAuth state"), + ( + "code=test-code&state=test-state&iss=first&iss=second", + "exactly one nonempty iss", + ), + ("code=test-code&state=test-state&iss=", "exactly one nonempty iss"), + ("state=test-state", "exactly one of"), + ], +) +def test_auth_adapter_rejects_ambiguous_oauth_callback_parameters( + query: str, + message: str, +) -> None: + redirect_uri = "http://127.0.0.1:32123/callback" + + with pytest.raises(AdapterFailure, match=message): + _validated_callback_url( + _authorization_url(redirect_uri), + f"{redirect_uri}?{query}", + ) + + +def test_auth_adapter_allows_a_single_error_callback_for_client_validation() -> None: + redirect_uri = "http://127.0.0.1:32123/callback" + callback_url = redirect_uri + "?error=access_denied&state=test-state&iss=issuer" + + assert ( + _validated_callback_url(_authorization_url(redirect_uri), callback_url) + == callback_url + ) + + +def test_official_runner_is_pinned_instead_of_using_npm_latest() -> None: + command = default_conformance_command() + + assert OFFICIAL_CONFORMANCE_GIT_REF in command[-1] + assert "@latest" not in command[-1] + + +def test_official_adapter_launcher_preserves_strict_production_auth_mode( + tmp_path: Path, +) -> None: + adapter = tmp_path / "show_automatic_auth.py" + adapter.write_text( + 'import os\nprint(os.environ.get("CODEX_CONFORMANCE_REQUIRE_AUTOMATIC_AUTH", ""))\n', + encoding="utf-8", + ) + launcher = _make_adapter_launcher(adapter, require_automatic_auth=True) + + try: + result = subprocess.run( + [str(launcher)], + capture_output=True, + check=True, + text=True, + ) + finally: + launcher.unlink() + launcher.parent.rmdir() + + assert result.stdout.strip() == "1" + + +@pytest.mark.parametrize("require_automatic_auth", [False, True]) +def test_windows_official_adapter_launcher_quotes_checkout_paths( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + require_automatic_auth: bool, +) -> None: + adapter = tmp_path / "checkout with spaces" / "adapter script.py" + adapter.parent.mkdir() + adapter.write_text("raise AssertionError('launcher generation only')\n") + monkeypatch.setattr(official_conformance.sys, "platform", "win32") + + launcher = _make_adapter_launcher( + adapter, + require_automatic_auth=require_automatic_auth, + ) + try: + contents = launcher.read_text(encoding="utf-8") + finally: + launcher.unlink() + launcher.parent.rmdir() + + expected_auth = "1" if require_automatic_auth else "0" + assert launcher.name == "client.cmd" + assert contents.startswith("@echo off\n") + assert ( + f'set "CODEX_CONFORMANCE_REQUIRE_AUTOMATIC_AUTH={expected_auth}"\n' in contents + ) + assert subprocess.list2cmdline([sys.executable, str(adapter)]) + " %*\n" in contents + + +@pytest.mark.parametrize( + ("force", "expected_command"), + [ + (False, ["taskkill", "/T", "/PID", "417"]), + (True, ["taskkill", "/F", "/T", "/PID", "417"]), + ], +) +def test_windows_official_timeout_terminates_the_entire_process_tree( + monkeypatch: pytest.MonkeyPatch, + force: bool, + expected_command: list[str], +) -> None: + observed: list[list[str]] = [] + + class FakeProcess: + pid = 417 + + def terminate(self) -> None: + raise AssertionError("successful taskkill must own tree termination") + + def kill(self) -> None: + raise AssertionError("successful taskkill must own tree termination") + + def fake_run( + command: list[str], + *, + check: bool, + stdout: int, + stderr: int, + ) -> subprocess.CompletedProcess[str]: + observed.append(command) + assert check is False + assert stdout == subprocess.DEVNULL + assert stderr == subprocess.DEVNULL + return subprocess.CompletedProcess(command, 0) + + monkeypatch.setattr(official_conformance.sys, "platform", "win32") + monkeypatch.setattr(official_conformance.subprocess, "run", fake_run) + + _terminate_process_group( + FakeProcess(), # type: ignore[arg-type] - validate the Popen process contract. + force=force, + ) + + assert observed == [expected_command] + + +@pytest.mark.parametrize( + "scenario", + ["auth/scope-step-up", "auth/authorization-server-migration"], +) +def test_strict_auth_does_not_inject_a_second_login( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + scenario: str, +) -> None: + manual_reauthorizations: list[object] = [] + + monkeypatch.setattr( + codex_conformance_adapter, + "_oauth_login", + lambda _client, **_kwargs: (True, None), + ) + monkeypatch.setattr(codex_conformance_adapter, "_reload_mcp", lambda _client: None) + monkeypatch.setattr( + codex_conformance_adapter, "_auth_inventory", lambda _client: {} + ) + + def fail_tool_call(_client: object, _workspace: Path) -> None: + raise AdapterFailure("reauthorization is required") + + monkeypatch.setattr(codex_conformance_adapter, "_auth_tool_call", fail_tool_call) + monkeypatch.setattr( + codex_conformance_adapter, + "_login_reload_and_call", + lambda *args, **kwargs: manual_reauthorizations.append((args, kwargs)), + ) + + with pytest.raises(AdapterFailure, match="reauthorization is required"): + _exercise_auth_scenario( + object(), # type: ignore[arg-type] + scenario=scenario, + workspace=tmp_path, + timeout_seconds=1, + require_automatic_auth=True, + ) + + assert manual_reauthorizations == [] + + +@pytest.mark.parametrize( + "scenario", + ["auth/scope-step-up", "auth/authorization-server-migration"], +) +def test_strict_auth_accepts_product_owned_reauthentication( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + scenario: str, +) -> None: + tool_calls: list[Path] = [] + manual_reauthorizations: list[object] = [] + + monkeypatch.setattr( + codex_conformance_adapter, + "_oauth_login", + lambda _client, **_kwargs: (True, None), + ) + monkeypatch.setattr(codex_conformance_adapter, "_reload_mcp", lambda _client: None) + monkeypatch.setattr( + codex_conformance_adapter, "_auth_inventory", lambda _client: {} + ) + monkeypatch.setattr( + codex_conformance_adapter, + "_auth_tool_call", + lambda _client, workspace: tool_calls.append(workspace), + ) + monkeypatch.setattr( + codex_conformance_adapter, + "_login_reload_and_call", + lambda *args, **kwargs: manual_reauthorizations.append((args, kwargs)), + ) + + detail = _exercise_auth_scenario( + object(), # type: ignore[arg-type] + scenario=scenario, + workspace=tmp_path, + timeout_seconds=1, + require_automatic_auth=True, + ) + + assert "automatically" in detail + assert tool_calls == [tmp_path] + assert manual_reauthorizations == [] + + +def test_strict_auth_does_not_invent_a_production_client_metadata_url() -> None: + assert ( + _oauth_client_id( + "auth/basic-cimd", + {}, + require_production_client_identity=True, + ) + is None + ) + + +@pytest.mark.parametrize("filename", ["config.toml", ".credentials.json"]) +def test_oauth_client_secret_persistence_is_detected_without_disclosing_it( + tmp_path: Path, + filename: str, +) -> None: + secret = "reviewer-confidential-client-secret" + (tmp_path / filename).write_text( + json.dumps({"client_secret": secret}), + encoding="utf-8", + ) + + with pytest.raises(AdapterFailure, match="client secret was persisted") as error: + _validate_oauth_secret_not_persisted(tmp_path, secret) + + assert filename in str(error.value) + assert secret not in str(error.value) + + +def test_oauth_client_secret_environment_reference_is_not_a_persisted_secret( + tmp_path: Path, +) -> None: + (tmp_path / "config.toml").write_text( + 'client_secret_env_var = "MCP_CONFORMANCE_CLIENT_SECRET"\n', + encoding="utf-8", + ) + + _validate_oauth_secret_not_persisted( + tmp_path, "reviewer-confidential-client-secret" + ) + + +def test_auth_adapter_selects_only_scenario_provided_client_ids() -> None: + assert _oauth_client_id("auth/basic-cimd", {}) == CIMD_CLIENT_METADATA_URL + assert ( + _oauth_client_id( + "auth/basic-cimd", + {}, + require_production_client_identity=True, + ) + is None + ) + assert ( + _oauth_client_id( + "auth/pre-registration", + {"client_id": "pre-registered", "client_secret": "do-not-log"}, + ) + == "pre-registered" + ) + assert _oauth_client_id("auth/metadata-default", {}) is None + with pytest.raises(AdapterFailure, match="client_id"): + _oauth_client_id("auth/pre-registration", {}) + + +def test_auth_registration_does_not_persist_context_secret(tmp_path: Path) -> None: + config_path = tmp_path / "config.toml" + config_path.write_text('mcp_oauth_credentials_store = "file"\n', encoding="utf-8") + + _write_auth_registration( + config_path, + server_url="http://127.0.0.1:8765/mcp", + oauth_client_id="pre-registered-client", + oauth_client_secret_env_var="MCP_CONFORMANCE_CLIENT_SECRET", + ) + + config = config_path.read_text(encoding="utf-8") + assert 'url = "http://127.0.0.1:8765/mcp"' in config + assert 'client_id = "pre-registered-client"' in config + assert 'client_secret_env_var = "MCP_CONFORMANCE_CLIENT_SECRET"' in config + assert "do-not-log" not in config + assert "client_secret =" not in config + + +def test_official_diagnostics_redact_oauth_secrets() -> None: + diagnostic = ( + 'With context: {"client_id":"visible","client_secret":"secret-value",' + '"private_key_pem":"private-value"}\n' + "Authorize at http://localhost/authorize?state=state-value&code_challenge=pkce-value\n" + "Authorization: Bearer token-value" + ) + + redacted = redact_sensitive_text(diagnostic) + + assert '"client_id":"visible"' in redacted + assert '"client_secret":"[REDACTED]"' in redacted + assert '"private_key_pem":"[REDACTED]"' in redacted + assert "state=[REDACTED]" in redacted + assert "code_challenge=[REDACTED]" in redacted + assert "Bearer [REDACTED]" in redacted + serialized = redact_sensitive_text(json.dumps({"diagnostic": diagnostic})) + assert json.loads(serialized)["diagnostic"] + for secret in ( + "secret-value", + "private-value", + "state-value", + "pkce-value", + "token-value", + ): + assert secret not in redacted + assert secret not in serialized + + +def test_retained_artifacts_remove_oauth_credential_store(tmp_path: Path) -> None: + codex_home = tmp_path / "codex-home" + codex_home.mkdir() + credentials = codex_home / ".credentials.json" + credentials.write_text('{"client_secret":"do-not-retain"}', encoding="utf-8") + stdout = tmp_path / "stdout.txt" + stdout.write_text("Bearer token-value", encoding="utf-8") + + _scrub_retained_artifacts(tmp_path) + + assert not credentials.exists() + assert stdout.read_text(encoding="utf-8") == "Bearer [REDACTED]" + + +def test_pinned_full_scenarios_are_selected_by_protocol_version() -> None: + shipping_legacy = scenarios_for_mode(SHIPPING_LEGACY_VERSION) + legacy = scenarios_for_mode(LEGACY_VERSION) + modern = scenarios_for_mode(MODERN_VERSION) + + assert shipping_legacy == ( + "initialize", + "tools_call", + "auth/token-endpoint-auth-basic", + "auth/token-endpoint-auth-post", + "auth/token-endpoint-auth-none", + ) + assert scenarios_for_mode(SHIPPING_LEGACY_VERSION, include_auth=False) == ( + "initialize", + "tools_call", + ) + + assert legacy[:4] == ( + "initialize", + "tools_call", + "elicitation-sep1034-client-defaults", + "sse-retry", + ) + assert len(legacy) == 18 + assert len([scenario for scenario in legacy if scenario.startswith("auth/")]) == 14 + assert "auth/pre-registration" in legacy + + assert modern[:7] == ( + "tools_call", + "request-metadata", + "sep-2322-client-request-state", + "http-standard-headers", + "http-custom-headers", + "http-invalid-tool-headers", + "json-schema-ref-no-deref", + ) + assert len(modern) == 32 + assert len([scenario for scenario in modern if scenario.startswith("auth/")]) == 25 + assert "auth/resource-mismatch" in modern + assert "auth/metadata-issuer-mismatch" in modern + + assert scenarios_for_mode(LEGACY_VERSION, include_auth=False) == legacy[:4] + assert scenarios_for_mode(MODERN_VERSION, include_auth=False) == modern[:7] + assert scenarios_for_mode( + MODERN_VERSION, + ["tools_call", "http-custom-headers"], + ) == ("tools_call", "http-custom-headers") + assert len(OFFICIAL_CONFORMANCE_GIT_REF) == 40 + + +@pytest.mark.parametrize( + ("mode", "scenario"), + [ + (SHIPPING_LEGACY_VERSION, "request-metadata"), + (LEGACY_VERSION, "http-standard-headers"), + (MODERN_VERSION, "elicitation-sep1034-client-defaults"), + ], +) +def test_versioned_scenario_selection_rejects_an_incompatible_protocol( + mode: str, scenario: str +) -> None: + with pytest.raises( + ValueError, + match=rf"unavailable for MCP protocol version {mode}.*{scenario}", + ): + scenarios_for_mode(mode, [scenario], include_auth=False) + + +def test_cross_version_auth_subset_remains_available_to_every_mode() -> None: + requested = ( + "auth/token-endpoint-auth-basic", + "auth/resource-mismatch", + "auth/authorization-server-migration", + ) + + assert scenarios_for_mode(SHIPPING_LEGACY_VERSION, requested) == ( + "auth/token-endpoint-auth-basic", + ) + assert scenarios_for_mode(LEGACY_VERSION, requested) == ( + "auth/token-endpoint-auth-basic", + ) + assert scenarios_for_mode(MODERN_VERSION, requested) == requested + + +def test_official_cli_rejects_an_empty_versioned_scenario_run( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + def unexpected_run(*_args: object, **_kwargs: object) -> None: + raise AssertionError("an incompatible selection must not execute conformance") + + monkeypatch.setattr(run_codex_compliance, "run_compliance", unexpected_run) + + exit_code = run_codex_compliance.main( + [ + sys.executable, + "--mode", + SHIPPING_LEGACY_VERSION, + "--transport", + "http", + "--official-scenario", + "request-metadata", + "--no-auth", + "--conformance-cli", + sys.executable, + ] + ) + + assert exit_code == 2 + assert ( + "requested scenarios are unavailable for MCP protocol version 2025-06-18: " + "request-metadata" + ) in capsys.readouterr().err + + +def test_official_driver_reads_checks_and_adapter_report(tmp_path: Path) -> None: + fake_cli = tmp_path / "fake_conformance.py" + fake_cli.write_text( + """ +import json +import os +import pathlib +import sys + +args = sys.argv[1:] +scenario = args[args.index("--scenario") + 1] +output = pathlib.Path(args[args.index("--output-dir") + 1]) +result_dir = output / f"{scenario}-timestamp" +result_dir.mkdir(parents=True) +(result_dir / "checks.json").write_text(json.dumps([ + { + "id": "request-trace", + "name": "RequestTrace", + "description": "not a conformance assertion", + "status": "INFO", + }, + { + "id": "official-check", + "name": "OfficialCheck", + "description": "observed", + "status": "SUCCESS", + }, +])) +pathlib.Path(os.environ["CODEX_CONFORMANCE_ADAPTER_REPORT"]).write_text( + json.dumps({"success": True, "steps": []}) +) +""".lstrip(), + encoding="utf-8", + ) + adapter = tmp_path / "adapter.py" + adapter.write_text("raise AssertionError('fake CLI should not run adapter')\n") + + results = run_official_mode( + conformance_command=[sys.executable, str(fake_cli)], + adapter_script=adapter, + codex_binary=Path("/opt/codex"), + mode=MODERN_VERSION, + scenarios=["tools_call"], + output_dir=tmp_path / "results", + timeout_seconds=1, + base_env={}, + ) + + assert len(results) == 1 + assert results[0].success + assert results[0].adapter_success + assert len(results[0].checks) == 1 + assert results[0].checks[0].check_id == "official-check" + + +def test_official_driver_terminates_timed_out_process_group(tmp_path: Path) -> None: + fake_cli = tmp_path / "hanging_conformance.py" + fake_cli.write_text( + """ +import subprocess +import sys +import time + +subprocess.Popen([sys.executable, "-c", "import time; time.sleep(60)"]) +time.sleep(60) +""".lstrip(), + encoding="utf-8", + ) + adapter = tmp_path / "adapter.py" + adapter.write_text("raise AssertionError('not reached')\n", encoding="utf-8") + launcher = _make_adapter_launcher(adapter) + try: + result = _run_scenario( + conformance_command=[sys.executable, str(fake_cli)], + adapter_launcher=launcher, + codex_binary=Path("/opt/codex"), + mode=MODERN_VERSION, + scenario="tools_call", + output_dir=tmp_path / "results", + timeout_seconds=0.05, + process_grace_seconds=0.05, + base_env={}, + ) + finally: + launcher.unlink() + launcher.parent.rmdir() + + assert not result.success + assert "timed out" in result.runner_detail + + +def test_official_driver_does_not_mislabel_runner_crash_as_adapter_failure( + tmp_path: Path, +) -> None: + fake_cli = tmp_path / "crashing_conformance.py" + fake_cli.write_text( + """ +import os +import subprocess +import sys + +script = ''' +import json +import os +import pathlib +import time + +time.sleep(0.1) +pathlib.Path(os.environ["CODEX_CONFORMANCE_ADAPTER_REPORT"]).write_text( + json.dumps({"success": True, "steps": []}) +) +''' +subprocess.Popen([sys.executable, "-c", script], env=os.environ) +sys.exit(1) +""".lstrip(), + encoding="utf-8", + ) + adapter = tmp_path / "adapter.py" + adapter.write_text("raise AssertionError('not reached')\n", encoding="utf-8") + launcher = _make_adapter_launcher(adapter) + try: + result = _run_scenario( + conformance_command=[sys.executable, str(fake_cli)], + adapter_launcher=launcher, + codex_binary=Path("/opt/codex"), + mode=MODERN_VERSION, + scenario="request-metadata", + output_dir=tmp_path / "results", + timeout_seconds=1, + process_grace_seconds=1, + base_env={}, + ) + finally: + launcher.unlink() + launcher.parent.rmdir() + + assert not result.success + assert result.adapter_success + assert "did not produce exactly one checks.json" in result.runner_detail diff --git a/scripts/mcp_conformance/test_review_regressions.py b/scripts/mcp_conformance/test_review_regressions.py new file mode 100644 index 0000000000..609769a3e5 --- /dev/null +++ b/scripts/mcp_conformance/test_review_regressions.py @@ -0,0 +1,1093 @@ +import json +import sys +import time +from copy import deepcopy +from pathlib import Path +from typing import Callable, Mapping, Sequence + +import pytest +from review_regressions import ( + CATALOG_BOUNDARY_PROFILES, + CATALOG_BOUNDARY_TRANSPORTS, + CATALOG_LIMIT_ERROR, + CATALOG_MAX_PROFILE, + CATALOG_OVER_LIMIT_PROFILE, + LEGACY_ENVIRONMENT_SENTINEL, + LEGACY_VERSION, + MAX_CATALOG_ITEMS, + MISMATCHED_DISCOVERY_ID_PROFILE, + MODERN_VERSION, + NULL_DISCOVERY_ID_PROFILE, + REPEATED_CURSOR_PROFILE, + REVIEW_EXACT_INTEGER, + REVIEW_BASELINE_KIND, + REVIEW_MODES, + REVIEW_PROFILE, + REVIEW_REPORT_SCHEMA_VERSION, + REVIEWER, + SHIPPING_LEGACY_VERSION, + SSE_COMMENT_FLOOD_PROFILE, + SSE_CR_COMMENTS_PROFILE, + TEST_SERVER_NAME, + AppServerError, + CaseResult, + _compact_review_regression_baseline, + _elicitation_schema, + _evaluate_review_regression_gate, + _exact_integer_property, + _mrtr_budget_is_bounded, + _parse_args, + _required_review_cases, + _required_review_checks, + _run_catalog_boundary_checks, + _review_elicitation_content, + _review_inventory_entry, + _review_registration_command, + _tool_input_schema, + _write_review_json, + main, + run_review_regressions, +) + + +def _integer_schema(value: int = REVIEW_EXACT_INTEGER) -> dict[str, object]: + return { + "type": "object", + "properties": { + "value": { + "type": "integer", + "minimum": value, + "maximum": 9_223_372_036_854_775_807, + "default": value, + } + }, + } + + +class _CatalogBoundaryClient: + def __init__( + self, + *, + status: str | None, + tool_count: int, + server_info: dict[str, object] | None, + error: str | None = None, + notification_name: str = TEST_SERVER_NAME, + preceding_statuses: Sequence[str] = (), + ) -> None: + self.events: list[dict[str, object]] = [] + self._status = status + self._error = error + self._notification_name = notification_name + self._preceding_statuses = preceding_statuses + self._entry: dict[str, object] = { + "name": TEST_SERVER_NAME, + "serverInfo": server_info, + "tools": { + f"catalog_boundary_{index:04d}": {} for index in range(tool_count) + }, + } + + def request( + self, method: str, params: Mapping[str, object] | None + ) -> dict[str, object]: + if method == "thread/start": + if self._status is not None: + for status in self._preceding_statuses: + self.events.append( + { + "method": "mcpServer/startupStatus/updated", + "params": { + "name": self._notification_name, + "status": status, + }, + } + ) + notification_params: dict[str, object] = { + "name": self._notification_name, + "status": self._status, + } + if self._error is not None: + notification_params["error"] = self._error + self.events.append( + { + "method": "mcpServer/startupStatus/updated", + "params": notification_params, + } + ) + return {"result": {"thread": {"id": "catalog-boundary-thread"}}} + if method == "mcpServerStatus/list": + return {"result": {"data": [self._entry]}} + raise AssertionError(f"unexpected app-server request: {method}") + + def wait_for_notification( + self, + method: str, + *, + predicate: Callable[[Mapping[str, object]], bool] | None = None, + after_event_index: int = 0, + ) -> dict[str, object]: + for event in self.events[after_event_index:]: + params = event.get("params") + if ( + event.get("method") == method + and isinstance(params, dict) + and (predicate is None or predicate(params)) + ): + return event + raise AppServerError(f"timed out waiting for app-server notification {method}") + + +def test_review_matrix_includes_the_actual_shipping_legacy_protocol() -> None: + assert REVIEW_MODES == ("2025-06-18", "2025-11-25", "2026-07-28") + assert SHIPPING_LEGACY_VERSION == "2025-06-18" + + +def test_exact_integer_schema_does_not_accept_float_rounded_values() -> None: + assert _exact_integer_property(_integer_schema()) + assert not _exact_integer_property(_integer_schema(REVIEW_EXACT_INTEGER - 1)) + + rounded = _integer_schema() + rounded["properties"]["value"]["minimum"] = float(REVIEW_EXACT_INTEGER) # type: ignore[index] + assert not _exact_integer_property(rounded) + + +def test_exact_integer_schema_does_not_accept_boolean_values() -> None: + schema = _integer_schema() + schema["properties"]["value"]["minimum"] = True # type: ignore[index] + + assert not _exact_integer_property(schema) + + +@pytest.mark.parametrize("key", ["requestedSchema", "requested_schema", "schema"]) +def test_extracts_the_actual_elicitation_requested_schema(key: str) -> None: + schema = _integer_schema() + + assert _elicitation_schema({key: schema}) == schema + assert _elicitation_schema({"params": {key: schema}}) == schema + + +def test_elicitation_response_preserves_the_exact_large_integer() -> None: + response = _review_elicitation_content({"requestedSchema": _integer_schema()}) + + assert response == {"value": 9_007_199_254_740_993} + + +@pytest.mark.parametrize( + ("request_count", "elapsed_seconds", "expected"), + [ + (0, 0.1, True), + (64, 1.0, True), + (65, 1.0, False), + (64, 10.1, False), + ], +) +def test_mrtr_budget_requires_both_request_and_time_limits( + request_count: int, + elapsed_seconds: float, + expected: bool, +) -> None: + assert ( + _mrtr_budget_is_bounded( + request_count, + elapsed_seconds=elapsed_seconds, + timeout_seconds=8, + ) + is expected + ) + + +def test_review_inventory_requires_the_registered_server() -> None: + entry = {"name": "mcp_spec_fixture", "tools": {}} + + assert _review_inventory_entry({"data": [entry]}) == entry + assert _review_inventory_entry({"data": [{"name": "another-server"}]}) is None + assert _review_inventory_entry({"data": "invalid"}) is None + + +@pytest.mark.parametrize("schema_key", ["inputSchema", "input_schema"]) +def test_review_inventory_recognizes_both_tool_schema_spellings( + schema_key: str, +) -> None: + schema = _integer_schema() + entry = {"tools": {"review_large_integer": {schema_key: schema}}} + + assert _tool_input_schema(entry, "review_large_integer") == schema + + +def test_shipping_legacy_registration_preserves_the_reserved_protocol_environment() -> ( + None +): + command = _review_registration_command( + Path("/opt/codex"), + Path("/src/server.py"), + transport="stdio", + mode=SHIPPING_LEGACY_VERSION, + profile=REVIEW_PROFILE, + http_url=None, + ) + + assert command == [ + "/opt/codex", + "mcp", + "add", + "mcp_spec_fixture", + "--env", + f"CODEX_MCP_PROTOCOL_VERSION={LEGACY_ENVIRONMENT_SENTINEL}", + "--", + sys.executable, + "/src/server.py", + "--mode", + SHIPPING_LEGACY_VERSION, + "--transport", + "stdio", + "--profile", + REVIEW_PROFILE, + ] + + +def test_modern_registration_explicitly_opts_into_the_modern_protocol() -> None: + command = _review_registration_command( + Path("/opt/codex"), + Path("/src/server.py"), + transport="stdio", + mode=MODERN_VERSION, + profile=REVIEW_PROFILE, + http_url=None, + ) + + assert f"CODEX_MCP_PROTOCOL_VERSION={MODERN_VERSION}" in command + + +def test_http_review_registration_requires_a_real_fixture_url() -> None: + with pytest.raises(ValueError, match="fixture URL"): + _review_registration_command( + Path("/opt/codex"), + Path("/src/server.py"), + transport="http", + mode=MODERN_VERSION, + profile=REVIEW_PROFILE, + http_url=None, + ) + + +def test_review_cli_supports_the_real_shipping_protocol_and_json_report() -> None: + args = _parse_args( + [ + "/opt/codex", + "--mode", + SHIPPING_LEGACY_VERSION, + "--report", + "/tmp/mcp-review-regressions.json", + ] + ) + + assert args.mode == "2025-06-18" + assert args.report == Path("/tmp/mcp-review-regressions.json") + + +def test_review_cli_reports_a_missing_codex_binary( + capsys: pytest.CaptureFixture[str], +) -> None: + assert main(["/path/to/nonexistent-review-codex"]) == 2 + assert "not executable" in capsys.readouterr().err + + +@pytest.mark.parametrize("mode", REVIEW_MODES) +def test_catalog_at_limit_requires_a_ready_server_and_all_tools( + mode: str, tmp_path: Path +) -> None: + case = CaseResult(transport="review-stdio:catalog-max", mode=mode) + client = _CatalogBoundaryClient( + status="ready", + tool_count=MAX_CATALOG_ITEMS, + server_info={"name": "catalog-boundary-server"}, + ) + + _run_catalog_boundary_checks( + case, + client, # type: ignore[arg-type] - exercise the real app-server client contract. + profile=CATALOG_MAX_PROFILE, + workspace=tmp_path, + ) + case.finish(time.monotonic()) + + assert case.success + assert [check.name for check in case.checks] == [ + "review/ephemeral-thread", + "review/catalog-at-limit/startup-ready", + "review/catalog-at-limit/configured-server", + "review/catalog-at-limit/server-info", + "review/catalog-at-limit/tool-count", + ] + + +@pytest.mark.parametrize("mode", REVIEW_MODES) +def test_clean_catalog_over_limit_rejection_is_an_expected_pass( + mode: str, tmp_path: Path +) -> None: + case = CaseResult(transport="review-http:catalog-over-limit", mode=mode) + client = _CatalogBoundaryClient( + status="failed", + tool_count=0, + server_info=None, + error=f"MCP startup failed: {CATALOG_LIMIT_ERROR}", + ) + + _run_catalog_boundary_checks( + case, + client, # type: ignore[arg-type] - exercise the real app-server client contract. + profile=CATALOG_OVER_LIMIT_PROFILE, + workspace=tmp_path, + ) + case.finish(time.monotonic()) + + assert case.success + assert [check.name for check in case.checks] == [ + "review/ephemeral-thread", + "review/catalog-over-limit/startup-failed", + "review/catalog-over-limit/exact-limit-error", + "review/catalog-over-limit/configured-server", + "review/catalog-over-limit/server-info", + "review/catalog-over-limit/tool-count", + ] + + +@pytest.mark.parametrize( + ("profile", "terminal_status", "tool_count"), + [ + (CATALOG_MAX_PROFILE, "ready", MAX_CATALOG_ITEMS), + (CATALOG_OVER_LIMIT_PROFILE, "failed", 0), + ], +) +def test_catalog_boundary_ignores_starting_until_the_real_terminal_notification( + profile: str, + terminal_status: str, + tool_count: int, + tmp_path: Path, +) -> None: + at_limit = profile == CATALOG_MAX_PROFILE + case = CaseResult(transport=f"review-http:{profile}", mode=MODERN_VERSION) + client = _CatalogBoundaryClient( + status=terminal_status, + tool_count=tool_count, + server_info={"name": "catalog-boundary-server"} if at_limit else None, + error=f"MCP startup failed: {CATALOG_LIMIT_ERROR}" if not at_limit else None, + preceding_statuses=("starting",), + ) + + _run_catalog_boundary_checks( + case, + client, # type: ignore[arg-type] - exercise the real app-server client contract. + profile=profile, + workspace=tmp_path, + ) + case.finish(time.monotonic()) + + assert case.success + assert client.events[0]["params"]["status"] == "starting" # type: ignore[index] + assert client.events[1]["params"]["status"] == terminal_status # type: ignore[index] + + +def test_catalog_over_limit_does_not_accept_an_unrelated_startup_failure( + tmp_path: Path, +) -> None: + case = CaseResult(transport="review-stdio:catalog-over-limit", mode=MODERN_VERSION) + client = _CatalogBoundaryClient( + status="failed", + tool_count=0, + server_info=None, + error="MCP startup failed: connection unexpectedly closed", + ) + + _run_catalog_boundary_checks( + case, + client, # type: ignore[arg-type] - exercise the real app-server client contract. + profile=CATALOG_OVER_LIMIT_PROFILE, + workspace=tmp_path, + ) + case.finish(time.monotonic()) + + assert not case.success + assert [check.name for check in case.checks if not check.success] == [ + "review/catalog-over-limit/exact-limit-error" + ] + + +@pytest.mark.parametrize("mode", REVIEW_MODES) +@pytest.mark.parametrize("transport", CATALOG_BOUNDARY_TRANSPORTS) +def test_catalog_over_limit_records_all_checks_when_startup_incorrectly_succeeds( + mode: str, transport: str, tmp_path: Path +) -> None: + case = CaseResult(transport=f"review-{transport}:catalog-over-limit", mode=mode) + client = _CatalogBoundaryClient( + status="ready", + tool_count=MAX_CATALOG_ITEMS + 1, + server_info={"name": "catalog-boundary-server"}, + ) + + _run_catalog_boundary_checks( + case, + client, # type: ignore[arg-type] - exercise the real app-server client contract. + profile=CATALOG_OVER_LIMIT_PROFILE, + workspace=tmp_path, + ) + case.finish(time.monotonic()) + + assert not case.success + assert [check.name for check in case.checks] == [ + "review/ephemeral-thread", + "review/catalog-over-limit/startup-failed", + "review/catalog-over-limit/exact-limit-error", + "review/catalog-over-limit/configured-server", + "review/catalog-over-limit/server-info", + "review/catalog-over-limit/tool-count", + ] + assert [check.name for check in case.checks if not check.success] == [ + "review/catalog-over-limit/startup-failed", + "review/catalog-over-limit/exact-limit-error", + "review/catalog-over-limit/server-info", + "review/catalog-over-limit/tool-count", + ] + + +def test_catalog_over_limit_does_not_treat_a_missing_notification_as_rejection( + tmp_path: Path, +) -> None: + case = CaseResult(transport="review-stdio:catalog-over-limit", mode=LEGACY_VERSION) + client = _CatalogBoundaryClient( + status=None, + tool_count=0, + server_info=None, + ) + + _run_catalog_boundary_checks( + case, + client, # type: ignore[arg-type] - exercise the real app-server client contract. + profile=CATALOG_OVER_LIMIT_PROFILE, + workspace=tmp_path, + ) + case.finish(time.monotonic()) + + assert not case.success + assert [check.name for check in case.checks if not check.success] == [ + "review/catalog-over-limit/startup-failed" + ] + + +@pytest.mark.parametrize( + ("profile", "status", "failure_name"), + [ + ( + CATALOG_MAX_PROFILE, + "ready", + "review/catalog-at-limit/startup-ready", + ), + ( + CATALOG_OVER_LIMIT_PROFILE, + "failed", + "review/catalog-over-limit/startup-failed", + ), + ], +) +def test_catalog_boundary_does_not_accept_another_servers_startup_notification( + profile: str, status: str, failure_name: str, tmp_path: Path +) -> None: + at_limit = profile == CATALOG_MAX_PROFILE + case = CaseResult(transport=f"review-stdio:{profile}", mode=MODERN_VERSION) + client = _CatalogBoundaryClient( + status=status, + tool_count=MAX_CATALOG_ITEMS if at_limit else 0, + server_info={"name": "catalog-boundary-server"} if at_limit else None, + error=CATALOG_LIMIT_ERROR if not at_limit else None, + notification_name="another-mcp-server", + ) + + _run_catalog_boundary_checks( + case, + client, # type: ignore[arg-type] - exercise the real app-server client contract. + profile=profile, + workspace=tmp_path, + ) + case.finish(time.monotonic()) + + assert not case.success + assert [check.name for check in case.checks if not check.success] == [failure_name] + + +@pytest.mark.parametrize("tool_count", [MAX_CATALOG_ITEMS - 1, MAX_CATALOG_ITEMS + 1]) +def test_catalog_at_limit_rejects_incomplete_or_excessive_discovery( + tool_count: int, tmp_path: Path +) -> None: + case = CaseResult(transport="review-stdio:catalog-max", mode=MODERN_VERSION) + client = _CatalogBoundaryClient( + status="ready", + tool_count=tool_count, + server_info={"name": "catalog-boundary-server"}, + ) + + _run_catalog_boundary_checks( + case, + client, # type: ignore[arg-type] - exercise the real app-server client contract. + profile=CATALOG_MAX_PROFILE, + workspace=tmp_path, + ) + case.finish(time.monotonic()) + + assert not case.success + assert [check.name for check in case.checks if not check.success] == [ + "review/catalog-at-limit/tool-count" + ] + + +@pytest.mark.parametrize("mode", REVIEW_MODES) +@pytest.mark.parametrize("profile", CATALOG_BOUNDARY_PROFILES) +def test_catalog_boundary_stdio_registration_uses_the_exact_fixture_profile( + mode: str, profile: str +) -> None: + command = _review_registration_command( + Path("/opt/codex"), + Path("/src/server.py"), + transport="stdio", + mode=mode, + profile=profile, + http_url=None, + ) + + assert command[-6:] == [ + "--mode", + mode, + "--transport", + "stdio", + "--profile", + profile, + ] + + +def test_review_matrix_appends_all_transport_catalog_boundaries( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + observed: list[tuple[str, str, str]] = [] + + def fake_run_review_case( + codex_binary: Path, + server_script: Path, + *, + mode: str, + profile: str, + transport: str, + case_home: Path, + timeout_seconds: float, + ) -> CaseResult: + observed.append((mode, profile, transport)) + case = CaseResult(transport=f"review-{transport}:{profile}", mode=mode) + case.check("review/fake-preserved-case", True, "ok") + case.finish(time.monotonic()) + return case + + monkeypatch.setattr("review_regressions._run_review_case", fake_run_review_case) + report = run_review_regressions( + Path("/opt/codex"), + modes=REVIEW_MODES, + server_script=Path("/src/server.py"), + artifact_parent=tmp_path, + ) + + assert observed[:9] == [ + (SHIPPING_LEGACY_VERSION, REVIEW_PROFILE, "stdio"), + (LEGACY_VERSION, REVIEW_PROFILE, "stdio"), + (MODERN_VERSION, REVIEW_PROFILE, "stdio"), + (MODERN_VERSION, REVIEW_PROFILE, "http"), + (MODERN_VERSION, REPEATED_CURSOR_PROFILE, "http"), + (MODERN_VERSION, MISMATCHED_DISCOVERY_ID_PROFILE, "http"), + (MODERN_VERSION, NULL_DISCOVERY_ID_PROFILE, "http"), + (MODERN_VERSION, SSE_CR_COMMENTS_PROFILE, "http"), + (MODERN_VERSION, SSE_COMMENT_FLOOD_PROFILE, "http"), + ] + assert observed[9:] == [ + (mode, profile, transport) + for mode in REVIEW_MODES + for transport in CATALOG_BOUNDARY_TRANSPORTS + for profile in CATALOG_BOUNDARY_PROFILES + ] + assert report["success"] is True + assert report["summary"] == { + "passed": 21, + "failed": 0, + "total": 21, + "casesPassed": 21, + "casesTotal": 21, + } + + +_KNOWN_REVIEW_FAILURES = { + (MODERN_VERSION, f"review-{transport}:{REVIEW_PROFILE}", check_id) + for transport in ("stdio", "http") + for check_id in ( + "review/exact-large-integer-elicitation-schema", + "review/exact-large-integer-elicitation-round-trip", + "review/mrtr-input-requests-bounded-to-64", + ) +} + +_KNOWN_MAIN_CATALOG_FAILURES = { + (mode, f"review-{transport}:{CATALOG_OVER_LIMIT_PROFILE}", check_id) + for mode in REVIEW_MODES + for transport in CATALOG_BOUNDARY_TRANSPORTS + for check_id in ( + "review/catalog-over-limit/startup-failed", + "review/catalog-over-limit/exact-limit-error", + "review/catalog-over-limit/server-info", + "review/catalog-over-limit/tool-count", + ) +} + + +def _refresh_review_gate_report(report: dict[str, object]) -> None: + cases = report["cases"] + assert isinstance(cases, list) + passed = 0 + failed = 0 + cases_passed = 0 + for case in cases: + assert isinstance(case, dict) + checks = case["checks"] + assert isinstance(checks, list) + successes = [] + for check in checks: + assert isinstance(check, dict) + success = check["success"] + assert isinstance(success, bool) + successes.append(success) + case_success = bool(checks) and all(successes) + case["success"] = case_success + passed += sum(successes) + failed += len(successes) - sum(successes) + cases_passed += case_success + report["success"] = bool(cases) and failed == 0 + report["summary"] = { + "passed": passed, + "failed": failed, + "total": passed + failed, + "casesPassed": cases_passed, + "casesTotal": len(cases), + } + + +def _review_gate_report( + *, + failures: set[tuple[str, str, str]] | None = None, +) -> dict[str, object]: + failures = _KNOWN_REVIEW_FAILURES if failures is None else failures + cases: list[dict[str, object]] = [] + for mode, transport in sorted(_required_review_cases()): + check_ids = ["review/mcp-registration"] + if transport.endswith(f":{CATALOG_OVER_LIMIT_PROFILE}"): + check_ids.extend( + ( + "review/catalog-over-limit/startup-failed", + "review/catalog-over-limit/exact-limit-error", + "review/catalog-over-limit/configured-server", + "review/catalog-over-limit/server-info", + "review/catalog-over-limit/tool-count", + ) + ) + if mode == MODERN_VERSION and transport in { + f"review-stdio:{REVIEW_PROFILE}", + f"review-http:{REVIEW_PROFILE}", + }: + check_ids.extend( + ( + "review/exact-large-integer-elicitation-schema", + "review/exact-large-integer-elicitation-round-trip", + "review/mrtr-input-requests-bounded-to-64", + ) + ) + cases.append( + { + "mode": mode, + "transport": transport, + "checks": [ + { + "name": check_id, + "success": (mode, transport, check_id) not in failures, + "detail": "synthetic real-probe identity", + } + for check_id in check_ids + ], + } + ) + report: dict[str, object] = { + "schemaVersion": REVIEW_REPORT_SCHEMA_VERSION, + "reviewer": REVIEWER, + "modes": list(REVIEW_MODES), + "cases": cases, + } + _refresh_review_gate_report(report) + return report + + +def test_reviewer_baseline_preserves_all_required_cases_and_known_failures() -> None: + baseline = _compact_review_regression_baseline(_review_gate_report()) + + assert baseline["baselineKind"] == REVIEW_BASELINE_KIND + assert baseline["requiredModes"] == list(REVIEW_MODES) + assert len(baseline["requiredCases"]) == 21 # type: ignore[arg-type] + checks = baseline["checks"] + assert isinstance(checks, dict) + failures = checks["failing"] + assert isinstance(failures, list) + assert { + (failure["mode"], failure["transport"], failure["check_id"]) + for failure in failures + } == _KNOWN_REVIEW_FAILURES + + +def test_committed_reviewer_baseline_records_all_real_production_checks() -> None: + path = Path(__file__).with_name("review-regression-baseline-v1.json") + baseline = json.loads(path.read_text(encoding="utf-8")) + errors: list[str] = [] + checks = _required_review_checks(baseline, label="baseline", errors=errors) + + assert errors == [] + assert len(checks) == 186 + assert sum(checks.values()) == 156 + assert { + (identity.mode, identity.transport, identity.check_id) + for identity, success in checks.items() + if not success + } == _KNOWN_REVIEW_FAILURES | _KNOWN_MAIN_CATALOG_FAILURES + assert baseline["summary"] == { + "passed": 156, + "failed": 30, + "total": 186, + "casesPassed": 13, + "casesTotal": 21, + } + + +def test_reviewer_gate_reports_known_failures_without_calling_them_passes() -> None: + report = _review_gate_report() + baseline = _compact_review_regression_baseline(report) + + gate = _evaluate_review_regression_gate(report, baseline) + + assert report["success"] is False + assert gate["success"] is True + assert gate["configurationErrors"] == [] + assert gate["newFailures"] == [] + assert gate["missingChecks"] == [] + assert gate["fixedChecks"] == [] + known = gate["knownFailures"] + assert isinstance(known, list) + assert { + (failure["mode"], failure["transport"], failure["check_id"]) + for failure in known + } == _KNOWN_REVIEW_FAILURES + + +def test_reviewer_gate_preserves_catalog_failures_observed_on_current_main() -> None: + main_failures = _KNOWN_REVIEW_FAILURES | _KNOWN_MAIN_CATALOG_FAILURES + report = _review_gate_report(failures=main_failures) + baseline = _compact_review_regression_baseline(report) + + gate = _evaluate_review_regression_gate(report, baseline) + + assert report["success"] is False + assert gate["success"] is True + assert gate["configurationErrors"] == [] + assert gate["newFailures"] == [] + assert gate["missingChecks"] == [] + known = gate["knownFailures"] + assert isinstance(known, list) + assert { + (failure["mode"], failure["transport"], failure["check_id"]) + for failure in known + } == main_failures + + +def test_reviewer_gate_reports_all_future_catalog_boundary_fixes() -> None: + main_failures = _KNOWN_REVIEW_FAILURES | _KNOWN_MAIN_CATALOG_FAILURES + baseline = _compact_review_regression_baseline( + _review_gate_report(failures=main_failures) + ) + candidate = _review_gate_report() + + gate = _evaluate_review_regression_gate(candidate, baseline) + + assert gate["success"] is True + assert gate["configurationErrors"] == [] + assert gate["newFailures"] == [] + assert gate["missingChecks"] == [] + fixed = gate["fixedChecks"] + assert isinstance(fixed, list) + assert { + (identity["mode"], identity["transport"], identity["check_id"]) + for identity in fixed + } == _KNOWN_MAIN_CATALOG_FAILURES + known = gate["knownFailures"] + assert isinstance(known, list) + assert { + (identity["mode"], identity["transport"], identity["check_id"]) + for identity in known + } == _KNOWN_REVIEW_FAILURES + + +def test_reviewer_gate_rejects_a_newly_failing_previously_passing_check() -> None: + baseline = _compact_review_regression_baseline(_review_gate_report()) + new_failure = ( + SHIPPING_LEGACY_VERSION, + f"review-stdio:{REVIEW_PROFILE}", + "review/mcp-registration", + ) + candidate = _review_gate_report(failures=_KNOWN_REVIEW_FAILURES | {new_failure}) + + gate = _evaluate_review_regression_gate(candidate, baseline) + + assert gate["success"] is False + assert gate["newFailures"] == [ + { + "mode": new_failure[0], + "transport": new_failure[1], + "check_id": new_failure[2], + } + ] + + +def test_reviewer_gate_reports_when_a_known_failure_is_fixed() -> None: + baseline = _compact_review_regression_baseline(_review_gate_report()) + fixed = sorted(_KNOWN_REVIEW_FAILURES)[0] + candidate = _review_gate_report(failures=_KNOWN_REVIEW_FAILURES - {fixed}) + + gate = _evaluate_review_regression_gate(candidate, baseline) + + assert gate["success"] is True + assert gate["fixedChecks"] == [ + {"mode": fixed[0], "transport": fixed[1], "check_id": fixed[2]} + ] + assert len(gate["knownFailures"]) == 5 # type: ignore[arg-type] + + +def test_reviewer_gate_rejects_an_omitted_required_case() -> None: + baseline = _compact_review_regression_baseline(_review_gate_report()) + candidate = _review_gate_report() + cases = candidate["cases"] + assert isinstance(cases, list) + omitted = cases.pop(0) + assert isinstance(omitted, dict) + _refresh_review_gate_report(candidate) + + gate = _evaluate_review_regression_gate(candidate, baseline) + + assert gate["success"] is False + assert any( + "missing required reviewer case" in error + for error in gate["configurationErrors"] # type: ignore[union-attr] + ) + assert any( + missing["mode"] == omitted["mode"] + and missing["transport"] == omitted["transport"] + for missing in gate["missingChecks"] # type: ignore[union-attr] + ) + + +def test_reviewer_gate_rejects_an_omitted_required_check() -> None: + baseline = _compact_review_regression_baseline(_review_gate_report()) + candidate = _review_gate_report() + cases = candidate["cases"] + assert isinstance(cases, list) + case = next( + case + for case in cases + if isinstance(case, dict) + and case["mode"] == MODERN_VERSION + and case["transport"] == f"review-stdio:{REVIEW_PROFILE}" + ) + checks = case["checks"] + assert isinstance(checks, list) + omitted = checks.pop(0) + assert isinstance(omitted, dict) + _refresh_review_gate_report(candidate) + + gate = _evaluate_review_regression_gate(candidate, baseline) + + assert gate["success"] is False + assert gate["missingChecks"] == [ + { + "mode": MODERN_VERSION, + "transport": f"review-stdio:{REVIEW_PROFILE}", + "check_id": omitted["name"], + } + ] + + +@pytest.mark.parametrize("field", ["schemaVersion", "reviewer", "requiredModes"]) +def test_reviewer_gate_rejects_an_incompatible_baseline(field: str) -> None: + baseline = _compact_review_regression_baseline(_review_gate_report()) + baseline[field] = [] if field == "requiredModes" else "wrong" + + gate = _evaluate_review_regression_gate(_review_gate_report(), baseline) + + assert gate["success"] is False + assert gate["configurationErrors"] + + +def test_reviewer_gate_rejects_duplicate_baseline_check_identities() -> None: + baseline = _compact_review_regression_baseline(_review_gate_report()) + buckets = baseline["checks"] + assert isinstance(buckets, dict) + passing = buckets["passing"] + assert isinstance(passing, list) + passing.append(deepcopy(passing[0])) + + gate = _evaluate_review_regression_gate(_review_gate_report(), baseline) + + assert gate["success"] is False + assert any( + "duplicate reviewer check" in error + for error in gate["configurationErrors"] # type: ignore[union-attr] + ) + + +def test_reviewer_gate_rejects_inconsistent_candidate_totals() -> None: + baseline = _compact_review_regression_baseline(_review_gate_report()) + candidate = _review_gate_report() + candidate["summary"] = {"passed": 999, "failed": 0, "total": 999} + + gate = _evaluate_review_regression_gate(candidate, baseline) + + assert gate["success"] is False + assert any( + "inconsistent reviewer check totals" in error + for error in gate["configurationErrors"] # type: ignore[union-attr] + ) + + +def test_review_json_is_pretty_sorted_and_newline_terminated(tmp_path: Path) -> None: + report_path = tmp_path / "nested" / "review.json" + + _write_review_json(report_path, {"z": 1, "a": {"second": 2, "first": 1}}) + + assert report_path.read_text(encoding="utf-8") == ( + '{\n "a": {\n "first": 1,\n "second": 2\n },\n "z": 1\n}\n' + ) + + +def test_review_cli_supports_a_reviewed_production_baseline() -> None: + args = _parse_args( + [ + "/opt/codex", + "--baseline-report", + "/tmp/review-regression-baseline-v1.json", + "--report", + "/tmp/mcp-review-regressions.json", + ] + ) + + assert args.baseline_report == Path("/tmp/review-regression-baseline-v1.json") + + +def test_review_cli_requires_a_source_report_when_extracting_a_baseline( + capsys: pytest.CaptureFixture[str], +) -> None: + result = main( + [sys.executable, "--extract-baseline", "/tmp/review-baseline-test.json"] + ) + + assert result == 2 + assert "--extract-baseline requires --baseline-report" in capsys.readouterr().err + + +def test_review_cli_exits_successfully_only_for_a_passing_regression_gate( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + candidate = _review_gate_report() + baseline_path = tmp_path / "review-baseline.json" + report_path = tmp_path / "review-report.json" + _write_review_json( + baseline_path, _compact_review_regression_baseline(deepcopy(candidate)) + ) + monkeypatch.setattr( + "review_regressions.run_review_regressions", + lambda *args, **kwargs: deepcopy(candidate), + ) + + result = main( + [ + sys.executable, + "--baseline-report", + str(baseline_path), + "--report", + str(report_path), + ] + ) + + report = json.loads(report_path.read_text(encoding="utf-8")) + assert result == 0 + assert report["success"] is False + assert report["regressionGate"]["success"] is True + assert len(report["regressionGate"]["knownFailures"]) == 6 + + +def test_review_cli_does_not_accept_new_failures_as_known_regressions( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + baseline_path = tmp_path / "review-baseline.json" + report_path = tmp_path / "review-report.json" + _write_review_json( + baseline_path, _compact_review_regression_baseline(_review_gate_report()) + ) + new_failure = ( + SHIPPING_LEGACY_VERSION, + f"review-stdio:{REVIEW_PROFILE}", + "review/mcp-registration", + ) + monkeypatch.setattr( + "review_regressions.run_review_regressions", + lambda *args, **kwargs: _review_gate_report( + failures=_KNOWN_REVIEW_FAILURES | {new_failure} + ), + ) + + result = main( + [ + sys.executable, + "--baseline-report", + str(baseline_path), + "--report", + str(report_path), + ] + ) + + report = json.loads(report_path.read_text(encoding="utf-8")) + assert result == 1 + assert report["success"] is False + assert report["regressionGate"]["success"] is False + assert len(report["regressionGate"]["newFailures"]) == 1 + + +def test_review_cli_extracts_a_deterministic_complete_baseline( + tmp_path: Path, +) -> None: + source_path = tmp_path / "full-review-report.json" + baseline_path = tmp_path / "review-regression-baseline-v1.json" + report = _review_gate_report() + _write_review_json(source_path, report) + + result = main( + [ + sys.executable, + "--baseline-report", + str(source_path), + "--extract-baseline", + str(baseline_path), + ] + ) + + expected = _compact_review_regression_baseline(report) + assert result == 0 + assert baseline_path.read_text(encoding="utf-8") == ( + json.dumps(expected, ensure_ascii=False, indent=2, sort_keys=True) + "\n" + ) diff --git a/scripts/mcp_conformance/test_server.py b/scripts/mcp_conformance/test_server.py new file mode 100644 index 0000000000..0828ff8db7 --- /dev/null +++ b/scripts/mcp_conformance/test_server.py @@ -0,0 +1,1590 @@ +import base64 +import http.client +import io +import json +import threading +from collections.abc import Iterator +from contextlib import contextmanager +from unittest.mock import patch + +import pytest + +from server import ( + CATALOG_MAX_PROFILE, + CATALOG_OVER_LIMIT_PROFILE, + DEFAULT_PROFILE, + HEADER_MISMATCH, + INVALID_PARAMS, + LEGACY_VERSION, + MAX_CATALOG_ITEMS, + MISMATCHED_DISCOVERY_ID_PROFILE, + MISSING_REQUIRED_CLIENT_CAPABILITY, + MODERN_VERSION, + NULL_DISCOVERY_ID_PROFILE, + REPEATED_CURSOR_PROFILE, + RESOURCE_URIS, + REVIEW_EXACT_INTEGER, + REVIEW_MAX_INTEGER, + REVIEW_MRTR_INPUT_REQUEST_COUNT, + REVIEW_PAGE_CURSOR, + REVIEW_PROFILE, + REVIEW_SSE_COMMENT_LINE_BYTES, + REVIEW_SSE_COMMENT_LINE_COUNT, + REVIEW_SSE_EVENT_LIMIT_BYTES, + SHIPPING_LEGACY_VERSION, + SSE_COMMENT_FLOOD_PROFILE, + SSE_CR_COMMENTS_PROFILE, + UNSUPPORTED_PROTOCOL_VERSION, + ConnectionState, + ProtocolServer, + ResponsePlan, + _encode_sse_messages, + _parse_args, + make_http_server, + run_stdio, +) + + +def modern_meta( + *, + version: str = MODERN_VERSION, + capabilities: dict[str, object] | None = None, +) -> dict[str, object]: + return { + "io.modelcontextprotocol/protocolVersion": version, + "io.modelcontextprotocol/clientInfo": { + "name": "fixture-test-client", + "version": "1.0.0", + }, + "io.modelcontextprotocol/clientCapabilities": capabilities or {}, + } + + +def request( + method: str, + *, + request_id: int = 1, + params: dict[str, object] | None = None, +) -> dict[str, object]: + return { + "jsonrpc": "2.0", + "id": request_id, + "method": method, + "params": {"_meta": modern_meta()} if params is None else params, + } + + +def result(plan: ResponsePlan) -> dict[str, object]: + response = plan.response + assert isinstance(response, dict) + value = response.get("result") + assert isinstance(value, dict) + return value + + +def error(plan: ResponsePlan) -> dict[str, object]: + response = plan.response + assert isinstance(response, dict) + value = response.get("error") + assert isinstance(value, dict) + return value + + +def initialize_legacy( + server: ProtocolServer, + state: ConnectionState, + *, + version: str = LEGACY_VERSION, +) -> None: + server.handle( + request( + "initialize", + params={ + "protocolVersion": version, + "capabilities": {}, + "clientInfo": {"name": "legacy-client", "version": "1.0.0"}, + }, + ), + state, + ) + server.handle( + { + "jsonrpc": "2.0", + "method": "notifications/initialized", + "params": {}, + }, + state, + ) + + +def test_modern_discovery_is_stateless_and_cacheable() -> None: + server = ProtocolServer(MODERN_VERSION) + + plan = server.handle(request("server/discover"), ConnectionState()) + value = result(plan) + + assert value["resultType"] == "complete" + assert value["supportedVersions"] == [MODERN_VERSION] + assert value["ttlMs"] == 60_000 + assert value["cacheScope"] == "public" + assert value["_meta"] == { + "io.modelcontextprotocol/serverInfo": { + "name": "openai-mcp-spec-test-server", + "version": "0.1.0", + } + } + + +def test_modern_request_requires_metadata() -> None: + server = ProtocolServer(MODERN_VERSION) + + plan = server.handle( + request("tools/list", params={}), + ConnectionState(), + ) + + assert error(plan)["code"] == INVALID_PARAMS + assert plan.http_status == 400 + + +def test_modern_request_reports_supported_version() -> None: + server = ProtocolServer(MODERN_VERSION) + message = request( + "tools/list", + params={"_meta": modern_meta(version=LEGACY_VERSION)}, + ) + + plan = server.handle(message, ConnectionState()) + + assert error(plan) == { + "code": UNSUPPORTED_PROTOCOL_VERSION, + "message": "Unsupported protocol version", + "data": { + "supported": [MODERN_VERSION], + "requested": LEGACY_VERSION, + }, + } + + +def test_legacy_probe_falls_back_then_uses_initialize() -> None: + server = ProtocolServer(LEGACY_VERSION) + state = ConnectionState() + + probe = server.handle(request("server/discover"), state) + assert error(probe)["code"] == -32601 + + initialize_legacy(server, state) + tools = result(server.handle(request("tools/list", params={}), state)) + + assert "resultType" not in tools + assert "ttlMs" not in tools + assert [tool["name"] for tool in tools["tools"]] == sorted( + tool["name"] for tool in tools["tools"] + ) + + +def test_shipping_legacy_mode_negotiates_the_real_product_protocol() -> None: + server = ProtocolServer(SHIPPING_LEGACY_VERSION) + state = ConnectionState() + + plan = server.handle( + request( + "initialize", + params={ + "protocolVersion": SHIPPING_LEGACY_VERSION, + "capabilities": {}, + "clientInfo": {"name": "shipping-client", "version": "1.0.0"}, + }, + ), + state, + ) + + assert result(plan)["protocolVersion"] == "2025-06-18" + assert result(plan)["instructions"] == "Legacy 2025-06-18 compatibility fixture." + assert "resultType" not in result(plan) + assert state.initialize_seen + + +def test_shipping_legacy_discovery_falls_back_to_initialized_legacy_tools() -> None: + server = ProtocolServer(SHIPPING_LEGACY_VERSION) + state = ConnectionState() + + probe = server.handle(request("server/discover"), state) + + assert error(probe)["code"] == -32601 + + initialize_legacy(server, state, version=SHIPPING_LEGACY_VERSION) + tools = result(server.handle(request("tools/list", params={}), state)) + + assert "resultType" not in tools + assert "ttlMs" not in tools + assert [tool["name"] for tool in tools["tools"]] == [ + "client_metadata", + "echo", + "fail", + "header_echo", + "progress", + ] + + +def test_cli_accepts_the_shipping_legacy_protocol_mode() -> None: + args = _parse_args(["--mode", SHIPPING_LEGACY_VERSION, "--transport", "stdio"]) + + assert args.mode == "2025-06-18" + assert args.transport == "stdio" + assert args.profile == DEFAULT_PROFILE + + +@pytest.mark.parametrize( + "mode", (SHIPPING_LEGACY_VERSION, LEGACY_VERSION, MODERN_VERSION) +) +@pytest.mark.parametrize("transport", ("stdio", "http")) +@pytest.mark.parametrize("profile", (CATALOG_MAX_PROFILE, CATALOG_OVER_LIMIT_PROFILE)) +def test_catalog_boundary_profiles_are_available_from_the_cli( + mode: str, transport: str, profile: str +) -> None: + args = _parse_args(["--mode", mode, "--transport", transport, "--profile", profile]) + + assert (args.mode, args.transport, args.profile) == (mode, transport, profile) + + +def test_mrtr_requires_capability_and_echoes_state() -> None: + server = ProtocolServer(MODERN_VERSION) + tools = result(server.handle(request("tools/list"), ConnectionState()))["tools"] + tools_by_name = {tool["name"]: tool for tool in tools} + assert "outputSchema" not in tools_by_name["request_input"] + assert "outputSchema" in tools_by_name["echo"] + + first = request( + "tools/call", + params={ + "_meta": modern_meta(), + "name": "request_input", + "arguments": {}, + }, + ) + + missing_capability = server.handle(first, ConnectionState()) + assert error(missing_capability)["code"] == MISSING_REQUIRED_CLIENT_CAPABILITY + + first["params"]["_meta"] = modern_meta(capabilities={"elicitation": {"form": {}}}) + interim = result(server.handle(first, ConnectionState())) + assert interim["resultType"] == "input_required" + assert interim["_meta"] == { + "io.modelcontextprotocol/serverInfo": { + "name": "openai-mcp-spec-test-server", + "version": "0.1.0", + } + } + assert "structuredContent" not in interim + assert interim["requestState"] == "opaque:request_input:v1" + assert interim["inputRequests"]["confirmation"]["method"] == "elicitation/create" + + retry = request( + "tools/call", + request_id=2, + params={ + "_meta": modern_meta(capabilities={"elicitation": {"form": {}}}), + "name": "request_input", + "arguments": {}, + "requestState": interim["requestState"], + "inputResponses": { + "confirmation": { + "action": "accept", + "content": {"confirmation": "confirmed"}, + } + }, + }, + ) + completed = result(server.handle(retry, ConnectionState())) + + assert completed["resultType"] == "complete" + assert completed["structuredContent"] == {"confirmation": "confirmed"} + + +def test_subscription_is_acknowledged_tagged_and_closed() -> None: + server = ProtocolServer(MODERN_VERSION) + plan = server.handle( + request( + "subscriptions/listen", + request_id=42, + params={ + "_meta": modern_meta(), + "notifications": { + "toolsListChanged": True, + "resourceSubscriptions": [RESOURCE_URIS[0], "test://unknown"], + }, + }, + ), + ConnectionState(), + ) + + assert plan.force_sse + assert plan.notifications[0]["method"] == ( + "notifications/subscriptions/acknowledged" + ) + for notification in plan.notifications: + assert ( + notification["params"]["_meta"]["io.modelcontextprotocol/subscriptionId"] + == 42 + ) + assert result(plan)["_meta"]["io.modelcontextprotocol/subscriptionId"] == 42 + + +def test_progress_and_logging_are_per_request_opt_ins() -> None: + server = ProtocolServer(MODERN_VERSION) + message = request( + "tools/call", + params={ + "_meta": { + **modern_meta(), + "progressToken": "progress-1", + "io.modelcontextprotocol/logLevel": "info", + }, + "name": "progress", + "arguments": {}, + }, + ) + + plan = server.handle(message, ConnectionState()) + + assert [notification["method"] for notification in plan.notifications] == [ + "notifications/progress", + "notifications/message", + ] + assert result(plan)["resultType"] == "complete" + + +def test_stdio_supports_both_eras() -> None: + modern_input = io.StringIO( + json.dumps(request("tools/list"), separators=(",", ":")) + "\n" + ) + modern_output = io.StringIO() + run_stdio(ProtocolServer(MODERN_VERSION), modern_input, modern_output) + modern_response = json.loads(modern_output.getvalue()) + assert modern_response["result"]["resultType"] == "complete" + + legacy_messages = [ + request( + "initialize", + params={ + "protocolVersion": LEGACY_VERSION, + "capabilities": {}, + "clientInfo": {"name": "legacy", "version": "1.0.0"}, + }, + ), + { + "jsonrpc": "2.0", + "method": "notifications/initialized", + "params": {}, + }, + request("tools/list", request_id=2, params={}), + ] + legacy_input = io.StringIO( + "".join( + json.dumps(item, separators=(",", ":")) + "\n" for item in legacy_messages + ) + ) + legacy_output = io.StringIO() + run_stdio(ProtocolServer(LEGACY_VERSION), legacy_input, legacy_output) + responses = [json.loads(line) for line in legacy_output.getvalue().splitlines()] + assert len(responses) == 2 + assert "resultType" not in responses[1]["result"] + + +def test_stdio_negotiates_and_lists_tools_for_the_real_shipping_legacy_protocol() -> ( + None +): + messages = [ + request( + "initialize", + params={ + "protocolVersion": SHIPPING_LEGACY_VERSION, + "capabilities": {}, + "clientInfo": {"name": "shipping-legacy", "version": "1.0.0"}, + }, + ), + { + "jsonrpc": "2.0", + "method": "notifications/initialized", + "params": {}, + }, + request("tools/list", request_id=2, params={}), + ] + stdin = io.StringIO( + "".join( + json.dumps(message, separators=(",", ":")) + "\n" for message in messages + ) + ) + stdout = io.StringIO() + + run_stdio(ProtocolServer(SHIPPING_LEGACY_VERSION), stdin, stdout) + + responses = [json.loads(line) for line in stdout.getvalue().splitlines()] + assert len(responses) == 2 + assert responses[0]["result"]["protocolVersion"] == "2025-06-18" + assert [tool["name"] for tool in responses[1]["result"]["tools"]] == [ + "client_metadata", + "echo", + "fail", + "header_echo", + "progress", + ] + assert "resultType" not in responses[1]["result"] + + +def test_default_profile_preserves_existing_legacy_and_modern_tool_lists() -> None: + modern = ProtocolServer(MODERN_VERSION) + modern_tools = result(modern.handle(request("tools/list"), ConnectionState())) + + assert modern.profile == DEFAULT_PROFILE + assert [tool["name"] for tool in modern_tools["tools"]] == [ + "client_metadata", + "echo", + "fail", + "header_echo", + "progress", + "request_input", + ] + assert "nextCursor" not in modern_tools + + legacy = ProtocolServer(LEGACY_VERSION) + legacy_state = ConnectionState() + initialize_legacy(legacy, legacy_state) + legacy_tools = result(legacy.handle(request("tools/list", params={}), legacy_state)) + + assert [tool["name"] for tool in legacy_tools["tools"]] == [ + "client_metadata", + "echo", + "fail", + "header_echo", + "progress", + ] + assert "nextCursor" not in legacy_tools + + +@pytest.mark.parametrize( + "mode", (SHIPPING_LEGACY_VERSION, LEGACY_VERSION, MODERN_VERSION) +) +@pytest.mark.parametrize( + ("profile", "expected_size"), + ( + (CATALOG_MAX_PROFILE, MAX_CATALOG_ITEMS), + (CATALOG_OVER_LIMIT_PROFILE, MAX_CATALOG_ITEMS + 1), + ), +) +def test_catalog_boundary_profiles_return_exact_valid_unique_tool_counts( + mode: str, profile: str, expected_size: int +) -> None: + server = ProtocolServer(mode, profile=profile) + state = ConnectionState() + if mode != MODERN_VERSION: + initialize_legacy(server, state, version=mode) + + params = {"_meta": modern_meta()} if mode == MODERN_VERSION else {} + catalog = result(server.handle(request("tools/list", params=params), state)) + tools = catalog["tools"] + names = [tool["name"] for tool in tools] + + assert len(tools) == expected_size + assert len(set(names)) == expected_size + assert names == sorted(names) + assert {"client_metadata", "echo", "fail", "header_echo", "progress"} <= set(names) + assert all(tool["inputSchema"]["type"] == "object" for tool in tools) + assert "nextCursor" not in catalog + if mode == MODERN_VERSION: + assert "request_input" in names + assert catalog["resultType"] == "complete" + else: + assert "resultType" not in catalog + + +@pytest.mark.parametrize( + "mode", (SHIPPING_LEGACY_VERSION, LEGACY_VERSION, MODERN_VERSION) +) +@pytest.mark.parametrize( + ("profile", "expected_size"), + ( + (CATALOG_MAX_PROFILE, MAX_CATALOG_ITEMS), + (CATALOG_OVER_LIMIT_PROFILE, MAX_CATALOG_ITEMS + 1), + ), +) +def test_catalog_boundary_profiles_round_trip_over_real_stdio( + mode: str, profile: str, expected_size: int +) -> None: + messages: list[dict[str, object]] = [] + if mode != MODERN_VERSION: + messages.extend( + [ + request( + "initialize", + params={ + "protocolVersion": mode, + "capabilities": {}, + "clientInfo": {"name": "catalog-boundary", "version": "1.0.0"}, + }, + ), + { + "jsonrpc": "2.0", + "method": "notifications/initialized", + "params": {}, + }, + ] + ) + messages.append( + request( + "tools/list", + request_id=2, + params={"_meta": modern_meta()} if mode == MODERN_VERSION else {}, + ) + ) + stdin = io.StringIO( + "".join( + json.dumps(message, separators=(",", ":")) + "\n" for message in messages + ) + ) + stdout = io.StringIO() + + run_stdio(ProtocolServer(mode, profile=profile), stdin, stdout) + + catalog = json.loads(stdout.getvalue().splitlines()[-1])["result"] + names = [tool["name"] for tool in catalog["tools"]] + assert len(names) == expected_size + assert len(set(names)) == expected_size + assert "nextCursor" not in catalog + assert ("resultType" in catalog) is (mode == MODERN_VERSION) + + +def test_review_profile_hides_regression_tools_on_the_second_page() -> None: + server = ProtocolServer(MODERN_VERSION, profile=REVIEW_PROFILE) + + first = result(server.handle(request("tools/list"), ConnectionState())) + assert [tool["name"] for tool in first["tools"]] == ["client_metadata", "echo"] + assert first["nextCursor"] == REVIEW_PAGE_CURSOR + assert first["resultType"] == "complete" + + second = result( + server.handle( + request( + "tools/list", + request_id=2, + params={"_meta": modern_meta(), "cursor": first["nextCursor"]}, + ), + ConnectionState(), + ) + ) + + assert [tool["name"] for tool in second["tools"]] == [ + "fail", + "header_echo", + "progress", + "request_input", + "review_integer_elicitation", + "review_large_integer", + "review_mrtr_cap", + "review_protocol_env", + ] + assert "nextCursor" not in second + + +def test_review_profile_rejects_an_invalid_tool_cursor() -> None: + server = ProtocolServer(MODERN_VERSION, profile=REVIEW_PROFILE) + + plan = server.handle( + request( + "tools/list", + params={"_meta": modern_meta(), "cursor": "not-a-review-cursor"}, + ), + ConnectionState(), + ) + + assert error(plan) == { + "code": INVALID_PARAMS, + "message": "Invalid tool cursor", + } + + +def test_repeated_cursor_profile_deterministically_repeats_the_cursor() -> None: + server = ProtocolServer(MODERN_VERSION, profile=REPEATED_CURSOR_PROFILE) + + first = result(server.handle(request("tools/list"), ConnectionState())) + second = result( + server.handle( + request( + "tools/list", + request_id=2, + params={"_meta": modern_meta(), "cursor": first["nextCursor"]}, + ), + ConnectionState(), + ) + ) + + assert first["nextCursor"] == REVIEW_PAGE_CURSOR + assert second["nextCursor"] == REVIEW_PAGE_CURSOR + assert any(tool["name"] == "review_large_integer" for tool in second["tools"]) + + +def test_review_large_integer_schema_preserves_exact_json_integer_boundaries() -> None: + server = ProtocolServer(MODERN_VERSION, profile=REVIEW_PROFILE) + page = result( + server.handle( + request( + "tools/list", + params={"_meta": modern_meta(), "cursor": REVIEW_PAGE_CURSOR}, + ), + ConnectionState(), + ) + ) + definition = next( + tool for tool in page["tools"] if tool["name"] == "review_large_integer" + ) + schema = definition["inputSchema"]["properties"]["value"] + + assert schema == { + "type": "integer", + "minimum": 9_007_199_254_740_993, + "maximum": 9_223_372_036_854_775_807, + "default": 9_007_199_254_740_993, + } + assert json.loads(json.dumps(schema)) == schema + assert REVIEW_EXACT_INTEGER > 2**53 + assert REVIEW_MAX_INTEGER == 2**63 - 1 + + +def test_review_large_integer_tool_echoes_without_float_rounding() -> None: + server = ProtocolServer(MODERN_VERSION, profile=REVIEW_PROFILE) + + plan = server.handle( + request( + "tools/call", + params={ + "_meta": modern_meta(), + "name": "review_large_integer", + "arguments": {"value": REVIEW_EXACT_INTEGER}, + }, + ), + ConnectionState(), + ) + + assert result(plan)["structuredContent"] == {"value": 9_007_199_254_740_993} + assert json.loads(result(plan)["content"][0]["text"]) == { + "value": 9_007_199_254_740_993 + } + + +def test_review_large_integer_rejects_floats_booleans_and_out_of_range_values() -> None: + server = ProtocolServer(MODERN_VERSION, profile=REVIEW_PROFILE) + + for invalid_value in ( + float(REVIEW_EXACT_INTEGER), + True, + REVIEW_EXACT_INTEGER - 1, + REVIEW_MAX_INTEGER + 1, + str(REVIEW_EXACT_INTEGER), + ): + plan = server.handle( + request( + "tools/call", + params={ + "_meta": modern_meta(), + "name": "review_large_integer", + "arguments": {"value": invalid_value}, + }, + ), + ConnectionState(), + ) + + assert error(plan)["code"] == INVALID_PARAMS + + +def test_review_large_integer_survives_stdio_json_serialization() -> None: + message = request( + "tools/call", + params={ + "_meta": modern_meta(), + "name": "review_large_integer", + "arguments": {"value": REVIEW_EXACT_INTEGER}, + }, + ) + stdin = io.StringIO(json.dumps(message, separators=(",", ":")) + "\n") + stdout = io.StringIO() + + run_stdio(ProtocolServer(MODERN_VERSION, profile=REVIEW_PROFILE), stdin, stdout) + + raw_response = stdout.getvalue() + assert "9007199254740993" in raw_response + assert json.loads(raw_response)["result"]["structuredContent"] == { + "value": 9_007_199_254_740_993 + } + + +def test_review_integer_elicitation_preserves_the_exact_requested_form_schema() -> None: + server = ProtocolServer(MODERN_VERSION, profile=REVIEW_PROFILE) + + plan = server.handle( + request( + "tools/call", + params={ + "_meta": modern_meta(capabilities={"elicitation": {"form": {}}}), + "name": "review_integer_elicitation", + "arguments": {}, + }, + ), + ConnectionState(), + ) + value = result(plan) + pending = value["inputRequests"]["large_integer"] + + assert value["resultType"] == "input_required" + assert value["requestState"] == "opaque:review_integer_elicitation:v1" + assert pending["method"] == "elicitation/create" + assert pending["params"]["requestedSchema"]["properties"]["value"] == { + "type": "integer", + "minimum": 9_007_199_254_740_993, + "maximum": 9_223_372_036_854_775_807, + "default": 9_007_199_254_740_993, + } + assert "9007199254740993" in json.dumps(value) + + +def test_review_integer_elicitation_round_trip_keeps_the_exact_integer() -> None: + server = ProtocolServer(MODERN_VERSION, profile=REVIEW_PROFILE) + meta = modern_meta(capabilities={"elicitation": {"form": {}}}) + + first = result( + server.handle( + request( + "tools/call", + params={ + "_meta": meta, + "name": "review_integer_elicitation", + "arguments": {}, + }, + ), + ConnectionState(), + ) + ) + completed = result( + server.handle( + request( + "tools/call", + request_id=2, + params={ + "_meta": meta, + "name": "review_integer_elicitation", + "arguments": {}, + "requestState": first["requestState"], + "inputResponses": { + "large_integer": { + "action": "accept", + "content": {"value": REVIEW_EXACT_INTEGER}, + } + }, + }, + ), + ConnectionState(), + ) + ) + + assert completed["resultType"] == "complete" + assert completed["structuredContent"] == {"value": 9_007_199_254_740_993} + assert completed["content"] == [{"type": "text", "text": "9007199254740993"}] + + +def test_review_integer_elicitation_rejects_float_rounded_form_response() -> None: + server = ProtocolServer(MODERN_VERSION, profile=REVIEW_PROFILE) + + plan = server.handle( + request( + "tools/call", + params={ + "_meta": modern_meta(capabilities={"elicitation": {"form": {}}}), + "name": "review_integer_elicitation", + "arguments": {}, + "requestState": "opaque:review_integer_elicitation:v1", + "inputResponses": { + "large_integer": { + "action": "accept", + "content": {"value": float(REVIEW_EXACT_INTEGER)}, + } + }, + }, + ), + ConnectionState(), + ) + + assert error(plan)["code"] == INVALID_PARAMS + + +def test_review_integer_elicitation_requires_advertised_form_capability() -> None: + server = ProtocolServer(MODERN_VERSION, profile=REVIEW_PROFILE) + + plan = server.handle( + request( + "tools/call", + params={ + "_meta": modern_meta(), + "name": "review_integer_elicitation", + "arguments": {}, + }, + ), + ConnectionState(), + ) + + assert error(plan)["code"] == MISSING_REQUIRED_CLIENT_CAPABILITY + assert plan.http_status == 400 + + +def test_review_mrtr_cap_returns_65_deterministic_pending_requests() -> None: + server = ProtocolServer(MODERN_VERSION, profile=REVIEW_PROFILE) + + plan = server.handle( + request( + "tools/call", + params={ + "_meta": modern_meta(capabilities={"elicitation": {"form": {}}}), + "name": "review_mrtr_cap", + "arguments": {}, + }, + ), + ConnectionState(), + ) + value = result(plan) + + assert value["resultType"] == "input_required" + assert value["requestState"] == "opaque:review_mrtr_cap:v1" + assert len(value["inputRequests"]) == REVIEW_MRTR_INPUT_REQUEST_COUNT == 65 + assert list(value["inputRequests"]) == [ + f"review-input-{index:02d}" for index in range(65) + ] + assert all( + pending["method"] == "elicitation/create" + for pending in value["inputRequests"].values() + ) + + +def test_review_mrtr_cap_requires_advertised_elicitation_capability() -> None: + server = ProtocolServer(MODERN_VERSION, profile=REVIEW_PROFILE) + + plan = server.handle( + request( + "tools/call", + params={ + "_meta": modern_meta(), + "name": "review_mrtr_cap", + "arguments": {}, + }, + ), + ConnectionState(), + ) + + assert error(plan)["code"] == MISSING_REQUIRED_CLIENT_CAPABILITY + assert plan.http_status == 400 + + +def test_review_protocol_env_is_available_to_a_legacy_client() -> None: + server = ProtocolServer(LEGACY_VERSION, profile=REVIEW_PROFILE) + state = ConnectionState() + initialize_legacy(server, state) + tools = result(server.handle(request("tools/list", params={}), state)) + + assert any(tool["name"] == "review_protocol_env" for tool in tools["tools"]) + + with patch.dict( + "os.environ", {"CODEX_MCP_PROTOCOL_VERSION": "legacy-review-sentinel"} + ): + plan = server.handle( + request( + "tools/call", + params={"name": "review_protocol_env", "arguments": {}}, + ), + state, + ) + + assert result(plan)["structuredContent"] == {"value": "legacy-review-sentinel"} + + +def test_review_protocol_env_is_available_to_the_shipping_legacy_client() -> None: + server = ProtocolServer(SHIPPING_LEGACY_VERSION, profile=REVIEW_PROFILE) + state = ConnectionState() + initialize_legacy(server, state, version=SHIPPING_LEGACY_VERSION) + tools = result(server.handle(request("tools/list", params={}), state)) + + assert any(tool["name"] == "review_protocol_env" for tool in tools["tools"]) + + with patch.dict( + "os.environ", {"CODEX_MCP_PROTOCOL_VERSION": "shipping-legacy-sentinel"} + ): + plan = server.handle( + request( + "tools/call", + params={"name": "review_protocol_env", "arguments": {}}, + ), + state, + ) + + assert result(plan)["structuredContent"] == {"value": "shipping-legacy-sentinel"} + + +def test_review_protocol_env_is_available_to_a_modern_client() -> None: + server = ProtocolServer(MODERN_VERSION, profile=REVIEW_PROFILE) + + with patch.dict("os.environ", {"CODEX_MCP_PROTOCOL_VERSION": MODERN_VERSION}): + plan = server.handle( + request( + "tools/call", + params={ + "_meta": modern_meta(), + "name": "review_protocol_env", + "arguments": {}, + }, + ), + ConnectionState(), + ) + + assert result(plan)["structuredContent"] == {"value": MODERN_VERSION} + + +def test_mismatched_discovery_profile_returns_the_wrong_jsonrpc_id() -> None: + server = ProtocolServer(MODERN_VERSION, profile=MISMATCHED_DISCOVERY_ID_PROFILE) + + plan = server.handle(request("server/discover", request_id=42), ConnectionState()) + + assert plan.response is not None + assert plan.response["id"] == "review-mismatched-42" + assert result(plan)["supportedVersions"] == [MODERN_VERSION] + + +def test_null_discovery_profile_returns_a_null_jsonrpc_id() -> None: + server = ProtocolServer(MODERN_VERSION, profile=NULL_DISCOVERY_ID_PROFILE) + + plan = server.handle(request("server/discover", request_id=42), ConnectionState()) + + assert plan.response is not None + assert plan.response["id"] is None + assert result(plan)["supportedVersions"] == [MODERN_VERSION] + + +def test_sse_comment_flood_profile_is_available_from_the_cli() -> None: + args = _parse_args( + [ + "--mode", + MODERN_VERSION, + "--transport", + "http", + "--profile", + SSE_COMMENT_FLOOD_PROFILE, + ] + ) + + assert args.profile == "sse-comment-flood" + + +def test_sse_comment_flood_is_valid_bounded_and_exceeds_the_data_event_limit() -> None: + message: dict[str, object] = {"jsonrpc": "2.0", "id": 42, "result": {}} + + body = _encode_sse_messages([message], profile=SSE_COMMENT_FLOOD_PROFILE) + + assert REVIEW_SSE_COMMENT_LINE_COUNT == 4_097 + assert len(body) > REVIEW_SSE_EVENT_LIMIT_BYTES + assert len(body) < ( + REVIEW_SSE_EVENT_LIMIT_BYTES + REVIEW_SSE_COMMENT_LINE_BYTES + 4_096 + ) + assert body.count(b": reviewer keepalive ") == REVIEW_SSE_COMMENT_LINE_COUNT + assert b"\n" not in body + assert body.endswith(b"\r\r") + assert json.loads(body.rsplit(b"data: ", 1)[1].removesuffix(b"\r\r")) == message + + +@contextmanager +def running_http_server( + mode: str, + *, + profile: str = DEFAULT_PROFILE, +) -> Iterator[tuple[str, int]]: + server = make_http_server(ProtocolServer(mode, profile=profile), "127.0.0.1", 0) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + host, port = server.server_address + yield str(host), int(port) + finally: + server.shutdown() + server.server_close() + thread.join(timeout=2) + + +def post_json( + address: tuple[str, int], + message: dict[str, object], + headers: dict[str, str], +) -> tuple[int, dict[str, str], dict[str, object] | None]: + connection = http.client.HTTPConnection(*address, timeout=2) + body = json.dumps(message, ensure_ascii=False, separators=(",", ":")).encode() + connection.request( + "POST", + "/mcp", + body=body, + headers={"Content-Type": "application/json", **headers}, + ) + response = connection.getresponse() + response_body = response.read() + response_headers = {name.lower(): value for name, value in response.getheaders()} + connection.close() + return ( + response.status, + response_headers, + json.loads(response_body) if response_body else None, + ) + + +def modern_headers( + method: str, + *, + name: str | None = None, + extra: dict[str, str] | None = None, +) -> dict[str, str]: + headers = { + "Accept": "application/json, text/event-stream", + "MCP-Protocol-Version": MODERN_VERSION, + "Mcp-Method": method, + } + if name is not None: + headers["Mcp-Name"] = name + if extra is not None: + headers.update(extra) + return headers + + +@pytest.mark.parametrize( + "mode", (SHIPPING_LEGACY_VERSION, LEGACY_VERSION, MODERN_VERSION) +) +@pytest.mark.parametrize( + ("profile", "expected_size"), + ( + (CATALOG_MAX_PROFILE, MAX_CATALOG_ITEMS), + (CATALOG_OVER_LIMIT_PROFILE, MAX_CATALOG_ITEMS + 1), + ), +) +def test_catalog_boundary_profiles_round_trip_over_localhost_http( + mode: str, profile: str, expected_size: int +) -> None: + with running_http_server(mode, profile=profile) as address: + if mode == MODERN_VERSION: + status, _, body = post_json( + address, + request("tools/list"), + modern_headers("tools/list"), + ) + else: + status, headers, body = post_json( + address, + request( + "initialize", + params={ + "protocolVersion": mode, + "capabilities": {}, + "clientInfo": {"name": "catalog-boundary", "version": "1.0.0"}, + }, + ), + {}, + ) + assert status == 200 + assert body is not None + assert body["result"]["protocolVersion"] == mode + session_headers = {"Mcp-Session-Id": headers["mcp-session-id"]} + + initialized_status, _, initialized_body = post_json( + address, + { + "jsonrpc": "2.0", + "method": "notifications/initialized", + "params": {}, + }, + session_headers, + ) + assert initialized_status == 202 + assert initialized_body is None + + status, _, body = post_json( + address, + request("tools/list", request_id=2, params={}), + session_headers, + ) + + assert status == 200 + assert body is not None + catalog = body["result"] + names = [tool["name"] for tool in catalog["tools"]] + assert len(names) == expected_size + assert len(set(names)) == expected_size + assert "nextCursor" not in catalog + assert ("resultType" in catalog) is (mode == MODERN_VERSION) + + +def test_modern_http_validates_standard_and_custom_headers() -> None: + with running_http_server(MODERN_VERSION) as address: + message = request( + "tools/call", + params={ + "_meta": modern_meta(), + "name": "header_echo", + "arguments": { + "region": "us-west1", + "attempt": 3, + "enabled": True, + "greeting": "Hello, 世界", + }, + }, + ) + encoded_greeting = base64.b64encode("Hello, 世界".encode()).decode() + headers = modern_headers( + "tools/call", + name="header_echo", + extra={ + "Mcp-Param-Region": "us-west1", + "Mcp-Param-Attempt": "3", + "Mcp-Param-Enabled": "true", + "Mcp-Param-Greeting": f"=?base64?{encoded_greeting}?=", + }, + ) + + status, _, body = post_json(address, message, headers) + assert status == 200 + assert body["result"]["resultType"] == "complete" + + headers["Mcp-Name"] = "different" + status, _, body = post_json(address, message, headers) + assert status == 400 + assert body["error"]["code"] == HEADER_MISMATCH + + +def test_modern_http_accepts_base64_mcp_name() -> None: + with running_http_server(MODERN_VERSION) as address: + uri = RESOURCE_URIS[1] + encoded_uri = base64.b64encode(uri.encode()).decode() + message = request( + "resources/read", + params={"_meta": modern_meta(), "uri": uri}, + ) + + status, _, body = post_json( + address, + message, + modern_headers( + "resources/read", + name=f"=?base64?{encoded_uri}?=", + ), + ) + + assert status == 200 + assert body["result"]["contents"][0]["uri"] == uri + + +def test_modern_http_streams_subscription_then_closes_gracefully() -> None: + with running_http_server(MODERN_VERSION) as address: + message = request( + "subscriptions/listen", + request_id=77, + params={ + "_meta": modern_meta(), + "notifications": {"toolsListChanged": True}, + }, + ) + connection = http.client.HTTPConnection(*address, timeout=2) + connection.request( + "POST", + "/mcp", + body=json.dumps(message, separators=(",", ":")), + headers={ + "Content-Type": "application/json", + **modern_headers("subscriptions/listen"), + }, + ) + + response = connection.getresponse() + body = response.read().decode() + connection.close() + events = [ + json.loads(block.removeprefix("data: ")) + for block in body.strip().split("\n\n") + ] + + assert response.status == 200 + assert response.getheader("Content-Type") == "text/event-stream" + assert events[0]["method"] == "notifications/subscriptions/acknowledged" + assert events[1]["method"] == "notifications/tools/list_changed" + assert events[-1]["id"] == 77 + assert events[-1]["result"]["resultType"] == "complete" + + +def test_modern_http_rejects_removed_methods_and_unsupported_versions() -> None: + with running_http_server(MODERN_VERSION) as address: + message = request( + "tools/list", + params={"_meta": modern_meta(version=LEGACY_VERSION)}, + ) + headers = modern_headers("tools/list") + headers["MCP-Protocol-Version"] = LEGACY_VERSION + + status, _, body = post_json(address, message, headers) + assert status == 400 + assert body["error"]["code"] == UNSUPPORTED_PROTOCOL_VERSION + + connection = http.client.HTTPConnection(*address, timeout=2) + connection.request("GET", "/mcp") + get_response = connection.getresponse() + get_response.read() + assert get_response.status == 405 + + connection.request("DELETE", "/mcp") + delete_response = connection.getresponse() + delete_response.read() + connection.close() + assert delete_response.status == 405 + + +def test_modern_http_accepts_notification_without_request_headers() -> None: + with running_http_server(MODERN_VERSION) as address: + status, _, body = post_json( + address, + { + "jsonrpc": "2.0", + "method": "notifications/cancelled", + "params": {"requestId": 1, "reason": "fixture test"}, + }, + {}, + ) + + assert status == 202 + assert body is None + + +def test_modern_http_rejects_untrusted_origin() -> None: + with running_http_server(MODERN_VERSION) as address: + status, _, body = post_json( + address, + request("server/discover"), + { + **modern_headers("server/discover"), + "Origin": "https://attacker.example", + }, + ) + + assert status == 403 + assert body["error"]["message"] == "Origin is not allowed" + + +def test_legacy_http_mints_and_terminates_session() -> None: + with running_http_server(LEGACY_VERSION) as address: + initialize = request( + "initialize", + params={ + "protocolVersion": LEGACY_VERSION, + "capabilities": {}, + "clientInfo": {"name": "legacy", "version": "1.0.0"}, + }, + ) + status, headers, body = post_json(address, initialize, {}) + assert status == 200 + assert body["result"]["protocolVersion"] == LEGACY_VERSION + session_id = headers["mcp-session-id"] + + initialized = { + "jsonrpc": "2.0", + "method": "notifications/initialized", + "params": {}, + } + status, _, body = post_json( + address, initialized, {"Mcp-Session-Id": session_id} + ) + assert status == 202 + assert body is None + + status, _, body = post_json( + address, + request("tools/list", request_id=2, params={}), + {"Mcp-Session-Id": session_id}, + ) + assert status == 200 + assert "resultType" not in body["result"] + + connection = http.client.HTTPConnection(*address, timeout=2) + connection.request( + "DELETE", + "/mcp", + headers={"Mcp-Session-Id": session_id}, + ) + response = connection.getresponse() + response.read() + connection.close() + assert response.status == 200 + + +def test_shipping_legacy_http_negotiates_and_retains_a_real_legacy_session() -> None: + with running_http_server(SHIPPING_LEGACY_VERSION) as address: + status, headers, body = post_json( + address, + request( + "initialize", + params={ + "protocolVersion": SHIPPING_LEGACY_VERSION, + "capabilities": {}, + "clientInfo": {"name": "shipping-legacy", "version": "1.0.0"}, + }, + ), + {}, + ) + + assert status == 200 + assert body is not None + assert body["result"]["protocolVersion"] == "2025-06-18" + session_id = headers["mcp-session-id"] + + status, _, body = post_json( + address, + { + "jsonrpc": "2.0", + "method": "notifications/initialized", + "params": {}, + }, + {"Mcp-Session-Id": session_id}, + ) + + assert status == 202 + assert body is None + + status, _, body = post_json( + address, + request("tools/list", request_id=2, params={}), + {"Mcp-Session-Id": session_id}, + ) + + assert status == 200 + assert body is not None + assert [tool["name"] for tool in body["result"]["tools"]] == [ + "client_metadata", + "echo", + "fail", + "header_echo", + "progress", + ] + assert "resultType" not in body["result"] + + +def test_review_http_lists_tools_across_two_pages() -> None: + with running_http_server(MODERN_VERSION, profile=REVIEW_PROFILE) as address: + status, _, first_body = post_json( + address, + request("tools/list"), + modern_headers("tools/list"), + ) + + assert status == 200 + assert first_body is not None + first = first_body["result"] + assert [tool["name"] for tool in first["tools"]] == ["client_metadata", "echo"] + assert first["nextCursor"] == REVIEW_PAGE_CURSOR + + status, _, second_body = post_json( + address, + request( + "tools/list", + request_id=2, + params={"_meta": modern_meta(), "cursor": REVIEW_PAGE_CURSOR}, + ), + modern_headers("tools/list"), + ) + + assert status == 200 + assert second_body is not None + assert any( + tool["name"] == "review_large_integer" + for tool in second_body["result"]["tools"] + ) + assert "nextCursor" not in second_body["result"] + + +def test_review_http_preserves_integer_precision_in_schema_and_tool_results() -> None: + with running_http_server(MODERN_VERSION, profile=REVIEW_PROFILE) as address: + status, _, tools_body = post_json( + address, + request( + "tools/list", + params={"_meta": modern_meta(), "cursor": REVIEW_PAGE_CURSOR}, + ), + modern_headers("tools/list"), + ) + + assert status == 200 + assert tools_body is not None + definition = next( + tool + for tool in tools_body["result"]["tools"] + if tool["name"] == "review_large_integer" + ) + assert definition["inputSchema"]["properties"]["value"]["minimum"] == ( + 9_007_199_254_740_993 + ) + + status, _, call_body = post_json( + address, + request( + "tools/call", + params={ + "_meta": modern_meta(), + "name": "review_large_integer", + "arguments": {"value": REVIEW_EXACT_INTEGER}, + }, + ), + modern_headers("tools/call", name="review_large_integer"), + ) + + assert status == 200 + assert call_body is not None + assert call_body["result"]["structuredContent"] == { + "value": 9_007_199_254_740_993 + } + + +def test_review_http_preserves_exact_integer_in_elicitation_form_schema() -> None: + with running_http_server(MODERN_VERSION, profile=REVIEW_PROFILE) as address: + status, _, body = post_json( + address, + request( + "tools/call", + params={ + "_meta": modern_meta(capabilities={"elicitation": {"form": {}}}), + "name": "review_integer_elicitation", + "arguments": {}, + }, + ), + modern_headers("tools/call", name="review_integer_elicitation"), + ) + + assert status == 200 + assert body is not None + pending = body["result"]["inputRequests"]["large_integer"] + assert pending["params"]["requestedSchema"]["properties"]["value"] == { + "type": "integer", + "minimum": 9_007_199_254_740_993, + "maximum": 9_223_372_036_854_775_807, + "default": 9_007_199_254_740_993, + } + + +def test_review_http_returns_mismatched_discovery_response_id() -> None: + with running_http_server( + MODERN_VERSION, + profile=MISMATCHED_DISCOVERY_ID_PROFILE, + ) as address: + status, _, body = post_json( + address, + request("server/discover", request_id=42), + modern_headers("server/discover"), + ) + + assert status == 200 + assert body is not None + assert body["id"] == "review-mismatched-42" + + +def test_review_http_returns_null_discovery_response_id() -> None: + with running_http_server( + MODERN_VERSION, profile=NULL_DISCOVERY_ID_PROFILE + ) as address: + status, _, body = post_json( + address, + request("server/discover", request_id=42), + modern_headers("server/discover"), + ) + + assert status == 200 + assert body is not None + assert body["id"] is None + + +def test_review_http_streams_sse_with_carriage_returns_and_comments() -> None: + with running_http_server( + MODERN_VERSION, profile=SSE_CR_COMMENTS_PROFILE + ) as address: + message = request( + "subscriptions/listen", + request_id=77, + params={ + "_meta": modern_meta(), + "notifications": {"toolsListChanged": True}, + }, + ) + connection = http.client.HTTPConnection(*address, timeout=2) + connection.request( + "POST", + "/mcp", + body=json.dumps(message, separators=(",", ":")), + headers={ + "Content-Type": "application/json", + **modern_headers("subscriptions/listen"), + }, + ) + + response = connection.getresponse() + body = response.read() + connection.close() + + assert response.status == 200 + assert response.getheader("Content-Type") == "text/event-stream" + assert body.startswith(b": reviewer keepalive\rdata: ") + assert b"\n" not in body + events = [ + json.loads(line.removeprefix(b"data: ")) + for line in body.split(b"\r") + if line.startswith(b"data: ") + ] + assert [event.get("method") for event in events[:-1]] == [ + "notifications/subscriptions/acknowledged", + "notifications/tools/list_changed", + ] + assert events[-1]["id"] == 77 + assert events[-1]["result"]["resultType"] == "complete" + + +def test_review_http_streams_bounded_keepalive_flood_before_valid_sse_events() -> None: + with running_http_server( + MODERN_VERSION, profile=SSE_COMMENT_FLOOD_PROFILE + ) as address: + message = request( + "subscriptions/listen", + request_id=77, + params={ + "_meta": modern_meta(), + "notifications": {"toolsListChanged": True}, + }, + ) + connection = http.client.HTTPConnection(*address, timeout=10) + connection.request( + "POST", + "/mcp", + body=json.dumps(message, separators=(",", ":")), + headers={ + "Content-Type": "application/json", + **modern_headers("subscriptions/listen"), + }, + ) + + response = connection.getresponse() + body = response.read() + connection.close() + + assert response.status == 200 + assert response.getheader("Content-Type") == "text/event-stream" + assert response.getheader("Content-Length") == str(len(body)) + assert ( + REVIEW_SSE_EVENT_LIMIT_BYTES + < len(body) + < (REVIEW_SSE_EVENT_LIMIT_BYTES + REVIEW_SSE_COMMENT_LINE_BYTES + 4_096) + ) + assert body.count(b": reviewer keepalive ") == REVIEW_SSE_COMMENT_LINE_COUNT + assert b"\n" not in body + completed = json.loads(body.rsplit(b"data: ", 1)[1].removesuffix(b"\r\r")) + assert completed["id"] == 77 + assert completed["result"]["resultType"] == "complete" diff --git a/sdk/python/tests/test_mcp_conformance_fixtures.py b/sdk/python/tests/test_mcp_conformance_fixtures.py new file mode 100644 index 0000000000..57f6bc1592 --- /dev/null +++ b/sdk/python/tests/test_mcp_conformance_fixtures.py @@ -0,0 +1,37 @@ +import os +import subprocess +import sys +from pathlib import Path + + +def test_mcp_conformance_fixture_self_tests() -> None: + repository_root = Path(__file__).resolve().parents[3] + candidates = ( + repository_root / "public" / "scripts" / "mcp_conformance", + repository_root / "scripts" / "mcp_conformance", + ) + fixture_directory = next((candidate for candidate in candidates if candidate.is_dir()), None) + assert fixture_directory is not None, ( + "MCP conformance fixtures are missing; checked " + + " and ".join(str(candidate) for candidate in candidates) + ) + + env = os.environ.copy() + env["PYTEST_DISABLE_PLUGIN_AUTOLOAD"] = "1" + result = subprocess.run( + [sys.executable, "-m", "pytest", "-q", str(fixture_directory)], + cwd=repository_root, + env=env, + capture_output=True, + text=True, + timeout=120, + check=False, + ) + + assert result.returncode == 0, ( + f"MCP conformance fixture self-tests failed with exit " + f"{result.returncode}.\n" + f"Fixture directory: {fixture_directory}\n" + f"STDOUT:\n{result.stdout}\n" + f"STDERR:\n{result.stderr}" + ) diff --git a/sdk/typescript/package.json b/sdk/typescript/package.json index 13a7eca028..d75c8f0317 100644 --- a/sdk/typescript/package.json +++ b/sdk/typescript/package.json @@ -45,6 +45,7 @@ "prepare": "pnpm run build" }, "devDependencies": { + "@modelcontextprotocol/conformance": "github:modelcontextprotocol/conformance#49103de6ed70804e940637bf3e9e29e4a3f54e64", "@modelcontextprotocol/sdk": "^1.24.0", "@types/jest": "^29.5.14", "@types/node": "^20.19.18", diff --git a/sdk/typescript/tests/mcpConformance.test.ts b/sdk/typescript/tests/mcpConformance.test.ts new file mode 100644 index 0000000000..28ad908f07 --- /dev/null +++ b/sdk/typescript/tests/mcpConformance.test.ts @@ -0,0 +1,158 @@ +import { spawnSync } from "node:child_process"; +import { existsSync } from "node:fs"; +import { createRequire } from "node:module"; +import os from "node:os"; +import path from "node:path"; + +import { describe, it } from "@jest/globals"; + +import { codexExecPath } from "./testCodex"; + +const conformanceTimeoutMs = 300_000; +const reviewerRegressionTimeoutMs = 180_000; + +function requireFile(filePath: string, description: string): string { + if (!existsSync(filePath)) { + throw new Error(`MCP conformance ${description} does not exist: ${filePath}`); + } + + return filePath; +} + +function conformanceDirectory(): string { + const repositoryRoot = path.resolve(process.cwd(), "../.."); + const candidates = [ + path.join(repositoryRoot, "public", "scripts", "mcp_conformance"), + path.join(repositoryRoot, "scripts", "mcp_conformance"), + ]; + + for (const candidate of candidates) { + if (existsSync(path.join(candidate, "run_codex_compliance.py"))) { + return candidate; + } + } + + throw new Error(`MCP conformance harness is missing; checked ${candidates.join(" and ")}`); +} + +describe("MCP client conformance", () => { + it( + "does not introduce shipping, intermediate, or modern official conformance regressions", + () => { + const directory = conformanceDirectory(); + const require = createRequire(import.meta.url); + const conformancePackage = require.resolve("@modelcontextprotocol/conformance/package.json"); + const officialConformanceCli = requireFile( + path.join(path.dirname(conformancePackage), "dist", "index.js"), + "pinned official CLI", + ); + const codexBinary = requireFile(codexExecPath, "built Codex binary"); + const baseline = requireFile( + path.join(directory, "regression-baseline-v1.json"), + "committed regression baseline", + ); + const reportDirectory = process.env.RUNNER_TEMP ?? os.tmpdir(); + const reportPath = path.join(reportDirectory, `codex-mcp-conformance-${process.pid}.json`); + const result = spawnSync( + "python3", + [ + path.join(directory, "run_codex_compliance.py"), + codexBinary, + "--mode", + "all", + "--transport", + "all", + "--auth", + "--conformance-cli", + officialConformanceCli, + "--baseline-report", + baseline, + "--report", + reportPath, + ], + { + encoding: "utf8", + maxBuffer: 16 * 1024 * 1024, + timeout: conformanceTimeoutMs - 10_000, + }, + ); + + if (result.error) { + throw new Error(`MCP conformance runner could not complete: ${result.error.message}`, { + cause: result.error, + }); + } + + if (result.status !== 0) { + throw new Error( + [ + `MCP conformance regression gate exited with ${result.status ?? result.signal}.`, + `Report: ${reportPath}`, + result.stdout, + result.stderr, + ] + .filter(Boolean) + .join("\n"), + ); + } + }, + conformanceTimeoutMs, + ); + + it( + "does not introduce production reviewer or catalog-boundary regressions", + () => { + const directory = conformanceDirectory(); + const codexBinary = requireFile(codexExecPath, "built Codex binary"); + const reviewer = requireFile( + path.join(directory, "review_regressions.py"), + "production reviewer regression runner", + ); + const baseline = requireFile( + path.join(directory, "review-regression-baseline-v1.json"), + "committed production reviewer regression baseline", + ); + const reportDirectory = process.env.RUNNER_TEMP ?? os.tmpdir(); + const reportPath = path.join(reportDirectory, `codex-mcp-review-${process.pid}.json`); + const result = spawnSync( + "python3", + [ + reviewer, + codexBinary, + "--mode", + "all", + "--baseline-report", + baseline, + "--report", + reportPath, + ], + { + encoding: "utf8", + maxBuffer: 16 * 1024 * 1024, + timeout: reviewerRegressionTimeoutMs - 10_000, + }, + ); + + if (result.error) { + throw new Error( + `MCP production reviewer regression runner could not complete: ${result.error.message}`, + { cause: result.error }, + ); + } + + if (result.status !== 0) { + throw new Error( + [ + `MCP production reviewer regression gate exited with ${result.status ?? result.signal}.`, + `Report: ${reportPath}`, + result.stdout, + result.stderr, + ] + .filter(Boolean) + .join("\n"), + ); + } + }, + reviewerRegressionTimeoutMs, + ); +});