import json import subprocess import sys import urllib.parse from pathlib import Path import codex_conformance_adapter import official_conformance import pytest import run_codex_compliance from codex_conformance_adapter import ( CIMD_CLIENT_METADATA_URL, CLIENT_REGISTRATION_OVERRIDE_ENV_VAR, AdapterFailure, _client_registration_override, _exercise_auth_scenario, _oauth_client_id, _validate_oauth_secret_not_persisted, _validated_callback_url, _write_auth_registration, ) from official_conformance import ( LEGACY_VERSION, MODERN_VERSION, OFFICIAL_CONFORMANCE_GIT_REF, SHIPPING_LEGACY_VERSION, _make_adapter_launcher, _run_scenario, _scrub_retained_artifacts, _terminate_process_group, default_conformance_command, redact_sensitive_text, run_official_mode, scenarios_for_mode, ) def _authorization_url(redirect_uri: str, state: str = "test-state") -> str: return "http://127.0.0.1:8765/authorize?" + urllib.parse.urlencode( { "client_id": "test-client", "redirect_uri": redirect_uri, "state": state, } ) def test_auth_adapter_only_accepts_exact_loopback_callback() -> None: redirect_uri = "http://127.0.0.1:32123/callback" authorization_url = _authorization_url(redirect_uri) callback_url = redirect_uri + "?code=test-code&state=test-state" assert _validated_callback_url(authorization_url, callback_url) == callback_url with pytest.raises(AdapterFailure, match="exact loopback"): _validated_callback_url( authorization_url, "http://127.0.0.1:32124/callback?code=test-code&state=test-state", ) with pytest.raises(AdapterFailure, match="preserve OAuth state"): _validated_callback_url( authorization_url, redirect_uri + "?code=test-code&state=wrong-state", ) with pytest.raises(AdapterFailure, match="safe loopback"): _validated_callback_url( _authorization_url("https://example.com/callback"), "https://example.com/callback?code=test-code&state=test-state", ) @pytest.mark.parametrize( ("query", "message"), [ ("code=first&code=second&state=test-state", "exactly one nonempty code"), ("code=&state=test-state", "exactly one nonempty code"), ("error=first&error=second&state=test-state", "exactly one nonempty error"), ("error=&state=test-state", "exactly one nonempty error"), ("code=test-code&error=access_denied&state=test-state", "exactly one of"), ("code=test-code&state=test-state&state=test-state", "preserve OAuth state"), ( "code=test-code&state=test-state&iss=first&iss=second", "exactly one nonempty iss", ), ("code=test-code&state=test-state&iss=", "exactly one nonempty iss"), ("state=test-state", "exactly one of"), ], ) def test_auth_adapter_rejects_ambiguous_oauth_callback_parameters( query: str, message: str, ) -> None: redirect_uri = "http://127.0.0.1:32123/callback" with pytest.raises(AdapterFailure, match=message): _validated_callback_url( _authorization_url(redirect_uri), f"{redirect_uri}?{query}", ) def test_auth_adapter_allows_a_single_error_callback_for_client_validation() -> None: redirect_uri = "http://127.0.0.1:32123/callback" callback_url = redirect_uri + "?error=access_denied&state=test-state&iss=issuer" assert ( _validated_callback_url(_authorization_url(redirect_uri), callback_url) == callback_url ) def test_official_runner_is_pinned_instead_of_using_npm_latest() -> None: command = default_conformance_command() assert OFFICIAL_CONFORMANCE_GIT_REF in command[-1] assert "@latest" not in command[-1] def test_official_adapter_launcher_preserves_strict_production_auth_mode( tmp_path: Path, ) -> None: adapter = tmp_path / "show_automatic_auth.py" adapter.write_text( 'import os\nprint(os.environ.get("CODEX_CONFORMANCE_REQUIRE_AUTOMATIC_AUTH", ""))\n', encoding="utf-8", ) launcher = _make_adapter_launcher(adapter, require_automatic_auth=True) try: result = subprocess.run( [str(launcher)], capture_output=True, check=True, text=True, ) finally: launcher.unlink() launcher.parent.rmdir() assert result.stdout.strip() == "1" @pytest.mark.parametrize("require_automatic_auth", [False, True]) def test_windows_official_adapter_launcher_quotes_checkout_paths( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, require_automatic_auth: bool, ) -> None: adapter = tmp_path / "checkout with spaces" / "adapter script.py" adapter.parent.mkdir() adapter.write_text("raise AssertionError('launcher generation only')\n") monkeypatch.setattr(official_conformance.sys, "platform", "win32") launcher = _make_adapter_launcher( adapter, require_automatic_auth=require_automatic_auth, ) try: contents = launcher.read_text(encoding="utf-8") finally: launcher.unlink() launcher.parent.rmdir() expected_auth = "1" if require_automatic_auth else "0" assert launcher.name == "client.cmd" assert contents.startswith("@echo off\n") assert ( f'set "CODEX_CONFORMANCE_REQUIRE_AUTOMATIC_AUTH={expected_auth}"\n' in contents ) assert subprocess.list2cmdline([sys.executable, str(adapter)]) + " %*\n" in contents @pytest.mark.parametrize( ("force", "expected_command"), [ (False, ["taskkill", "/T", "/PID", "417"]), (True, ["taskkill", "/F", "/T", "/PID", "417"]), ], ) def test_windows_official_timeout_terminates_the_entire_process_tree( monkeypatch: pytest.MonkeyPatch, force: bool, expected_command: list[str], ) -> None: observed: list[list[str]] = [] class FakeProcess: pid = 417 def terminate(self) -> None: raise AssertionError("successful taskkill must own tree termination") def kill(self) -> None: raise AssertionError("successful taskkill must own tree termination") def fake_run( command: list[str], *, check: bool, stdout: int, stderr: int, ) -> subprocess.CompletedProcess[str]: observed.append(command) assert check is False assert stdout == subprocess.DEVNULL assert stderr == subprocess.DEVNULL return subprocess.CompletedProcess(command, 0) monkeypatch.setattr(official_conformance.sys, "platform", "win32") monkeypatch.setattr(official_conformance.subprocess, "run", fake_run) _terminate_process_group( FakeProcess(), # type: ignore[arg-type] - validate the Popen process contract. force=force, ) assert observed == [expected_command] @pytest.mark.parametrize( "scenario", ["auth/scope-step-up", "auth/authorization-server-migration"], ) def test_strict_auth_does_not_inject_a_second_login( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, scenario: str, ) -> None: manual_reauthorizations: list[object] = [] monkeypatch.setattr( codex_conformance_adapter, "_oauth_login", lambda _client, **_kwargs: (True, None), ) monkeypatch.setattr(codex_conformance_adapter, "_reload_mcp", lambda _client: None) monkeypatch.setattr( codex_conformance_adapter, "_auth_inventory", lambda _client: {} ) def fail_tool_call(_client: object, _workspace: Path) -> None: raise AdapterFailure("reauthorization is required") monkeypatch.setattr(codex_conformance_adapter, "_auth_tool_call", fail_tool_call) monkeypatch.setattr( codex_conformance_adapter, "_login_reload_and_call", lambda *args, **kwargs: manual_reauthorizations.append((args, kwargs)), ) with pytest.raises(AdapterFailure, match="reauthorization is required"): _exercise_auth_scenario( object(), # type: ignore[arg-type] scenario=scenario, workspace=tmp_path, timeout_seconds=1, require_automatic_auth=True, ) assert manual_reauthorizations == [] @pytest.mark.parametrize( "scenario", ["auth/scope-step-up", "auth/authorization-server-migration"], ) def test_strict_auth_accepts_product_owned_reauthentication( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, scenario: str, ) -> None: tool_calls: list[Path] = [] manual_reauthorizations: list[object] = [] monkeypatch.setattr( codex_conformance_adapter, "_oauth_login", lambda _client, **_kwargs: (True, None), ) monkeypatch.setattr(codex_conformance_adapter, "_reload_mcp", lambda _client: None) monkeypatch.setattr( codex_conformance_adapter, "_auth_inventory", lambda _client: {} ) monkeypatch.setattr( codex_conformance_adapter, "_auth_tool_call", lambda _client, workspace: tool_calls.append(workspace), ) monkeypatch.setattr( codex_conformance_adapter, "_login_reload_and_call", lambda *args, **kwargs: manual_reauthorizations.append((args, kwargs)), ) detail = _exercise_auth_scenario( object(), # type: ignore[arg-type] scenario=scenario, workspace=tmp_path, timeout_seconds=1, require_automatic_auth=True, ) assert "automatically" in detail assert tool_calls == [tmp_path] assert manual_reauthorizations == [] @pytest.mark.parametrize("filename", ["config.toml", ".credentials.json"]) def test_oauth_client_secret_persistence_is_detected_without_disclosing_it( tmp_path: Path, filename: str, ) -> None: secret = "reviewer-confidential-client-secret" (tmp_path / filename).write_text( json.dumps({"client_secret": secret}), encoding="utf-8", ) with pytest.raises(AdapterFailure, match="client secret was persisted") as error: _validate_oauth_secret_not_persisted(tmp_path, secret) assert filename in str(error.value) assert secret not in str(error.value) def test_oauth_client_secret_environment_reference_is_not_a_persisted_secret( tmp_path: Path, ) -> None: (tmp_path / "config.toml").write_text( 'client_secret_env_var = "MCP_CONFORMANCE_CLIENT_SECRET"\n', encoding="utf-8", ) _validate_oauth_secret_not_persisted( tmp_path, "reviewer-confidential-client-secret" ) def test_auth_adapter_selects_only_scenario_provided_client_ids() -> None: assert _oauth_client_id("auth/basic-cimd", {}) == CIMD_CLIENT_METADATA_URL assert ( _oauth_client_id( "auth/pre-registration", {"client_id": "pre-registered", "client_secret": "do-not-log"}, ) == "pre-registered" ) assert _oauth_client_id("auth/metadata-default", {}) is None with pytest.raises(AdapterFailure, match="client_id"): _oauth_client_id("auth/pre-registration", {}) @pytest.mark.parametrize( ("scenario", "require_automatic_auth", "override", "expected"), [ ("auth/offline-access-not-supported", False, None, None), ("auth/offline-access-scope", False, None, "dcr"), ("auth/offline-access-scope", True, None, None), ("auth/offline-access-scope", False, "cimd", "cimd"), ], ) def test_auth_adapter_scopes_client_registration_overrides( monkeypatch: pytest.MonkeyPatch, scenario: str, require_automatic_auth: bool, override: str | None, expected: str | None, ) -> None: if override is None: monkeypatch.delenv(CLIENT_REGISTRATION_OVERRIDE_ENV_VAR, raising=False) else: monkeypatch.setenv(CLIENT_REGISTRATION_OVERRIDE_ENV_VAR, override) assert ( _client_registration_override( scenario, require_automatic_auth=require_automatic_auth, ) == expected ) def test_auth_registration_does_not_persist_context_secret(tmp_path: Path) -> None: config_path = tmp_path / "config.toml" config_path.write_text('mcp_oauth_credentials_store = "file"\n', encoding="utf-8") _write_auth_registration( config_path, server_url="http://127.0.0.1:8765/mcp", oauth_client_id="pre-registered-client", oauth_client_secret_env_var="MCP_CONFORMANCE_CLIENT_SECRET", ) config = config_path.read_text(encoding="utf-8") assert 'url = "http://127.0.0.1:8765/mcp"' in config assert 'client_id = "pre-registered-client"' in config assert 'client_secret_env_var = "MCP_CONFORMANCE_CLIENT_SECRET"' in config assert "do-not-log" not in config assert "client_secret =" not in config def test_official_diagnostics_redact_oauth_secrets() -> None: diagnostic = ( 'With context: {"client_id":"visible","client_secret":"secret-value",' '"private_key_pem":"private-value"}\n' "Authorize at http://localhost/authorize?state=state-value&code_challenge=pkce-value\n" "Authorization: Bearer token-value" ) redacted = redact_sensitive_text(diagnostic) assert '"client_id":"visible"' in redacted assert '"client_secret":"[REDACTED]"' in redacted assert '"private_key_pem":"[REDACTED]"' in redacted assert "state=[REDACTED]" in redacted assert "code_challenge=[REDACTED]" in redacted assert "Bearer [REDACTED]" in redacted serialized = redact_sensitive_text(json.dumps({"diagnostic": diagnostic})) assert json.loads(serialized)["diagnostic"] for secret in ( "secret-value", "private-value", "state-value", "pkce-value", "token-value", ): assert secret not in redacted assert secret not in serialized def test_retained_artifacts_remove_oauth_credential_store(tmp_path: Path) -> None: codex_home = tmp_path / "codex-home" codex_home.mkdir() credentials = codex_home / ".credentials.json" credentials.write_text('{"client_secret":"do-not-retain"}', encoding="utf-8") stdout = tmp_path / "stdout.txt" stdout.write_text("Bearer token-value", encoding="utf-8") _scrub_retained_artifacts(tmp_path) assert not credentials.exists() assert stdout.read_text(encoding="utf-8") == "Bearer [REDACTED]" def test_pinned_full_scenarios_are_selected_by_protocol_version() -> None: shipping_legacy = scenarios_for_mode(SHIPPING_LEGACY_VERSION) legacy = scenarios_for_mode(LEGACY_VERSION) modern = scenarios_for_mode(MODERN_VERSION) assert shipping_legacy == ( "initialize", "tools_call", "auth/token-endpoint-auth-basic", "auth/token-endpoint-auth-post", "auth/token-endpoint-auth-none", ) assert scenarios_for_mode(SHIPPING_LEGACY_VERSION, include_auth=False) == ( "initialize", "tools_call", ) assert legacy[:4] == ( "initialize", "tools_call", "elicitation-sep1034-client-defaults", "sse-retry", ) assert len(legacy) == 18 assert len([scenario for scenario in legacy if scenario.startswith("auth/")]) == 14 assert "auth/pre-registration" in legacy assert modern[:7] == ( "tools_call", "request-metadata", "sep-2322-client-request-state", "http-standard-headers", "http-custom-headers", "http-invalid-tool-headers", "json-schema-ref-no-deref", ) assert len(modern) == 32 assert len([scenario for scenario in modern if scenario.startswith("auth/")]) == 25 assert "auth/resource-mismatch" in modern assert "auth/metadata-issuer-mismatch" in modern assert scenarios_for_mode(LEGACY_VERSION, include_auth=False) == legacy[:4] assert scenarios_for_mode(MODERN_VERSION, include_auth=False) == modern[:7] assert scenarios_for_mode( MODERN_VERSION, ["tools_call", "http-custom-headers"], ) == ("tools_call", "http-custom-headers") assert len(OFFICIAL_CONFORMANCE_GIT_REF) == 40 @pytest.mark.parametrize( ("mode", "scenario"), [ (SHIPPING_LEGACY_VERSION, "request-metadata"), (LEGACY_VERSION, "http-standard-headers"), (MODERN_VERSION, "elicitation-sep1034-client-defaults"), ], ) def test_versioned_scenario_selection_rejects_an_incompatible_protocol( mode: str, scenario: str ) -> None: with pytest.raises( ValueError, match=rf"unavailable for MCP protocol version {mode}.*{scenario}", ): scenarios_for_mode(mode, [scenario], include_auth=False) def test_cross_version_auth_subset_remains_available_to_every_mode() -> None: requested = ( "auth/token-endpoint-auth-basic", "auth/resource-mismatch", "auth/authorization-server-migration", ) assert scenarios_for_mode(SHIPPING_LEGACY_VERSION, requested) == ( "auth/token-endpoint-auth-basic", ) assert scenarios_for_mode(LEGACY_VERSION, requested) == ( "auth/token-endpoint-auth-basic", ) assert scenarios_for_mode(MODERN_VERSION, requested) == requested def test_official_cli_rejects_an_empty_versioned_scenario_run( monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], ) -> None: def unexpected_run(*_args: object, **_kwargs: object) -> None: raise AssertionError("an incompatible selection must not execute conformance") monkeypatch.setattr(run_codex_compliance, "run_compliance", unexpected_run) exit_code = run_codex_compliance.main( [ sys.executable, "--mode", SHIPPING_LEGACY_VERSION, "--transport", "http", "--official-scenario", "request-metadata", "--no-auth", "--conformance-cli", sys.executable, ] ) assert exit_code == 2 assert ( "requested scenarios are unavailable for MCP protocol version 2025-06-18: " "request-metadata" ) in capsys.readouterr().err def test_official_driver_reads_checks_and_adapter_report(tmp_path: Path) -> None: fake_cli = tmp_path / "fake_conformance.py" fake_cli.write_text( """ import json import os import pathlib import sys args = sys.argv[1:] scenario = args[args.index("--scenario") + 1] output = pathlib.Path(args[args.index("--output-dir") + 1]) result_dir = output / f"{scenario}-timestamp" result_dir.mkdir(parents=True) (result_dir / "checks.json").write_text(json.dumps([ { "id": "request-trace", "name": "RequestTrace", "description": "not a conformance assertion", "status": "INFO", }, { "id": "official-check", "name": "OfficialCheck", "description": "observed", "status": "SUCCESS", }, ])) pathlib.Path(os.environ["CODEX_CONFORMANCE_ADAPTER_REPORT"]).write_text( json.dumps({"success": True, "steps": []}) ) """.lstrip(), encoding="utf-8", ) adapter = tmp_path / "adapter.py" adapter.write_text("raise AssertionError('fake CLI should not run adapter')\n") results = run_official_mode( conformance_command=[sys.executable, str(fake_cli)], adapter_script=adapter, codex_binary=Path("/opt/codex"), mode=MODERN_VERSION, scenarios=["tools_call"], output_dir=tmp_path / "results", timeout_seconds=1, base_env={}, ) assert len(results) == 1 assert results[0].success assert results[0].adapter_success assert len(results[0].checks) == 1 assert results[0].checks[0].check_id == "official-check" def test_official_driver_terminates_timed_out_process_group(tmp_path: Path) -> None: fake_cli = tmp_path / "hanging_conformance.py" fake_cli.write_text( """ import subprocess import sys import time subprocess.Popen([sys.executable, "-c", "import time; time.sleep(60)"]) time.sleep(60) """.lstrip(), encoding="utf-8", ) adapter = tmp_path / "adapter.py" adapter.write_text("raise AssertionError('not reached')\n", encoding="utf-8") launcher = _make_adapter_launcher(adapter) try: result = _run_scenario( conformance_command=[sys.executable, str(fake_cli)], adapter_launcher=launcher, codex_binary=Path("/opt/codex"), mode=MODERN_VERSION, scenario="tools_call", output_dir=tmp_path / "results", timeout_seconds=0.05, process_grace_seconds=0.05, base_env={}, ) finally: launcher.unlink() launcher.parent.rmdir() assert not result.success assert "timed out" in result.runner_detail def test_official_driver_does_not_mislabel_runner_crash_as_adapter_failure( tmp_path: Path, ) -> None: fake_cli = tmp_path / "crashing_conformance.py" fake_cli.write_text( """ import os import subprocess import sys script = ''' import json import os import pathlib import time time.sleep(0.1) pathlib.Path(os.environ["CODEX_CONFORMANCE_ADAPTER_REPORT"]).write_text( json.dumps({"success": True, "steps": []}) ) ''' subprocess.Popen([sys.executable, "-c", script], env=os.environ) sys.exit(1) """.lstrip(), encoding="utf-8", ) adapter = tmp_path / "adapter.py" adapter.write_text("raise AssertionError('not reached')\n", encoding="utf-8") launcher = _make_adapter_launcher(adapter) try: result = _run_scenario( conformance_command=[sys.executable, str(fake_cli)], adapter_launcher=launcher, codex_binary=Path("/opt/codex"), mode=MODERN_VERSION, scenario="request-metadata", output_dir=tmp_path / "results", timeout_seconds=1, process_grace_seconds=1, base_env={}, ) finally: launcher.unlink() launcher.parent.rmdir() assert not result.success assert result.adapter_success assert "did not produce exactly one checks.json" in result.runner_detail