mirror of
https://github.com/openai/codex.git
synced 2026-11-12 21:20:34 +00:00
Add MCP client conformance regression gates (#36810)
## What changed - Add a harness that runs the Codex executable against a pinned official MCP client conformance suite across shipping, intermediate, and modern protocol versions, HTTP and stdio transports, and OAuth scenarios. - Add a separate app-server regression matrix for transport, security, schema, pagination, SSE, multi-round request, and catalog-boundary behavior. - Check both suites against committed baselines so previously passing or required checks cannot disappear or newly fail, while keeping known failures visible in complete reports. - Run the fixture self-tests and both executable-level gates from the Python and TypeScript SDK test workflows. ## Testing - Add unit tests for the fixture server, official-suite adapter, conformance runner, baseline comparison, and reviewer regression runner. GitOrigin-RevId: de59f039294e34ed72873d9f6940b52e89172c0d
This commit is contained in:
@@ -0,0 +1,37 @@
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def test_mcp_conformance_fixture_self_tests() -> None:
|
||||
repository_root = Path(__file__).resolve().parents[3]
|
||||
candidates = (
|
||||
repository_root / "public" / "scripts" / "mcp_conformance",
|
||||
repository_root / "scripts" / "mcp_conformance",
|
||||
)
|
||||
fixture_directory = next((candidate for candidate in candidates if candidate.is_dir()), None)
|
||||
assert fixture_directory is not None, (
|
||||
"MCP conformance fixtures are missing; checked "
|
||||
+ " and ".join(str(candidate) for candidate in candidates)
|
||||
)
|
||||
|
||||
env = os.environ.copy()
|
||||
env["PYTEST_DISABLE_PLUGIN_AUTOLOAD"] = "1"
|
||||
result = subprocess.run(
|
||||
[sys.executable, "-m", "pytest", "-q", str(fixture_directory)],
|
||||
cwd=repository_root,
|
||||
env=env,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=120,
|
||||
check=False,
|
||||
)
|
||||
|
||||
assert result.returncode == 0, (
|
||||
f"MCP conformance fixture self-tests failed with exit "
|
||||
f"{result.returncode}.\n"
|
||||
f"Fixture directory: {fixture_directory}\n"
|
||||
f"STDOUT:\n{result.stdout}\n"
|
||||
f"STDERR:\n{result.stderr}"
|
||||
)
|
||||
Reference in New Issue
Block a user