mirror of
https://github.com/openai/codex.git
synced 2026-08-23 13:09:46 +00:00
Add MCP client conformance regression gates (#36810)
## What changed - Add a harness that runs the Codex executable against a pinned official MCP client conformance suite across shipping, intermediate, and modern protocol versions, HTTP and stdio transports, and OAuth scenarios. - Add a separate app-server regression matrix for transport, security, schema, pagination, SSE, multi-round request, and catalog-boundary behavior. - Check both suites against committed baselines so previously passing or required checks cannot disappear or newly fail, while keeping known failures visible in complete reports. - Run the fixture self-tests and both executable-level gates from the Python and TypeScript SDK test workflows. ## Testing - Add unit tests for the fixture server, official-suite adapter, conformance runner, baseline comparison, and reviewer regression runner. GitOrigin-RevId: de59f039294e34ed72873d9f6940b52e89172c0d
This commit is contained in:
thomas
253
pnpm-lock.yaml
generated
253
pnpm-lock.yaml
generated
@@ -36,6 +36,9 @@ importers:
|
||||
|
||||
sdk/typescript:
|
||||
devDependencies:
|
||||
'@modelcontextprotocol/conformance':
|
||||
specifier: github:modelcontextprotocol/conformance#49103de6ed70804e940637bf3e9e29e4a3f54e64
|
||||
version: https://codeload.github.com/modelcontextprotocol/conformance/tar.gz/49103de6ed70804e940637bf3e9e29e4a3f54e64
|
||||
'@modelcontextprotocol/sdk':
|
||||
specifier: 1.26.0
|
||||
version: 1.26.0(zod@3.25.76)
|
||||
@@ -74,7 +77,7 @@ importers:
|
||||
version: 10.9.2(@types/node@20.19.18)(typescript@5.9.2)
|
||||
tsup:
|
||||
specifier: ^8.5.0
|
||||
version: 8.5.0(postcss@8.5.6)(typescript@5.9.2)(yaml@2.8.1)
|
||||
version: 8.5.0(postcss@8.5.6)(typescript@5.9.2)(yaml@2.9.0)
|
||||
typescript:
|
||||
specifier: ^5.9.2
|
||||
version: 5.9.2
|
||||
@@ -572,6 +575,11 @@ packages:
|
||||
'@jridgewell/trace-mapping@0.3.9':
|
||||
resolution: {integrity: sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==}
|
||||
|
||||
'@modelcontextprotocol/conformance@https://codeload.github.com/modelcontextprotocol/conformance/tar.gz/49103de6ed70804e940637bf3e9e29e4a3f54e64':
|
||||
resolution: {tarball: https://codeload.github.com/modelcontextprotocol/conformance/tar.gz/49103de6ed70804e940637bf3e9e29e4a3f54e64}
|
||||
version: 0.2.0-alpha.10
|
||||
hasBin: true
|
||||
|
||||
'@modelcontextprotocol/sdk@1.26.0':
|
||||
resolution: {integrity: sha512-Y5RmPncpiDtTXDbLKswIJzTqu2hyBKxTNsgKqKclDbhIgg1wgtf1fRuvxgTnRfcnxtvvgbIEcqUOzZrJ6iSReg==}
|
||||
engines: {node: '>=18'}
|
||||
@@ -594,6 +602,58 @@ packages:
|
||||
resolution: {integrity: sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==}
|
||||
engines: {node: '>= 8'}
|
||||
|
||||
'@octokit/auth-token@6.0.0':
|
||||
resolution: {integrity: sha512-P4YJBPdPSpWTQ1NU4XYdvHvXJJDxM6YwpS0FZHRgP7YFkdVxsWcpWGy/NVqlAA7PcPCnMacXlRm1y2PFZRWL/w==}
|
||||
engines: {node: '>= 20'}
|
||||
|
||||
'@octokit/core@7.0.6':
|
||||
resolution: {integrity: sha512-DhGl4xMVFGVIyMwswXeyzdL4uXD5OGILGX5N8Y+f6W7LhC1Ze2poSNrkF/fedpVDHEEZ+PHFW0vL14I+mm8K3Q==}
|
||||
engines: {node: '>= 20'}
|
||||
|
||||
'@octokit/endpoint@11.0.3':
|
||||
resolution: {integrity: sha512-FWFlNxghg4HrXkD3ifYbS/IdL/mDHjh9QcsNyhQjN8dplUoZbejsdpmuqdA76nxj2xoWPs7p8uX2SNr9rYu0Ag==}
|
||||
engines: {node: '>= 20'}
|
||||
|
||||
'@octokit/graphql@9.0.3':
|
||||
resolution: {integrity: sha512-grAEuupr/C1rALFnXTv6ZQhFuL1D8G5y8CN04RgrO4FIPMrtm+mcZzFG7dcBm+nq+1ppNixu+Jd78aeJOYxlGA==}
|
||||
engines: {node: '>= 20'}
|
||||
|
||||
'@octokit/openapi-types@27.0.0':
|
||||
resolution: {integrity: sha512-whrdktVs1h6gtR+09+QsNk2+FO+49j6ga1c55YZudfEG+oKJVvJLQi3zkOm5JjiUXAagWK2tI2kTGKJ2Ys7MGA==}
|
||||
|
||||
'@octokit/plugin-paginate-rest@14.0.0':
|
||||
resolution: {integrity: sha512-fNVRE7ufJiAA3XUrha2omTA39M6IXIc6GIZLvlbsm8QOQCYvpq/LkMNGyFlB1d8hTDzsAXa3OKtybdMAYsV/fw==}
|
||||
engines: {node: '>= 20'}
|
||||
peerDependencies:
|
||||
'@octokit/core': '>=6'
|
||||
|
||||
'@octokit/plugin-request-log@6.0.0':
|
||||
resolution: {integrity: sha512-UkOzeEN3W91/eBq9sPZNQ7sUBvYCqYbrrD8gTbBuGtHEuycE4/awMXcYvx6sVYo7LypPhmQwwpUe4Yyu4QZN5Q==}
|
||||
engines: {node: '>= 20'}
|
||||
peerDependencies:
|
||||
'@octokit/core': '>=6'
|
||||
|
||||
'@octokit/plugin-rest-endpoint-methods@17.0.0':
|
||||
resolution: {integrity: sha512-B5yCyIlOJFPqUUeiD0cnBJwWJO8lkJs5d8+ze9QDP6SvfiXSz1BF+91+0MeI1d2yxgOhU/O+CvtiZ9jSkHhFAw==}
|
||||
engines: {node: '>= 20'}
|
||||
peerDependencies:
|
||||
'@octokit/core': '>=6'
|
||||
|
||||
'@octokit/request-error@7.1.0':
|
||||
resolution: {integrity: sha512-KMQIfq5sOPpkQYajXHwnhjCC0slzCNScLHs9JafXc4RAJI+9f+jNDlBNaIMTvazOPLgb4BnlhGJOTbnN0wIjPw==}
|
||||
engines: {node: '>= 20'}
|
||||
|
||||
'@octokit/request@10.0.11':
|
||||
resolution: {integrity: sha512-+s7HUxjfFqOMS9VlIwDffq0MikjSAK0gSpG73W+meAvVAvX4MBrHYTK5Bj3Uot55qFT4gzUtfzE4mGWY4Br8/Q==}
|
||||
engines: {node: '>= 20'}
|
||||
|
||||
'@octokit/rest@22.0.1':
|
||||
resolution: {integrity: sha512-Jzbhzl3CEexhnivb1iQ0KJ7s5vvjMWcmRtq5aUsKmKDrRW6z3r84ngmiFKFvpZjpiU/9/S6ITPFRpn5s/3uQJw==}
|
||||
engines: {node: '>= 20'}
|
||||
|
||||
'@octokit/types@16.0.0':
|
||||
resolution: {integrity: sha512-sKq+9r1Mm4efXW1FCk7hFSeJo4QKreL/tTbR0rz/qx/r1Oa2VV83LTA/H/MuCOX7uCIJmQVRKBcbmWoySjAnSg==}
|
||||
|
||||
'@pkgjs/parseargs@0.11.0':
|
||||
resolution: {integrity: sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==}
|
||||
engines: {node: '>=14'}
|
||||
@@ -893,6 +953,9 @@ packages:
|
||||
ajv@8.17.1:
|
||||
resolution: {integrity: sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==}
|
||||
|
||||
ajv@8.20.0:
|
||||
resolution: {integrity: sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==}
|
||||
|
||||
ansi-escapes@4.3.2:
|
||||
resolution: {integrity: sha512-gKXj5ALrKWQLsYG9jlTRmR/xKluxHV+Z9QEwNIgCfM1/uwPMCuzVVnh5mwTd+OuBZcwSIMbqssNWRm1lE51QaQ==}
|
||||
engines: {node: '>=8'}
|
||||
@@ -969,6 +1032,9 @@ packages:
|
||||
resolution: {integrity: sha512-hY/u2lxLrbecMEWSB0IpGzGyDyeoMFQhCvZd2jGFSE5I17Fh01sYUBPCJtkWERw7zrac9+cIghxm/ytJa2X8iA==}
|
||||
hasBin: true
|
||||
|
||||
before-after-hook@4.0.0:
|
||||
resolution: {integrity: sha512-q6tR3RPqIB1pMiTRMFcZwuG5T8vwp+vUvEG0vuI6B+Rikh5BfPp2fQ82c925FOs+b0lcFQ8CFrL+KbilfZFhOQ==}
|
||||
|
||||
body-parser@2.2.2:
|
||||
resolution: {integrity: sha512-oP5VkATKlNwcgvxi0vM0p/D3n2C3EReYVX+DNYs5TjZFn/oQt2j+4sVJtSMr18pdRr8wjTcBl6LoV+FUwzPmNA==}
|
||||
engines: {node: '>=18'}
|
||||
@@ -1073,6 +1139,10 @@ packages:
|
||||
color-name@1.1.4:
|
||||
resolution: {integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==}
|
||||
|
||||
commander@14.0.3:
|
||||
resolution: {integrity: sha512-H+y0Jo/T1RZ9qPP4Eh1pkcQcLRglraJaSLoyOtHxu6AapkjWVCy2Sit1QQ4x3Dng8qDlSsZEet7g5Pq06MvTgw==}
|
||||
engines: {node: '>=20'}
|
||||
|
||||
commander@4.1.1:
|
||||
resolution: {integrity: sha512-NOKm8xhkzAjzFx8B2v5OAHT+u5pRQc2UCa2Vq9jYL/31o2wi9mxBA7LIFs3sV5VSC49z6pEhfbMULvShKj26WA==}
|
||||
engines: {node: '>= 6'}
|
||||
@@ -1095,6 +1165,10 @@ packages:
|
||||
resolution: {integrity: sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==}
|
||||
engines: {node: '>= 0.6'}
|
||||
|
||||
content-type@2.0.0:
|
||||
resolution: {integrity: sha512-j/O/d7GcZCyNl7/hwZAb606rzqkyvaDctLmckbxLzHvFBzTJHuGEdodATcP3yIRoDrLHkIATJuvzbFlp/ki2cQ==}
|
||||
engines: {node: '>=18'}
|
||||
|
||||
convert-source-map@2.0.0:
|
||||
resolution: {integrity: sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==}
|
||||
|
||||
@@ -1304,6 +1378,10 @@ packages:
|
||||
resolution: {integrity: sha512-Vo1ab+QXPzZ4tCa8SwIHJFaSzy4R6SHf7BY79rFBDf0idraZWAkYrDjDj8uWaSm3S2TK+hJ7/t1CEmZ7jXw+pg==}
|
||||
engines: {node: '>=18.0.0'}
|
||||
|
||||
eventsource-parser@3.1.0:
|
||||
resolution: {integrity: sha512-kJezFj9YFAMLeORyi7aCLxLbD5/qWMQnoMVlVPyHIll7lgRJCc3JVln9Vgl9nwQi0YkMnhdGTMNn7CkRRAptMg==}
|
||||
engines: {node: '>=18.0.0'}
|
||||
|
||||
eventsource@3.0.7:
|
||||
resolution: {integrity: sha512-CRT1WTyuQoD771GW56XEZFQ/ZoSfWid1alKGDYMmkt2yl8UXrVR4pspqWNEcqKvVIzg6PAltWjxcSSPrboA4iA==}
|
||||
engines: {node: '>=18.0.0'}
|
||||
@@ -1453,7 +1531,7 @@ packages:
|
||||
|
||||
glob@7.2.3:
|
||||
resolution: {integrity: sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==}
|
||||
deprecated: Glob versions prior to v9 are no longer supported
|
||||
deprecated: Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me
|
||||
|
||||
globals@14.0.0:
|
||||
resolution: {integrity: sha512-oahGvuMGQlPw/ivIYBjVSrWAfWLBeku5tpPE2fOPLi+WHffIWbuh2tCjhyQhTBPMf5E9jDEH4FOmTYgYwbKwtQ==}
|
||||
@@ -1775,6 +1853,9 @@ packages:
|
||||
json-stable-stringify-without-jsonify@1.0.1:
|
||||
resolution: {integrity: sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==}
|
||||
|
||||
json-with-bigint@3.5.10:
|
||||
resolution: {integrity: sha512-Vcx+JVNEBts/xfcoCS69sKrOhOk/3TVlvlT+XzUOefVKnnrbYSCKpDCm10pohsJFtsJVYnwa/cXRZ4eElzaM6w==}
|
||||
|
||||
json5@2.2.3:
|
||||
resolution: {integrity: sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==}
|
||||
engines: {node: '>=6'}
|
||||
@@ -2093,10 +2174,6 @@ packages:
|
||||
pure-rand@6.1.0:
|
||||
resolution: {integrity: sha512-bVWawvoZoBYpp6yIoQtQXHZjmz35RSVHnUOTefl8Vcjr8snTPY1wnpSPMWekcFwbxI6gtmT7rSYPFvz71ldiOA==}
|
||||
|
||||
qs@6.14.0:
|
||||
resolution: {integrity: sha512-YWWTjgABSKcvs/nWBi9PycY/JiPJqOD4JA6o9Sej2AtvSGarXxKC3OQSk4pAarbdQlKAh5D4FCQkJNkW+GAn3w==}
|
||||
engines: {node: '>=0.6'}
|
||||
|
||||
qs@6.15.1:
|
||||
resolution: {integrity: sha512-6YHEFRL9mfgcAvql/XhwTvf5jKcOiiupt2FiJxHkiX1z4j7WL8J/jRHYLluORvc1XxB5rV20KoeK00gVJamspg==}
|
||||
engines: {node: '>=0.6'}
|
||||
@@ -2457,6 +2534,13 @@ packages:
|
||||
undici-types@6.21.0:
|
||||
resolution: {integrity: sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==}
|
||||
|
||||
undici@7.28.0:
|
||||
resolution: {integrity: sha512-cRZYrTDwWznlnRiPjggAGxZXanty6M8RV1ff8Wm4LWXBp7/IG8v5DnOm74DtUBp9OONpK75YlPnIjQqX0dBDtA==}
|
||||
engines: {node: '>=20.18.1'}
|
||||
|
||||
universal-user-agent@7.0.3:
|
||||
resolution: {integrity: sha512-TmnEAEAsBJVZM/AADELsK76llnwcf9vMKuPz8JflO1frO8Lchitr0fNaN9d+Ap0BjKtqWqd/J17qeDnXh8CL2A==}
|
||||
|
||||
unpipe@1.0.0:
|
||||
resolution: {integrity: sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==}
|
||||
engines: {node: '>= 0.8'}
|
||||
@@ -2524,8 +2608,8 @@ packages:
|
||||
yallist@3.1.1:
|
||||
resolution: {integrity: sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==}
|
||||
|
||||
yaml@2.8.1:
|
||||
resolution: {integrity: sha512-lcYcMxX2PO9XMGvAJkJ3OsNMw+/7FKes7/hgerGUYWIoWu5j/+YQqcZr5JnPZWzOsEBgMbSbiSTn/dv/69Mkpw==}
|
||||
yaml@2.9.0:
|
||||
resolution: {integrity: sha512-2AvhNX3mb8zd6Zy7INTtSpl1F15HW6Wnqj0srWlkKLcpYl/gMIMJiyuGq2KeI2YFxUPjdlB+3Lc10seMLtL4cA==}
|
||||
engines: {node: '>= 14.6'}
|
||||
hasBin: true
|
||||
|
||||
@@ -2558,6 +2642,9 @@ packages:
|
||||
zod@3.25.76:
|
||||
resolution: {integrity: sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==}
|
||||
|
||||
zod@4.4.3:
|
||||
resolution: {integrity: sha512-ytENFjIJFl2UwYglde2jchW2Hwm4GJFLDiSXWdTrJQBIN9Fcyp7n4DhxJEiWNAJMV1/BqWfW/kkg71UDcHJyTQ==}
|
||||
|
||||
snapshots:
|
||||
|
||||
'@babel/code-frame@7.27.1':
|
||||
@@ -3095,6 +3182,23 @@ snapshots:
|
||||
'@jridgewell/resolve-uri': 3.1.2
|
||||
'@jridgewell/sourcemap-codec': 1.5.5
|
||||
|
||||
'@modelcontextprotocol/conformance@https://codeload.github.com/modelcontextprotocol/conformance/tar.gz/49103de6ed70804e940637bf3e9e29e4a3f54e64':
|
||||
dependencies:
|
||||
'@modelcontextprotocol/sdk': 1.26.0(zod@4.4.3)
|
||||
'@octokit/rest': 22.0.1
|
||||
ajv: 8.20.0
|
||||
ajv-formats: 3.0.1(ajv@8.20.0)
|
||||
commander: 14.0.3
|
||||
eventsource-parser: 3.1.0
|
||||
express: 5.2.1
|
||||
jose: 6.1.3
|
||||
undici: 7.28.0
|
||||
yaml: 2.9.0
|
||||
zod: 4.4.3
|
||||
transitivePeerDependencies:
|
||||
- '@cfworker/json-schema'
|
||||
- supports-color
|
||||
|
||||
'@modelcontextprotocol/sdk@1.26.0(zod@3.25.76)':
|
||||
dependencies:
|
||||
'@hono/node-server': 1.19.13(hono@4.12.25)
|
||||
@@ -3117,6 +3221,28 @@ snapshots:
|
||||
transitivePeerDependencies:
|
||||
- supports-color
|
||||
|
||||
'@modelcontextprotocol/sdk@1.26.0(zod@4.4.3)':
|
||||
dependencies:
|
||||
'@hono/node-server': 1.19.13(hono@4.12.25)
|
||||
ajv: 8.17.1
|
||||
ajv-formats: 3.0.1(ajv@8.17.1)
|
||||
content-type: 1.0.5
|
||||
cors: 2.8.5
|
||||
cross-spawn: 7.0.6
|
||||
eventsource: 3.0.7
|
||||
eventsource-parser: 3.0.6
|
||||
express: 5.2.1
|
||||
express-rate-limit: 8.3.2(express@5.2.1)
|
||||
hono: 4.12.25
|
||||
jose: 6.1.3
|
||||
json-schema-typed: 8.0.2
|
||||
pkce-challenge: 5.0.0
|
||||
raw-body: 3.0.1
|
||||
zod: 4.4.3
|
||||
zod-to-json-schema: 3.25.2(zod@4.4.3)
|
||||
transitivePeerDependencies:
|
||||
- supports-color
|
||||
|
||||
'@nodelib/fs.scandir@2.1.5':
|
||||
dependencies:
|
||||
'@nodelib/fs.stat': 2.0.5
|
||||
@@ -3129,6 +3255,69 @@ snapshots:
|
||||
'@nodelib/fs.scandir': 2.1.5
|
||||
fastq: 1.19.1
|
||||
|
||||
'@octokit/auth-token@6.0.0': {}
|
||||
|
||||
'@octokit/core@7.0.6':
|
||||
dependencies:
|
||||
'@octokit/auth-token': 6.0.0
|
||||
'@octokit/graphql': 9.0.3
|
||||
'@octokit/request': 10.0.11
|
||||
'@octokit/request-error': 7.1.0
|
||||
'@octokit/types': 16.0.0
|
||||
before-after-hook: 4.0.0
|
||||
universal-user-agent: 7.0.3
|
||||
|
||||
'@octokit/endpoint@11.0.3':
|
||||
dependencies:
|
||||
'@octokit/types': 16.0.0
|
||||
universal-user-agent: 7.0.3
|
||||
|
||||
'@octokit/graphql@9.0.3':
|
||||
dependencies:
|
||||
'@octokit/request': 10.0.11
|
||||
'@octokit/types': 16.0.0
|
||||
universal-user-agent: 7.0.3
|
||||
|
||||
'@octokit/openapi-types@27.0.0': {}
|
||||
|
||||
'@octokit/plugin-paginate-rest@14.0.0(@octokit/core@7.0.6)':
|
||||
dependencies:
|
||||
'@octokit/core': 7.0.6
|
||||
'@octokit/types': 16.0.0
|
||||
|
||||
'@octokit/plugin-request-log@6.0.0(@octokit/core@7.0.6)':
|
||||
dependencies:
|
||||
'@octokit/core': 7.0.6
|
||||
|
||||
'@octokit/plugin-rest-endpoint-methods@17.0.0(@octokit/core@7.0.6)':
|
||||
dependencies:
|
||||
'@octokit/core': 7.0.6
|
||||
'@octokit/types': 16.0.0
|
||||
|
||||
'@octokit/request-error@7.1.0':
|
||||
dependencies:
|
||||
'@octokit/types': 16.0.0
|
||||
|
||||
'@octokit/request@10.0.11':
|
||||
dependencies:
|
||||
'@octokit/endpoint': 11.0.3
|
||||
'@octokit/request-error': 7.1.0
|
||||
'@octokit/types': 16.0.0
|
||||
content-type: 2.0.0
|
||||
json-with-bigint: 3.5.10
|
||||
universal-user-agent: 7.0.3
|
||||
|
||||
'@octokit/rest@22.0.1':
|
||||
dependencies:
|
||||
'@octokit/core': 7.0.6
|
||||
'@octokit/plugin-paginate-rest': 14.0.0(@octokit/core@7.0.6)
|
||||
'@octokit/plugin-request-log': 6.0.0(@octokit/core@7.0.6)
|
||||
'@octokit/plugin-rest-endpoint-methods': 17.0.0(@octokit/core@7.0.6)
|
||||
|
||||
'@octokit/types@16.0.0':
|
||||
dependencies:
|
||||
'@octokit/openapi-types': 27.0.0
|
||||
|
||||
'@pkgjs/parseargs@0.11.0':
|
||||
optional: true
|
||||
|
||||
@@ -3393,6 +3582,10 @@ snapshots:
|
||||
optionalDependencies:
|
||||
ajv: 8.17.1
|
||||
|
||||
ajv-formats@3.0.1(ajv@8.20.0):
|
||||
optionalDependencies:
|
||||
ajv: 8.20.0
|
||||
|
||||
ajv@6.12.6:
|
||||
dependencies:
|
||||
fast-deep-equal: 3.1.3
|
||||
@@ -3407,6 +3600,13 @@ snapshots:
|
||||
json-schema-traverse: 1.0.0
|
||||
require-from-string: 2.0.2
|
||||
|
||||
ajv@8.20.0:
|
||||
dependencies:
|
||||
fast-deep-equal: 3.1.3
|
||||
fast-uri: 3.1.1
|
||||
json-schema-traverse: 1.0.0
|
||||
require-from-string: 2.0.2
|
||||
|
||||
ansi-escapes@4.3.2:
|
||||
dependencies:
|
||||
type-fest: 0.21.3
|
||||
@@ -3499,6 +3699,8 @@ snapshots:
|
||||
|
||||
baseline-browser-mapping@2.8.9: {}
|
||||
|
||||
before-after-hook@4.0.0: {}
|
||||
|
||||
body-parser@2.2.2:
|
||||
dependencies:
|
||||
bytes: 3.1.2
|
||||
@@ -3602,6 +3804,8 @@ snapshots:
|
||||
|
||||
color-name@1.1.4: {}
|
||||
|
||||
commander@14.0.3: {}
|
||||
|
||||
commander@4.1.1: {}
|
||||
|
||||
concat-map@0.0.1: {}
|
||||
@@ -3616,6 +3820,8 @@ snapshots:
|
||||
|
||||
content-type@1.0.5: {}
|
||||
|
||||
content-type@2.0.0: {}
|
||||
|
||||
convert-source-map@2.0.0: {}
|
||||
|
||||
cookie-signature@1.2.2: {}
|
||||
@@ -3829,6 +4035,8 @@ snapshots:
|
||||
|
||||
eventsource-parser@3.0.6: {}
|
||||
|
||||
eventsource-parser@3.1.0: {}
|
||||
|
||||
eventsource@3.0.7:
|
||||
dependencies:
|
||||
eventsource-parser: 3.0.6
|
||||
@@ -3882,7 +4090,7 @@ snapshots:
|
||||
once: 1.4.0
|
||||
parseurl: 1.3.3
|
||||
proxy-addr: 2.0.7
|
||||
qs: 6.14.0
|
||||
qs: 6.15.1
|
||||
range-parser: 1.2.1
|
||||
router: 2.2.0
|
||||
send: 1.2.0
|
||||
@@ -4510,6 +4718,8 @@ snapshots:
|
||||
|
||||
json-stable-stringify-without-jsonify@1.0.1: {}
|
||||
|
||||
json-with-bigint@3.5.10: {}
|
||||
|
||||
json5@2.2.3: {}
|
||||
|
||||
keyv@4.5.4:
|
||||
@@ -4729,12 +4939,12 @@ snapshots:
|
||||
mlly: 1.8.0
|
||||
pathe: 2.0.3
|
||||
|
||||
postcss-load-config@6.0.1(postcss@8.5.6)(yaml@2.8.1):
|
||||
postcss-load-config@6.0.1(postcss@8.5.6)(yaml@2.9.0):
|
||||
dependencies:
|
||||
lilconfig: 3.1.3
|
||||
optionalDependencies:
|
||||
postcss: 8.5.6
|
||||
yaml: 2.8.1
|
||||
yaml: 2.9.0
|
||||
|
||||
postcss@8.5.6:
|
||||
dependencies:
|
||||
@@ -4769,10 +4979,6 @@ snapshots:
|
||||
|
||||
pure-rand@6.1.0: {}
|
||||
|
||||
qs@6.14.0:
|
||||
dependencies:
|
||||
side-channel: 1.1.0
|
||||
|
||||
qs@6.15.1:
|
||||
dependencies:
|
||||
side-channel: 1.1.0
|
||||
@@ -5093,7 +5299,7 @@ snapshots:
|
||||
v8-compile-cache-lib: 3.0.1
|
||||
yn: 3.1.1
|
||||
|
||||
tsup@8.5.0(postcss@8.5.6)(typescript@5.9.2)(yaml@2.8.1):
|
||||
tsup@8.5.0(postcss@8.5.6)(typescript@5.9.2)(yaml@2.9.0):
|
||||
dependencies:
|
||||
bundle-require: 5.1.0(esbuild@0.28.1)
|
||||
cac: 6.7.14
|
||||
@@ -5104,7 +5310,7 @@ snapshots:
|
||||
fix-dts-default-cjs-exports: 1.0.1
|
||||
joycon: 3.1.1
|
||||
picocolors: 1.1.1
|
||||
postcss-load-config: 6.0.1(postcss@8.5.6)(yaml@2.8.1)
|
||||
postcss-load-config: 6.0.1(postcss@8.5.6)(yaml@2.9.0)
|
||||
resolve-from: 5.0.0
|
||||
rollup: 4.59.0
|
||||
source-map: 0.8.0-beta.0
|
||||
@@ -5157,6 +5363,10 @@ snapshots:
|
||||
|
||||
undici-types@6.21.0: {}
|
||||
|
||||
undici@7.28.0: {}
|
||||
|
||||
universal-user-agent@7.0.3: {}
|
||||
|
||||
unpipe@1.0.0: {}
|
||||
|
||||
update-browserslist-db@1.1.3(browserslist@4.26.2):
|
||||
@@ -5222,8 +5432,7 @@ snapshots:
|
||||
|
||||
yallist@3.1.1: {}
|
||||
|
||||
yaml@2.8.1:
|
||||
optional: true
|
||||
yaml@2.9.0: {}
|
||||
|
||||
yargs-parser@21.1.1: {}
|
||||
|
||||
@@ -5249,4 +5458,10 @@ snapshots:
|
||||
dependencies:
|
||||
zod: 3.25.76
|
||||
|
||||
zod-to-json-schema@3.25.2(zod@4.4.3):
|
||||
dependencies:
|
||||
zod: 4.4.3
|
||||
|
||||
zod@3.25.76: {}
|
||||
|
||||
zod@4.4.3: {}
|
||||
|
||||
@@ -14,4 +14,6 @@ strictDepBuilds: true
|
||||
trustPolicy: no-downgrade
|
||||
trustPolicyIgnoreAfter: 10080
|
||||
trustPolicyExclude: []
|
||||
allowBuilds: {}
|
||||
allowBuilds:
|
||||
# Build only the official MCP conformance CLI pinned by the TypeScript SDK.
|
||||
"@modelcontextprotocol/conformance": true
|
||||
|
||||
121
scripts/mcp_conformance/README.md
Normal file
121
scripts/mcp_conformance/README.md
Normal file
@@ -0,0 +1,121 @@
|
||||
# MCP client conformance
|
||||
|
||||
This directory tests the actual Codex executable against the official Model
|
||||
Context Protocol client conformance suite. It exercises the shipping legacy,
|
||||
intermediate `2025-11-25`, and modern `2026-07-28` protocols, localhost HTTP,
|
||||
stdio, OAuth, and additional transport and security regression fixtures.
|
||||
|
||||
The official upstream suite is pinned to
|
||||
`modelcontextprotocol/conformance@49103de6ed70804e940637bf3e9e29e4a3f54e64`.
|
||||
Use Node.js 22 and Python 3.10 or later.
|
||||
|
||||
## Run the conformance gate
|
||||
|
||||
First install the frozen workspace dependencies and build Codex:
|
||||
|
||||
```bash
|
||||
pnpm install --frozen-lockfile
|
||||
cargo build --locked --manifest-path codex-rs/Cargo.toml -p codex-cli --bin codex
|
||||
```
|
||||
|
||||
From a published Codex checkout, run:
|
||||
|
||||
```bash
|
||||
python3 scripts/mcp_conformance/run_codex_compliance.py \
|
||||
codex-rs/target/debug/codex \
|
||||
--conformance-cli node_modules/@modelcontextprotocol/conformance/dist/index.js \
|
||||
--baseline-report scripts/mcp_conformance/regression-baseline-v1.json \
|
||||
--report /tmp/codex-mcp-conformance.json
|
||||
```
|
||||
|
||||
The positional executable can also point to an already built Codex binary.
|
||||
`--conformance-cli` selects the exact, lockfile-installed upstream JavaScript
|
||||
runner instead of downloading a moving version during a test.
|
||||
|
||||
## What the baseline means
|
||||
|
||||
`regression-baseline-v1.json` is a compact, reviewed snapshot of the upstream
|
||||
revision, required protocol versions, HTTP and stdio transports, enabled modern
|
||||
feature, OAuth coverage, and individual passing and failing check identities.
|
||||
|
||||
The gate exits successfully only when:
|
||||
|
||||
- The upstream suite and modern feature match the committed baseline.
|
||||
- The shipping legacy, intermediate, and modern protocols are actually tested.
|
||||
- The required HTTP, stdio, and authentication scenarios are actually run.
|
||||
- Every previously passing check still passes.
|
||||
- No additional check fails.
|
||||
|
||||
Existing known failures remain visible in the complete JSON report. In
|
||||
particular, `success` describes complete upstream conformance and
|
||||
`regressionGate.success` describes the no-new-regressions merge gate; the gate
|
||||
does not relabel an existing failure as a pass.
|
||||
|
||||
Create a compact baseline from a reviewed complete report without contacting
|
||||
the upstream suite again:
|
||||
|
||||
```bash
|
||||
python3 scripts/mcp_conformance/run_codex_compliance.py \
|
||||
/absolute/path/to/codex \
|
||||
--baseline-report /absolute/path/to/full-conformance-report.json \
|
||||
--extract-baseline /tmp/mcp-conformance-regression-baseline-v1.json
|
||||
```
|
||||
|
||||
Alternatively, add `--write-baseline /tmp/mcp-conformance-regression-baseline-v1.json`
|
||||
to a complete conformance run. Review every baseline change; do not regenerate
|
||||
it to conceal a regression.
|
||||
|
||||
## Run the production reviewer regression gate
|
||||
|
||||
The separate reviewer gate tests the real Codex app-server across all three
|
||||
shipping, legacy, and modern protocol modes. It covers stdio and localhost
|
||||
HTTP, exact-integer tool and elicitation schemas, bounded multi-round requests,
|
||||
malformed discovery response IDs, repeated pagination cursors, SSE framing and
|
||||
keepalives, and catalog boundaries. In a published Codex checkout, run:
|
||||
|
||||
```bash
|
||||
python3 scripts/mcp_conformance/review_regressions.py \
|
||||
/absolute/path/to/codex \
|
||||
--mode all \
|
||||
--baseline-report scripts/mcp_conformance/review-regression-baseline-v1.json \
|
||||
--report /tmp/codex-mcp-review-regressions.json
|
||||
```
|
||||
|
||||
A complete, main-derived baseline records all 186 real check identities and all
|
||||
21 required cases. Existing failures remain explicitly visible in the complete
|
||||
report; `regressionGate.success: true` means there are no newly failing or
|
||||
missing checks. Improvements are recorded under `fixedChecks`. The gate never
|
||||
classifies an existing failure as a passing check.
|
||||
|
||||
Extract a compact deterministic reviewer baseline from a reviewed complete
|
||||
production report without rerunning the client:
|
||||
|
||||
```bash
|
||||
python3 scripts/mcp_conformance/review_regressions.py \
|
||||
/absolute/path/to/codex \
|
||||
--baseline-report /absolute/path/to/full-review-regressions.json \
|
||||
--extract-baseline /tmp/review-regression-baseline-v1.json
|
||||
```
|
||||
|
||||
Review every baseline update. Do not regenerate a baseline to hide a regression.
|
||||
|
||||
## Run the fixture self-tests
|
||||
|
||||
```bash
|
||||
env PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 \
|
||||
python3 -m pytest -q scripts/mcp_conformance
|
||||
```
|
||||
|
||||
## Run the required SDK integration
|
||||
|
||||
The existing required SDK workflow runs the complete Python fixture self-tests.
|
||||
Its TypeScript job builds the actual Codex executable, sets `CODEX_EXEC_PATH`,
|
||||
installs the pinned upstream conformance runner, and runs both the official
|
||||
authenticated suite and the separate production reviewer regression matrix.
|
||||
Neither gate can be skipped. To reproduce the focused integration locally:
|
||||
|
||||
```bash
|
||||
CODEX_EXEC_PATH=/absolute/path/to/codex \
|
||||
pnpm --filter @openai/codex-sdk test -- \
|
||||
--runInBand tests/mcpConformance.test.ts
|
||||
```
|
||||
976
scripts/mcp_conformance/codex_conformance_adapter.py
Normal file
976
scripts/mcp_conformance/codex_conformance_adapter.py
Normal file
@@ -0,0 +1,976 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Adapt one official MCP client scenario to Codex app-server requests."""
|
||||
|
||||
import ipaddress
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import traceback
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
from dataclasses import asdict, dataclass
|
||||
from pathlib import Path
|
||||
from typing import Mapping, Sequence
|
||||
from urllib.request import HTTPRedirectHandler, ProxyHandler, Request, build_opener # noqa: TID251
|
||||
|
||||
_MODULE_DIR = Path(__file__).resolve().parent
|
||||
if str(_MODULE_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(_MODULE_DIR))
|
||||
|
||||
from run_codex_compliance import ( # noqa: E402 - direct scripts must first add their sibling directory.
|
||||
MODERN_VERSION,
|
||||
TEST_SERVER_NAME,
|
||||
AppServerClient,
|
||||
AppServerError,
|
||||
_call_tool,
|
||||
_command_detail,
|
||||
_isolated_environment,
|
||||
_response_result,
|
||||
_run_command,
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class Step:
|
||||
name: str
|
||||
success: bool
|
||||
detail: str
|
||||
|
||||
|
||||
class AdapterFailure(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
CIMD_CLIENT_METADATA_URL = "https://conformance-test.local/client-metadata.json"
|
||||
PRE_REGISTERED_CLIENT_SECRET_ENV_VAR = "MCP_CONFORMANCE_CLIENT_SECRET"
|
||||
AUTH_COMPLETION_METHOD = "mcpServer/oauthLogin/completed"
|
||||
EXPECTED_AUTH_REJECTION_SCENARIOS = frozenset(
|
||||
{
|
||||
"auth/resource-mismatch",
|
||||
"auth/iss-supported-missing",
|
||||
"auth/iss-wrong-issuer",
|
||||
"auth/iss-unexpected",
|
||||
"auth/iss-normalized",
|
||||
"auth/metadata-issuer-mismatch",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _required_path(name: str) -> Path:
|
||||
value = os.environ.get(name)
|
||||
if not value:
|
||||
raise AdapterFailure(f"{name} is required")
|
||||
return Path(value).expanduser().resolve()
|
||||
|
||||
|
||||
def _required_env(name: str) -> str:
|
||||
value = os.environ.get(name)
|
||||
if not value:
|
||||
raise AdapterFailure(f"{name} is required")
|
||||
return value
|
||||
|
||||
|
||||
def _conformance_context() -> dict[str, object]:
|
||||
raw = os.environ.get("MCP_CONFORMANCE_CONTEXT")
|
||||
if not raw:
|
||||
return {}
|
||||
try:
|
||||
decoded = json.loads(raw)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise AdapterFailure(f"invalid MCP_CONFORMANCE_CONTEXT: {exc}") from exc
|
||||
if not isinstance(decoded, dict):
|
||||
raise AdapterFailure("MCP_CONFORMANCE_CONTEXT was not an object")
|
||||
return decoded
|
||||
|
||||
|
||||
def _result_or_raise(
|
||||
response: Mapping[str, object],
|
||||
operation: str,
|
||||
) -> dict[str, object]:
|
||||
result, detail = _response_result(response)
|
||||
if result is None:
|
||||
raise AdapterFailure(f"{operation}: {detail}")
|
||||
return result
|
||||
|
||||
|
||||
def _server_entry(inventory: Mapping[str, object]) -> dict[str, object]:
|
||||
entries = inventory.get("data")
|
||||
if not isinstance(entries, list):
|
||||
raise AdapterFailure("mcpServerStatus/list returned no data array")
|
||||
for entry in entries:
|
||||
if isinstance(entry, dict) and entry.get("name") == TEST_SERVER_NAME:
|
||||
return entry
|
||||
raise AdapterFailure(f"{TEST_SERVER_NAME!r} was absent from MCP status")
|
||||
|
||||
|
||||
def _thread_id(client: AppServerClient, workspace: Path) -> str:
|
||||
result = _result_or_raise(
|
||||
client.request(
|
||||
"thread/start",
|
||||
{"cwd": str(workspace), "ephemeral": True},
|
||||
),
|
||||
"thread/start",
|
||||
)
|
||||
thread = result.get("thread")
|
||||
if not isinstance(thread, dict) or not isinstance(thread.get("id"), str):
|
||||
raise AdapterFailure("thread/start did not return a thread id")
|
||||
return str(thread["id"])
|
||||
|
||||
|
||||
def _call(
|
||||
client: AppServerClient,
|
||||
*,
|
||||
thread_id: str,
|
||||
tool: str,
|
||||
arguments: Mapping[str, object],
|
||||
) -> None:
|
||||
result, detail = _call_tool(
|
||||
client,
|
||||
thread_id=thread_id,
|
||||
tool=tool,
|
||||
arguments=arguments,
|
||||
)
|
||||
if result is None:
|
||||
raise AdapterFailure(f"tool {tool}: {detail}")
|
||||
|
||||
|
||||
def _elicitation_content(
|
||||
scenario: str,
|
||||
_params: Mapping[str, object],
|
||||
) -> Mapping[str, object]:
|
||||
if scenario == "elicitation-sep1034-client-defaults":
|
||||
# The official scenario deliberately supplies no values. Codex, as the
|
||||
# MCP client under test, must materialize the JSON Schema defaults.
|
||||
return {}
|
||||
if scenario == "sep-2322-client-request-state":
|
||||
return {"confirmed": True}
|
||||
return {"confirmation": "confirmed"}
|
||||
|
||||
|
||||
def _context_tool_calls() -> list[tuple[str, dict[str, object]]]:
|
||||
context = _conformance_context()
|
||||
if not context:
|
||||
raise AdapterFailure(
|
||||
"official scenario did not provide MCP_CONFORMANCE_CONTEXT"
|
||||
)
|
||||
calls = context.get("toolCalls")
|
||||
if not isinstance(calls, list):
|
||||
raise AdapterFailure("official scenario context did not contain toolCalls")
|
||||
result: list[tuple[str, dict[str, object]]] = []
|
||||
for call in calls:
|
||||
if (
|
||||
not isinstance(call, dict)
|
||||
or not isinstance(call.get("name"), str)
|
||||
or not isinstance(call.get("arguments"), dict)
|
||||
):
|
||||
raise AdapterFailure("official scenario contained an invalid tool call")
|
||||
result.append((str(call["name"]), dict(call["arguments"])))
|
||||
return result
|
||||
|
||||
|
||||
def _is_loopback_hostname(hostname: str | None) -> bool:
|
||||
if hostname is None:
|
||||
return False
|
||||
if hostname.lower() == "localhost":
|
||||
return True
|
||||
try:
|
||||
return ipaddress.ip_address(hostname).is_loopback
|
||||
except ValueError:
|
||||
return False
|
||||
|
||||
|
||||
def _validated_callback_url(authorization_url: str, location: str) -> str:
|
||||
authorization = urllib.parse.urlsplit(authorization_url)
|
||||
query = urllib.parse.parse_qs(authorization.query, keep_blank_values=True)
|
||||
redirect_values = query.get("redirect_uri")
|
||||
state_values = query.get("state")
|
||||
if not redirect_values or len(redirect_values) != 1:
|
||||
raise AdapterFailure(
|
||||
"authorization request did not contain exactly one redirect_uri"
|
||||
)
|
||||
if not state_values or len(state_values) != 1:
|
||||
raise AdapterFailure("authorization request did not contain exactly one state")
|
||||
|
||||
redirect = urllib.parse.urlsplit(redirect_values[0])
|
||||
if (
|
||||
redirect.scheme != "http"
|
||||
or not _is_loopback_hostname(redirect.hostname)
|
||||
or redirect.username is not None
|
||||
or redirect.password is not None
|
||||
or redirect.port is None
|
||||
or redirect.query
|
||||
or redirect.fragment
|
||||
):
|
||||
raise AdapterFailure("OAuth redirect_uri was not a safe loopback HTTP endpoint")
|
||||
|
||||
callback_url = urllib.parse.urljoin(authorization_url, location)
|
||||
callback = urllib.parse.urlsplit(callback_url)
|
||||
expected_endpoint = (
|
||||
redirect.scheme,
|
||||
redirect.hostname,
|
||||
redirect.port,
|
||||
redirect.path,
|
||||
)
|
||||
actual_endpoint = (
|
||||
callback.scheme,
|
||||
callback.hostname,
|
||||
callback.port,
|
||||
callback.path,
|
||||
)
|
||||
if (
|
||||
actual_endpoint != expected_endpoint
|
||||
or callback.username is not None
|
||||
or callback.password is not None
|
||||
or callback.fragment
|
||||
):
|
||||
raise AdapterFailure(
|
||||
"authorization server redirect did not target Codex's exact loopback callback"
|
||||
)
|
||||
|
||||
callback_query = urllib.parse.parse_qs(callback.query, keep_blank_values=True)
|
||||
callback_states = callback_query.get("state")
|
||||
if callback_states != state_values:
|
||||
raise AdapterFailure(
|
||||
"authorization server redirect did not preserve OAuth state"
|
||||
)
|
||||
for parameter in ("code", "error", "iss"):
|
||||
values = callback_query.get(parameter)
|
||||
if values is not None and (len(values) != 1 or not values[0]):
|
||||
raise AdapterFailure(
|
||||
f"authorization server redirect must contain exactly one nonempty {parameter}"
|
||||
)
|
||||
if ("code" in callback_query) == ("error" in callback_query):
|
||||
raise AdapterFailure(
|
||||
"authorization server redirect must contain exactly one of code or error"
|
||||
)
|
||||
return callback_url
|
||||
|
||||
|
||||
class _NoRedirect(HTTPRedirectHandler):
|
||||
def redirect_request(
|
||||
self,
|
||||
req: Request,
|
||||
fp: object,
|
||||
code: int,
|
||||
msg: str,
|
||||
headers: Mapping[str, str],
|
||||
newurl: str,
|
||||
) -> Request | None:
|
||||
del req, fp, code, msg, headers, newurl
|
||||
return None
|
||||
|
||||
|
||||
def _open_without_redirects(url: str, timeout_seconds: float) -> tuple[int, str | None]:
|
||||
opener = build_opener(
|
||||
ProxyHandler({}),
|
||||
_NoRedirect(),
|
||||
)
|
||||
request = Request(
|
||||
url,
|
||||
method="GET",
|
||||
headers={"Accept": "text/html,application/xhtml+xml"},
|
||||
)
|
||||
try:
|
||||
with opener.open(request, timeout=timeout_seconds) as response:
|
||||
return response.status, response.headers.get("Location")
|
||||
except urllib.error.HTTPError as exc:
|
||||
# With redirects disabled urllib represents 3xx as HTTPError. Closing
|
||||
# the body promptly prevents an authorization page from being retained.
|
||||
try:
|
||||
return exc.code, exc.headers.get("Location")
|
||||
finally:
|
||||
exc.close()
|
||||
|
||||
|
||||
def _drive_headless_authorization(
|
||||
authorization_url: str,
|
||||
*,
|
||||
timeout_seconds: float,
|
||||
) -> None:
|
||||
status, location = _open_without_redirects(authorization_url, timeout_seconds)
|
||||
if status not in {301, 302, 303, 307, 308} or not location:
|
||||
raise AdapterFailure(
|
||||
"authorization endpoint did not issue the expected callback redirect"
|
||||
)
|
||||
callback_url = _validated_callback_url(authorization_url, location)
|
||||
callback_status, _ = _open_without_redirects(callback_url, timeout_seconds)
|
||||
# Error callbacks may intentionally return a 4xx after notifying the OAuth
|
||||
# waiter. The completion notification is the authoritative outcome.
|
||||
if not 200 <= callback_status < 500:
|
||||
raise AdapterFailure("Codex OAuth callback endpoint returned an invalid status")
|
||||
|
||||
|
||||
def _oauth_client_id(
|
||||
scenario: str,
|
||||
context: Mapping[str, object],
|
||||
*,
|
||||
require_production_client_identity: bool = False,
|
||||
) -> str | None:
|
||||
if scenario == "auth/basic-cimd":
|
||||
return None if require_production_client_identity else CIMD_CLIENT_METADATA_URL
|
||||
if scenario == "auth/pre-registration":
|
||||
client_id = context.get("client_id")
|
||||
if not isinstance(client_id, str) or not client_id:
|
||||
raise AdapterFailure("pre-registration context did not contain client_id")
|
||||
return client_id
|
||||
return None
|
||||
|
||||
|
||||
def _write_auth_registration(
|
||||
config_path: Path,
|
||||
*,
|
||||
server_url: str,
|
||||
oauth_client_id: str | None,
|
||||
oauth_client_secret_env_var: str | None = None,
|
||||
) -> None:
|
||||
existing = config_path.read_text(encoding="utf-8") if config_path.exists() else ""
|
||||
block = f"\n[mcp_servers.{TEST_SERVER_NAME}]\nurl = {json.dumps(server_url, ensure_ascii=False)}\n"
|
||||
if oauth_client_id is not None:
|
||||
block += (
|
||||
f"\n[mcp_servers.{TEST_SERVER_NAME}.oauth]\n"
|
||||
f"client_id = {json.dumps(oauth_client_id, ensure_ascii=False)}\n"
|
||||
)
|
||||
if oauth_client_secret_env_var is not None:
|
||||
block += (
|
||||
"client_secret_env_var = "
|
||||
f"{json.dumps(oauth_client_secret_env_var, ensure_ascii=False)}\n"
|
||||
)
|
||||
config_path.write_text(existing.rstrip() + "\n" + block, encoding="utf-8")
|
||||
|
||||
|
||||
def _validate_oauth_secret_not_persisted(codex_home: Path, client_secret: str) -> None:
|
||||
if not client_secret:
|
||||
return
|
||||
|
||||
candidates = [codex_home / "config.toml", *codex_home.rglob(".credentials.json")]
|
||||
secret_bytes = client_secret.encode("utf-8")
|
||||
for candidate in dict.fromkeys(candidates):
|
||||
if not candidate.is_file():
|
||||
continue
|
||||
if secret_bytes in candidate.read_bytes():
|
||||
raise AdapterFailure(
|
||||
"environment-provided OAuth client secret was persisted in "
|
||||
f"{candidate.relative_to(codex_home)}"
|
||||
)
|
||||
|
||||
|
||||
def _oauth_login(
|
||||
client: AppServerClient,
|
||||
*,
|
||||
scopes: Sequence[str] | None,
|
||||
timeout_seconds: float,
|
||||
) -> tuple[bool, str | None]:
|
||||
event_index = len(client.events)
|
||||
params: dict[str, object] = {
|
||||
"name": TEST_SERVER_NAME,
|
||||
"timeoutSecs": max(1, round(timeout_seconds)),
|
||||
}
|
||||
if scopes is not None:
|
||||
params["scopes"] = list(scopes)
|
||||
response = client.request("mcpServer/oauth/login", params)
|
||||
result, detail = _response_result(response)
|
||||
if result is None:
|
||||
return False, detail
|
||||
authorization_url = result.get("authorizationUrl")
|
||||
if not isinstance(authorization_url, str):
|
||||
raise AdapterFailure("mcpServer/oauth/login returned no authorization URL")
|
||||
|
||||
_drive_headless_authorization(
|
||||
authorization_url,
|
||||
timeout_seconds=timeout_seconds,
|
||||
)
|
||||
event = client.wait_for_notification(
|
||||
AUTH_COMPLETION_METHOD,
|
||||
predicate=lambda params: params.get("name") == TEST_SERVER_NAME,
|
||||
after_event_index=event_index,
|
||||
)
|
||||
params_value = event.get("params")
|
||||
if not isinstance(params_value, dict):
|
||||
raise AdapterFailure("OAuth completion notification did not contain params")
|
||||
success = params_value.get("success") is True
|
||||
error = params_value.get("error")
|
||||
return success, str(error) if error is not None else None
|
||||
|
||||
|
||||
def _reload_mcp(client: AppServerClient) -> None:
|
||||
_result_or_raise(
|
||||
client.request("config/mcpServer/reload", None),
|
||||
"config/mcpServer/reload",
|
||||
)
|
||||
|
||||
|
||||
def _auth_inventory(client: AppServerClient) -> dict[str, object]:
|
||||
inventory = _result_or_raise(
|
||||
client.request("mcpServerStatus/list", {"detail": "full"}),
|
||||
"mcpServerStatus/list",
|
||||
)
|
||||
_server_entry(inventory)
|
||||
return inventory
|
||||
|
||||
|
||||
def _auth_tool_call(client: AppServerClient, workspace: Path) -> None:
|
||||
thread_id = _thread_id(client, workspace)
|
||||
_call(
|
||||
client,
|
||||
thread_id=thread_id,
|
||||
tool="test-tool",
|
||||
arguments={},
|
||||
)
|
||||
|
||||
|
||||
def _login_reload_and_call(
|
||||
client: AppServerClient,
|
||||
*,
|
||||
workspace: Path,
|
||||
timeout_seconds: float,
|
||||
scopes: Sequence[str] | None = None,
|
||||
) -> None:
|
||||
success, error = _oauth_login(
|
||||
client,
|
||||
scopes=scopes,
|
||||
timeout_seconds=timeout_seconds,
|
||||
)
|
||||
if not success:
|
||||
raise AdapterFailure(f"OAuth login failed: {error or 'unknown error'}")
|
||||
_reload_mcp(client)
|
||||
_auth_inventory(client)
|
||||
_auth_tool_call(client, workspace)
|
||||
|
||||
|
||||
def _exercise_auth_scenario(
|
||||
client: AppServerClient,
|
||||
*,
|
||||
scenario: str,
|
||||
workspace: Path,
|
||||
timeout_seconds: float,
|
||||
require_automatic_auth: bool = False,
|
||||
) -> str:
|
||||
if scenario in EXPECTED_AUTH_REJECTION_SCENARIOS:
|
||||
success, error = _oauth_login(
|
||||
client,
|
||||
scopes=None,
|
||||
timeout_seconds=timeout_seconds,
|
||||
)
|
||||
if success:
|
||||
raise AdapterFailure(
|
||||
"OAuth flow unexpectedly accepted authorization metadata that must be rejected"
|
||||
)
|
||||
return f"rejected invalid authorization flow: {error or 'request rejected'}"
|
||||
|
||||
if scenario == "auth/scope-step-up":
|
||||
success, error = _oauth_login(
|
||||
client,
|
||||
scopes=None,
|
||||
timeout_seconds=timeout_seconds,
|
||||
)
|
||||
if not success:
|
||||
raise AdapterFailure(
|
||||
f"initial OAuth login failed: {error or 'unknown error'}"
|
||||
)
|
||||
_reload_mcp(client)
|
||||
_auth_inventory(client)
|
||||
if require_automatic_auth:
|
||||
_auth_tool_call(client, workspace)
|
||||
return "Codex automatically recovered from the challenged OAuth scope"
|
||||
try:
|
||||
_auth_tool_call(client, workspace)
|
||||
except AdapterFailure:
|
||||
pass
|
||||
else:
|
||||
raise AdapterFailure(
|
||||
"scope-step-up tool call did not request additional scope"
|
||||
)
|
||||
|
||||
# The resource server challenges with only the missing scope. The Rust
|
||||
# client must union it with the previously granted scope.
|
||||
_login_reload_and_call(
|
||||
client,
|
||||
workspace=workspace,
|
||||
timeout_seconds=timeout_seconds,
|
||||
scopes=("mcp:write",),
|
||||
)
|
||||
return "completed initial and scope-upgrade authorization flows"
|
||||
|
||||
if scenario == "auth/scope-retry-limit":
|
||||
if require_automatic_auth:
|
||||
success, error = _oauth_login(
|
||||
client,
|
||||
scopes=None,
|
||||
timeout_seconds=timeout_seconds,
|
||||
)
|
||||
if not success:
|
||||
raise AdapterFailure(
|
||||
f"initial OAuth login failed: {error or 'unknown error'}"
|
||||
)
|
||||
_reload_mcp(client)
|
||||
_auth_inventory(client)
|
||||
try:
|
||||
_auth_tool_call(client, workspace)
|
||||
except AdapterFailure:
|
||||
return "observed Codex's production OAuth retry-limit behavior"
|
||||
raise AdapterFailure(
|
||||
"retry-limit scenario unexpectedly completed the tool call"
|
||||
)
|
||||
for attempt in range(3):
|
||||
success, error = _oauth_login(
|
||||
client,
|
||||
scopes=None if attempt == 0 else ("mcp:write",),
|
||||
timeout_seconds=timeout_seconds,
|
||||
)
|
||||
if not success:
|
||||
raise AdapterFailure(f"OAuth retry failed: {error or 'unknown error'}")
|
||||
_reload_mcp(client)
|
||||
_auth_inventory(client)
|
||||
try:
|
||||
_auth_tool_call(client, workspace)
|
||||
except AdapterFailure:
|
||||
continue
|
||||
raise AdapterFailure(
|
||||
"retry-limit scenario unexpectedly completed the tool call"
|
||||
)
|
||||
return "stopped after three unsuccessful authorization attempts"
|
||||
|
||||
if scenario == "auth/authorization-server-migration":
|
||||
success, error = _oauth_login(
|
||||
client,
|
||||
scopes=None,
|
||||
timeout_seconds=timeout_seconds,
|
||||
)
|
||||
if not success:
|
||||
raise AdapterFailure(
|
||||
f"initial OAuth login failed: {error or 'unknown error'}"
|
||||
)
|
||||
_reload_mcp(client)
|
||||
_auth_inventory(client)
|
||||
if require_automatic_auth:
|
||||
_auth_tool_call(client, workspace)
|
||||
return (
|
||||
"Codex automatically registered with the migrated authorization server"
|
||||
)
|
||||
try:
|
||||
_auth_tool_call(client, workspace)
|
||||
except AdapterFailure:
|
||||
pass
|
||||
else:
|
||||
raise AdapterFailure("migration scenario did not require re-authorization")
|
||||
_login_reload_and_call(
|
||||
client,
|
||||
workspace=workspace,
|
||||
timeout_seconds=timeout_seconds,
|
||||
)
|
||||
return (
|
||||
"re-authorized after the protected resource changed authorization servers"
|
||||
)
|
||||
|
||||
_login_reload_and_call(
|
||||
client,
|
||||
workspace=workspace,
|
||||
timeout_seconds=timeout_seconds,
|
||||
)
|
||||
return "completed OAuth login, authenticated discovery, and tool call"
|
||||
|
||||
|
||||
def _exercise_scenario(
|
||||
client: AppServerClient,
|
||||
*,
|
||||
scenario: str,
|
||||
workspace: Path,
|
||||
inventory: Mapping[str, object],
|
||||
) -> str:
|
||||
thread_id = _thread_id(client, workspace)
|
||||
|
||||
if scenario == "tools_call":
|
||||
_call(
|
||||
client,
|
||||
thread_id=thread_id,
|
||||
tool="add_numbers",
|
||||
arguments={"a": 2, "b": 3},
|
||||
)
|
||||
return "called add_numbers"
|
||||
|
||||
if scenario == "elicitation-sep1034-client-defaults":
|
||||
_call(
|
||||
client,
|
||||
thread_id=thread_id,
|
||||
tool="test_client_elicitation_defaults",
|
||||
arguments={},
|
||||
)
|
||||
return "completed legacy elicitation with omitted optional fields"
|
||||
|
||||
if scenario == "sse-retry":
|
||||
_call(
|
||||
client,
|
||||
thread_id=thread_id,
|
||||
tool="test_reconnection",
|
||||
arguments={},
|
||||
)
|
||||
return "completed the SSE reconnection tool call"
|
||||
|
||||
if scenario == "sep-2322-client-request-state":
|
||||
for tool in (
|
||||
"test_mrtr_unrelated",
|
||||
"test_mrtr_no_result_type",
|
||||
"test_mrtr_echo_state",
|
||||
"test_mrtr_no_state",
|
||||
):
|
||||
_call(client, thread_id=thread_id, tool=tool, arguments={})
|
||||
return "completed all four MRTR flows"
|
||||
|
||||
if scenario == "http-standard-headers":
|
||||
_call(
|
||||
client,
|
||||
thread_id=thread_id,
|
||||
tool="test_headers",
|
||||
arguments={},
|
||||
)
|
||||
entry = _server_entry(inventory)
|
||||
resources = entry.get("resources")
|
||||
if not isinstance(resources, list) or not resources:
|
||||
raise AdapterFailure("standard-header scenario exposed no resources")
|
||||
first = resources[0]
|
||||
if not isinstance(first, dict) or not isinstance(first.get("uri"), str):
|
||||
raise AdapterFailure("standard-header scenario resource had no URI")
|
||||
_result_or_raise(
|
||||
client.request(
|
||||
"mcpServer/resource/read",
|
||||
{
|
||||
"threadId": thread_id,
|
||||
"server": TEST_SERVER_NAME,
|
||||
"uri": first["uri"],
|
||||
},
|
||||
),
|
||||
"mcpServer/resource/read",
|
||||
)
|
||||
return "called a tool and read a resource"
|
||||
|
||||
if scenario == "http-custom-headers":
|
||||
for tool, arguments in _context_tool_calls():
|
||||
_call(
|
||||
client,
|
||||
thread_id=thread_id,
|
||||
tool=tool,
|
||||
arguments=arguments,
|
||||
)
|
||||
return "called both custom-header tools with official values"
|
||||
|
||||
if scenario == "http-invalid-tool-headers":
|
||||
_call(
|
||||
client,
|
||||
thread_id=thread_id,
|
||||
tool="valid_tool",
|
||||
arguments={"region": "us-west1"},
|
||||
)
|
||||
return "called the valid tool after filtering malformed definitions"
|
||||
|
||||
if scenario in {
|
||||
"initialize",
|
||||
"request-metadata",
|
||||
"json-schema-ref-no-deref",
|
||||
}:
|
||||
# Discovery performed by mcpServerStatus/list is the behavior these
|
||||
# scenarios observe. Starting a thread also exercises the initialized
|
||||
# server through the same public app-server interface as the other
|
||||
# scenarios.
|
||||
return "completed MCP discovery"
|
||||
|
||||
raise AdapterFailure(f"unsupported official scenario: {scenario}")
|
||||
|
||||
|
||||
def run_adapter(server_url: str) -> dict[str, object]:
|
||||
codex_binary = _required_path("CODEX_CONFORMANCE_BINARY")
|
||||
codex_home = _required_path("CODEX_CONFORMANCE_HOME")
|
||||
scenario = _required_env("MCP_CONFORMANCE_SCENARIO")
|
||||
protocol_version = _required_env("MCP_CONFORMANCE_PROTOCOL_VERSION")
|
||||
timeout_seconds = float(os.environ.get("CODEX_CONFORMANCE_TIMEOUT", "30"))
|
||||
enable_modern_feature = (
|
||||
os.environ.get("CODEX_CONFORMANCE_ENABLE_MODERN_FEATURE", "1") != "0"
|
||||
)
|
||||
require_automatic_auth = (
|
||||
os.environ.get("CODEX_CONFORMANCE_REQUIRE_AUTOMATIC_AUTH", "0") == "1"
|
||||
)
|
||||
context = _conformance_context()
|
||||
|
||||
codex_home.mkdir(parents=True, exist_ok=True)
|
||||
workspace = codex_home / "workspace"
|
||||
workspace.mkdir()
|
||||
env = _isolated_environment(codex_home)
|
||||
# Scenario context can contain ephemeral OAuth client secrets. The adapter
|
||||
# consumes it directly and does not expose the full blob to Codex.
|
||||
env.pop("MCP_CONFORMANCE_CONTEXT", None)
|
||||
steps: list[Step] = []
|
||||
registered = False
|
||||
error: str | None = None
|
||||
|
||||
try:
|
||||
if scenario.startswith("auth/"):
|
||||
config_path = codex_home / "config.toml"
|
||||
config_path.write_text(
|
||||
'mcp_oauth_credentials_store = "file"\n',
|
||||
encoding="utf-8",
|
||||
)
|
||||
steps.append(
|
||||
Step(
|
||||
"oauth_store_configuration",
|
||||
True,
|
||||
"configured the isolated file OAuth credential store",
|
||||
)
|
||||
)
|
||||
|
||||
if protocol_version == MODERN_VERSION and enable_modern_feature:
|
||||
feature = _run_command(
|
||||
[
|
||||
str(codex_binary),
|
||||
"features",
|
||||
"enable",
|
||||
"mcp_2026_07_28",
|
||||
],
|
||||
env=env,
|
||||
cwd=workspace,
|
||||
timeout_seconds=timeout_seconds,
|
||||
)
|
||||
feature_configured = feature.returncode == 0
|
||||
steps.append(
|
||||
Step(
|
||||
"modern_feature_configuration",
|
||||
feature_configured,
|
||||
"configured mcp_2026_07_28 before MCP startup"
|
||||
if feature_configured
|
||||
else _command_detail(feature),
|
||||
)
|
||||
)
|
||||
if not feature_configured:
|
||||
raise AdapterFailure("could not configure the modern MCP feature")
|
||||
|
||||
oauth_client_id = _oauth_client_id(
|
||||
scenario,
|
||||
context,
|
||||
require_production_client_identity=require_automatic_auth,
|
||||
)
|
||||
oauth_client_secret = context.get("client_secret")
|
||||
oauth_client_secret_env_var = None
|
||||
if (
|
||||
scenario == "auth/pre-registration"
|
||||
and isinstance(oauth_client_secret, str)
|
||||
and oauth_client_secret
|
||||
):
|
||||
env[PRE_REGISTERED_CLIENT_SECRET_ENV_VAR] = oauth_client_secret
|
||||
oauth_client_secret_env_var = PRE_REGISTERED_CLIENT_SECRET_ENV_VAR
|
||||
if scenario.startswith("auth/"):
|
||||
_write_auth_registration(
|
||||
codex_home / "config.toml",
|
||||
server_url=server_url,
|
||||
oauth_client_id=oauth_client_id,
|
||||
oauth_client_secret_env_var=oauth_client_secret_env_var,
|
||||
)
|
||||
registered = True
|
||||
steps.append(
|
||||
Step(
|
||||
"mcp_registration",
|
||||
True,
|
||||
"wrote isolated registration without triggering CLI auto-login",
|
||||
)
|
||||
)
|
||||
else:
|
||||
add = _run_command(
|
||||
[
|
||||
str(codex_binary),
|
||||
"mcp",
|
||||
"add",
|
||||
TEST_SERVER_NAME,
|
||||
"--url",
|
||||
server_url,
|
||||
],
|
||||
env=env,
|
||||
cwd=workspace,
|
||||
timeout_seconds=timeout_seconds,
|
||||
)
|
||||
registered = add.returncode == 0
|
||||
steps.append(Step("mcp_add", registered, _command_detail(add)))
|
||||
if not registered:
|
||||
raise AdapterFailure("codex mcp add failed")
|
||||
|
||||
get = _run_command(
|
||||
[
|
||||
str(codex_binary),
|
||||
"mcp",
|
||||
"get",
|
||||
TEST_SERVER_NAME,
|
||||
"--json",
|
||||
],
|
||||
env=env,
|
||||
cwd=workspace,
|
||||
timeout_seconds=timeout_seconds,
|
||||
)
|
||||
registration_ok = False
|
||||
if get.returncode == 0:
|
||||
try:
|
||||
decoded = json.loads(get.stdout)
|
||||
transport = (
|
||||
decoded.get("transport") if isinstance(decoded, dict) else None
|
||||
)
|
||||
registration_ok = (
|
||||
isinstance(transport, dict)
|
||||
and transport.get("type") == "streamable_http"
|
||||
and transport.get("url") == server_url
|
||||
)
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
steps.append(
|
||||
Step(
|
||||
"mcp_get",
|
||||
registration_ok,
|
||||
"registered official scenario URL"
|
||||
if registration_ok
|
||||
else _command_detail(get),
|
||||
)
|
||||
)
|
||||
if not registration_ok:
|
||||
raise AdapterFailure("Codex registration did not preserve the scenario URL")
|
||||
|
||||
with AppServerClient(
|
||||
codex_binary,
|
||||
env=env,
|
||||
cwd=workspace,
|
||||
timeout_seconds=timeout_seconds,
|
||||
elicitation_content=lambda params: _elicitation_content(scenario, params),
|
||||
) as client:
|
||||
initialize = _result_or_raise(
|
||||
client.request(
|
||||
"initialize",
|
||||
{
|
||||
"clientInfo": {
|
||||
"name": "official-mcp-conformance-adapter",
|
||||
"title": "Official MCP conformance adapter",
|
||||
"version": "1.0.0",
|
||||
},
|
||||
"capabilities": {
|
||||
"experimentalApi": True,
|
||||
"requestAttestation": False,
|
||||
"mcpServerOpenaiFormElicitation": True,
|
||||
},
|
||||
},
|
||||
),
|
||||
"app-server initialize",
|
||||
)
|
||||
steps.append(Step("app_server_initialize", bool(initialize), "initialized"))
|
||||
client.notify("initialized")
|
||||
|
||||
if protocol_version == MODERN_VERSION and enable_modern_feature:
|
||||
feature = _result_or_raise(
|
||||
client.request(
|
||||
"experimentalFeature/enablement/set",
|
||||
{"enablement": {"mcp_2026_07_28": True}},
|
||||
),
|
||||
"experimentalFeature/enablement/set",
|
||||
)
|
||||
enabled = (
|
||||
isinstance(feature.get("enablement"), dict)
|
||||
and feature["enablement"].get("mcp_2026_07_28") is True
|
||||
)
|
||||
steps.append(
|
||||
Step(
|
||||
"modern_feature_enablement",
|
||||
enabled,
|
||||
"enabled mcp_2026_07_28"
|
||||
if enabled
|
||||
else f"unexpected response: {feature!r}",
|
||||
)
|
||||
)
|
||||
if not enabled:
|
||||
raise AdapterFailure("could not enable the modern MCP feature")
|
||||
|
||||
if scenario.startswith("auth/"):
|
||||
detail = _exercise_auth_scenario(
|
||||
client,
|
||||
scenario=scenario,
|
||||
workspace=workspace,
|
||||
timeout_seconds=timeout_seconds,
|
||||
require_automatic_auth=require_automatic_auth,
|
||||
)
|
||||
if scenario == "auth/pre-registration" and isinstance(
|
||||
oauth_client_secret,
|
||||
str,
|
||||
):
|
||||
_validate_oauth_secret_not_persisted(
|
||||
codex_home, oauth_client_secret
|
||||
)
|
||||
steps.append(
|
||||
Step(
|
||||
"oauth_client_secret_not_persisted",
|
||||
True,
|
||||
"environment-provided confidential-client secret was not "
|
||||
"written to configuration or file-backed credentials",
|
||||
)
|
||||
)
|
||||
steps.append(Step("authentication", True, detail))
|
||||
else:
|
||||
inventory = _result_or_raise(
|
||||
client.request("mcpServerStatus/list", {"detail": "full"}),
|
||||
"mcpServerStatus/list",
|
||||
)
|
||||
_server_entry(inventory)
|
||||
steps.append(Step("inventory", True, "official server discovered"))
|
||||
detail = _exercise_scenario(
|
||||
client,
|
||||
scenario=scenario,
|
||||
workspace=workspace,
|
||||
inventory=inventory,
|
||||
)
|
||||
steps.append(Step("scenario", True, detail))
|
||||
except (AdapterFailure, AppServerError, OSError, ValueError) as exc:
|
||||
error = str(exc)
|
||||
if not steps or steps[-1].success:
|
||||
steps.append(Step("scenario", False, error))
|
||||
finally:
|
||||
if registered:
|
||||
remove = _run_command(
|
||||
[str(codex_binary), "mcp", "remove", TEST_SERVER_NAME],
|
||||
env=env,
|
||||
cwd=workspace,
|
||||
timeout_seconds=timeout_seconds,
|
||||
)
|
||||
steps.append(
|
||||
Step("mcp_remove", remove.returncode == 0, _command_detail(remove))
|
||||
)
|
||||
|
||||
success = bool(steps) and all(step.success for step in steps)
|
||||
return {
|
||||
"success": success,
|
||||
"scenario": scenario,
|
||||
"protocolVersion": protocol_version,
|
||||
"automaticAuthRequired": require_automatic_auth,
|
||||
"serverUrl": server_url,
|
||||
"steps": [asdict(step) for step in steps],
|
||||
"error": error,
|
||||
}
|
||||
|
||||
|
||||
def main(argv: Sequence[str] | None = None) -> int:
|
||||
values = list(sys.argv[1:] if argv is None else argv)
|
||||
report_path_value = os.environ.get("CODEX_CONFORMANCE_ADAPTER_REPORT")
|
||||
report: dict[str, object]
|
||||
try:
|
||||
if len(values) != 1:
|
||||
raise AdapterFailure("expected exactly one official scenario server URL")
|
||||
report = run_adapter(values[0])
|
||||
except Exception as exc: # Preserve a diagnostic artifact for the parent.
|
||||
report = {
|
||||
"success": False,
|
||||
"error": str(exc),
|
||||
"traceback": traceback.format_exc(limit=20),
|
||||
"steps": [],
|
||||
}
|
||||
|
||||
if report_path_value:
|
||||
report_path = Path(report_path_value)
|
||||
report_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
report_path.write_text(
|
||||
json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
print(json.dumps(report, ensure_ascii=False, sort_keys=True))
|
||||
return 0 if report.get("success") is True else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
518
scripts/mcp_conformance/official_conformance.py
Normal file
518
scripts/mcp_conformance/official_conformance.py
Normal file
@@ -0,0 +1,518 @@
|
||||
"""Driver for the upstream MCP client conformance suite.
|
||||
|
||||
The upstream suite owns the localhost HTTP server and the wire-level checks.
|
||||
This module deliberately invokes one scenario at a time because the upstream
|
||||
parallel suite path does not currently propagate the client adapter result.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shlex
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Mapping, Sequence
|
||||
|
||||
SHIPPING_LEGACY_VERSION = "2025-06-18"
|
||||
LEGACY_VERSION = "2025-11-25"
|
||||
MODERN_VERSION = "2026-07-28"
|
||||
|
||||
# Reviewed on 2026-07-29. Pinning a commit instead of an npm prerelease keeps
|
||||
# the scenario definitions and result semantics reproducible.
|
||||
OFFICIAL_CONFORMANCE_GIT_REF = "49103de6ed70804e940637bf3e9e29e4a3f54e64"
|
||||
OFFICIAL_CONFORMANCE_REPOSITORY = "modelcontextprotocol/conformance"
|
||||
|
||||
OFFICIAL_NON_AUTH_SCENARIOS: Mapping[str, tuple[str, ...]] = {
|
||||
SHIPPING_LEGACY_VERSION: (
|
||||
"initialize",
|
||||
"tools_call",
|
||||
),
|
||||
LEGACY_VERSION: (
|
||||
"initialize",
|
||||
"tools_call",
|
||||
"elicitation-sep1034-client-defaults",
|
||||
"sse-retry",
|
||||
),
|
||||
MODERN_VERSION: (
|
||||
"tools_call",
|
||||
"request-metadata",
|
||||
"sep-2322-client-request-state",
|
||||
"http-standard-headers",
|
||||
"http-custom-headers",
|
||||
"http-invalid-tool-headers",
|
||||
"json-schema-ref-no-deref",
|
||||
),
|
||||
}
|
||||
|
||||
OFFICIAL_AUTH_SCENARIOS: Mapping[str, tuple[str, ...]] = {
|
||||
SHIPPING_LEGACY_VERSION: (
|
||||
"auth/token-endpoint-auth-basic",
|
||||
"auth/token-endpoint-auth-post",
|
||||
"auth/token-endpoint-auth-none",
|
||||
),
|
||||
LEGACY_VERSION: (
|
||||
"auth/metadata-default",
|
||||
"auth/metadata-var1",
|
||||
"auth/metadata-var2",
|
||||
"auth/metadata-var3",
|
||||
"auth/basic-cimd",
|
||||
"auth/scope-from-www-authenticate",
|
||||
"auth/scope-from-scopes-supported",
|
||||
"auth/scope-omitted-when-undefined",
|
||||
"auth/scope-step-up",
|
||||
"auth/scope-retry-limit",
|
||||
"auth/token-endpoint-auth-basic",
|
||||
"auth/token-endpoint-auth-post",
|
||||
"auth/token-endpoint-auth-none",
|
||||
"auth/pre-registration",
|
||||
),
|
||||
MODERN_VERSION: (
|
||||
"auth/metadata-default",
|
||||
"auth/metadata-var1",
|
||||
"auth/metadata-var2",
|
||||
"auth/metadata-var3",
|
||||
"auth/basic-cimd",
|
||||
"auth/scope-from-www-authenticate",
|
||||
"auth/scope-from-scopes-supported",
|
||||
"auth/scope-omitted-when-undefined",
|
||||
"auth/scope-step-up",
|
||||
"auth/scope-retry-limit",
|
||||
"auth/token-endpoint-auth-basic",
|
||||
"auth/token-endpoint-auth-post",
|
||||
"auth/token-endpoint-auth-none",
|
||||
"auth/pre-registration",
|
||||
"auth/resource-mismatch",
|
||||
"auth/offline-access-scope",
|
||||
"auth/offline-access-not-supported",
|
||||
"auth/authorization-server-migration",
|
||||
"auth/iss-supported",
|
||||
"auth/iss-not-advertised",
|
||||
"auth/iss-supported-missing",
|
||||
"auth/iss-wrong-issuer",
|
||||
"auth/iss-unexpected",
|
||||
"auth/iss-normalized",
|
||||
"auth/metadata-issuer-mismatch",
|
||||
),
|
||||
}
|
||||
|
||||
# These scenarios cover separately negotiated protocol extensions rather than
|
||||
# either dated release. Keep them visible for future adapters, but do not mix
|
||||
# them into versioned release conformance percentages.
|
||||
OFFICIAL_AUTH_EXTENSION_SCENARIOS: tuple[str, ...] = (
|
||||
"auth/client-credentials-jwt",
|
||||
"auth/client-credentials-basic",
|
||||
"auth/enterprise-managed-authorization",
|
||||
"auth/dpop",
|
||||
"auth/dpop-nonce",
|
||||
"auth/wif-jwt-bearer",
|
||||
)
|
||||
|
||||
_SENSITIVE_JSON_FIELD = re.compile(
|
||||
r'("(?:client_secret|private_key_pem|valid_jwt|wrong_audience_jwt|'
|
||||
r'expired_jwt|idp_id_token)"\s*:\s*)"(?:\\.|[^"\\])*"'
|
||||
)
|
||||
_SENSITIVE_ESCAPED_JSON_FIELD = re.compile(
|
||||
r'(\\"(?:client_secret|private_key_pem|valid_jwt|wrong_audience_jwt|'
|
||||
r'expired_jwt|idp_id_token)\\"\s*:\s*)\\"(?:\\\\.|[^"\\])*\\"'
|
||||
)
|
||||
_SENSITIVE_URL_PARAMETER = re.compile(
|
||||
r"([?&](?:code|state|code_challenge|code_verifier|access_token|"
|
||||
r"""refresh_token|client_secret)=)[^&\s"'\\<>]+""",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_BEARER_TOKEN = re.compile(r"(\bBearer\s+)[A-Za-z0-9._~+/=-]+", re.IGNORECASE)
|
||||
|
||||
|
||||
def redact_sensitive_text(value: str) -> str:
|
||||
value = _SENSITIVE_JSON_FIELD.sub(r'\1"[REDACTED]"', value)
|
||||
value = _SENSITIVE_ESCAPED_JSON_FIELD.sub(r'\1\\"[REDACTED]\\"', value)
|
||||
value = _SENSITIVE_URL_PARAMETER.sub(r"\1[REDACTED]", value)
|
||||
return _BEARER_TOKEN.sub(r"\1[REDACTED]", value)
|
||||
|
||||
|
||||
def _scrub_retained_artifacts(scenario_dir: Path) -> None:
|
||||
for name in ("stdout.txt", "stderr.txt"):
|
||||
for path in scenario_dir.rglob(name):
|
||||
try:
|
||||
original = path.read_text(encoding="utf-8")
|
||||
redacted = redact_sensitive_text(original)
|
||||
if redacted != original:
|
||||
path.write_text(redacted, encoding="utf-8")
|
||||
except OSError:
|
||||
continue
|
||||
for path in scenario_dir.rglob(".credentials.json"):
|
||||
try:
|
||||
path.unlink(missing_ok=True)
|
||||
except OSError:
|
||||
continue
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class OfficialCheck:
|
||||
scenario: str
|
||||
check_id: str
|
||||
name: str
|
||||
status: str
|
||||
description: str
|
||||
error_message: str | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class OfficialScenarioResult:
|
||||
scenario: str
|
||||
success: bool
|
||||
checks: list[OfficialCheck] = field(default_factory=list)
|
||||
adapter_success: bool = False
|
||||
adapter_detail: str = ""
|
||||
runner_detail: str = ""
|
||||
|
||||
|
||||
def default_conformance_command() -> list[str]:
|
||||
return [
|
||||
"npx",
|
||||
"--yes",
|
||||
(f"github:{OFFICIAL_CONFORMANCE_REPOSITORY}#{OFFICIAL_CONFORMANCE_GIT_REF}"),
|
||||
]
|
||||
|
||||
|
||||
def scenarios_for_mode(
|
||||
mode: str,
|
||||
requested: Sequence[str] | None = None,
|
||||
*,
|
||||
include_auth: bool = True,
|
||||
) -> tuple[str, ...]:
|
||||
non_auth = OFFICIAL_NON_AUTH_SCENARIOS.get(mode)
|
||||
auth = OFFICIAL_AUTH_SCENARIOS.get(mode)
|
||||
if non_auth is None or auth is None:
|
||||
raise ValueError(f"unsupported MCP protocol version: {mode}")
|
||||
available = non_auth + (auth if include_auth else ())
|
||||
if not requested:
|
||||
return available
|
||||
all_scenarios = {
|
||||
scenario
|
||||
for scenario_map in (
|
||||
OFFICIAL_NON_AUTH_SCENARIOS,
|
||||
OFFICIAL_AUTH_SCENARIOS,
|
||||
)
|
||||
for mode_scenarios in scenario_map.values()
|
||||
for scenario in mode_scenarios
|
||||
}
|
||||
unknown = sorted(set(requested) - all_scenarios)
|
||||
if unknown:
|
||||
raise ValueError(
|
||||
"scenarios are not part of the pinned versioned client suite: "
|
||||
+ ", ".join(unknown)
|
||||
)
|
||||
disabled = sorted(
|
||||
set(requested)
|
||||
& {
|
||||
scenario
|
||||
for mode_scenarios in OFFICIAL_AUTH_SCENARIOS.values()
|
||||
for scenario in mode_scenarios
|
||||
}
|
||||
if not include_auth
|
||||
else ()
|
||||
)
|
||||
if disabled:
|
||||
raise ValueError(
|
||||
"authentication scenarios require auth coverage to be enabled: "
|
||||
+ ", ".join(disabled)
|
||||
)
|
||||
requested_set = set(requested)
|
||||
selected = tuple(scenario for scenario in available if scenario in requested_set)
|
||||
if not selected:
|
||||
raise ValueError(
|
||||
f"requested scenarios are unavailable for MCP protocol version {mode}: "
|
||||
+ ", ".join(sorted(requested_set))
|
||||
)
|
||||
return selected
|
||||
|
||||
|
||||
def _safe_scenario_name(scenario: str) -> str:
|
||||
return re.sub(r"[^A-Za-z0-9_.-]+", "-", scenario)
|
||||
|
||||
|
||||
def _make_adapter_launcher(
|
||||
adapter_script: Path,
|
||||
*,
|
||||
require_automatic_auth: bool = False,
|
||||
) -> Path:
|
||||
# The official runner currently splits --command on spaces before spawning
|
||||
# it. Put a one-token launcher in the system temp directory so arbitrary
|
||||
# checkout paths (including paths with spaces) remain supported.
|
||||
launcher_dir = Path(tempfile.mkdtemp(prefix="mcp-conformance-adapter-"))
|
||||
automatic_auth = "1" if require_automatic_auth else "0"
|
||||
if sys.platform == "win32":
|
||||
launcher = launcher_dir / "client.cmd"
|
||||
adapter_command = subprocess.list2cmdline([sys.executable, str(adapter_script)])
|
||||
launcher.write_text(
|
||||
"@echo off\n"
|
||||
f'set "CODEX_CONFORMANCE_REQUIRE_AUTOMATIC_AUTH={automatic_auth}"\n'
|
||||
f"{adapter_command} %*\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
else:
|
||||
launcher = launcher_dir / "client"
|
||||
launcher.write_text(
|
||||
"#!/bin/sh\n"
|
||||
f"export CODEX_CONFORMANCE_REQUIRE_AUTOMATIC_AUTH={automatic_auth}\n"
|
||||
f'exec {shlex.quote(sys.executable)} {shlex.quote(str(adapter_script))} "$@"\n',
|
||||
encoding="utf-8",
|
||||
)
|
||||
launcher.chmod(0o755)
|
||||
return launcher
|
||||
|
||||
|
||||
def _terminate_process_group(
|
||||
process: subprocess.Popen[str],
|
||||
*,
|
||||
force: bool = False,
|
||||
) -> None:
|
||||
if sys.platform == "win32":
|
||||
command = ["taskkill", "/T", "/PID", str(process.pid)]
|
||||
if force:
|
||||
command.insert(1, "/F")
|
||||
try:
|
||||
result = subprocess.run(
|
||||
command,
|
||||
check=False,
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
)
|
||||
if result.returncode == 0:
|
||||
return
|
||||
except OSError:
|
||||
pass
|
||||
try:
|
||||
if force:
|
||||
process.kill()
|
||||
else:
|
||||
process.terminate()
|
||||
except (OSError, ProcessLookupError):
|
||||
pass
|
||||
return
|
||||
|
||||
try:
|
||||
os.killpg(process.pid, signal.SIGKILL if force else signal.SIGTERM)
|
||||
except (OSError, ProcessLookupError):
|
||||
pass
|
||||
|
||||
|
||||
def _load_checks(result_dir: Path, scenario: str) -> list[OfficialCheck]:
|
||||
check_files = sorted(result_dir.rglob("checks.json"))
|
||||
if len(check_files) != 1:
|
||||
return []
|
||||
try:
|
||||
decoded = json.loads(check_files[0].read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError):
|
||||
return []
|
||||
if not isinstance(decoded, list):
|
||||
return []
|
||||
|
||||
checks: list[OfficialCheck] = []
|
||||
for index, raw in enumerate(decoded, start=1):
|
||||
if not isinstance(raw, dict):
|
||||
continue
|
||||
check_id = str(raw.get("id") or f"unnamed-{index}")
|
||||
name = str(raw.get("name") or check_id)
|
||||
status = str(raw.get("status") or "FAILURE").upper()
|
||||
# INFO entries are the suite's HTTP trace, not conformance assertions.
|
||||
# They remain available in the retained upstream checks.json artifact.
|
||||
if status == "INFO":
|
||||
continue
|
||||
checks.append(
|
||||
OfficialCheck(
|
||||
scenario=scenario,
|
||||
check_id=check_id,
|
||||
name=name,
|
||||
status=status,
|
||||
description=str(raw.get("description") or name),
|
||||
error_message=(
|
||||
redact_sensitive_text(str(raw["errorMessage"]))
|
||||
if raw.get("errorMessage") is not None
|
||||
else None
|
||||
),
|
||||
)
|
||||
)
|
||||
return checks
|
||||
|
||||
|
||||
def _load_adapter_report(report_path: Path) -> tuple[bool, str]:
|
||||
try:
|
||||
decoded = json.loads(report_path.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError) as exc:
|
||||
return False, f"Codex adapter did not write a valid report: {exc}"
|
||||
if not isinstance(decoded, dict):
|
||||
return False, "Codex adapter report was not an object"
|
||||
success = decoded.get("success") is True
|
||||
steps = decoded.get("steps")
|
||||
failed_steps: list[str] = []
|
||||
if isinstance(steps, list):
|
||||
for step in steps:
|
||||
if isinstance(step, dict) and step.get("success") is not True:
|
||||
failed_steps.append(
|
||||
f"{step.get('name')}: {step.get('detail', 'failed')}"
|
||||
)
|
||||
if success:
|
||||
return True, "Codex adapter completed the scenario"
|
||||
if failed_steps:
|
||||
return False, redact_sensitive_text("; ".join(failed_steps))[-4_000:]
|
||||
return False, redact_sensitive_text(
|
||||
str(decoded.get("error") or "Codex adapter failed")
|
||||
)[-4_000:]
|
||||
|
||||
|
||||
def _run_scenario(
|
||||
*,
|
||||
conformance_command: Sequence[str],
|
||||
adapter_launcher: Path,
|
||||
codex_binary: Path,
|
||||
mode: str,
|
||||
scenario: str,
|
||||
output_dir: Path,
|
||||
timeout_seconds: float,
|
||||
base_env: Mapping[str, str],
|
||||
process_grace_seconds: float = 15,
|
||||
) -> OfficialScenarioResult:
|
||||
scenario_dir = output_dir / _safe_scenario_name(scenario)
|
||||
scenario_dir.mkdir(parents=True, exist_ok=True)
|
||||
adapter_report = scenario_dir / "codex-adapter.json"
|
||||
adapter_home = scenario_dir / "codex-home"
|
||||
adapter_home.mkdir()
|
||||
|
||||
env = dict(base_env)
|
||||
env.update(
|
||||
{
|
||||
"CODEX_CONFORMANCE_BINARY": str(codex_binary),
|
||||
"CODEX_CONFORMANCE_HOME": str(adapter_home),
|
||||
"CODEX_CONFORMANCE_ADAPTER_REPORT": str(adapter_report),
|
||||
}
|
||||
)
|
||||
command = [
|
||||
*conformance_command,
|
||||
"client",
|
||||
"--command",
|
||||
str(adapter_launcher),
|
||||
"--scenario",
|
||||
scenario,
|
||||
"--spec-version",
|
||||
mode,
|
||||
"--timeout",
|
||||
str(round(timeout_seconds * 1_000)),
|
||||
"--output-dir",
|
||||
str(scenario_dir),
|
||||
]
|
||||
process: subprocess.Popen[str] | None = None
|
||||
try:
|
||||
process = subprocess.Popen(
|
||||
command,
|
||||
env=env,
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
start_new_session=sys.platform != "win32",
|
||||
creationflags=(
|
||||
getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0)
|
||||
if sys.platform == "win32"
|
||||
else 0
|
||||
),
|
||||
)
|
||||
try:
|
||||
process_timeout = timeout_seconds + process_grace_seconds
|
||||
stdout, stderr = process.communicate(timeout=process_timeout)
|
||||
runner_detail = redact_sensitive_text((stderr or stdout).strip())[-8_000:]
|
||||
returncode = process.returncode
|
||||
except subprocess.TimeoutExpired:
|
||||
_terminate_process_group(process)
|
||||
try:
|
||||
stdout, stderr = process.communicate(timeout=3)
|
||||
except subprocess.TimeoutExpired:
|
||||
_terminate_process_group(process, force=True)
|
||||
stdout, stderr = process.communicate()
|
||||
runner_detail = redact_sensitive_text(
|
||||
(stderr or stdout)
|
||||
+ f"\nofficial scenario timed out after {process_timeout:g}s"
|
||||
).strip()[-8_000:]
|
||||
returncode = 124
|
||||
except OSError as exc:
|
||||
runner_detail = redact_sensitive_text(str(exc))
|
||||
returncode = 127
|
||||
|
||||
# A crashing upstream runner can exit before its still-running adapter has
|
||||
# atomically written the diagnostic report. Give that report a short grace
|
||||
# period so an upstream fixture crash is not mislabeled as an adapter
|
||||
# failure, then terminate any orphaned descendants in the runner's process
|
||||
# group.
|
||||
if returncode != 0 and not adapter_report.is_file():
|
||||
report_deadline = time.monotonic() + min(process_grace_seconds, 10)
|
||||
while time.monotonic() < report_deadline and not adapter_report.is_file():
|
||||
time.sleep(0.05)
|
||||
if returncode != 0 and process is not None:
|
||||
_terminate_process_group(process)
|
||||
|
||||
_scrub_retained_artifacts(scenario_dir)
|
||||
checks = _load_checks(scenario_dir, scenario)
|
||||
adapter_success, adapter_detail = _load_adapter_report(adapter_report)
|
||||
official_failure = any(check.status in {"FAILURE", "WARNING"} for check in checks)
|
||||
success = (
|
||||
returncode == 0 and bool(checks) and adapter_success and not official_failure
|
||||
)
|
||||
if not checks:
|
||||
runner_detail = (
|
||||
"official runner did not produce exactly one checks.json; " + runner_detail
|
||||
).strip()
|
||||
elif returncode != 0 and not official_failure and adapter_success:
|
||||
runner_detail = (
|
||||
f"official runner exited with code {returncode}; " + runner_detail
|
||||
).strip()
|
||||
return OfficialScenarioResult(
|
||||
scenario=scenario,
|
||||
success=success,
|
||||
checks=checks,
|
||||
adapter_success=adapter_success,
|
||||
adapter_detail=adapter_detail,
|
||||
runner_detail=runner_detail,
|
||||
)
|
||||
|
||||
|
||||
def run_official_mode(
|
||||
*,
|
||||
conformance_command: Sequence[str],
|
||||
adapter_script: Path,
|
||||
codex_binary: Path,
|
||||
mode: str,
|
||||
scenarios: Sequence[str],
|
||||
output_dir: Path,
|
||||
timeout_seconds: float,
|
||||
base_env: Mapping[str, str],
|
||||
) -> list[OfficialScenarioResult]:
|
||||
launcher = _make_adapter_launcher(
|
||||
adapter_script,
|
||||
require_automatic_auth=base_env.get("CODEX_CONFORMANCE_REQUIRE_AUTOMATIC_AUTH")
|
||||
== "1",
|
||||
)
|
||||
try:
|
||||
return [
|
||||
_run_scenario(
|
||||
conformance_command=conformance_command,
|
||||
adapter_launcher=launcher,
|
||||
codex_binary=codex_binary,
|
||||
mode=mode,
|
||||
scenario=scenario,
|
||||
output_dir=output_dir,
|
||||
timeout_seconds=timeout_seconds,
|
||||
base_env=base_env,
|
||||
)
|
||||
for scenario in scenarios
|
||||
]
|
||||
finally:
|
||||
try:
|
||||
launcher.unlink()
|
||||
launcher.parent.rmdir()
|
||||
except OSError:
|
||||
pass
|
||||
4208
scripts/mcp_conformance/regression-baseline-v1.json
Normal file
4208
scripts/mcp_conformance/regression-baseline-v1.json
Normal file
File diff suppressed because it is too large
Load Diff
1039
scripts/mcp_conformance/review-regression-baseline-v1.json
Normal file
1039
scripts/mcp_conformance/review-regression-baseline-v1.json
Normal file
File diff suppressed because it is too large
Load Diff
1294
scripts/mcp_conformance/review_regressions.py
Normal file
1294
scripts/mcp_conformance/review_regressions.py
Normal file
File diff suppressed because it is too large
Load Diff
2201
scripts/mcp_conformance/run_codex_compliance.py
Normal file
2201
scripts/mcp_conformance/run_codex_compliance.py
Normal file
File diff suppressed because it is too large
Load Diff
1744
scripts/mcp_conformance/server.py
Normal file
1744
scripts/mcp_conformance/server.py
Normal file
File diff suppressed because it is too large
Load Diff
1202
scripts/mcp_conformance/test_codex_compliance.py
Normal file
1202
scripts/mcp_conformance/test_codex_compliance.py
Normal file
File diff suppressed because it is too large
Load Diff
685
scripts/mcp_conformance/test_official_conformance.py
Normal file
685
scripts/mcp_conformance/test_official_conformance.py
Normal file
@@ -0,0 +1,685 @@
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import urllib.parse
|
||||
from pathlib import Path
|
||||
|
||||
import codex_conformance_adapter
|
||||
import official_conformance
|
||||
import pytest
|
||||
import run_codex_compliance
|
||||
from codex_conformance_adapter import (
|
||||
CIMD_CLIENT_METADATA_URL,
|
||||
AdapterFailure,
|
||||
_exercise_auth_scenario,
|
||||
_oauth_client_id,
|
||||
_validate_oauth_secret_not_persisted,
|
||||
_validated_callback_url,
|
||||
_write_auth_registration,
|
||||
)
|
||||
from official_conformance import (
|
||||
LEGACY_VERSION,
|
||||
MODERN_VERSION,
|
||||
OFFICIAL_CONFORMANCE_GIT_REF,
|
||||
SHIPPING_LEGACY_VERSION,
|
||||
_make_adapter_launcher,
|
||||
_run_scenario,
|
||||
_scrub_retained_artifacts,
|
||||
_terminate_process_group,
|
||||
default_conformance_command,
|
||||
redact_sensitive_text,
|
||||
run_official_mode,
|
||||
scenarios_for_mode,
|
||||
)
|
||||
|
||||
|
||||
def _authorization_url(redirect_uri: str, state: str = "test-state") -> str:
|
||||
return "http://127.0.0.1:8765/authorize?" + urllib.parse.urlencode(
|
||||
{
|
||||
"client_id": "test-client",
|
||||
"redirect_uri": redirect_uri,
|
||||
"state": state,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def test_auth_adapter_only_accepts_exact_loopback_callback() -> None:
|
||||
redirect_uri = "http://127.0.0.1:32123/callback"
|
||||
authorization_url = _authorization_url(redirect_uri)
|
||||
callback_url = redirect_uri + "?code=test-code&state=test-state"
|
||||
|
||||
assert _validated_callback_url(authorization_url, callback_url) == callback_url
|
||||
|
||||
with pytest.raises(AdapterFailure, match="exact loopback"):
|
||||
_validated_callback_url(
|
||||
authorization_url,
|
||||
"http://127.0.0.1:32124/callback?code=test-code&state=test-state",
|
||||
)
|
||||
with pytest.raises(AdapterFailure, match="preserve OAuth state"):
|
||||
_validated_callback_url(
|
||||
authorization_url,
|
||||
redirect_uri + "?code=test-code&state=wrong-state",
|
||||
)
|
||||
with pytest.raises(AdapterFailure, match="safe loopback"):
|
||||
_validated_callback_url(
|
||||
_authorization_url("https://example.com/callback"),
|
||||
"https://example.com/callback?code=test-code&state=test-state",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("query", "message"),
|
||||
[
|
||||
("code=first&code=second&state=test-state", "exactly one nonempty code"),
|
||||
("code=&state=test-state", "exactly one nonempty code"),
|
||||
("error=first&error=second&state=test-state", "exactly one nonempty error"),
|
||||
("error=&state=test-state", "exactly one nonempty error"),
|
||||
("code=test-code&error=access_denied&state=test-state", "exactly one of"),
|
||||
("code=test-code&state=test-state&state=test-state", "preserve OAuth state"),
|
||||
(
|
||||
"code=test-code&state=test-state&iss=first&iss=second",
|
||||
"exactly one nonempty iss",
|
||||
),
|
||||
("code=test-code&state=test-state&iss=", "exactly one nonempty iss"),
|
||||
("state=test-state", "exactly one of"),
|
||||
],
|
||||
)
|
||||
def test_auth_adapter_rejects_ambiguous_oauth_callback_parameters(
|
||||
query: str,
|
||||
message: str,
|
||||
) -> None:
|
||||
redirect_uri = "http://127.0.0.1:32123/callback"
|
||||
|
||||
with pytest.raises(AdapterFailure, match=message):
|
||||
_validated_callback_url(
|
||||
_authorization_url(redirect_uri),
|
||||
f"{redirect_uri}?{query}",
|
||||
)
|
||||
|
||||
|
||||
def test_auth_adapter_allows_a_single_error_callback_for_client_validation() -> None:
|
||||
redirect_uri = "http://127.0.0.1:32123/callback"
|
||||
callback_url = redirect_uri + "?error=access_denied&state=test-state&iss=issuer"
|
||||
|
||||
assert (
|
||||
_validated_callback_url(_authorization_url(redirect_uri), callback_url)
|
||||
== callback_url
|
||||
)
|
||||
|
||||
|
||||
def test_official_runner_is_pinned_instead_of_using_npm_latest() -> None:
|
||||
command = default_conformance_command()
|
||||
|
||||
assert OFFICIAL_CONFORMANCE_GIT_REF in command[-1]
|
||||
assert "@latest" not in command[-1]
|
||||
|
||||
|
||||
def test_official_adapter_launcher_preserves_strict_production_auth_mode(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
adapter = tmp_path / "show_automatic_auth.py"
|
||||
adapter.write_text(
|
||||
'import os\nprint(os.environ.get("CODEX_CONFORMANCE_REQUIRE_AUTOMATIC_AUTH", ""))\n',
|
||||
encoding="utf-8",
|
||||
)
|
||||
launcher = _make_adapter_launcher(adapter, require_automatic_auth=True)
|
||||
|
||||
try:
|
||||
result = subprocess.run(
|
||||
[str(launcher)],
|
||||
capture_output=True,
|
||||
check=True,
|
||||
text=True,
|
||||
)
|
||||
finally:
|
||||
launcher.unlink()
|
||||
launcher.parent.rmdir()
|
||||
|
||||
assert result.stdout.strip() == "1"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("require_automatic_auth", [False, True])
|
||||
def test_windows_official_adapter_launcher_quotes_checkout_paths(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path: Path,
|
||||
require_automatic_auth: bool,
|
||||
) -> None:
|
||||
adapter = tmp_path / "checkout with spaces" / "adapter script.py"
|
||||
adapter.parent.mkdir()
|
||||
adapter.write_text("raise AssertionError('launcher generation only')\n")
|
||||
monkeypatch.setattr(official_conformance.sys, "platform", "win32")
|
||||
|
||||
launcher = _make_adapter_launcher(
|
||||
adapter,
|
||||
require_automatic_auth=require_automatic_auth,
|
||||
)
|
||||
try:
|
||||
contents = launcher.read_text(encoding="utf-8")
|
||||
finally:
|
||||
launcher.unlink()
|
||||
launcher.parent.rmdir()
|
||||
|
||||
expected_auth = "1" if require_automatic_auth else "0"
|
||||
assert launcher.name == "client.cmd"
|
||||
assert contents.startswith("@echo off\n")
|
||||
assert (
|
||||
f'set "CODEX_CONFORMANCE_REQUIRE_AUTOMATIC_AUTH={expected_auth}"\n' in contents
|
||||
)
|
||||
assert subprocess.list2cmdline([sys.executable, str(adapter)]) + " %*\n" in contents
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("force", "expected_command"),
|
||||
[
|
||||
(False, ["taskkill", "/T", "/PID", "417"]),
|
||||
(True, ["taskkill", "/F", "/T", "/PID", "417"]),
|
||||
],
|
||||
)
|
||||
def test_windows_official_timeout_terminates_the_entire_process_tree(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
force: bool,
|
||||
expected_command: list[str],
|
||||
) -> None:
|
||||
observed: list[list[str]] = []
|
||||
|
||||
class FakeProcess:
|
||||
pid = 417
|
||||
|
||||
def terminate(self) -> None:
|
||||
raise AssertionError("successful taskkill must own tree termination")
|
||||
|
||||
def kill(self) -> None:
|
||||
raise AssertionError("successful taskkill must own tree termination")
|
||||
|
||||
def fake_run(
|
||||
command: list[str],
|
||||
*,
|
||||
check: bool,
|
||||
stdout: int,
|
||||
stderr: int,
|
||||
) -> subprocess.CompletedProcess[str]:
|
||||
observed.append(command)
|
||||
assert check is False
|
||||
assert stdout == subprocess.DEVNULL
|
||||
assert stderr == subprocess.DEVNULL
|
||||
return subprocess.CompletedProcess(command, 0)
|
||||
|
||||
monkeypatch.setattr(official_conformance.sys, "platform", "win32")
|
||||
monkeypatch.setattr(official_conformance.subprocess, "run", fake_run)
|
||||
|
||||
_terminate_process_group(
|
||||
FakeProcess(), # type: ignore[arg-type] - validate the Popen process contract.
|
||||
force=force,
|
||||
)
|
||||
|
||||
assert observed == [expected_command]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"scenario",
|
||||
["auth/scope-step-up", "auth/authorization-server-migration"],
|
||||
)
|
||||
def test_strict_auth_does_not_inject_a_second_login(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path: Path,
|
||||
scenario: str,
|
||||
) -> None:
|
||||
manual_reauthorizations: list[object] = []
|
||||
|
||||
monkeypatch.setattr(
|
||||
codex_conformance_adapter,
|
||||
"_oauth_login",
|
||||
lambda _client, **_kwargs: (True, None),
|
||||
)
|
||||
monkeypatch.setattr(codex_conformance_adapter, "_reload_mcp", lambda _client: None)
|
||||
monkeypatch.setattr(
|
||||
codex_conformance_adapter, "_auth_inventory", lambda _client: {}
|
||||
)
|
||||
|
||||
def fail_tool_call(_client: object, _workspace: Path) -> None:
|
||||
raise AdapterFailure("reauthorization is required")
|
||||
|
||||
monkeypatch.setattr(codex_conformance_adapter, "_auth_tool_call", fail_tool_call)
|
||||
monkeypatch.setattr(
|
||||
codex_conformance_adapter,
|
||||
"_login_reload_and_call",
|
||||
lambda *args, **kwargs: manual_reauthorizations.append((args, kwargs)),
|
||||
)
|
||||
|
||||
with pytest.raises(AdapterFailure, match="reauthorization is required"):
|
||||
_exercise_auth_scenario(
|
||||
object(), # type: ignore[arg-type]
|
||||
scenario=scenario,
|
||||
workspace=tmp_path,
|
||||
timeout_seconds=1,
|
||||
require_automatic_auth=True,
|
||||
)
|
||||
|
||||
assert manual_reauthorizations == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"scenario",
|
||||
["auth/scope-step-up", "auth/authorization-server-migration"],
|
||||
)
|
||||
def test_strict_auth_accepts_product_owned_reauthentication(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path: Path,
|
||||
scenario: str,
|
||||
) -> None:
|
||||
tool_calls: list[Path] = []
|
||||
manual_reauthorizations: list[object] = []
|
||||
|
||||
monkeypatch.setattr(
|
||||
codex_conformance_adapter,
|
||||
"_oauth_login",
|
||||
lambda _client, **_kwargs: (True, None),
|
||||
)
|
||||
monkeypatch.setattr(codex_conformance_adapter, "_reload_mcp", lambda _client: None)
|
||||
monkeypatch.setattr(
|
||||
codex_conformance_adapter, "_auth_inventory", lambda _client: {}
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
codex_conformance_adapter,
|
||||
"_auth_tool_call",
|
||||
lambda _client, workspace: tool_calls.append(workspace),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
codex_conformance_adapter,
|
||||
"_login_reload_and_call",
|
||||
lambda *args, **kwargs: manual_reauthorizations.append((args, kwargs)),
|
||||
)
|
||||
|
||||
detail = _exercise_auth_scenario(
|
||||
object(), # type: ignore[arg-type]
|
||||
scenario=scenario,
|
||||
workspace=tmp_path,
|
||||
timeout_seconds=1,
|
||||
require_automatic_auth=True,
|
||||
)
|
||||
|
||||
assert "automatically" in detail
|
||||
assert tool_calls == [tmp_path]
|
||||
assert manual_reauthorizations == []
|
||||
|
||||
|
||||
def test_strict_auth_does_not_invent_a_production_client_metadata_url() -> None:
|
||||
assert (
|
||||
_oauth_client_id(
|
||||
"auth/basic-cimd",
|
||||
{},
|
||||
require_production_client_identity=True,
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("filename", ["config.toml", ".credentials.json"])
|
||||
def test_oauth_client_secret_persistence_is_detected_without_disclosing_it(
|
||||
tmp_path: Path,
|
||||
filename: str,
|
||||
) -> None:
|
||||
secret = "reviewer-confidential-client-secret"
|
||||
(tmp_path / filename).write_text(
|
||||
json.dumps({"client_secret": secret}),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
with pytest.raises(AdapterFailure, match="client secret was persisted") as error:
|
||||
_validate_oauth_secret_not_persisted(tmp_path, secret)
|
||||
|
||||
assert filename in str(error.value)
|
||||
assert secret not in str(error.value)
|
||||
|
||||
|
||||
def test_oauth_client_secret_environment_reference_is_not_a_persisted_secret(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
(tmp_path / "config.toml").write_text(
|
||||
'client_secret_env_var = "MCP_CONFORMANCE_CLIENT_SECRET"\n',
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
_validate_oauth_secret_not_persisted(
|
||||
tmp_path, "reviewer-confidential-client-secret"
|
||||
)
|
||||
|
||||
|
||||
def test_auth_adapter_selects_only_scenario_provided_client_ids() -> None:
|
||||
assert _oauth_client_id("auth/basic-cimd", {}) == CIMD_CLIENT_METADATA_URL
|
||||
assert (
|
||||
_oauth_client_id(
|
||||
"auth/basic-cimd",
|
||||
{},
|
||||
require_production_client_identity=True,
|
||||
)
|
||||
is None
|
||||
)
|
||||
assert (
|
||||
_oauth_client_id(
|
||||
"auth/pre-registration",
|
||||
{"client_id": "pre-registered", "client_secret": "do-not-log"},
|
||||
)
|
||||
== "pre-registered"
|
||||
)
|
||||
assert _oauth_client_id("auth/metadata-default", {}) is None
|
||||
with pytest.raises(AdapterFailure, match="client_id"):
|
||||
_oauth_client_id("auth/pre-registration", {})
|
||||
|
||||
|
||||
def test_auth_registration_does_not_persist_context_secret(tmp_path: Path) -> None:
|
||||
config_path = tmp_path / "config.toml"
|
||||
config_path.write_text('mcp_oauth_credentials_store = "file"\n', encoding="utf-8")
|
||||
|
||||
_write_auth_registration(
|
||||
config_path,
|
||||
server_url="http://127.0.0.1:8765/mcp",
|
||||
oauth_client_id="pre-registered-client",
|
||||
oauth_client_secret_env_var="MCP_CONFORMANCE_CLIENT_SECRET",
|
||||
)
|
||||
|
||||
config = config_path.read_text(encoding="utf-8")
|
||||
assert 'url = "http://127.0.0.1:8765/mcp"' in config
|
||||
assert 'client_id = "pre-registered-client"' in config
|
||||
assert 'client_secret_env_var = "MCP_CONFORMANCE_CLIENT_SECRET"' in config
|
||||
assert "do-not-log" not in config
|
||||
assert "client_secret =" not in config
|
||||
|
||||
|
||||
def test_official_diagnostics_redact_oauth_secrets() -> None:
|
||||
diagnostic = (
|
||||
'With context: {"client_id":"visible","client_secret":"secret-value",'
|
||||
'"private_key_pem":"private-value"}\n'
|
||||
"Authorize at http://localhost/authorize?state=state-value&code_challenge=pkce-value\n"
|
||||
"Authorization: Bearer token-value"
|
||||
)
|
||||
|
||||
redacted = redact_sensitive_text(diagnostic)
|
||||
|
||||
assert '"client_id":"visible"' in redacted
|
||||
assert '"client_secret":"[REDACTED]"' in redacted
|
||||
assert '"private_key_pem":"[REDACTED]"' in redacted
|
||||
assert "state=[REDACTED]" in redacted
|
||||
assert "code_challenge=[REDACTED]" in redacted
|
||||
assert "Bearer [REDACTED]" in redacted
|
||||
serialized = redact_sensitive_text(json.dumps({"diagnostic": diagnostic}))
|
||||
assert json.loads(serialized)["diagnostic"]
|
||||
for secret in (
|
||||
"secret-value",
|
||||
"private-value",
|
||||
"state-value",
|
||||
"pkce-value",
|
||||
"token-value",
|
||||
):
|
||||
assert secret not in redacted
|
||||
assert secret not in serialized
|
||||
|
||||
|
||||
def test_retained_artifacts_remove_oauth_credential_store(tmp_path: Path) -> None:
|
||||
codex_home = tmp_path / "codex-home"
|
||||
codex_home.mkdir()
|
||||
credentials = codex_home / ".credentials.json"
|
||||
credentials.write_text('{"client_secret":"do-not-retain"}', encoding="utf-8")
|
||||
stdout = tmp_path / "stdout.txt"
|
||||
stdout.write_text("Bearer token-value", encoding="utf-8")
|
||||
|
||||
_scrub_retained_artifacts(tmp_path)
|
||||
|
||||
assert not credentials.exists()
|
||||
assert stdout.read_text(encoding="utf-8") == "Bearer [REDACTED]"
|
||||
|
||||
|
||||
def test_pinned_full_scenarios_are_selected_by_protocol_version() -> None:
|
||||
shipping_legacy = scenarios_for_mode(SHIPPING_LEGACY_VERSION)
|
||||
legacy = scenarios_for_mode(LEGACY_VERSION)
|
||||
modern = scenarios_for_mode(MODERN_VERSION)
|
||||
|
||||
assert shipping_legacy == (
|
||||
"initialize",
|
||||
"tools_call",
|
||||
"auth/token-endpoint-auth-basic",
|
||||
"auth/token-endpoint-auth-post",
|
||||
"auth/token-endpoint-auth-none",
|
||||
)
|
||||
assert scenarios_for_mode(SHIPPING_LEGACY_VERSION, include_auth=False) == (
|
||||
"initialize",
|
||||
"tools_call",
|
||||
)
|
||||
|
||||
assert legacy[:4] == (
|
||||
"initialize",
|
||||
"tools_call",
|
||||
"elicitation-sep1034-client-defaults",
|
||||
"sse-retry",
|
||||
)
|
||||
assert len(legacy) == 18
|
||||
assert len([scenario for scenario in legacy if scenario.startswith("auth/")]) == 14
|
||||
assert "auth/pre-registration" in legacy
|
||||
|
||||
assert modern[:7] == (
|
||||
"tools_call",
|
||||
"request-metadata",
|
||||
"sep-2322-client-request-state",
|
||||
"http-standard-headers",
|
||||
"http-custom-headers",
|
||||
"http-invalid-tool-headers",
|
||||
"json-schema-ref-no-deref",
|
||||
)
|
||||
assert len(modern) == 32
|
||||
assert len([scenario for scenario in modern if scenario.startswith("auth/")]) == 25
|
||||
assert "auth/resource-mismatch" in modern
|
||||
assert "auth/metadata-issuer-mismatch" in modern
|
||||
|
||||
assert scenarios_for_mode(LEGACY_VERSION, include_auth=False) == legacy[:4]
|
||||
assert scenarios_for_mode(MODERN_VERSION, include_auth=False) == modern[:7]
|
||||
assert scenarios_for_mode(
|
||||
MODERN_VERSION,
|
||||
["tools_call", "http-custom-headers"],
|
||||
) == ("tools_call", "http-custom-headers")
|
||||
assert len(OFFICIAL_CONFORMANCE_GIT_REF) == 40
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("mode", "scenario"),
|
||||
[
|
||||
(SHIPPING_LEGACY_VERSION, "request-metadata"),
|
||||
(LEGACY_VERSION, "http-standard-headers"),
|
||||
(MODERN_VERSION, "elicitation-sep1034-client-defaults"),
|
||||
],
|
||||
)
|
||||
def test_versioned_scenario_selection_rejects_an_incompatible_protocol(
|
||||
mode: str, scenario: str
|
||||
) -> None:
|
||||
with pytest.raises(
|
||||
ValueError,
|
||||
match=rf"unavailable for MCP protocol version {mode}.*{scenario}",
|
||||
):
|
||||
scenarios_for_mode(mode, [scenario], include_auth=False)
|
||||
|
||||
|
||||
def test_cross_version_auth_subset_remains_available_to_every_mode() -> None:
|
||||
requested = (
|
||||
"auth/token-endpoint-auth-basic",
|
||||
"auth/resource-mismatch",
|
||||
"auth/authorization-server-migration",
|
||||
)
|
||||
|
||||
assert scenarios_for_mode(SHIPPING_LEGACY_VERSION, requested) == (
|
||||
"auth/token-endpoint-auth-basic",
|
||||
)
|
||||
assert scenarios_for_mode(LEGACY_VERSION, requested) == (
|
||||
"auth/token-endpoint-auth-basic",
|
||||
)
|
||||
assert scenarios_for_mode(MODERN_VERSION, requested) == requested
|
||||
|
||||
|
||||
def test_official_cli_rejects_an_empty_versioned_scenario_run(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
capsys: pytest.CaptureFixture[str],
|
||||
) -> None:
|
||||
def unexpected_run(*_args: object, **_kwargs: object) -> None:
|
||||
raise AssertionError("an incompatible selection must not execute conformance")
|
||||
|
||||
monkeypatch.setattr(run_codex_compliance, "run_compliance", unexpected_run)
|
||||
|
||||
exit_code = run_codex_compliance.main(
|
||||
[
|
||||
sys.executable,
|
||||
"--mode",
|
||||
SHIPPING_LEGACY_VERSION,
|
||||
"--transport",
|
||||
"http",
|
||||
"--official-scenario",
|
||||
"request-metadata",
|
||||
"--no-auth",
|
||||
"--conformance-cli",
|
||||
sys.executable,
|
||||
]
|
||||
)
|
||||
|
||||
assert exit_code == 2
|
||||
assert (
|
||||
"requested scenarios are unavailable for MCP protocol version 2025-06-18: "
|
||||
"request-metadata"
|
||||
) in capsys.readouterr().err
|
||||
|
||||
|
||||
def test_official_driver_reads_checks_and_adapter_report(tmp_path: Path) -> None:
|
||||
fake_cli = tmp_path / "fake_conformance.py"
|
||||
fake_cli.write_text(
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
args = sys.argv[1:]
|
||||
scenario = args[args.index("--scenario") + 1]
|
||||
output = pathlib.Path(args[args.index("--output-dir") + 1])
|
||||
result_dir = output / f"{scenario}-timestamp"
|
||||
result_dir.mkdir(parents=True)
|
||||
(result_dir / "checks.json").write_text(json.dumps([
|
||||
{
|
||||
"id": "request-trace",
|
||||
"name": "RequestTrace",
|
||||
"description": "not a conformance assertion",
|
||||
"status": "INFO",
|
||||
},
|
||||
{
|
||||
"id": "official-check",
|
||||
"name": "OfficialCheck",
|
||||
"description": "observed",
|
||||
"status": "SUCCESS",
|
||||
},
|
||||
]))
|
||||
pathlib.Path(os.environ["CODEX_CONFORMANCE_ADAPTER_REPORT"]).write_text(
|
||||
json.dumps({"success": True, "steps": []})
|
||||
)
|
||||
""".lstrip(),
|
||||
encoding="utf-8",
|
||||
)
|
||||
adapter = tmp_path / "adapter.py"
|
||||
adapter.write_text("raise AssertionError('fake CLI should not run adapter')\n")
|
||||
|
||||
results = run_official_mode(
|
||||
conformance_command=[sys.executable, str(fake_cli)],
|
||||
adapter_script=adapter,
|
||||
codex_binary=Path("/opt/codex"),
|
||||
mode=MODERN_VERSION,
|
||||
scenarios=["tools_call"],
|
||||
output_dir=tmp_path / "results",
|
||||
timeout_seconds=1,
|
||||
base_env={},
|
||||
)
|
||||
|
||||
assert len(results) == 1
|
||||
assert results[0].success
|
||||
assert results[0].adapter_success
|
||||
assert len(results[0].checks) == 1
|
||||
assert results[0].checks[0].check_id == "official-check"
|
||||
|
||||
|
||||
def test_official_driver_terminates_timed_out_process_group(tmp_path: Path) -> None:
|
||||
fake_cli = tmp_path / "hanging_conformance.py"
|
||||
fake_cli.write_text(
|
||||
"""
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
subprocess.Popen([sys.executable, "-c", "import time; time.sleep(60)"])
|
||||
time.sleep(60)
|
||||
""".lstrip(),
|
||||
encoding="utf-8",
|
||||
)
|
||||
adapter = tmp_path / "adapter.py"
|
||||
adapter.write_text("raise AssertionError('not reached')\n", encoding="utf-8")
|
||||
launcher = _make_adapter_launcher(adapter)
|
||||
try:
|
||||
result = _run_scenario(
|
||||
conformance_command=[sys.executable, str(fake_cli)],
|
||||
adapter_launcher=launcher,
|
||||
codex_binary=Path("/opt/codex"),
|
||||
mode=MODERN_VERSION,
|
||||
scenario="tools_call",
|
||||
output_dir=tmp_path / "results",
|
||||
timeout_seconds=0.05,
|
||||
process_grace_seconds=0.05,
|
||||
base_env={},
|
||||
)
|
||||
finally:
|
||||
launcher.unlink()
|
||||
launcher.parent.rmdir()
|
||||
|
||||
assert not result.success
|
||||
assert "timed out" in result.runner_detail
|
||||
|
||||
|
||||
def test_official_driver_does_not_mislabel_runner_crash_as_adapter_failure(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
fake_cli = tmp_path / "crashing_conformance.py"
|
||||
fake_cli.write_text(
|
||||
"""
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
script = '''
|
||||
import json
|
||||
import os
|
||||
import pathlib
|
||||
import time
|
||||
|
||||
time.sleep(0.1)
|
||||
pathlib.Path(os.environ["CODEX_CONFORMANCE_ADAPTER_REPORT"]).write_text(
|
||||
json.dumps({"success": True, "steps": []})
|
||||
)
|
||||
'''
|
||||
subprocess.Popen([sys.executable, "-c", script], env=os.environ)
|
||||
sys.exit(1)
|
||||
""".lstrip(),
|
||||
encoding="utf-8",
|
||||
)
|
||||
adapter = tmp_path / "adapter.py"
|
||||
adapter.write_text("raise AssertionError('not reached')\n", encoding="utf-8")
|
||||
launcher = _make_adapter_launcher(adapter)
|
||||
try:
|
||||
result = _run_scenario(
|
||||
conformance_command=[sys.executable, str(fake_cli)],
|
||||
adapter_launcher=launcher,
|
||||
codex_binary=Path("/opt/codex"),
|
||||
mode=MODERN_VERSION,
|
||||
scenario="request-metadata",
|
||||
output_dir=tmp_path / "results",
|
||||
timeout_seconds=1,
|
||||
process_grace_seconds=1,
|
||||
base_env={},
|
||||
)
|
||||
finally:
|
||||
launcher.unlink()
|
||||
launcher.parent.rmdir()
|
||||
|
||||
assert not result.success
|
||||
assert result.adapter_success
|
||||
assert "did not produce exactly one checks.json" in result.runner_detail
|
||||
1093
scripts/mcp_conformance/test_review_regressions.py
Normal file
1093
scripts/mcp_conformance/test_review_regressions.py
Normal file
File diff suppressed because it is too large
Load Diff
1590
scripts/mcp_conformance/test_server.py
Normal file
1590
scripts/mcp_conformance/test_server.py
Normal file
File diff suppressed because it is too large
Load Diff
37
sdk/python/tests/test_mcp_conformance_fixtures.py
Normal file
37
sdk/python/tests/test_mcp_conformance_fixtures.py
Normal file
@@ -0,0 +1,37 @@
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def test_mcp_conformance_fixture_self_tests() -> None:
|
||||
repository_root = Path(__file__).resolve().parents[3]
|
||||
candidates = (
|
||||
repository_root / "public" / "scripts" / "mcp_conformance",
|
||||
repository_root / "scripts" / "mcp_conformance",
|
||||
)
|
||||
fixture_directory = next((candidate for candidate in candidates if candidate.is_dir()), None)
|
||||
assert fixture_directory is not None, (
|
||||
"MCP conformance fixtures are missing; checked "
|
||||
+ " and ".join(str(candidate) for candidate in candidates)
|
||||
)
|
||||
|
||||
env = os.environ.copy()
|
||||
env["PYTEST_DISABLE_PLUGIN_AUTOLOAD"] = "1"
|
||||
result = subprocess.run(
|
||||
[sys.executable, "-m", "pytest", "-q", str(fixture_directory)],
|
||||
cwd=repository_root,
|
||||
env=env,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=120,
|
||||
check=False,
|
||||
)
|
||||
|
||||
assert result.returncode == 0, (
|
||||
f"MCP conformance fixture self-tests failed with exit "
|
||||
f"{result.returncode}.\n"
|
||||
f"Fixture directory: {fixture_directory}\n"
|
||||
f"STDOUT:\n{result.stdout}\n"
|
||||
f"STDERR:\n{result.stderr}"
|
||||
)
|
||||
@@ -45,6 +45,7 @@
|
||||
"prepare": "pnpm run build"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@modelcontextprotocol/conformance": "github:modelcontextprotocol/conformance#49103de6ed70804e940637bf3e9e29e4a3f54e64",
|
||||
"@modelcontextprotocol/sdk": "^1.24.0",
|
||||
"@types/jest": "^29.5.14",
|
||||
"@types/node": "^20.19.18",
|
||||
|
||||
158
sdk/typescript/tests/mcpConformance.test.ts
Normal file
158
sdk/typescript/tests/mcpConformance.test.ts
Normal file
@@ -0,0 +1,158 @@
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { existsSync } from "node:fs";
|
||||
import { createRequire } from "node:module";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
|
||||
import { describe, it } from "@jest/globals";
|
||||
|
||||
import { codexExecPath } from "./testCodex";
|
||||
|
||||
const conformanceTimeoutMs = 300_000;
|
||||
const reviewerRegressionTimeoutMs = 180_000;
|
||||
|
||||
function requireFile(filePath: string, description: string): string {
|
||||
if (!existsSync(filePath)) {
|
||||
throw new Error(`MCP conformance ${description} does not exist: ${filePath}`);
|
||||
}
|
||||
|
||||
return filePath;
|
||||
}
|
||||
|
||||
function conformanceDirectory(): string {
|
||||
const repositoryRoot = path.resolve(process.cwd(), "../..");
|
||||
const candidates = [
|
||||
path.join(repositoryRoot, "public", "scripts", "mcp_conformance"),
|
||||
path.join(repositoryRoot, "scripts", "mcp_conformance"),
|
||||
];
|
||||
|
||||
for (const candidate of candidates) {
|
||||
if (existsSync(path.join(candidate, "run_codex_compliance.py"))) {
|
||||
return candidate;
|
||||
}
|
||||
}
|
||||
|
||||
throw new Error(`MCP conformance harness is missing; checked ${candidates.join(" and ")}`);
|
||||
}
|
||||
|
||||
describe("MCP client conformance", () => {
|
||||
it(
|
||||
"does not introduce shipping, intermediate, or modern official conformance regressions",
|
||||
() => {
|
||||
const directory = conformanceDirectory();
|
||||
const require = createRequire(import.meta.url);
|
||||
const conformancePackage = require.resolve("@modelcontextprotocol/conformance/package.json");
|
||||
const officialConformanceCli = requireFile(
|
||||
path.join(path.dirname(conformancePackage), "dist", "index.js"),
|
||||
"pinned official CLI",
|
||||
);
|
||||
const codexBinary = requireFile(codexExecPath, "built Codex binary");
|
||||
const baseline = requireFile(
|
||||
path.join(directory, "regression-baseline-v1.json"),
|
||||
"committed regression baseline",
|
||||
);
|
||||
const reportDirectory = process.env.RUNNER_TEMP ?? os.tmpdir();
|
||||
const reportPath = path.join(reportDirectory, `codex-mcp-conformance-${process.pid}.json`);
|
||||
const result = spawnSync(
|
||||
"python3",
|
||||
[
|
||||
path.join(directory, "run_codex_compliance.py"),
|
||||
codexBinary,
|
||||
"--mode",
|
||||
"all",
|
||||
"--transport",
|
||||
"all",
|
||||
"--auth",
|
||||
"--conformance-cli",
|
||||
officialConformanceCli,
|
||||
"--baseline-report",
|
||||
baseline,
|
||||
"--report",
|
||||
reportPath,
|
||||
],
|
||||
{
|
||||
encoding: "utf8",
|
||||
maxBuffer: 16 * 1024 * 1024,
|
||||
timeout: conformanceTimeoutMs - 10_000,
|
||||
},
|
||||
);
|
||||
|
||||
if (result.error) {
|
||||
throw new Error(`MCP conformance runner could not complete: ${result.error.message}`, {
|
||||
cause: result.error,
|
||||
});
|
||||
}
|
||||
|
||||
if (result.status !== 0) {
|
||||
throw new Error(
|
||||
[
|
||||
`MCP conformance regression gate exited with ${result.status ?? result.signal}.`,
|
||||
`Report: ${reportPath}`,
|
||||
result.stdout,
|
||||
result.stderr,
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join("\n"),
|
||||
);
|
||||
}
|
||||
},
|
||||
conformanceTimeoutMs,
|
||||
);
|
||||
|
||||
it(
|
||||
"does not introduce production reviewer or catalog-boundary regressions",
|
||||
() => {
|
||||
const directory = conformanceDirectory();
|
||||
const codexBinary = requireFile(codexExecPath, "built Codex binary");
|
||||
const reviewer = requireFile(
|
||||
path.join(directory, "review_regressions.py"),
|
||||
"production reviewer regression runner",
|
||||
);
|
||||
const baseline = requireFile(
|
||||
path.join(directory, "review-regression-baseline-v1.json"),
|
||||
"committed production reviewer regression baseline",
|
||||
);
|
||||
const reportDirectory = process.env.RUNNER_TEMP ?? os.tmpdir();
|
||||
const reportPath = path.join(reportDirectory, `codex-mcp-review-${process.pid}.json`);
|
||||
const result = spawnSync(
|
||||
"python3",
|
||||
[
|
||||
reviewer,
|
||||
codexBinary,
|
||||
"--mode",
|
||||
"all",
|
||||
"--baseline-report",
|
||||
baseline,
|
||||
"--report",
|
||||
reportPath,
|
||||
],
|
||||
{
|
||||
encoding: "utf8",
|
||||
maxBuffer: 16 * 1024 * 1024,
|
||||
timeout: reviewerRegressionTimeoutMs - 10_000,
|
||||
},
|
||||
);
|
||||
|
||||
if (result.error) {
|
||||
throw new Error(
|
||||
`MCP production reviewer regression runner could not complete: ${result.error.message}`,
|
||||
{ cause: result.error },
|
||||
);
|
||||
}
|
||||
|
||||
if (result.status !== 0) {
|
||||
throw new Error(
|
||||
[
|
||||
`MCP production reviewer regression gate exited with ${result.status ?? result.signal}.`,
|
||||
`Report: ${reportPath}`,
|
||||
result.stdout,
|
||||
result.stderr,
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join("\n"),
|
||||
);
|
||||
}
|
||||
},
|
||||
reviewerRegressionTimeoutMs,
|
||||
);
|
||||
});
|
||||
Reference in New Issue
Block a user