diff --git a/.github/scripts/resolve_python_cli_release.py b/.github/scripts/resolve_python_cli_release.py new file mode 100644 index 0000000000..4b230b56f2 --- /dev/null +++ b/.github/scripts/resolve_python_cli_release.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python3 +"""Bind downstream Python publication to a published stable CLI release.""" + +import argparse +import itertools +import json +import re +import subprocess +from pathlib import Path + + +def github_api(path: str): + return json.loads(subprocess.check_output(["gh", "api", path], text=True)) + + +def github_items(path: str, key: str | None = None): + separator = "&" if "?" in path else "?" + for page in itertools.count(1): + response = github_api(f"{path}{separator}per_page=100&page={page}") + items = response[key] if key else response + yield from items + if len(items) < 100: + return + + +def resolve_release( + repository: str, run_id: str, *, run: dict | None = None +) -> dict[str, str] | None: + if repository != "openai/codex" or not re.fullmatch(r"[1-9][0-9]*", run_id): + raise ValueError("Expected an openai/codex Rust release run ID") + prefix = f"repos/{repository}" + if run is None: + run = github_api(f"{prefix}/actions/runs/{run_id}") + if ( + str(run["id"]) != run_id + or run["path"] != ".github/workflows/rust-release.yml" + or run["event"] != "push" + or run["status"] != "completed" + or run["head_repository"]["full_name"] != repository + ): + raise ValueError("Python publication requires a completed Rust release run") + tag = run["head_branch"] + if not isinstance(tag, str) or not tag.startswith("rust-v"): + raise ValueError("The Rust release run must identify a release tag") + match = re.fullmatch(r"rust-v([0-9]+\.[0-9]+\.[0-9]+)", tag) + if match is None: + return None # CLI prereleases do not publish the Python SDK. + revision = run["head_sha"] + if not re.fullmatch(r"[0-9a-f]{40}", revision): + raise ValueError("The Rust release run must identify an exact commit") + + # A partial rerun of an unrelated publisher may not contain the release job. + # Inspect all attempts and use the last release job that actually ran. + release_job = max( + ( + job + for job in github_items( + f"{prefix}/actions/runs/{run_id}/jobs?filter=all", "jobs" + ) + if job["name"] == "release" and job["conclusion"] != "skipped" + ), + key=lambda job: (job["run_attempt"], job["id"]), + default=None, + ) + if release_job is None or release_job["conclusion"] != "success": + raise ValueError("Python publication requires a successful release job") + + target = github_api(f"{prefix}/git/ref/tags/{tag}")["object"] + for _ in range(8): + if target["type"] != "tag": + break + target = github_api(f"{prefix}/git/tags/{target['sha']}")["object"] + if target.get("type") != "commit" or target.get("sha") != revision: + raise ValueError("The release tag no longer matches the successful CLI run") + release = github_api(f"{prefix}/releases/tags/{tag}") + if release["draft"] or release["prerelease"] or release["tag_name"] != tag: + raise ValueError("The CLI release must be published and stable") + + # The runtime builder downloads these six wheels and builds the two musl + # wheels from the release's package archives. + required_assets = { + f"openai_codex_cli_bin-{match[1]}-py3-none-{platform}.whl" + for platform in ( + "macosx_10_9_x86_64", + "macosx_11_0_arm64", + "manylinux_2_17_aarch64", + "manylinux_2_17_x86_64", + "win_amd64", + "win_arm64", + ) + } | { + f"codex-package-{arch}-unknown-linux-musl.tar.gz" + for arch in ("aarch64", "x86_64") + } + available_assets = { + asset["name"] + for asset in github_items(f"{prefix}/releases/{release['id']}/assets") + if asset["state"] == "uploaded" and asset["size"] > 0 + } + if missing := required_assets - available_assets: + raise ValueError(f"The CLI release is missing Python inputs: {sorted(missing)}") + return {"source_sha": revision, "release_tag": tag, "version": match[1]} + + +def main(argv: list[str] | None = None) -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("run_id") + parser.add_argument("--repository", required=True) + parser.add_argument("--event-path", type=Path) + parser.add_argument("--github-output", type=Path, required=True) + args = parser.parse_args(argv) + event = json.loads(args.event_path.read_text()) if args.event_path else {} + release = resolve_release( + args.repository, args.run_id, run=event.get("workflow_run") + ) + with args.github_output.open("a") as output: + print(f"publish={str(release is not None).lower()}", file=output) + for name, value in (release or {}).items(): + print(f"{name}={value}", file=output) + + +if __name__ == "__main__": + main() diff --git a/.github/scripts/test_resolve_python_cli_release.py b/.github/scripts/test_resolve_python_cli_release.py new file mode 100644 index 0000000000..c7b44e7626 --- /dev/null +++ b/.github/scripts/test_resolve_python_cli_release.py @@ -0,0 +1,250 @@ +import copy +import json +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +import resolve_python_cli_release as resolver + + +class ResolvePythonCliReleaseTest(unittest.TestCase): + def setUp(self) -> None: + self.revision = "a" * 40 + self.run = { + "id": 123, + "path": ".github/workflows/rust-release.yml", + "event": "push", + "status": "completed", + "conclusion": "success", + "head_repository": {"full_name": "openai/codex"}, + "head_branch": "rust-v1.2.3", + "head_sha": self.revision, + } + self.job = { + "id": 456, + "run_attempt": 1, + "name": "release", + "conclusion": "success", + } + self.commit = {"type": "commit", "sha": self.revision} + self.release = { + "id": 789, + "draft": False, + "prerelease": False, + "tag_name": "rust-v1.2.3", + } + self.assets = [ + {"name": name, "state": "uploaded", "size": 100} + for name in [ + "openai_codex_cli_bin-1.2.3-py3-none-macosx_10_9_x86_64.whl", + "openai_codex_cli_bin-1.2.3-py3-none-macosx_11_0_arm64.whl", + "openai_codex_cli_bin-1.2.3-py3-none-manylinux_2_17_aarch64.whl", + "openai_codex_cli_bin-1.2.3-py3-none-manylinux_2_17_x86_64.whl", + "openai_codex_cli_bin-1.2.3-py3-none-win_amd64.whl", + "openai_codex_cli_bin-1.2.3-py3-none-win_arm64.whl", + "codex-package-aarch64-unknown-linux-musl.tar.gz", + "codex-package-x86_64-unknown-linux-musl.tar.gz", + ] + ] + self.expected = { + "source_sha": self.revision, + "release_tag": "rust-v1.2.3", + "version": "1.2.3", + } + + def responses(self) -> list: + return copy.deepcopy( + [ + self.run, + {"jobs": [self.job]}, + {"object": self.commit}, + self.release, + self.assets, + ] + ) + + def test_resolves_lightweight_and_annotated_tags_to_the_run_revision(self) -> None: + for annotated in (False, True): + with self.subTest(annotated=annotated): + responses = self.responses() + if annotated: + responses.insert(2, {"object": {"type": "tag", "sha": "b" * 40}}) + with patch.object(resolver, "github_api", side_effect=responses): + self.assertEqual( + resolver.resolve_release("openai/codex", "123"), self.expected + ) + + def test_skips_cli_prereleases_before_reading_jobs_or_assets(self) -> None: + for suffix in ("-alpha", "-alpha.1", "-alpha.1.2", "-beta", "-beta.1"): + with self.subTest(suffix=suffix): + run = {**self.run, "head_branch": f"rust-v1.2.3{suffix}"} + with patch.object(resolver, "github_api", return_value=run) as api: + self.assertIsNone(resolver.resolve_release("openai/codex", "123")) + api.assert_called_once_with("repos/openai/codex/actions/runs/123") + + def test_rejects_unrelated_or_incomplete_runs(self) -> None: + for field, value in ( + ("id", 999), + ("path", ".github/workflows/sdk.yml"), + ("event", "pull_request"), + ("status", "in_progress"), + ("head_branch", "main"), + ("head_sha", "main"), + ("head_repository", {"full_name": "other/codex"}), + ): + with self.subTest(field=field, value=value): + with patch.object( + resolver, "github_api", return_value={**self.run, field: value} + ): + with self.assertRaises(ValueError): + resolver.resolve_release("openai/codex", "123") + + def test_accepts_failed_ancillary_publisher_and_its_partial_rerun(self) -> None: + for overall in ("failure", "cancelled"): + with self.subTest(overall=overall): + responses = self.responses() + responses[0]["conclusion"] = overall + responses[1]["jobs"].extend( + [ + { + "id": 999, + "run_attempt": 2, + "name": "publish-winget", + "conclusion": "failure", + }, + { + **self.job, + "id": 998, + "run_attempt": 2, + "conclusion": "skipped", + }, + ] + ) + with patch.object(resolver, "github_api", side_effect=responses): + self.assertEqual( + resolver.resolve_release("openai/codex", "123"), self.expected + ) + + def test_requires_success_from_latest_executed_release_job(self) -> None: + for jobs in ( + [], + [{**self.job, "conclusion": "skipped"}], + [{**self.job, "conclusion": "failure"}], + [{**self.job, "conclusion": "cancelled"}], + [ + {**self.job, "id": 999, "run_attempt": 2, "conclusion": "failure"}, + self.job, + ], + ): + with self.subTest(jobs=jobs): + with patch.object( + resolver, "github_api", side_effect=[self.run, {"jobs": jobs}] + ): + with self.assertRaisesRegex(ValueError, "successful release job"): + resolver.resolve_release("openai/codex", "123") + + def test_reads_all_pages_of_jobs_and_assets(self) -> None: + responses = self.responses() + responses.insert(1, {"jobs": [{**self.job, "name": "other"}] * 100}) + responses.insert(-1, [{"name": "other", "state": "uploaded", "size": 1}] * 100) + with patch.object(resolver, "github_api", side_effect=responses) as api: + self.assertEqual( + resolver.resolve_release("openai/codex", "123"), self.expected + ) + api.assert_any_call( + "repos/openai/codex/actions/runs/123/jobs?filter=all&per_page=100&page=2" + ) + api.assert_any_call( + "repos/openai/codex/releases/789/assets?per_page=100&page=2" + ) + + def test_rejects_a_moved_tag(self) -> None: + responses = self.responses() + responses[2] = {"object": {"type": "commit", "sha": "b" * 40}} + with patch.object(resolver, "github_api", side_effect=responses): + with self.assertRaisesRegex(ValueError, "no longer matches"): + resolver.resolve_release("openai/codex", "123") + + def test_bounds_annotated_tag_resolution(self) -> None: + tag = {"object": {"type": "tag", "sha": "b" * 40}} + with patch.object( + resolver, + "github_api", + side_effect=[self.run, {"jobs": [self.job]}, *([tag] * 9)], + ) as api: + with self.assertRaisesRegex(ValueError, "no longer matches"): + resolver.resolve_release("openai/codex", "123") + self.assertEqual(api.call_count, 11) + + def test_requires_a_published_stable_release_for_the_tag(self) -> None: + for field, value in ( + ("draft", True), + ("prerelease", True), + ("tag_name", "rust-v9.9.9"), + ): + responses = self.responses() + responses[3][field] = value + with self.subTest(field=field): + with patch.object(resolver, "github_api", side_effect=responses): + with self.assertRaisesRegex(ValueError, "published and stable"): + resolver.resolve_release("openai/codex", "123") + + def test_requires_all_runtime_inputs_to_be_uploaded_and_nonempty(self) -> None: + for assets in ( + self.assets[:-1], + [*self.assets[:-1], {**self.assets[-1], "size": 0}], + [*self.assets[:-1], {**self.assets[-1], "state": "new"}], + ): + with self.subTest(assets=assets): + responses = self.responses() + responses[-1] = assets + with patch.object(resolver, "github_api", side_effect=responses): + with self.assertRaisesRegex(ValueError, "missing Python inputs"): + resolver.resolve_release("openai/codex", "123") + + def test_rejects_invalid_inputs_before_github_access(self) -> None: + with patch.object(resolver, "github_api") as api: + for repository, run_id in ( + ("other/codex", "123"), + ("openai/codex", "../123"), + ): + with self.assertRaises(ValueError): + resolver.resolve_release(repository, run_id) + api.assert_not_called() + + def test_event_and_manual_retry_emit_the_same_revision_and_version(self) -> None: + with tempfile.TemporaryDirectory() as directory: + output = Path(directory) / "output" + event = Path(directory) / "event.json" + for automatic in (True, False): + output.write_text("") + event.write_text( + json.dumps({"workflow_run": self.run} if automatic else {}) + ) + responses = self.responses()[1:] if automatic else self.responses() + with patch.object(resolver, "github_api", side_effect=responses) as api: + resolver.main( + [ + "123", + "--repository", + "openai/codex", + "--event-path", + str(event), + "--github-output", + str(output), + ] + ) + self.assertEqual( + output.read_text(), + f"publish=true\nsource_sha={self.revision}\nrelease_tag=rust-v1.2.3\nversion=1.2.3\n", + ) + if automatic: + self.assertNotIn( + unittest.mock.call("repos/openai/codex/actions/runs/123"), + api.call_args_list, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/workflows/python-runtime-build.yml b/.github/workflows/python-runtime-build.yml index 338525c90c..748ff2f113 100644 --- a/.github/workflows/python-runtime-build.yml +++ b/.github/workflows/python-runtime-build.yml @@ -3,6 +3,10 @@ name: python-runtime-build on: workflow_call: inputs: + source_ref: + description: "Source revision to build; defaults to the calling workflow revision." + required: false + type: string runtime_version: description: "Runtime version to build, for example 0.136.0, 0.136.0a2, or 0.136.0a2.post1." required: true @@ -25,6 +29,7 @@ jobs: - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: + ref: ${{ inputs.source_ref || github.sha }} persist-credentials: false - name: Validate and resolve Python runtime release diff --git a/.github/workflows/python-sdk-build.yml b/.github/workflows/python-sdk-build.yml index c6aac56464..66d89bc96c 100644 --- a/.github/workflows/python-sdk-build.yml +++ b/.github/workflows/python-sdk-build.yml @@ -3,6 +3,10 @@ name: python-sdk-build on: workflow_call: inputs: + source_ref: + description: "Source revision to build; defaults to the calling workflow revision." + required: false + type: string sdk_version: description: "Python SDK version to build." required: true @@ -24,6 +28,7 @@ jobs: - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: + ref: ${{ inputs.source_ref || github.sha }} persist-credentials: false - name: Setup Python diff --git a/.github/workflows/python-sdk-cli-release.yml b/.github/workflows/python-sdk-cli-release.yml new file mode 100644 index 0000000000..2f7f6a7f19 --- /dev/null +++ b/.github/workflows/python-sdk-cli-release.yml @@ -0,0 +1,156 @@ +name: python-sdk-cli-release + +on: + workflow_run: + workflows: [rust-release] + types: [completed] + workflow_dispatch: + inputs: + cli_run_id: + description: "Stable rust-release run to publish or retry." + required: true + type: string + +permissions: + contents: read + +concurrency: + group: python-sdk-cli-release-${{ github.event.workflow_run.id || inputs.cli_run_id }} + cancel-in-progress: false + +jobs: + resolve-cli-release: + if: github.repository == 'openai/codex' + runs-on: ubuntu-latest + permissions: + contents: read + actions: read + outputs: + publish: ${{ steps.release.outputs.publish }} + version: ${{ steps.release.outputs.version }} + source_sha: ${{ steps.release.outputs.source_sha }} + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - name: Resolve the completed CLI release + id: release + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + CLI_RUN_ID: ${{ github.event.workflow_run.id || inputs.cli_run_id }} + run: | + python3 .github/scripts/resolve_python_cli_release.py "$CLI_RUN_ID" \ + --repository "$GITHUB_REPOSITORY" \ + --event-path "$GITHUB_EVENT_PATH" \ + --github-output "$GITHUB_OUTPUT" + + prepare-python-runtime: + name: prepare-python-runtime + needs: resolve-cli-release + if: needs.resolve-cli-release.outputs.publish == 'true' + permissions: + contents: read + uses: ./.github/workflows/python-runtime-build.yml + with: + runtime_version: ${{ needs.resolve-cli-release.outputs.version }} + source_ref: ${{ needs.resolve-cli-release.outputs.source_sha }} + + build-python-sdk: + name: build-python-sdk + needs: resolve-cli-release + if: needs.resolve-cli-release.outputs.publish == 'true' + permissions: + contents: read + uses: ./.github/workflows/python-sdk-build.yml + with: + sdk_version: ${{ needs.resolve-cli-release.outputs.version }} + runtime_version: ${{ needs.resolve-cli-release.outputs.version }} + source_ref: ${{ needs.resolve-cli-release.outputs.source_sha }} + + # Publish from the top-level workflow: PyPI does not support reusable + # workflows as Trusted Publishers. The runtime must be available before + # publishing the SDK that depends on it. + publish-python-runtime: + if: github.repository == 'openai/codex' + name: publish-python-runtime + needs: + - prepare-python-runtime + - build-python-sdk + - resolve-cli-release + runs-on: ubuntu-latest + environment: pypi + permissions: + contents: read + id-token: write # Required for PyPI trusted publishing. + + steps: + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + ref: ${{ needs.resolve-cli-release.outputs.source_sha }} + persist-credentials: false + + - name: Download Python runtime wheels + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: python-runtime-wheels + path: dist/python-runtime + + - name: Publish Python runtime wheels to PyPI + uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2 + with: + packages-dir: dist/python-runtime + skip-existing: true + + - name: Install uv + uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 + with: + version: "0.11.3" + + - name: Verify Python runtime wheels are available on PyPI + env: + PYTHON_RUNTIME_VERSION: ${{ needs.resolve-cli-release.outputs.version }} + run: | + uv run --no-project --with packaging==26.2 python .github/scripts/verify_pypi_release.py \ + openai-codex-cli-bin "$PYTHON_RUNTIME_VERSION" + + publish-python-sdk: + name: publish-python-sdk + needs: + - build-python-sdk + - publish-python-runtime + - resolve-cli-release + runs-on: ubuntu-latest + environment: pypi + permissions: + contents: read + id-token: write # Required for PyPI trusted publishing. + + steps: + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + ref: ${{ needs.resolve-cli-release.outputs.source_sha }} + persist-credentials: false + + - name: Download Python SDK package + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: python-sdk-package + path: dist/python-sdk + + - name: Publish Python SDK to PyPI + uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2 + with: + packages-dir: dist/python-sdk + skip-existing: true + + - name: Install uv + uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 + with: + version: "0.11.3" + + - name: Verify Python SDK is available on PyPI + env: + SDK_VERSION: ${{ needs.resolve-cli-release.outputs.version }} + run: uv run --no-project --with packaging==26.2 python .github/scripts/verify_pypi_release.py openai-codex "$SDK_VERSION" diff --git a/.github/workflows/repo-checks.yml b/.github/workflows/repo-checks.yml index b025de718d..efcc043de8 100644 --- a/.github/workflows/repo-checks.yml +++ b/.github/workflows/repo-checks.yml @@ -31,6 +31,9 @@ jobs: with: version: "0.11.3" + - name: Test downstream Python release resolution + run: python3 -m unittest discover -s .github/scripts -p 'test_resolve_python_cli_release.py' + - name: Test PyPI release verification run: uv run --no-project --with packaging==26.2 python -m unittest discover -s .github/scripts -p 'test_verify_pypi_release.py' diff --git a/sdk/python/RELEASING.md b/sdk/python/RELEASING.md new file mode 100644 index 0000000000..0eba5e9fd4 --- /dev/null +++ b/sdk/python/RELEASING.md @@ -0,0 +1,38 @@ +# Python SDK releases + +Stable CLI releases trigger `python-sdk-cli-release.yml` after `rust-release` +finishes. Publication requires a successful `release` job and all required runtime +assets on the published stable release; unrelated publisher failures, such as +winget, do not block it. The downstream workflow checks that the release tag still +points to that run's commit, builds the SDK and runtime from that revision, then +publishes and verifies all runtime wheels before publishing and verifying the SDK. +For CLI version X, the SDK version and its exact runtime dependency are both X. +CLI prereleases do not trigger Python publication. Failures in this downstream +workflow do not block CLI completion or `latest-alpha-cli`. + +The downstream workflow must exist on the default branch before GitHub can trigger +it. The CLI release revision must contain the Python build scripts and generated +contracts. It never substitutes newer SDK sources from the default branch. + +To retry independently, rerun the failed downstream jobs, or dispatch +`python-sdk-cli-release.yml` with `cli_run_id` set to the Rust release's GitHub +Actions run ID. The resolver checks the effective release job across run attempts +and verifies the same tag, commit, and required assets again. Existing +PyPI uploads are accepted, and verification still requires the complete release. +Use a new version if already-published package contents need to change. + +Before enabling publication, configure both `openai-codex` and +`openai-codex-cli-bin` on PyPI to trust owner `openai`, repository `codex`, workflow +`python-sdk-cli-release.yml`, environment `pypi`. The GitHub environment must permit +this workflow on the default branch, which is the ref used by `workflow_run` and +manual dispatch even though the build checks out the release commit. These PyPI +and GitHub environment settings are managed outside the repository. + +Independent SDK releases remain available through `python-v*` tags and +`python-sdk-release.yml`. Stable SDK versions must match the checked-in runtime +pin; beta SDK versions such as `python-v0.1.0b1` can use an independently versioned +runtime. Retain the existing trusted-publisher entries for this manual workflow +and `python-runtime-release.yml`. + +See GitHub's [workflow_run documentation](https://docs.github.com/en/actions/reference/workflows-and-actions/events-that-trigger-workflows#workflow_run) +and PyPI's [trusted-publisher setup](https://docs.pypi.org/trusted-publishers/adding-a-publisher/). diff --git a/sdk/python/docs/faq.md b/sdk/python/docs/faq.md index c21624a283..b633ea420a 100644 --- a/sdk/python/docs/faq.md +++ b/sdk/python/docs/faq.md @@ -7,8 +7,11 @@ ## Why does the SDK install a runtime package? -The SDK version tracks the corresponding Codex CLI release. Each SDK release -pins and installs its matching runtime dependency automatically. +Stable CLI releases publish the SDK with the same version and an exact runtime +pin. CLI prereleases do not trigger Python package publishing. Independent SDK +beta releases can still be published manually with a different version number, +but must pin a compatible runtime. The dependency is installed automatically. +See [Python SDK releases](../RELEASING.md) for publishing and retry instructions. ## Thread vs turn diff --git a/sdk/python/docs/getting-started.md b/sdk/python/docs/getting-started.md index af7b71026f..9ea561a6d5 100644 --- a/sdk/python/docs/getting-started.md +++ b/sdk/python/docs/getting-started.md @@ -17,7 +17,7 @@ Requirements: - An existing Codex account session, or one of the login flows below The SDK installs its matching `openai-codex-cli-bin` runtime dependency -automatically. SDK release versions track the corresponding Codex CLI release. +automatically. Stable SDK releases track the corresponding stable Codex CLI release. ## 2. Authenticate When Needed