diff --git a/.github/scripts/check_github_canary.py b/.github/scripts/check_github_canary.py deleted file mode 100644 index b8b74f4a1e..0000000000 --- a/.github/scripts/check_github_canary.py +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env python3 -"""Compare a publishing release with the canary Cargo manifest from stdin. - -Usage: check_github_canary.py RELEASE_VERSION -""" - -import sys -import tomllib - -from releases import should_update_version - - -def main() -> None: - try: - current_version = tomllib.loads(sys.stdin.read())["workspace"]["package"][ - "version" - ] - except (tomllib.TOMLDecodeError, KeyError, TypeError): - current_version = "" - if not isinstance(current_version, str): - current_version = "" - print(str(should_update_version(sys.argv[1], current_version)).lower()) - - -if __name__ == "__main__": - main() diff --git a/.github/scripts/npm_alpha_tag.py b/.github/scripts/npm_alpha_tag.py index cef206ff83..47e3498fe0 100644 --- a/.github/scripts/npm_alpha_tag.py +++ b/.github/scripts/npm_alpha_tag.py @@ -1,4 +1,4 @@ -"""Choose a forward-only npm alpha dist-tag under rust-release's concurrency lock.""" +"""Choose forward-only npm dist-tags under rust-release's publication lock.""" import json from pathlib import Path @@ -6,12 +6,12 @@ import subprocess import sys import tarfile -from releases import should_update_version +from releases import is_valid_stable_release_version, should_update_version def npm_alpha_tag(tarball: Path, version: str, tag: str) -> str: # Read the actual package name: platform archives share @openai/codex, - # while the SDK and proxy have independent alpha pointers. + # while the SDK and proxy have independent latest/alpha pointers. with tarfile.open(tarball, "r:gz") as archive: with archive.extractfile("package/package.json") as manifest: package = json.load(manifest)["name"] @@ -23,14 +23,19 @@ def npm_alpha_tag(tarball: Path, version: str, tag: str) -> str: text=True, ) current = json.loads(result.stdout).get(tag, "") - if tag != "alpha": + if tag not in ("alpha", "latest"): # build_npm_package.py appends - to the release version. current = current.removesuffix(f"-{tag.removeprefix('alpha-')}") + # Stable tags fail closed on unreadable metadata. Alpha tags keep their + # existing recovery behavior, as with the other prerelease channels. + is_alpha = tag == "alpha" or tag.startswith("alpha-") + if current and not is_alpha and not is_valid_stable_release_version(current): + raise ValueError(f"Cannot compare current {package}@{tag} version: {current}") if should_update_version(version, current): return tag # npm publish always writes a tag. Keep this version installable without - # changing alpha, alpha-, or latest. Prefix avoids semver tags, + # changing latest, alpha, or their platform tags. Prefix avoids semver tags, # which npm rejects. Include the platform for versions of the same package. print( f"Keeping {package}@{tag} at {current}; publishing {version} separately", diff --git a/.github/scripts/publish_r2_release.py b/.github/scripts/publish_r2_release.py index 5b6ded1331..456cab2cdb 100755 --- a/.github/scripts/publish_r2_release.py +++ b/.github/scripts/publish_r2_release.py @@ -10,8 +10,9 @@ versioned prefix includes every release asset plus installer-facing objects are verified, the same metadata advances ``codex/channels/latest`` when the release is marked latest and ``codex/channels/prerelease`` for prereleases that are newer than its current version, or when that version is unreadable. -Stable releases also update the mutable ``codex/install.sh`` and +Latest stable releases also update the mutable ``codex/install.sh`` and ``codex/install.ps1`` bootstrap aliases from their verified versioned assets. +An older stable release cannot move these aliases or the latest channel back. """ import argparse @@ -27,13 +28,16 @@ from pathlib import Path from typing import Any, NamedTuple, NoReturn from urllib.parse import quote -from releases import is_valid_release_version, should_update_version +from releases import ( + is_valid_release_version, + is_valid_stable_release_version, + should_update_version, +) BUCKET = "releases" PREFIX = "codex" REPOSITORY = "openai/codex" RELEASE_METADATA_NAME = "release.json" -PRERELEASE_CHANNEL_KEY = f"{PREFIX}/channels/prerelease" INSTALLER_NAMES = ("install.sh", "install.ps1") MAX_UPLOAD_WORKERS = 8 CRC64_RE = re.compile(r"^[A-Za-z0-9+/]{11}=$") @@ -263,27 +267,39 @@ def verify_remote( ) -def should_update_prerelease(endpoint: str, version: str) -> bool: +def should_update_channel(endpoint: str, version: str, channel: str) -> bool: # Read R2 directly: the public CDN may still serve an older version. - url = f"s3://{BUCKET}/{PRERELEASE_CHANNEL_KEY}" + key = f"{PREFIX}/channels/{channel}" + url = f"s3://{BUCKET}/{key}" try: raw = run_command(["aws", "s3", "cp", url, "-", "--endpoint-url", endpoint]) except subprocess.CalledProcessError as error: if MISSING_OBJECT_RE.search(error.stderr or ""): return True - raise_s3("read", PRERELEASE_CHANNEL_KEY, error, (error.stderr or "").strip()) + raise_s3("read", key, error, (error.stderr or "").strip()) except OSError as error: - raise_s3("read", PRERELEASE_CHANNEL_KEY, error) + raise_s3("read", key, error) try: metadata = json.loads(raw) - except json.JSONDecodeError: + except json.JSONDecodeError as error: + if channel == "latest": + raise PublishError(f"invalid channel metadata for {key}") from error return True tag = metadata.get("tag_name") if isinstance(metadata, dict) else None if not isinstance(tag, str) or not tag.startswith("rust-v"): + if channel == "latest": + raise PublishError(f"invalid channel metadata for {key}") return True - return should_update_version(version, tag.removeprefix("rust-v")) + current = tag.removeprefix("rust-v") + if channel == "latest": + if not is_valid_stable_release_version(current): + raise PublishError(f"invalid stable version in channel metadata for {key}") + # Reruns can repair installer writes interrupted before channel finalization. + if version == current: + return True + return should_update_version(version, current) def publish_installers(endpoint: str, tag: str, assets: list[ReleaseAsset]) -> None: @@ -435,7 +451,7 @@ def main() -> int: asset for asset in assets if asset.path.name not in previously_published ] required_downloads = list(remaining) - if args.prerelease == "false": + if args.prerelease == "false" and args.make_latest == "true": required_downloads.extend( asset for asset in assets @@ -497,15 +513,17 @@ def main() -> int: f"size={metadata_size} sha256={metadata_sha256}", file=sys.stderr, ) - if args.prerelease == "false": - publish_installers(endpoint, args.tag, assets) channels = [] - if args.make_latest == "true": + if args.make_latest == "true" and should_update_channel( + endpoint, version, "latest" + ): channels.append("latest") - if args.prerelease == "true" and should_update_prerelease( - endpoint, version + if args.prerelease == "true" and should_update_channel( + endpoint, version, "prerelease" ): channels.append("prerelease") + if "latest" in channels: + publish_installers(endpoint, args.tag, assets) for channel in channels: channel_key = f"{PREFIX}/channels/{channel}" put_object( diff --git a/.github/scripts/releases.py b/.github/scripts/releases.py index efb0bba116..bc0bb30985 100644 --- a/.github/scripts/releases.py +++ b/.github/scripts/releases.py @@ -1,5 +1,7 @@ -"""Validate and compare the version formats used by Codex releases.""" +"""Validate Codex releases and check whether their GitHub channel pointers can advance.""" +import argparse +import json import re import sys @@ -13,6 +15,10 @@ def is_valid_release_version(version: str) -> bool: return _VERSION_RE.fullmatch(version) is not None +def is_valid_stable_release_version(version: str) -> bool: + return is_valid_release_version(version) and "-" not in version + + def _version_key(version: str) -> tuple[tuple[int, ...], int, tuple[int, ...]]: base, _, prerelease = version.partition("-") channel, _, suffix = prerelease.partition(".") @@ -34,5 +40,46 @@ def should_update_version(version: str, current_version: str) -> bool: return _version_key(version) > _version_key(current_version) +def should_make_latest(version: str, latest: dict) -> bool: + if not is_valid_stable_release_version(version): + raise ValueError(f"expected a stable release version: {version}") + tag = latest.get("tag_name", "") + current = tag.removeprefix("rust-v") if isinstance(tag, str) else "" + if ( + not isinstance(tag, str) + or not tag.startswith("rust-v") + or not is_valid_stable_release_version(current) + or latest.get("draft") is not False + or latest.get("prerelease") is not False + ): + raise ValueError("Cannot determine the current GitHub latest stable release") + # An equal version must be able to finish a partially completed publication. + return version == current or should_update_version(version, current) + + if __name__ == "__main__": - raise SystemExit(0 if is_valid_release_version(sys.argv[1]) else 1) + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("version") + parser.add_argument( + "--check-github", + choices=("latest", "canary"), + help="Compare with the latest release JSON or canary Cargo.toml on stdin.", + ) + args = parser.parse_args() + if args.check_github is None: + raise SystemExit(0 if is_valid_release_version(args.version) else 1) + if args.check_github == "latest": + update = should_make_latest(args.version, json.load(sys.stdin)) + else: + import tomllib + + try: + current_version = tomllib.loads(sys.stdin.read())["workspace"]["package"][ + "version" + ] + except (tomllib.TOMLDecodeError, KeyError, TypeError): + current_version = "" + if not isinstance(current_version, str): + current_version = "" + update = should_update_version(args.version, current_version) + print(str(update).lower()) diff --git a/.github/scripts/test_releases.py b/.github/scripts/test_releases.py index 2133724e63..b2fa6a5163 100644 --- a/.github/scripts/test_releases.py +++ b/.github/scripts/test_releases.py @@ -8,7 +8,8 @@ import unittest from unittest.mock import patch from npm_alpha_tag import npm_alpha_tag -from releases import is_valid_release_version, should_update_version +import publish_r2_release as r2 +from releases import is_valid_release_version, should_make_latest, should_update_version class VersionComparisonTest(unittest.TestCase): @@ -61,10 +62,32 @@ class VersionComparisonTest(unittest.TestCase): class NpmAlphaTagTest(unittest.TestCase): + def write_package(self, tarball: Path, package: str) -> None: + content = json.dumps({"name": package}).encode() + with tarfile.open(tarball, "w:gz") as archive: + info = tarfile.TarInfo("package/package.json") + info.size = len(content) + archive.addfile(info, io.BytesIO(content)) + def test_published_tags_across_packages_and_platforms(self) -> None: # Each package/tag reads its own registry value, even after a partially # published release. Also exercise a missing pointer and dotted hotfix. cases = [ + ( + "@openai/codex-sdk", + "latest", + "0.100.0", + "0.99.9", + "release-0.99.9-latest", + ), + ( + "@openai/codex", + "linux-x64", + "0.99.9-linux-x64", + "0.100.0", + "linux-x64", + ), + ("@openai/codex-responses-api-proxy", "latest", None, "0.100.0", "latest"), ( "@openai/codex", "alpha", @@ -101,16 +124,19 @@ class NpmAlphaTagTest(unittest.TestCase): "release-0.158.0-alpha.10.1-alpha", ), ("@openai/codex-sdk", "alpha", None, "0.158.0-alpha.10", "alpha"), + ( + "@openai/codex", + "alpha-linux-x64", + "broken-linux-x64", + "0.158.0-alpha.10", + "alpha-linux-x64", + ), ] with tempfile.TemporaryDirectory() as tmpdir: tarball = Path(tmpdir) / "package.tgz" for package, tag, current, version, expected in cases: with self.subTest(package=package, tag=tag, version=version): - content = json.dumps({"name": package}).encode() - with tarfile.open(tarball, "w:gz") as archive: - info = tarfile.TarInfo("package/package.json") - info.size = len(content) - archive.addfile(info, io.BytesIO(content)) + self.write_package(tarball, package) tags = {tag: current} if current is not None else {} result = subprocess.CompletedProcess([], 0, json.dumps(tags)) with patch( @@ -131,14 +157,26 @@ class NpmAlphaTagTest(unittest.TestCase): text=True, ) + def test_stable_tags_reject_malformed_and_prerelease_versions(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + tarball = Path(tmpdir) / "package.tgz" + self.write_package(tarball, "@openai/codex") + for tag, current in ( + ("latest", "broken"), + ("linux-x64", "0.100.0-alpha.1-linux-x64"), + ): + with self.subTest(tag=tag, current=current): + result = subprocess.CompletedProcess( + [], 0, json.dumps({tag: current}) + ) + with patch("npm_alpha_tag.subprocess.run", return_value=result): + with self.assertRaises(ValueError): + npm_alpha_tag(tarball, "0.99.9", tag) + def test_registry_failure_prevents_publish(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: tarball = Path(tmpdir) / "package.tgz" - content = b'{"name": "@openai/codex"}' - with tarfile.open(tarball, "w:gz") as archive: - info = tarfile.TarInfo("package/package.json") - info.size = len(content) - archive.addfile(info, io.BytesIO(content)) + self.write_package(tarball, "@openai/codex") with patch( "npm_alpha_tag.subprocess.run", side_effect=subprocess.CalledProcessError(1, "npm"), @@ -147,5 +185,39 @@ class NpmAlphaTagTest(unittest.TestCase): npm_alpha_tag(tarball, "0.158.0-alpha.10", "alpha") +class StablePublicationTests(unittest.TestCase): + def test_github_latest_orders_numerically_and_allows_equal_reruns(self): + latest = {"tag_name": "rust-v0.100.0", "draft": False, "prerelease": False} + self.assertFalse(should_make_latest("0.99.9", latest)) + self.assertTrue(should_make_latest("0.100.0", latest)) + self.assertTrue(should_make_latest("0.101.0", latest)) + + def test_github_latest_rejects_unpublished_or_unreadable_metadata(self): + for latest in ( + {}, + {"tag_name": "rust-vunknown", "draft": False, "prerelease": False}, + {"tag_name": "rust-v0.100.0", "draft": True, "prerelease": False}, + {"tag_name": "rust-v0.100.0", "draft": False, "prerelease": True}, + ): + with self.subTest(latest=latest): + with self.assertRaises(ValueError): + should_make_latest("0.99.9", latest) + + def test_r2_checks_its_channel_and_only_repairs_unreadable_prereleases(self): + with patch.object(r2, "run_command") as command: + command.return_value = '{"tag_name":"rust-v0.100.0"}' + self.assertFalse(r2.should_update_channel("endpoint", "0.99.9", "latest")) + self.assertTrue(r2.should_update_channel("endpoint", "0.100.0", "latest")) + self.assertIn( + "s3://releases/codex/channels/latest", command.call_args.args[0] + ) + command.return_value = "invalid" + with self.assertRaises(r2.PublishError): + r2.should_update_channel("endpoint", "0.99.9", "latest") + self.assertTrue( + r2.should_update_channel("endpoint", "0.99.9-alpha.1", "prerelease") + ) + + if __name__ == "__main__": unittest.main() diff --git a/.github/workflows/rust-release.yml b/.github/workflows/rust-release.yml index 13e58ee28e..6dd30bce85 100644 --- a/.github/workflows/rust-release.yml +++ b/.github/workflows/rust-release.yml @@ -1639,7 +1639,10 @@ jobs: - name: Define release name id: release_name + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | + set -euo pipefail # Extract the version from the tag name, which is in the format # "rust-v0.1.0". version="${GITHUB_REF_NAME#rust-v}" @@ -1648,7 +1651,9 @@ jobs: echo "make_latest=false" >> $GITHUB_OUTPUT echo "prerelease=true" >> $GITHUB_OUTPUT else - echo "make_latest=true" >> $GITHUB_OUTPUT + make_latest="$(gh api "repos/${GITHUB_REPOSITORY}/releases/latest" \ + | python3 .github/scripts/releases.py "$version" --check-github latest)" + echo "make_latest=${make_latest}" >> $GITHUB_OUTPUT echo "prerelease=false" >> $GITHUB_OUTPUT fi @@ -1903,11 +1908,9 @@ jobs: ;; esac - # The workflow concurrency group holds the lock from this lookup - # through publish, including concurrent alpha/hotfix release runs. - if [[ "${NPM_TAG}" == "alpha" ]]; then - tag="$(python3 .github/scripts/npm_alpha_tag.py "${tarball}" "${VERSION}" "${tag}")" - fi + # The publication workflow holds the lock from this lookup through + # publish, including concurrent releases from other version lines. + tag="$(python3 .github/scripts/npm_alpha_tag.py "${tarball}" "${VERSION}" "${tag:-latest}")" publish_cmd=(npm publish "${GITHUB_WORKSPACE}/${tarball}") if [[ -n "${tag}" ]]; then @@ -2059,7 +2062,7 @@ jobs: # is JSON metadata with the file contents encoded as base64. update="$(gh api "repos/$GITHUB_REPOSITORY/contents/codex-rs/Cargo.toml?ref=latest-alpha-cli" \ -H 'Accept: application/vnd.github.raw+json' \ - | python3 .github/scripts/check_github_canary.py "$RELEASE_VERSION")" + | python3 .github/scripts/releases.py "$RELEASE_VERSION" --check-github canary)" echo "update=$update" >> "$GITHUB_OUTPUT" - name: Update latest-alpha-cli branch if: success() && steps.github_canary.outputs.update == 'true'