From 40d3c8cbd88efd90622572d8c9f23f63c7b04154 Mon Sep 17 00:00:00 2001 From: Zsolt Dollenstein Date: Tue, 14 Jul 2026 15:54:16 +0200 Subject: [PATCH] smoke-test r2 publisher --- .github/scripts/publish_r2_release.py | 252 ++++++++++++++++++++++++++ .github/workflows/r2-release.yml | 45 +++++ .github/workflows/r2-smoke.yml | 16 ++ 3 files changed, 313 insertions(+) create mode 100755 .github/scripts/publish_r2_release.py create mode 100644 .github/workflows/r2-release.yml create mode 100644 .github/workflows/r2-smoke.yml diff --git a/.github/scripts/publish_r2_release.py b/.github/scripts/publish_r2_release.py new file mode 100755 index 0000000000..cd9496403b --- /dev/null +++ b/.github/scripts/publish_r2_release.py @@ -0,0 +1,252 @@ +#!/usr/bin/env python3 +"""Mirror a Codex GitHub Release to Cloudflare R2. + +Cloudflare R2 exposes an S3-compatible API, so the built-in AWS CLI uses +standard AWS credentials and the R2 endpoint from ``AWS_ENDPOINT_URL``. +Objects are created under ``codex/releases//`` and read back before +the run succeeds. The versioned prefix includes every release asset plus +installer-facing ``release.json`` metadata derived from the verified downloads. +""" + +import argparse +import hashlib +import json +import os +import re +import subprocess +import sys +import tempfile +from pathlib import Path +from typing import Any, NoReturn +from urllib.parse import quote + +BUCKET = "releases" +PREFIX = "codex" +REPOSITORY = "openai/codex" +RELEASE_METADATA_NAME = "release.json" +VERSION_RE = re.compile(r"^[0-9]+\.[0-9]+\.[0-9]+(?:-(?:alpha|beta)(?:\.[0-9]+)?)?$") + + +class PublishError(RuntimeError): + pass + + +def run_command(args: list[str]) -> None: + result = subprocess.run( + args, + stderr=subprocess.PIPE, + text=True, + ) + if result.stderr: + print(result.stderr, end="", file=sys.stderr) + result.check_returncode() + + +def download_assets(tag: str, directory: Path) -> list[Path]: + try: + run_command( + [ + "gh", + "release", + "download", + tag, + "--repo", + REPOSITORY, + "--dir", + str(directory), + ] + ) + except (OSError, subprocess.CalledProcessError) as error: + raise PublishError( + f"GitHub release download failed for {tag}: {error}" + ) from error + + assets = sorted(directory.iterdir(), key=lambda path: path.name) + if not assets: + raise PublishError(f"GitHub Release {tag} has no assets") + if any(not path.is_file() or path.name == RELEASE_METADATA_NAME for path in assets): + raise PublishError("GitHub returned invalid release assets") + return assets + + +def stream_digest(source: Any) -> tuple[int, str]: + digest = hashlib.sha256() + size = 0 + while chunk := source.read(1024 * 1024): + digest.update(chunk) + size += len(chunk) + return size, digest.hexdigest() + + +def raise_s3( + action: str, key: str, error: Exception, detail: str | None = None +) -> NoReturn: + raise PublishError( + f"could not {action} s3://{BUCKET}/{key}: {detail or error}" + ) from error + + +def put_immutable(endpoint: str, key: str, path: Path) -> None: + try: + run_command( + [ + "aws", + "s3api", + "put-object", + "--bucket", + BUCKET, + "--key", + key, + "--body", + str(path), + "--if-none-match", + "*", + "--endpoint-url", + endpoint, + ] + ) + except subprocess.CalledProcessError as error: + if any( + code in (error.stderr or "") + for code in ("PreconditionFailed", "ConditionalRequestConflict") + ): + return + raise_s3("upload", key, error, (error.stderr or "").strip()) + except OSError as error: + raise_s3("upload", key, error) + + +def verify_remote( + endpoint: str, + key: str, + expected_size: int, + expected_sha256: str, + path: Path, +) -> None: + try: + run_command( + [ + "aws", + "s3api", + "get-object", + "--bucket", + BUCKET, + "--key", + key, + "--endpoint-url", + endpoint, + str(path), + ] + ) + with path.open("rb") as source: + size, sha256 = stream_digest(source) + except subprocess.CalledProcessError as error: + raise_s3("read back", key, error, (error.stderr or "").strip()) + except OSError as error: + raise_s3("read back", key, error) + finally: + path.unlink(missing_ok=True) + if size != expected_size or sha256 != expected_sha256: + raise PublishError( + f"read-back mismatch for {key}: expected size={expected_size} " + f"sha256={expected_sha256}, got size={size} sha256={sha256}" + ) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--tag", required=True) + return parser.parse_args() + + +def main() -> int: + args = parse_args() + try: + endpoint = os.environ.get("AWS_ENDPOINT_URL") + if not os.environ.get("GH_TOKEN"): + raise PublishError("GH_TOKEN is required") + if not endpoint: + raise PublishError("AWS_ENDPOINT_URL is required for the R2 S3 endpoint") + + version = args.tag.removeprefix("rust-v") + if args.tag == version or not VERSION_RE.fullmatch(version): + raise PublishError(f"invalid rust release tag: {args.tag}") + published = [] + metadata_assets = [] + with tempfile.TemporaryDirectory() as temp_dir: + assets_directory = Path(temp_dir) / "assets" + assets_directory.mkdir() + readback_path = Path(temp_dir) / "readback" + for path in download_assets(args.tag, assets_directory): + with path.open("rb") as source: + size, sha256 = stream_digest(source) + key = f"{PREFIX}/releases/{version}/{path.name}" + put_immutable(endpoint, key, path) + verify_remote(endpoint, key, size, sha256, readback_path) + published.append( + { + "key": key, + "sha256": sha256, + "size": size, + } + ) + metadata_assets.append( + { + "name": path.name, + "digest": f"sha256:{sha256}", + "browser_download_url": ( + f"https://releases.openai.com/{PREFIX}/releases/" + f"{version}/{quote(path.name, safe='')}" + ), + } + ) + + metadata_path = Path(temp_dir) / RELEASE_METADATA_NAME + metadata_path.write_text( + json.dumps( + { + "assets": metadata_assets, + "tag_name": args.tag, + }, + indent=2, + ) + + "\n", + encoding="utf-8", + ) + with metadata_path.open("rb") as source: + metadata_size, metadata_sha256 = stream_digest(source) + metadata_key = f"{PREFIX}/releases/{version}/{RELEASE_METADATA_NAME}" + put_immutable(endpoint, metadata_key, metadata_path) + verify_remote( + endpoint, + metadata_key, + metadata_size, + metadata_sha256, + readback_path, + ) + + print( + json.dumps( + { + "assetCount": len(published), + "assets": published, + "releaseMetadata": { + "key": metadata_key, + "sha256": metadata_sha256, + "size": metadata_size, + }, + "releasePrefix": f"{PREFIX}/releases/{version}/", + "tag": args.tag, + "version": version, + }, + sort_keys=True, + ) + ) + return 0 + except PublishError as error: + print(f"publish failed: {error}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/workflows/r2-release.yml b/.github/workflows/r2-release.yml new file mode 100644 index 0000000000..194343bd67 --- /dev/null +++ b/.github/workflows/r2-release.yml @@ -0,0 +1,45 @@ +name: publish-r2-release + +on: + workflow_call: + inputs: + tag: + required: true + type: string + +permissions: {} + +jobs: + publish: + runs-on: ubuntu-latest + timeout-minutes: 60 + environment: codex-r2-publisher + permissions: + contents: read + + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Check disk before publication + run: df -h + + - name: Publish release assets and metadata to R2 + # R2 exposes an S3-compatible API, so the AWS CLI reads AWS-named variables. + env: + AWS_ACCESS_KEY_ID: ${{ secrets.CODEX_R2_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.CODEX_R2_SECRET_ACCESS_KEY }} + AWS_ENDPOINT_URL: ${{ vars.CODEX_R2_ENDPOINT_URL }} + AWS_REGION: ${{ vars.CODEX_R2_REGION }} + GH_TOKEN: ${{ github.token }} + RELEASE_TAG: ${{ inputs.tag }} + run: | + set -euo pipefail + python3 .github/scripts/publish_r2_release.py \ + --tag "${RELEASE_TAG}" + + - name: Check disk after publication + if: always() + run: df -h diff --git a/.github/workflows/r2-smoke.yml b/.github/workflows/r2-smoke.yml new file mode 100644 index 0000000000..ca25011fa8 --- /dev/null +++ b/.github/workflows/r2-smoke.yml @@ -0,0 +1,16 @@ +name: r2-smoke + +on: + push: + branches: + - zsol/r2-smoke-20260714 + +permissions: {} + +jobs: + smoke: + permissions: + contents: read + uses: ./.github/workflows/r2-release.yml + with: + tag: rust-v0.145.0-alpha.10