diff --git a/.github/scripts/publish_r2_release.py b/.github/scripts/publish_r2_release.py new file mode 100755 index 0000000000..ab83c18684 --- /dev/null +++ b/.github/scripts/publish_r2_release.py @@ -0,0 +1,338 @@ +#!/usr/bin/env python3 +"""Mirror a Codex GitHub Release to Cloudflare R2. + +Cloudflare R2 exposes an S3-compatible API, so the built-in AWS CLI uses +standard AWS credentials and the R2 endpoint from ``AWS_ENDPOINT_URL``. +Objects are created under ``codex/releases//`` with a validated upload +checksum and checked using object metadata before the run succeeds. The +versioned prefix includes every release asset plus installer-facing +``release.json`` metadata derived from the verified downloads. +""" + +import argparse +import hashlib +import json +import os +import re +import subprocess +import sys +import tempfile +from pathlib import Path +from typing import Any, NamedTuple, NoReturn +from urllib.parse import quote + +BUCKET = "releases" +PREFIX = "codex" +REPOSITORY = "openai/codex" +RELEASE_METADATA_NAME = "release.json" +VERSION_RE = re.compile(r"^[0-9]+\.[0-9]+\.[0-9]+(?:-(?:alpha|beta)(?:\.[0-9]+)?)?$") +CRC64_RE = re.compile(r"^[A-Za-z0-9+/]{11}=$") +SHA256_RE = re.compile(r"^sha256:([0-9a-f]{64})$") + + +class PublishError(RuntimeError): + pass + + +class ReleaseAsset(NamedTuple): + path: Path + size: int + sha256: str + + +def run_command(args: list[str]) -> str: + result = subprocess.run( + args, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + if result.stdout: + print(result.stdout, end="", file=sys.stderr) + if result.stderr: + print(result.stderr, end="", file=sys.stderr) + result.check_returncode() + return result.stdout or "" + + +def download_assets(tag: str, directory: Path) -> list[ReleaseAsset]: + try: + metadata = json.loads( + run_command( + [ + "gh", + "release", + "view", + tag, + "--repo", + REPOSITORY, + "--json", + "assets", + "--jq", + "[.assets[] | {name, size, state, digest}]", + ] + ) + ) + run_command( + [ + "gh", + "release", + "download", + tag, + "--repo", + REPOSITORY, + "--dir", + str(directory), + ] + ) + except (OSError, subprocess.CalledProcessError) as error: + raise PublishError( + f"GitHub release download failed for {tag}: {error}" + ) from error + except json.JSONDecodeError as error: + raise PublishError( + f"invalid GitHub release metadata for {tag}: {error}" + ) from error + + expected = {} + if not isinstance(metadata, list): + raise PublishError(f"GitHub returned invalid release metadata for {tag}") + for asset in metadata: + if not isinstance(asset, dict): + raise PublishError( + f"GitHub returned invalid release metadata for {tag}: {asset!r}" + ) + name = asset.get("name") + size = asset.get("size") + digest = asset.get("digest") + match = SHA256_RE.fullmatch(digest) if isinstance(digest, str) else None + if ( + not isinstance(name, str) + or not name + or name == RELEASE_METADATA_NAME + or name in expected + or type(size) is not int + or size < 0 + or asset.get("state") != "uploaded" + or match is None + ): + raise PublishError( + f"GitHub returned invalid release metadata for {tag}: {asset!r}" + ) + expected[name] = ReleaseAsset(directory / name, size, match.group(1)) + + assets = sorted(directory.iterdir(), key=lambda path: path.name) + if not assets: + raise PublishError(f"GitHub Release {tag} has no assets") + if any(not path.is_file() for path in assets) or { + path.name for path in assets + } != set(expected): + raise PublishError("GitHub returned invalid release assets") + return [expected[path.name] for path in assets] + + +def stream_digest(source: Any) -> tuple[int, str]: + digest = hashlib.sha256() + size = 0 + while chunk := source.read(1024 * 1024): + digest.update(chunk) + size += len(chunk) + return size, digest.hexdigest() + + +def raise_s3( + action: str, key: str, error: Exception, detail: str | None = None +) -> NoReturn: + raise PublishError( + f"could not {action} s3://{BUCKET}/{key}: {detail or error}" + ) from error + + +def put_immutable(endpoint: str, key: str, path: Path, sha256: str) -> None: + try: + run_command( + [ + "aws", + "s3", + "cp", + str(path), + f"s3://{BUCKET}/{key}", + "--no-overwrite", + "--checksum-algorithm", + "CRC64NVME", + "--metadata", + f"sha256={sha256}", + "--endpoint-url", + endpoint, + ] + ) + except subprocess.CalledProcessError as error: + raise_s3("upload", key, error, (error.stderr or "").strip()) + except OSError as error: + raise_s3("upload", key, error) + + +def verify_remote( + endpoint: str, + key: str, + expected_size: int, + expected_sha256: str, +) -> None: + try: + response = json.loads( + run_command( + [ + "aws", + "s3api", + "head-object", + "--bucket", + BUCKET, + "--key", + key, + "--checksum-mode", + "ENABLED", + "--endpoint-url", + endpoint, + ] + ) + ) + except subprocess.CalledProcessError as error: + raise_s3("inspect", key, error, (error.stderr or "").strip()) + except OSError as error: + raise_s3("inspect", key, error) + except json.JSONDecodeError as error: + raise PublishError(f"invalid object metadata for {key}: {error}") from error + + metadata = response.get("Metadata") if isinstance(response, dict) else None + size = response.get("ContentLength") if isinstance(response, dict) else None + crc64 = response.get("ChecksumCRC64NVME") if isinstance(response, dict) else None + sha256 = metadata.get("sha256") if isinstance(metadata, dict) else None + if ( + size != expected_size + or sha256 != expected_sha256 + or not isinstance(crc64, str) + or not CRC64_RE.fullmatch(crc64) + ): + raise PublishError( + f"object metadata mismatch for {key}: expected size={expected_size} " + f"sha256={expected_sha256}, got size={size} sha256={sha256} " + f"crc64nvme={crc64}" + ) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--tag", required=True) + return parser.parse_args() + + +def main() -> int: + args = parse_args() + try: + endpoint = os.environ.get("AWS_ENDPOINT_URL") + if not os.environ.get("GH_TOKEN"): + raise PublishError("GH_TOKEN is required") + if not endpoint: + raise PublishError("AWS_ENDPOINT_URL is required for the R2 S3 endpoint") + + version = args.tag.removeprefix("rust-v") + if args.tag == version or not VERSION_RE.fullmatch(version): + raise PublishError(f"invalid rust release tag: {args.tag}") + published = [] + metadata_assets = [] + with tempfile.TemporaryDirectory() as temp_dir: + assets_directory = Path(temp_dir) / "assets" + assets_directory.mkdir() + assets = download_assets(args.tag, assets_directory) + for asset in assets: + with asset.path.open("rb") as source: + size, sha256 = stream_digest(source) + if size != asset.size or sha256 != asset.sha256: + raise PublishError( + f"GitHub asset mismatch for {asset.path.name}: expected " + f"size={asset.size} sha256={asset.sha256}, got " + f"size={size} sha256={sha256}" + ) + for asset in assets: + path = asset.path + size = asset.size + sha256 = asset.sha256 + key = f"{PREFIX}/releases/{version}/{path.name}" + put_immutable(endpoint, key, path, sha256) + verify_remote(endpoint, key, size, sha256) + print( + f"published and verified s3://{BUCKET}/{key} " + f"size={size} sha256={sha256}", + file=sys.stderr, + ) + published.append( + { + "key": key, + "sha256": sha256, + "size": size, + } + ) + metadata_assets.append( + { + "name": path.name, + "digest": f"sha256:{sha256}", + "browser_download_url": ( + f"https://releases.openai.com/{PREFIX}/releases/" + f"{version}/{quote(path.name, safe='')}" + ), + } + ) + + metadata_path = Path(temp_dir) / RELEASE_METADATA_NAME + metadata_path.write_text( + json.dumps( + { + "assets": metadata_assets, + "tag_name": args.tag, + }, + indent=2, + ) + + "\n", + encoding="utf-8", + ) + with metadata_path.open("rb") as source: + metadata_size, metadata_sha256 = stream_digest(source) + metadata_key = f"{PREFIX}/releases/{version}/{RELEASE_METADATA_NAME}" + put_immutable(endpoint, metadata_key, metadata_path, metadata_sha256) + verify_remote( + endpoint, + metadata_key, + metadata_size, + metadata_sha256, + ) + print( + f"published and verified s3://{BUCKET}/{metadata_key} " + f"size={metadata_size} sha256={metadata_sha256}", + file=sys.stderr, + ) + + print( + json.dumps( + { + "assetCount": len(published), + "assets": published, + "releaseMetadata": { + "key": metadata_key, + "sha256": metadata_sha256, + "size": metadata_size, + }, + "releasePrefix": f"{PREFIX}/releases/{version}/", + "tag": args.tag, + "version": version, + }, + sort_keys=True, + ) + ) + return 0 + except PublishError as error: + print(f"publish failed: {error}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/workflows/r2-release.yml b/.github/workflows/r2-release.yml new file mode 100644 index 0000000000..ea6a0ea2a5 --- /dev/null +++ b/.github/workflows/r2-release.yml @@ -0,0 +1,38 @@ +name: publish-r2-release + +on: + workflow_call: + inputs: + tag: + required: true + type: string + +permissions: {} + +jobs: + publish: + runs-on: ubuntu-latest + timeout-minutes: 60 + environment: codex-r2-publisher + permissions: + contents: read + + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Publish release assets and metadata to R2 + # R2 exposes an S3-compatible API, so the AWS CLI reads AWS-named variables. + env: + AWS_ACCESS_KEY_ID: ${{ secrets.CODEX_R2_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.CODEX_R2_SECRET_ACCESS_KEY }} + AWS_ENDPOINT_URL: ${{ vars.CODEX_R2_ENDPOINT_URL }} + AWS_REGION: ${{ vars.CODEX_R2_REGION }} + GH_TOKEN: ${{ github.token }} + RELEASE_TAG: ${{ inputs.tag }} + run: | + set -euo pipefail + python3 .github/scripts/publish_r2_release.py \ + --tag "${RELEASE_TAG}" diff --git a/.github/workflows/rust-release.yml b/.github/workflows/rust-release.yml index f90b594b4e..805d231e98 100644 --- a/.github/workflows/rust-release.yml +++ b/.github/workflows/rust-release.yml @@ -19,7 +19,7 @@ env: concurrency: group: ${{ github.workflow }} - cancel-in-progress: true + cancel-in-progress: false jobs: tag-check: @@ -1273,6 +1273,16 @@ jobs: # (e.g. -alpha, -beta). Otherwise publish a normal release. prerelease: ${{ contains(steps.release_name.outputs.name, '-') }} + publish-r2: + name: publish-r2 + needs: [release, publish-dotslash] + permissions: + contents: read + uses: ./.github/workflows/r2-release.yml + secrets: inherit + with: + tag: ${{ needs.release.outputs.tag }} + publish-dotslash: name: publish-dotslash needs: release