From 3c8bf7c3ebbedbd7ddd517233cabb675560eb9a2 Mon Sep 17 00:00:00 2001 From: Channing Conger Date: Tue, 7 Jul 2026 01:01:40 +0000 Subject: [PATCH] ci(v8): decouple artifact tags from bindings --- .github/actions/setup-rusty-v8/action.yml | 3 +- .github/scripts/rusty_v8_bazel.py | 91 +++++++++++++++++++ .github/scripts/test_rusty_v8_bazel.py | 105 ++++++++++++++++++++++ .github/workflows/rusty-v8-release.yml | 11 ++- .github/workflows/v8-canary.yml | 6 ++ scripts/codex_package/test_v8.py | 46 ++++++++++ scripts/codex_package/v8.py | 38 ++++++-- third_party/v8/README.md | 27 +++--- 8 files changed, 307 insertions(+), 20 deletions(-) create mode 100644 scripts/codex_package/test_v8.py diff --git a/.github/actions/setup-rusty-v8/action.yml b/.github/actions/setup-rusty-v8/action.yml index d9c4484657..564792ce41 100644 --- a/.github/actions/setup-rusty-v8/action.yml +++ b/.github/actions/setup-rusty-v8/action.yml @@ -15,8 +15,7 @@ runs: run: | set -euo pipefail - version="$(python3 "${GITHUB_WORKSPACE}/.github/scripts/rusty_v8_bazel.py" resolved-v8-crate-version)" - release_tag="rusty-v8-v${version}" + release_tag="$(python3 "${GITHUB_WORKSPACE}/.github/scripts/rusty_v8_bazel.py" rusty-v8-release-tag)" base_url="https://github.com/openai/codex/releases/download/${release_tag}" binding_dir="${RUNNER_TEMP}/rusty_v8" archive_path="${binding_dir}/librusty_v8_release_${TARGET}.a.gz" diff --git a/.github/scripts/rusty_v8_bazel.py b/.github/scripts/rusty_v8_bazel.py index 329d3f6c54..4813b65d40 100644 --- a/.github/scripts/rusty_v8_bazel.py +++ b/.github/scripts/rusty_v8_bazel.py @@ -27,6 +27,10 @@ RUSTY_V8_CHECKSUMS_DIR = ROOT / "third_party" / "v8" RELEASE_ARTIFACT_PROFILE = "release" SANDBOX_ARTIFACT_PROFILE = "ptrcomp_sandbox_release" ARTIFACT_BAZEL_CONFIGS = ["rusty-v8-upstream-libcxx"] +V8_SOURCE_ARCHIVE_PATTERN = re.compile( + r"https://github\.com/v8/v8/archive/refs/tags/" + r"([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+)\.tar\.gz" +) def bazel_execroot() -> Path: @@ -170,6 +174,26 @@ def resolved_v8_crate_version() -> str: return matches[0] +def resolved_v8_source_version(module_bazel: Path | None = None) -> str: + module_bazel = module_bazel or MODULE_BAZEL + matches = sorted(set(V8_SOURCE_ARCHIVE_PATTERN.findall(module_bazel.read_text()))) + if len(matches) != 1: + raise SystemExit( + "expected exactly one pinned V8 source version in MODULE.bazel, " + f"found: {matches}" + ) + return matches[0] + + +def rusty_v8_release_tag( + crate_version: str | None = None, + source_version: str | None = None, +) -> str: + crate_version = crate_version or resolved_v8_crate_version() + source_version = source_version or resolved_v8_source_version() + return f"rusty-v8-v{crate_version}-v8-{source_version}" + + def rusty_v8_checksum_manifest_path(version: str) -> Path: return RUSTY_V8_CHECKSUMS_DIR / f"rusty_v8_{version.replace('.', '_')}.sha256" @@ -258,6 +282,53 @@ def stage_artifacts( print(staged_checksums) +def v8_checkout_version(source_root: Path) -> str: + version_header = source_root / "v8" / "include" / "v8-version.h" + values: dict[str, str] = {} + for name, value in re.findall( + r"^#define V8_(MAJOR_VERSION|MINOR_VERSION|BUILD_NUMBER|PATCH_LEVEL) ([0-9]+)$", + version_header.read_text(), + flags=re.MULTILINE, + ): + values[name] = value + expected_names = {"MAJOR_VERSION", "MINOR_VERSION", "BUILD_NUMBER", "PATCH_LEVEL"} + if values.keys() != expected_names: + raise SystemExit(f"could not resolve V8 version from {version_header}") + return ".".join( + values[name] + for name in ["MAJOR_VERSION", "MINOR_VERSION", "BUILD_NUMBER", "PATCH_LEVEL"] + ) + + +def sync_upstream_v8_source( + source_root: Path, + source_version: str | None = None, +) -> None: + source_root = source_root.resolve() + source_version = source_version or resolved_v8_source_version() + v8_root = source_root / "v8" + subprocess.run( + ["git", "fetch", "--depth=1", "origin", f"refs/tags/{source_version}"], + cwd=v8_root, + check=True, + ) + subprocess.run( + ["git", "checkout", "--detach", "FETCH_HEAD"], + cwd=v8_root, + check=True, + ) + subprocess.run( + [sys.executable, "tools/update_deps.py"], + cwd=source_root, + check=True, + ) + actual_version = v8_checkout_version(source_root) + if actual_version != source_version: + raise SystemExit( + f"expected V8 {source_version} after source sync, found {actual_version}" + ) + + def upstream_release_pair_paths(source_root: Path, target: str) -> tuple[Path, Path]: lib_name = ( "rusty_v8.lib" if target.endswith("-pc-windows-msvc") else "librusty_v8.a" @@ -336,7 +407,15 @@ def parse_args() -> argparse.Namespace: stage_upstream_release_pair_parser.add_argument("--output-dir", required=True) stage_upstream_release_pair_parser.add_argument("--sandbox", action="store_true") + sync_upstream_source_parser = subparsers.add_parser("sync-upstream-source") + sync_upstream_source_parser.add_argument( + "--source-root", type=Path, required=True + ) + sync_upstream_source_parser.add_argument("--v8-version") + subparsers.add_parser("resolved-v8-crate-version") + subparsers.add_parser("resolved-v8-source-version") + subparsers.add_parser("rusty-v8-release-tag") check_module_bazel_parser = subparsers.add_parser("check-module-bazel") check_module_bazel_parser.add_argument("--version") @@ -379,9 +458,21 @@ def main() -> int: sandbox=args.sandbox, ) return 0 + if args.command == "sync-upstream-source": + sync_upstream_v8_source( + source_root=args.source_root, + source_version=args.v8_version, + ) + return 0 if args.command == "resolved-v8-crate-version": print(resolved_v8_crate_version()) return 0 + if args.command == "resolved-v8-source-version": + print(resolved_v8_source_version()) + return 0 + if args.command == "rusty-v8-release-tag": + print(rusty_v8_release_tag()) + return 0 if args.command == "check-module-bazel": version = command_version(args.version) manifest_path = command_manifest_path(args.manifest, version) diff --git a/.github/scripts/test_rusty_v8_bazel.py b/.github/scripts/test_rusty_v8_bazel.py index 0b5c03f436..7ca8364b6a 100644 --- a/.github/scripts/test_rusty_v8_bazel.py +++ b/.github/scripts/test_rusty_v8_bazel.py @@ -2,6 +2,7 @@ from __future__ import annotations +import subprocess import textwrap import unittest from os import environ @@ -54,6 +55,110 @@ class RustyV8BazelTest(unittest.TestCase): build_bazel, ) + def test_release_tag_combines_crate_and_source_versions(self) -> None: + self.assertEqual( + "rusty-v8-v149.2.0-v8-14.9.207.35", + rusty_v8_bazel.rusty_v8_release_tag( + crate_version="149.2.0", + source_version="14.9.207.35", + ), + ) + + def test_resolved_v8_source_version_reads_archive_override(self) -> None: + with TemporaryDirectory() as temp_dir: + module_bazel = Path(temp_dir) / "MODULE.bazel" + module_bazel.write_text( + textwrap.dedent( + """\ + archive_override( + module_name = "v8", + urls = ["https://github.com/v8/v8/archive/refs/tags/14.9.207.35.tar.gz"], + ) + """ + ) + ) + + self.assertEqual( + "14.9.207.35", + rusty_v8_bazel.resolved_v8_source_version(module_bazel), + ) + + def test_v8_checkout_version_reads_version_header(self) -> None: + with TemporaryDirectory() as temp_dir: + source_root = Path(temp_dir) + version_header = source_root / "v8" / "include" / "v8-version.h" + version_header.parent.mkdir(parents=True) + version_header.write_text( + textwrap.dedent( + """\ + #define V8_MAJOR_VERSION 14 + #define V8_MINOR_VERSION 9 + #define V8_BUILD_NUMBER 207 + #define V8_PATCH_LEVEL 35 + """ + ) + ) + + self.assertEqual( + "14.9.207.35", + rusty_v8_bazel.v8_checkout_version(source_root), + ) + + def test_sync_upstream_v8_source_checks_out_requested_tag(self) -> None: + with TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + remote = root / "remote" + version_header = remote / "include" / "v8-version.h" + version_header.parent.mkdir(parents=True) + version_header.write_text( + textwrap.dedent( + """\ + #define V8_MAJOR_VERSION 14 + #define V8_MINOR_VERSION 9 + #define V8_BUILD_NUMBER 207 + #define V8_PATCH_LEVEL 35 + """ + ) + ) + git = ["git", "-C", str(remote)] + subprocess.run(["git", "init", str(remote)], check=True, capture_output=True) + subprocess.run([*git, "add", "."], check=True, capture_output=True) + subprocess.run( + [ + *git, + "-c", + "user.name=Codex Test", + "-c", + "user.email=codex@example.com", + "commit", + "-m", + "V8 source", + ], + check=True, + capture_output=True, + ) + subprocess.run( + [*git, "tag", "14.9.207.35"], check=True, capture_output=True + ) + + source_root = root / "rusty_v8" + source_root.mkdir() + subprocess.run( + ["git", "clone", str(remote), str(source_root / "v8")], + check=True, + capture_output=True, + ) + tools = source_root / "tools" + tools.mkdir() + (tools / "update_deps.py").write_text("") + + rusty_v8_bazel.sync_upstream_v8_source(source_root, "14.9.207.35") + + self.assertEqual( + "14.9.207.35", + rusty_v8_bazel.v8_checkout_version(source_root), + ) + def test_command_version_tracks_remaining_http_file_assets(self) -> None: with TemporaryDirectory() as temp_dir: module_bazel = Path(temp_dir) / "MODULE.bazel" diff --git a/.github/workflows/rusty-v8-release.yml b/.github/workflows/rusty-v8-release.yml index 92be3761dd..ff3c38ede4 100644 --- a/.github/workflows/rusty-v8-release.yml +++ b/.github/workflows/rusty-v8-release.yml @@ -3,7 +3,7 @@ name: rusty-v8-release on: push: tags: - - "rusty-v8-v*.*.*" + - "rusty-v8-v*.*.*-v8-*.*.*.*" # Cargo's libgit2 transport has been flaky when fetching git dependencies with # nested submodules. Prefer the system git CLI for Cargo smoke tests. @@ -43,12 +43,11 @@ jobs: id: release_tag env: GITHUB_REF_NAME: ${{ github.ref_name }} - V8_VERSION: ${{ steps.v8_version.outputs.version }} shell: bash run: | set -euo pipefail - expected_release_tag="rusty-v8-v${V8_VERSION}" + expected_release_tag="$(python3 .github/scripts/rusty_v8_bazel.py rusty-v8-release-tag)" release_tag="${GITHUB_REF_NAME}" if [[ "${release_tag}" != "${expected_release_tag}" ]]; then echo "Tag ${release_tag} does not match expected release tag ${expected_release_tag}." >&2 @@ -323,6 +322,12 @@ jobs: python-version: "3.11" architecture: x64 + - name: Check out pinned V8 source and dependencies + shell: bash + run: | + python3 .github/scripts/rusty_v8_bazel.py sync-upstream-source \ + --source-root upstream-rusty-v8 + - name: Set up Codex Rust toolchain for Cargo smoke uses: dtolnay/rust-toolchain@e081816240890017053eacbb1bdf337761dc5582 # 1.95.0 with: diff --git a/.github/workflows/v8-canary.yml b/.github/workflows/v8-canary.yml index 53d9585108..5e77151b34 100644 --- a/.github/workflows/v8-canary.yml +++ b/.github/workflows/v8-canary.yml @@ -343,6 +343,12 @@ jobs: python-version: "3.11" architecture: x64 + - name: Check out pinned V8 source and dependencies + shell: bash + run: | + python3 .github/scripts/rusty_v8_bazel.py sync-upstream-source \ + --source-root upstream-rusty-v8 + - name: Set up Codex Rust toolchain for Cargo smoke uses: dtolnay/rust-toolchain@e081816240890017053eacbb1bdf337761dc5582 # 1.95.0 with: diff --git a/scripts/codex_package/test_v8.py b/scripts/codex_package/test_v8.py new file mode 100644 index 0000000000..bd397a74e1 --- /dev/null +++ b/scripts/codex_package/test_v8.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 + +from pathlib import Path +import sys +import tempfile +import textwrap +import unittest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from codex_package.v8 import resolved_v8_source_version +from codex_package.v8 import rusty_v8_release_tag + + +class RustyV8ReleaseTagTest(unittest.TestCase): + def test_release_tag_combines_crate_and_source_versions(self) -> None: + self.assertEqual( + rusty_v8_release_tag( + crate_version="149.2.0", + source_version="14.9.207.35", + ), + "rusty-v8-v149.2.0-v8-14.9.207.35", + ) + + def test_resolved_v8_source_version_reads_archive_override(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + module_bazel = Path(temp_dir) / "MODULE.bazel" + module_bazel.write_text( + textwrap.dedent( + """\ + archive_override( + module_name = "v8", + urls = ["https://github.com/v8/v8/archive/refs/tags/14.9.207.35.tar.gz"], + ) + """ + ) + ) + + self.assertEqual( + resolved_v8_source_version(module_bazel), + "14.9.207.35", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/codex_package/v8.py b/scripts/codex_package/v8.py index 4033e6822f..c049c70f62 100644 --- a/scripts/codex_package/v8.py +++ b/scripts/codex_package/v8.py @@ -4,6 +4,7 @@ from __future__ import annotations import hashlib import os +import re import shutil import tempfile from collections.abc import Mapping @@ -16,6 +17,10 @@ from .targets import TargetSpec DOWNLOAD_TIMEOUT_SECS = 120 +V8_SOURCE_ARCHIVE_PATTERN = re.compile( + r"https://github\.com/v8/v8/archive/refs/tags/" + r"([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+)\.tar\.gz" +) @dataclass(frozen=True) @@ -56,7 +61,7 @@ def resolve_codex_v8_cargo_env( def fetch_codex_v8_artifacts( spec: TargetSpec, *, - version: str | None = None, + crate_version: str | None = None, cache_root: Path | None = None, ) -> RustyV8ArtifactPair: if spec.is_windows: @@ -64,12 +69,14 @@ def fetch_codex_v8_artifacts( f"No Codex-built V8 release artifacts for target: {spec.target}" ) - version = version or resolved_v8_crate_version() - release_url = ( - f"https://github.com/openai/codex/releases/download/rusty-v8-v{version}" + crate_version = crate_version or resolved_v8_crate_version() + release_tag = rusty_v8_release_tag( + crate_version=crate_version, + source_version=resolved_v8_source_version(), ) + release_url = f"https://github.com/openai/codex/releases/download/{release_tag}" target = spec.target - cache_dir = (cache_root or default_cache_root()) / f"rusty-v8-{version}-{target}" + cache_dir = (cache_root or default_cache_root()) / f"{release_tag}-{target}" archive = cache_dir / f"librusty_v8_release_{target}.a.gz" binding = cache_dir / f"src_binding_release_{target}.rs" checksums = cache_dir / f"rusty_v8_release_{target}.sha256" @@ -104,6 +111,27 @@ def resolved_v8_crate_version() -> str: return versions[0] +def resolved_v8_source_version(module_bazel: Path | None = None) -> str: + module_bazel = module_bazel or REPO_ROOT / "MODULE.bazel" + matches = sorted(set(V8_SOURCE_ARCHIVE_PATTERN.findall(module_bazel.read_text()))) + if len(matches) != 1: + raise RuntimeError( + "Expected exactly one pinned V8 source version in MODULE.bazel, " + f"found: {matches}" + ) + return matches[0] + + +def rusty_v8_release_tag( + *, + crate_version: str | None = None, + source_version: str | None = None, +) -> str: + crate_version = crate_version or resolved_v8_crate_version() + source_version = source_version or resolved_v8_source_version() + return f"rusty-v8-v{crate_version}-v8-{source_version}" + + def default_cache_root() -> Path: return Path(tempfile.gettempdir()) / "codex-package" diff --git a/third_party/v8/README.md b/third_party/v8/README.md index 7fee412a69..5da0e0f589 100644 --- a/third_party/v8/README.md +++ b/third_party/v8/README.md @@ -21,16 +21,20 @@ Current pinned versions: - Rust crate: `v8 = =149.2.0` - Embedded upstream V8 source for Bazel-produced release builds: `14.9.207.35` (the revision used by Chromium `149.0.7827.201`) +- Codex artifact release tag: + `rusty-v8-v149.2.0-v8-14.9.207.35` ## Updating to a new `v8` release Use this as the maintainer flow for a version bump: -1. Bump the `v8` crate version and refresh `codex-rs/Cargo.lock`. -2. Update the Bazel versioned inputs in `MODULE.bazel`, then refresh the - matching checksum manifest and generated checksums as described below. +1. Update the `v8` crate and `codex-rs/Cargo.lock` when the Rust binding + changes. Update the embedded V8 source in `MODULE.bazel` independently. +2. Refresh the matching checksum manifest and generated checksums when the + remaining prebuilt inputs change, as described below. 3. Publish a release-candidate PR and validate that `v8-canary` passes. -4. If the canary is green, publish the release tag and release build. +4. If the canary is green, publish the release tag returned by + `python3 .github/scripts/rusty_v8_bazel.py rusty-v8-release-tag`. 5. Once the release build completes, rerun the build on the candidate branch and verify that the final artifact builds and tests pass. @@ -53,7 +57,7 @@ The consumer-facing selectors are: Published release assets are expected at the tag: -- `rusty-v8-v` +- `rusty-v8-v-v8-` with these raw asset names: @@ -89,8 +93,10 @@ The same run also builds the matching sandbox pair targets: The workflow also builds sandbox-enabled `x86_64-pc-windows-msvc` and `aarch64-pc-windows-msvc` archive/binding pairs -from upstream `rusty_v8` source. Those ABI-specific outputs cannot be produced -by Codex's Bazel Windows GNU toolchain. +from the matching upstream `rusty_v8` crate source, after synchronizing its V8 +checkout and Chromium dependencies to the source version pinned in +`MODULE.bazel`. Those ABI-specific outputs cannot be produced by Codex's Bazel +Windows GNU toolchain. The Bazel graph pins the same libc++, libc++abi, and llvm-libc source revisions used by `rusty_v8 v149.2.0`, compiles published artifact targets with @@ -111,7 +117,8 @@ Release and CI Cargo builds for Darwin and Linux use `RUSTY_V8_ARCHIVE` plus a downloaded `RUSTY_V8_SRC_BINDING_PATH` to point at those `openai/codex` release assets directly. We do not use `RUSTY_V8_MIRROR` because the upstream `v8` crate hardcodes a `v` tag layout, while our artifacts are published -under `rusty-v8-v`. +under `rusty-v8-v-v8-`. -Do not mix artifacts across crate versions. The archive and binding must match -the exact resolved `v8` crate version in `codex-rs/Cargo.lock`. +Do not mix artifacts across crate or source versions. The archive and binding +must match both the exact resolved `v8` crate version in `codex-rs/Cargo.lock` +and the embedded V8 source version in `MODULE.bazel`.