openai/codex
1
0
Fork 0
mirror of https://github.com/openai/codex.git synced 2026-09-05 15:18:41 +00:00
Code Issues Packages Projects Releases Wiki Activity

ci(v8): decouple artifact tags from bindings

Browse Source
This commit is contained in:
Channing Conger
2026-07-07 01:01:40 +00:00
parent 62868fa7aa
commit 3c8bf7c3eb
8 changed files with 307 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
.github/actions/setup-rusty-v8/action.yml vendored
View File

@@ -15,8 +15,7 @@ runs:
run: |
set -euo pipefail
version="$(python3 "${GITHUB_WORKSPACE}/.github/scripts/rusty_v8_bazel.py" resolved-v8-crate-version)"
release_tag="rusty-v8-v${version}"
release_tag="$(python3 "${GITHUB_WORKSPACE}/.github/scripts/rusty_v8_bazel.py" rusty-v8-release-tag)"
base_url="https://github.com/openai/codex/releases/download/${release_tag}"
binding_dir="${RUNNER_TEMP}/rusty_v8"
archive_path="${binding_dir}/librusty_v8_release_${TARGET}.a.gz"

91
.github/scripts/rusty_v8_bazel.py vendored
View File

@@ -27,6 +27,10 @@ RUSTY_V8_CHECKSUMS_DIR = ROOT / "third_party" / "v8"
RELEASE_ARTIFACT_PROFILE = "release"
SANDBOX_ARTIFACT_PROFILE = "ptrcomp_sandbox_release"
ARTIFACT_BAZEL_CONFIGS = ["rusty-v8-upstream-libcxx"]
V8_SOURCE_ARCHIVE_PATTERN = re.compile(
r"https://github\.com/v8/v8/archive/refs/tags/"
r"([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+)\.tar\.gz"
)
def bazel_execroot() -> Path:
@@ -170,6 +174,26 @@ def resolved_v8_crate_version() -> str:
return matches[0]
def resolved_v8_source_version(module_bazel: Path | None = None) -> str:
module_bazel = module_bazel or MODULE_BAZEL
matches = sorted(set(V8_SOURCE_ARCHIVE_PATTERN.findall(module_bazel.read_text())))
if len(matches) != 1:
raise SystemExit(
"expected exactly one pinned V8 source version in MODULE.bazel, "
f"found: {matches}"
)
return matches[0]
def rusty_v8_release_tag(
crate_version: str | None = None,
source_version: str | None = None,
) -> str:
crate_version = crate_version or resolved_v8_crate_version()
source_version = source_version or resolved_v8_source_version()
return f"rusty-v8-v{crate_version}-v8-{source_version}"
def rusty_v8_checksum_manifest_path(version: str) -> Path:
return RUSTY_V8_CHECKSUMS_DIR / f"rusty_v8_{version.replace('.', '_')}.sha256"
@@ -258,6 +282,53 @@ def stage_artifacts(
print(staged_checksums)
def v8_checkout_version(source_root: Path) -> str:
version_header = source_root / "v8" / "include" / "v8-version.h"
values: dict[str, str] = {}
for name, value in re.findall(
r"^#define V8_(MAJOR_VERSION|MINOR_VERSION|BUILD_NUMBER|PATCH_LEVEL) ([0-9]+)$",
version_header.read_text(),
flags=re.MULTILINE,
):
values[name] = value
expected_names = {"MAJOR_VERSION", "MINOR_VERSION", "BUILD_NUMBER", "PATCH_LEVEL"}
if values.keys() != expected_names:
raise SystemExit(f"could not resolve V8 version from {version_header}")
return ".".join(
values[name]
for name in ["MAJOR_VERSION", "MINOR_VERSION", "BUILD_NUMBER", "PATCH_LEVEL"]
)
def sync_upstream_v8_source(
source_root: Path,
source_version: str | None = None,
) -> None:
source_root = source_root.resolve()
source_version = source_version or resolved_v8_source_version()
v8_root = source_root / "v8"
subprocess.run(
["git", "fetch", "--depth=1", "origin", f"refs/tags/{source_version}"],
cwd=v8_root,
check=True,
)
subprocess.run(
["git", "checkout", "--detach", "FETCH_HEAD"],
cwd=v8_root,
check=True,
)
subprocess.run(
[sys.executable, "tools/update_deps.py"],
cwd=source_root,
check=True,
)
actual_version = v8_checkout_version(source_root)
if actual_version != source_version:
raise SystemExit(
f"expected V8 {source_version} after source sync, found {actual_version}"
)
def upstream_release_pair_paths(source_root: Path, target: str) -> tuple[Path, Path]:
lib_name = (
"rusty_v8.lib" if target.endswith("-pc-windows-msvc") else "librusty_v8.a"
@@ -336,7 +407,15 @@ def parse_args() -> argparse.Namespace:
stage_upstream_release_pair_parser.add_argument("--output-dir", required=True)
stage_upstream_release_pair_parser.add_argument("--sandbox", action="store_true")
sync_upstream_source_parser = subparsers.add_parser("sync-upstream-source")
sync_upstream_source_parser.add_argument(
"--source-root", type=Path, required=True
)
sync_upstream_source_parser.add_argument("--v8-version")
subparsers.add_parser("resolved-v8-crate-version")
subparsers.add_parser("resolved-v8-source-version")
subparsers.add_parser("rusty-v8-release-tag")
check_module_bazel_parser = subparsers.add_parser("check-module-bazel")
check_module_bazel_parser.add_argument("--version")
@@ -379,9 +458,21 @@ def main() -> int:
sandbox=args.sandbox,
)
return 0
if args.command == "sync-upstream-source":
sync_upstream_v8_source(
source_root=args.source_root,
source_version=args.v8_version,
)
return 0
if args.command == "resolved-v8-crate-version":
print(resolved_v8_crate_version())
return 0
if args.command == "resolved-v8-source-version":
print(resolved_v8_source_version())
return 0
if args.command == "rusty-v8-release-tag":
print(rusty_v8_release_tag())
return 0
if args.command == "check-module-bazel":
version = command_version(args.version)
manifest_path = command_manifest_path(args.manifest, version)

105
.github/scripts/test_rusty_v8_bazel.py vendored
View File

@@ -2,6 +2,7 @@
from __future__ import annotations
import subprocess
import textwrap
import unittest
from os import environ
@@ -54,6 +55,110 @@ class RustyV8BazelTest(unittest.TestCase):
build_bazel,
)
def test_release_tag_combines_crate_and_source_versions(self) -> None:
self.assertEqual(
"rusty-v8-v149.2.0-v8-14.9.207.35",
rusty_v8_bazel.rusty_v8_release_tag(
crate_version="149.2.0",
source_version="14.9.207.35",
),
)
def test_resolved_v8_source_version_reads_archive_override(self) -> None:
with TemporaryDirectory() as temp_dir:
module_bazel = Path(temp_dir) / "MODULE.bazel"
module_bazel.write_text(
textwrap.dedent(
"""\
archive_override(
module_name = "v8",
urls = ["https://github.com/v8/v8/archive/refs/tags/14.9.207.35.tar.gz"],
)
"""
)
)
self.assertEqual(
"14.9.207.35",
rusty_v8_bazel.resolved_v8_source_version(module_bazel),
)
def test_v8_checkout_version_reads_version_header(self) -> None:
with TemporaryDirectory() as temp_dir:
source_root = Path(temp_dir)
version_header = source_root / "v8" / "include" / "v8-version.h"
version_header.parent.mkdir(parents=True)
version_header.write_text(
textwrap.dedent(
"""\
#define V8_MAJOR_VERSION 14
#define V8_MINOR_VERSION 9
#define V8_BUILD_NUMBER 207
#define V8_PATCH_LEVEL 35
"""
)
)
self.assertEqual(
"14.9.207.35",
rusty_v8_bazel.v8_checkout_version(source_root),
)
def test_sync_upstream_v8_source_checks_out_requested_tag(self) -> None:
with TemporaryDirectory() as temp_dir:
root = Path(temp_dir)
remote = root / "remote"
version_header = remote / "include" / "v8-version.h"
version_header.parent.mkdir(parents=True)
version_header.write_text(
textwrap.dedent(
"""\
#define V8_MAJOR_VERSION 14
#define V8_MINOR_VERSION 9
#define V8_BUILD_NUMBER 207
#define V8_PATCH_LEVEL 35
"""
)
)
git = ["git", "-C", str(remote)]
subprocess.run(["git", "init", str(remote)], check=True, capture_output=True)
subprocess.run([*git, "add", "."], check=True, capture_output=True)
subprocess.run(
[
*git,
"-c",
"user.name=Codex Test",
"-c",
"user.email=codex@example.com",
"commit",
"-m",
"V8 source",
],
check=True,
capture_output=True,
)
subprocess.run(
[*git, "tag", "14.9.207.35"], check=True, capture_output=True
)
source_root = root / "rusty_v8"
source_root.mkdir()
subprocess.run(
["git", "clone", str(remote), str(source_root / "v8")],
check=True,
capture_output=True,
)
tools = source_root / "tools"
tools.mkdir()
(tools / "update_deps.py").write_text("")
rusty_v8_bazel.sync_upstream_v8_source(source_root, "14.9.207.35")
self.assertEqual(
"14.9.207.35",
rusty_v8_bazel.v8_checkout_version(source_root),
)
def test_command_version_tracks_remaining_http_file_assets(self) -> None:
with TemporaryDirectory() as temp_dir:
module_bazel = Path(temp_dir) / "MODULE.bazel"

11
.github/workflows/rusty-v8-release.yml vendored
View File

@@ -3,7 +3,7 @@ name: rusty-v8-release
on:
push:
tags:
- "rusty-v8-v*.*.*"
- "rusty-v8-v*.*.*-v8-*.*.*.*"
# Cargo's libgit2 transport has been flaky when fetching git dependencies with
# nested submodules. Prefer the system git CLI for Cargo smoke tests.
@@ -43,12 +43,11 @@ jobs:
id: release_tag
env:
GITHUB_REF_NAME: ${{ github.ref_name }}
V8_VERSION: ${{ steps.v8_version.outputs.version }}
shell: bash
run: |
set -euo pipefail
expected_release_tag="rusty-v8-v${V8_VERSION}"
expected_release_tag="$(python3 .github/scripts/rusty_v8_bazel.py rusty-v8-release-tag)"
release_tag="${GITHUB_REF_NAME}"
if [[ "${release_tag}" != "${expected_release_tag}" ]]; then
echo "Tag ${release_tag} does not match expected release tag ${expected_release_tag}." >&2
@@ -323,6 +322,12 @@ jobs:
python-version: "3.11"
architecture: x64
- name: Check out pinned V8 source and dependencies
shell: bash
run: |
python3 .github/scripts/rusty_v8_bazel.py sync-upstream-source \
--source-root upstream-rusty-v8
- name: Set up Codex Rust toolchain for Cargo smoke
uses: dtolnay/rust-toolchain@e081816240890017053eacbb1bdf337761dc5582 # 1.95.0
with:

6
.github/workflows/v8-canary.yml vendored
View File

@@ -343,6 +343,12 @@ jobs:
python-version: "3.11"
architecture: x64
- name: Check out pinned V8 source and dependencies
shell: bash
run: |
python3 .github/scripts/rusty_v8_bazel.py sync-upstream-source \
--source-root upstream-rusty-v8
- name: Set up Codex Rust toolchain for Cargo smoke
uses: dtolnay/rust-toolchain@e081816240890017053eacbb1bdf337761dc5582 # 1.95.0
with:

46
scripts/codex_package/test_v8.py Normal file
View File

@@ -0,0 +1,46 @@
#!/usr/bin/env python3
from pathlib import Path
import sys
import tempfile
import textwrap
import unittest
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from codex_package.v8 import resolved_v8_source_version
from codex_package.v8 import rusty_v8_release_tag
class RustyV8ReleaseTagTest(unittest.TestCase):
def test_release_tag_combines_crate_and_source_versions(self) -> None:
self.assertEqual(
rusty_v8_release_tag(
crate_version="149.2.0",
source_version="14.9.207.35",
),
"rusty-v8-v149.2.0-v8-14.9.207.35",
)
def test_resolved_v8_source_version_reads_archive_override(self) -> None:
with tempfile.TemporaryDirectory() as temp_dir:
module_bazel = Path(temp_dir) / "MODULE.bazel"
module_bazel.write_text(
textwrap.dedent(
"""\
archive_override(
module_name = "v8",
urls = ["https://github.com/v8/v8/archive/refs/tags/14.9.207.35.tar.gz"],
)
"""
)
)
self.assertEqual(
resolved_v8_source_version(module_bazel),
"14.9.207.35",
)
if __name__ == "__main__":
unittest.main()

38
scripts/codex_package/v8.py
View File

@@ -4,6 +4,7 @@ from __future__ import annotations
import hashlib
import os
import re
import shutil
import tempfile
from collections.abc import Mapping
@@ -16,6 +17,10 @@ from .targets import TargetSpec
DOWNLOAD_TIMEOUT_SECS = 120
V8_SOURCE_ARCHIVE_PATTERN = re.compile(
r"https://github\.com/v8/v8/archive/refs/tags/"
r"([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+)\.tar\.gz"
)
@dataclass(frozen=True)
@@ -56,7 +61,7 @@ def resolve_codex_v8_cargo_env(
def fetch_codex_v8_artifacts(
spec: TargetSpec,
*,
version: str | None = None,
crate_version: str | None = None,
cache_root: Path | None = None,
) -> RustyV8ArtifactPair:
if spec.is_windows:
@@ -64,12 +69,14 @@ def fetch_codex_v8_artifacts(
f"No Codex-built V8 release artifacts for target: {spec.target}"
)
version = version or resolved_v8_crate_version()
release_url = (
f"https://github.com/openai/codex/releases/download/rusty-v8-v{version}"
crate_version = crate_version or resolved_v8_crate_version()
release_tag = rusty_v8_release_tag(
crate_version=crate_version,
source_version=resolved_v8_source_version(),
)
release_url = f"https://github.com/openai/codex/releases/download/{release_tag}"
target = spec.target
cache_dir = (cache_root or default_cache_root()) / f"rusty-v8-{version}-{target}"
cache_dir = (cache_root or default_cache_root()) / f"{release_tag}-{target}"
archive = cache_dir / f"librusty_v8_release_{target}.a.gz"
binding = cache_dir / f"src_binding_release_{target}.rs"
checksums = cache_dir / f"rusty_v8_release_{target}.sha256"
@@ -104,6 +111,27 @@ def resolved_v8_crate_version() -> str:
return versions[0]
def resolved_v8_source_version(module_bazel: Path | None = None) -> str:
module_bazel = module_bazel or REPO_ROOT / "MODULE.bazel"
matches = sorted(set(V8_SOURCE_ARCHIVE_PATTERN.findall(module_bazel.read_text())))
if len(matches) != 1:
raise RuntimeError(
"Expected exactly one pinned V8 source version in MODULE.bazel, "
f"found: {matches}"
)
return matches[0]
def rusty_v8_release_tag(
*,
crate_version: str | None = None,
source_version: str | None = None,
) -> str:
crate_version = crate_version or resolved_v8_crate_version()
source_version = source_version or resolved_v8_source_version()
return f"rusty-v8-v{crate_version}-v8-{source_version}"
def default_cache_root() -> Path:
return Path(tempfile.gettempdir()) / "codex-package"

27
third_party/v8/README.md vendored
View File

@@ -21,16 +21,20 @@ Current pinned versions:
- Rust crate: `v8 = =149.2.0`
- Embedded upstream V8 source for Bazel-produced release builds: `14.9.207.35`
(the revision used by Chromium `149.0.7827.201`)
- Codex artifact release tag:
`rusty-v8-v149.2.0-v8-14.9.207.35`
## Updating to a new `v8` release
Use this as the maintainer flow for a version bump:
1. Bump the `v8` crate version and refresh `codex-rs/Cargo.lock`.
2. Update the Bazel versioned inputs in `MODULE.bazel`, then refresh the
matching checksum manifest and generated checksums as described below.
1. Update the `v8` crate and `codex-rs/Cargo.lock` when the Rust binding
changes. Update the embedded V8 source in `MODULE.bazel` independently.
2. Refresh the matching checksum manifest and generated checksums when the
remaining prebuilt inputs change, as described below.
3. Publish a release-candidate PR and validate that `v8-canary` passes.
4. If the canary is green, publish the release tag and release build.
4. If the canary is green, publish the release tag returned by
`python3 .github/scripts/rusty_v8_bazel.py rusty-v8-release-tag`.
5. Once the release build completes, rerun the build on the candidate branch
and verify that the final artifact builds and tests pass.
@@ -53,7 +57,7 @@ The consumer-facing selectors are:
Published release assets are expected at the tag:
- `rusty-v8-v<crate_version>`
- `rusty-v8-v<crate_version>-v8-<source_version>`
with these raw asset names:
@@ -89,8 +93,10 @@ The same run also builds the matching sandbox pair targets:
The workflow also builds sandbox-enabled
`x86_64-pc-windows-msvc` and `aarch64-pc-windows-msvc` archive/binding pairs
from upstream `rusty_v8` source. Those ABI-specific outputs cannot be produced
by Codex's Bazel Windows GNU toolchain.
from the matching upstream `rusty_v8` crate source, after synchronizing its V8
checkout and Chromium dependencies to the source version pinned in
`MODULE.bazel`. Those ABI-specific outputs cannot be produced by Codex's Bazel
Windows GNU toolchain.
The Bazel graph pins the same libc++, libc++abi, and llvm-libc source revisions
used by `rusty_v8 v149.2.0`, compiles published artifact targets with
@@ -111,7 +117,8 @@ Release and CI Cargo builds for Darwin and Linux use `RUSTY_V8_ARCHIVE` plus a
downloaded `RUSTY_V8_SRC_BINDING_PATH` to point at those `openai/codex` release
assets directly. We do not use `RUSTY_V8_MIRROR` because the upstream `v8` crate
hardcodes a `v<crate_version>` tag layout, while our artifacts are published
under `rusty-v8-v<crate_version>`.
under `rusty-v8-v<crate_version>-v8-<source_version>`.
Do not mix artifacts across crate versions. The archive and binding must match
the exact resolved `v8` crate version in `codex-rs/Cargo.lock`.
Do not mix artifacts across crate or source versions. The archive and binding
must match both the exact resolved `v8` crate version in `codex-rs/Cargo.lock`
and the embedded V8 source version in `MODULE.bazel`.