mirror of
https://github.com/openai/codex.git
synced 2026-08-23 13:09:46 +00:00
## Why Windows Cargo and Bazel jobs spend significant time in filesystem-heavy build and cache directories. Route those directories through one CI build root so Windows can use its Dev Drive and Unix can use a stable cache root. ## What - Have `setup-ci` define `CI_BUILD_ROOT`, `CARGO_TARGET_DIR`, Bazel cache/output paths, and temp paths. - Require Windows to find or provision a verified Dev Drive instead of falling back to `C:`. - Pass the shared Bazel output base to `setup-bazel` so its explicit `output_base` does not defeat Dev Drive routing. - Point nextest, release, and V8 source-build paths at the shared environment contract. ## Benchmark results One-off cold-cache WPR/ETW traces show the explicit Bazel output-base routing removes the dominant `C:` traffic: | sample | `C:\_bazel` | summed `C:` traffic | traced test step | |---|---:|---:|---:| | shard 1 before | 62.2 GiB | 85.2 GiB | 16m22s | | shard 1 updated | 0 | 16.5 GiB | 12m05s | | shard 3 before | 67.2 GiB | 84.6 GiB | 16m48s | | shard 3 updated | 0 | 13.5 GiB | 11m08s | For a cold x64 V8 source build, the retained build-tail sample showed `D:\cargo-target` at ~1.29 GiB while measured `C:` roots totaled ~0.45 GiB (`C:\Users` ~0.33 GiB, `C:\Program Files` ~0.06 GiB, `C:\Windows` ~0.03 GiB). The full cold build took 2h20m36s. The Bazel timing improvement is directional because both refreshed shards failed tests. The V8 trace is a bounded build-tail sample, not the full build. All final samples had zero lost ETW events; VHDX traffic was excluded from the optimization ranking. Runs: [baseline Bazel](https://github.com/openai/codex/actions/runs/28911908527), [updated Bazel](https://github.com/openai/codex/actions/runs/28917133701), [V8 build tail](https://github.com/openai/codex/actions/runs/28933626678). ## Manual validation - Ran `just fmt`. - Ran `just test-github-scripts` (35 tests). - Parsed GitHub Actions YAML with `yq`. - Ran `git diff --check`. ## Stack - [#31332](https://github.com/openai/codex/pull/31332) — parameterize Cargo target paths - [#31356](https://github.com/openai/codex/pull/31356) — Windows 2025 runner bump - [#31357](https://github.com/openai/codex/pull/31357) — Dev Drive I/O routing
196 lines
6.3 KiB
Python
196 lines
6.3 KiB
Python
#!/usr/bin/env python3
|
|
|
|
"""Decide which V8 canary work is needed for a commit range.
|
|
|
|
The workflow deliberately has no trigger-level path filters because it is both
|
|
directly triggered for pull requests and called by postmerge-ci. Keeping the
|
|
patterns here gives those entrypoints one source of truth; unrelated events
|
|
still run metadata but skip the expensive build matrices.
|
|
"""
|
|
|
|
import argparse
|
|
import subprocess
|
|
import tomllib
|
|
from fnmatch import fnmatchcase
|
|
from pathlib import Path
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
# These patterns replace the old pull_request/push path filters. Include parent
|
|
# workflow changes because they can alter whether the canary is invoked.
|
|
CANARY_PATH_PATTERNS = {
|
|
".bazelrc",
|
|
".github/actions/setup-bazel-ci/**",
|
|
".github/actions/setup-ci/**",
|
|
".github/scripts/run_bazel_with_buildbuddy.py",
|
|
".github/scripts/rusty_v8_bazel.py",
|
|
".github/scripts/rusty_v8_module_bazel.py",
|
|
".github/scripts/setup-dev-drive.ps1",
|
|
".github/scripts/v8_canary_changes.py",
|
|
".github/workflows/postmerge-ci.yml",
|
|
".github/workflows/rusty-v8-release.yml",
|
|
".github/workflows/v8-canary.yml",
|
|
"MODULE.bazel",
|
|
"MODULE.bazel.lock",
|
|
"codex-rs/Cargo.toml",
|
|
"patches/BUILD.bazel",
|
|
"patches/llvm_*.patch",
|
|
"patches/rules_cc_*.patch",
|
|
"patches/v8_*.patch",
|
|
"third_party/v8/**",
|
|
}
|
|
# Windows source builds are a narrower, more expensive subset of the canary.
|
|
# A V8 version change also requires them even when no path below changed.
|
|
WINDOWS_SOURCE_BUILD_PATHS = {
|
|
".github/actions/setup-ci/**",
|
|
".github/scripts/rusty_v8_bazel.py",
|
|
".github/scripts/rusty_v8_module_bazel.py",
|
|
".github/scripts/setup-dev-drive.ps1",
|
|
".github/scripts/v8_canary_changes.py",
|
|
".github/workflows/rusty-v8-release.yml",
|
|
".github/workflows/v8-canary.yml",
|
|
}
|
|
|
|
|
|
def matching_canary_paths(changed_files: set[str]) -> set[str]:
|
|
"""Return changed paths that require the general V8 build matrix."""
|
|
return {
|
|
path
|
|
for path in changed_files
|
|
if any(fnmatchcase(path, pattern) for pattern in CANARY_PATH_PATTERNS)
|
|
}
|
|
|
|
|
|
def canary_required(
|
|
changed_files: set[str],
|
|
base_v8_version: str,
|
|
head_v8_version: str,
|
|
*,
|
|
force: bool = False,
|
|
) -> bool:
|
|
"""Return whether the general V8 build matrix should run."""
|
|
return (
|
|
force
|
|
or base_v8_version != head_v8_version
|
|
or bool(matching_canary_paths(changed_files))
|
|
)
|
|
|
|
|
|
def matching_windows_source_paths(changed_files: set[str]) -> set[str]:
|
|
"""Return changed paths that require Windows rusty_v8 source builds."""
|
|
return {
|
|
path
|
|
for path in changed_files
|
|
if any(fnmatchcase(path, pattern) for pattern in WINDOWS_SOURCE_BUILD_PATHS)
|
|
}
|
|
|
|
|
|
def resolved_v8_version(cargo_lock: bytes) -> str:
|
|
versions = sorted(
|
|
{
|
|
package["version"]
|
|
for package in tomllib.loads(cargo_lock.decode())["package"]
|
|
if package["name"] == "v8"
|
|
}
|
|
)
|
|
if len(versions) != 1:
|
|
raise ValueError(f"expected exactly one resolved v8 version, found: {versions}")
|
|
return versions[0]
|
|
|
|
|
|
def windows_source_required(
|
|
changed_files: set[str],
|
|
base_v8_version: str,
|
|
head_v8_version: str,
|
|
*,
|
|
force: bool = False,
|
|
) -> bool:
|
|
"""Return whether Windows must rebuild rusty_v8 from source."""
|
|
return (
|
|
force
|
|
or base_v8_version != head_v8_version
|
|
or bool(matching_windows_source_paths(changed_files))
|
|
)
|
|
|
|
|
|
def git_output(*args: str, root: Path = ROOT) -> bytes:
|
|
return subprocess.check_output(["git", *args], cwd=root)
|
|
|
|
|
|
def v8_version_at_revision(revision: str, *, root: Path = ROOT) -> str:
|
|
return resolved_v8_version(
|
|
git_output("show", f"{revision}:codex-rs/Cargo.lock", root=root)
|
|
)
|
|
|
|
|
|
def merge_base(base: str, head: str, *, root: Path = ROOT) -> str:
|
|
return git_output("merge-base", base, head, root=root).decode().strip()
|
|
|
|
|
|
def changed_files(base: str, head: str, *, root: Path = ROOT) -> set[str]:
|
|
# Three-dot diff gives PRs merge-base semantics while remaining equivalent
|
|
# to before/after for ordinary linear pushes to main.
|
|
output = git_output(
|
|
"diff",
|
|
"--name-only",
|
|
"--no-renames",
|
|
f"{base}...{head}",
|
|
root=root,
|
|
)
|
|
return set(output.decode().splitlines())
|
|
|
|
|
|
def parse_args() -> argparse.Namespace:
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--base")
|
|
parser.add_argument("--head")
|
|
parser.add_argument("--force", action="store_true")
|
|
return parser.parse_args()
|
|
|
|
|
|
def main() -> None:
|
|
args = parse_args()
|
|
if args.force:
|
|
# workflow_dispatch has no comparison range, and callers use it as a
|
|
# manual retry path, so it intentionally runs every variant.
|
|
canary = True
|
|
canary_reason = "manual workflow dispatch"
|
|
windows_source = True
|
|
windows_source_reason = "manual workflow dispatch"
|
|
elif not args.base or not args.head:
|
|
raise SystemExit("--base and --head are required unless --force is set")
|
|
else:
|
|
files = changed_files(args.base, args.head)
|
|
base_version = v8_version_at_revision(merge_base(args.base, args.head))
|
|
head_version = v8_version_at_revision(args.head)
|
|
|
|
matched_canary_paths = sorted(matching_canary_paths(files))
|
|
canary = canary_required(files, base_version, head_version)
|
|
windows_source = windows_source_required(files, base_version, head_version)
|
|
if base_version != head_version:
|
|
canary_reason = (
|
|
f"v8 version changed from {base_version} to {head_version}"
|
|
)
|
|
windows_source_reason = canary_reason
|
|
else:
|
|
canary_reason = (
|
|
", ".join(matched_canary_paths)
|
|
if matched_canary_paths
|
|
else "no relevant changes"
|
|
)
|
|
matched_windows_paths = sorted(matching_windows_source_paths(files))
|
|
windows_source_reason = (
|
|
", ".join(matched_windows_paths)
|
|
if matched_windows_paths
|
|
else "no relevant changes"
|
|
)
|
|
|
|
print(f"canary_required={str(canary).lower()}")
|
|
print(f"canary_reason={canary_reason}")
|
|
print(f"windows_source_required={str(windows_source).lower()}")
|
|
print(f"windows_source_reason={windows_source_reason}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|