openai/codex
1
0
Fork 0
mirror of https://github.com/openai/codex.git synced 2026-09-05 15:18:41 +00:00
Code Issues Packages Projects Releases Wiki Activity

build: remove MSVC from Windows Bazel build actions

Browse Source
This commit is contained in:
Adam Perry
2026-06-20 02:47:11 +00:00
parent df81e569e1
commit fbecab78a8
19 changed files with 242 additions and 421 deletions
Download Patch File Download Diff File Expand all files Collapse all files

138
.github/scripts/compute-bazel-windows-path.ps1 vendored
View File

@@ -1,113 +1,59 @@
<#
BuildBuddy cache keys include the action and test environment, so Bazel should
not inherit the full hosted-runner PATH on Windows. That PATH includes volatile
tool entries, such as Maven, that can change independently of this repo and
cause avoidable cache misses.
This script derives a smaller, cache-stable PATH that keeps the Windows
toolchain entries Bazel-backed CI tasks need: MSVC and Windows SDK paths,
MinGW runtime DLL paths for gnullvm-built tests, Git, PowerShell, Node, Python,
DotSlash, and the standard Windows system directories.
`setup-bazel-ci` runs this after exporting the MSVC environment, and the script
publishes the result via `GITHUB_ENV` as `CODEX_BAZEL_WINDOWS_PATH` so later
steps can pass that explicit PATH to Bazel.
Bazel build actions must not inherit the hosted-runner PATH. Keep their
execution substrate fixed to Windows and the Git-for-Windows shell utilities
that Bazel genrules already use. Test actions get a separate fixed path with
the product runtimes exercised by Windows tests (Git, PowerShell, and
DotSlash). Compiler, SDK, MinGW, hosted Python, and hosted Node directories are
intentionally absent from both values.
#>
$stablePathEntries = New-Object System.Collections.Generic.List[string]
$seenEntries = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::OrdinalIgnoreCase)
$windowsAppsPath = if ([string]::IsNullOrWhiteSpace($env:LOCALAPPDATA)) {
$null
} else {
"$($env:LOCALAPPDATA)\Microsoft\WindowsApps"
}
$windowsDir = if ($env:WINDIR) {
$env:WINDIR
} elseif ($env:SystemRoot) {
$env:SystemRoot
} else {
$null
throw 'WINDIR or SystemRoot must be set.'
}
function Add-StablePathEntry {
param([string]$PathEntry)
if ([string]::IsNullOrWhiteSpace($PathEntry)) {
return
}
if ($seenEntries.Add($PathEntry)) {
[void]$stablePathEntries.Add($PathEntry)
}
if ([string]::IsNullOrWhiteSpace($env:ProgramFiles)) {
throw 'ProgramFiles must be set.'
}
foreach ($pathEntry in ($env:PATH -split ';')) {
if ([string]::IsNullOrWhiteSpace($pathEntry)) {
continue
}
if (
$pathEntry -like '*Microsoft Visual Studio*' -or
$pathEntry -like '*Windows Kits*' -or
$pathEntry -like '*Microsoft SDKs*' -or
$pathEntry -eq 'C:\mingw64\bin' -or
$pathEntry -like 'C:\msys64\*\bin' -or
$pathEntry -like 'C:\Program Files\Git\*' -or
$pathEntry -like 'C:\Program Files\PowerShell\*' -or
$pathEntry -like 'C:\hostedtoolcache\windows\node\*' -or
$pathEntry -like 'C:\hostedtoolcache\windows\Python\*' -or
$pathEntry -eq 'D:\a\_temp\install-dotslash\bin' -or
($windowsDir -and ($pathEntry -eq $windowsDir -or $pathEntry -like "${windowsDir}\*"))
) {
Add-StablePathEntry $pathEntry
}
if ([string]::IsNullOrWhiteSpace($env:LOCALAPPDATA)) {
throw 'LOCALAPPDATA must be set.'
}
$gitCommand = Get-Command git -ErrorAction SilentlyContinue
if ($gitCommand) {
Add-StablePathEntry (Split-Path $gitCommand.Source -Parent)
}
$nodeCommand = Get-Command node -ErrorAction SilentlyContinue
if ($nodeCommand) {
Add-StablePathEntry (Split-Path $nodeCommand.Source -Parent)
}
$python3Command = Get-Command python3 -ErrorAction SilentlyContinue
if ($python3Command) {
Add-StablePathEntry (Split-Path $python3Command.Source -Parent)
}
$pythonCommand = Get-Command python -ErrorAction SilentlyContinue
if ($pythonCommand) {
Add-StablePathEntry (Split-Path $pythonCommand.Source -Parent)
}
$pwshCommand = Get-Command pwsh -ErrorAction SilentlyContinue
if ($pwshCommand) {
Add-StablePathEntry (Split-Path $pwshCommand.Source -Parent)
}
foreach ($mingwPath in @('C:\mingw64\bin', 'C:\msys64\mingw64\bin', 'C:\msys64\ucrt64\bin')) {
if (Test-Path $mingwPath) {
Add-StablePathEntry $mingwPath
}
}
if ($windowsAppsPath) {
Add-StablePathEntry $windowsAppsPath
}
if ($stablePathEntries.Count -eq 0) {
throw 'Failed to derive cache-stable Windows PATH.'
}
if ([string]::IsNullOrWhiteSpace($env:GITHUB_ENV)) {
throw 'GITHUB_ENV must be set.'
}
$stablePath = $stablePathEntries -join ';'
Write-Host 'Derived CODEX_BAZEL_WINDOWS_PATH entries:'
foreach ($pathEntry in $stablePathEntries) {
Write-Host " $pathEntry"
$gitRoot = Join-Path $env:ProgramFiles 'Git'
$executionPathEntries = @(
(Join-Path $gitRoot 'usr\bin'),
(Join-Path $windowsDir 'System32'),
$windowsDir
)
$testPathEntries = @(
(Join-Path $env:ProgramFiles 'PowerShell\7'),
(Join-Path $gitRoot 'bin'),
(Join-Path $gitRoot 'usr\bin'),
(Join-Path $env:LOCALAPPDATA 'Microsoft\WindowsApps'),
(Join-Path $windowsDir 'System32'),
$windowsDir
)
$requiredPathEntries = ($executionPathEntries + $testPathEntries) | Select-Object -Unique
foreach ($pathEntry in $requiredPathEntries) {
if (-not (Test-Path $pathEntry)) {
throw "Required Windows Bazel substrate path does not exist: $pathEntry"
}
}
"CODEX_BAZEL_WINDOWS_PATH=$stablePath" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
$executionPath = $executionPathEntries -join ';'
$testPath = $testPathEntries -join ';'
Write-Host 'Frozen CODEX_BAZEL_WINDOWS_EXECUTION_PATH entries:'
$executionPathEntries | ForEach-Object { Write-Host " $_" }
Write-Host 'Frozen CODEX_BAZEL_WINDOWS_TEST_PATH entries:'
$testPathEntries | ForEach-Object { Write-Host " $_" }
"CODEX_BAZEL_WINDOWS_EXECUTION_PATH=$executionPath" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
"CODEX_BAZEL_WINDOWS_TEST_PATH=$testPath" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append

112
.github/scripts/run-bazel-ci.sh vendored
View File

@@ -5,7 +5,6 @@ set -euo pipefail
print_failed_bazel_test_logs=0
print_failed_bazel_action_summary=0
remote_download_toplevel=0
windows_msvc_host_platform=0
windows_cross_compile=0
windows_hybrid_execution=0
@@ -23,10 +22,6 @@ while [[ $# -gt 0 ]]; do
remote_download_toplevel=1
shift
;;
--windows-msvc-host-platform)
windows_msvc_host_platform=1
shift
;;
--windows-cross-compile)
windows_cross_compile=1
shift
@@ -47,7 +42,7 @@ while [[ $# -gt 0 ]]; do
done
if [[ $# -eq 0 ]]; then
echo "Usage: $0 [--print-failed-test-logs] [--print-failed-action-summary] [--remote-download-toplevel] [--windows-msvc-host-platform] [--windows-cross-compile] [--windows-hybrid-execution] -- <bazel args> -- <targets>" >&2
echo "Usage: $0 [--print-failed-test-logs] [--print-failed-action-summary] [--remote-download-toplevel] [--windows-cross-compile] [--windows-hybrid-execution] -- <bazel args> -- <targets>" >&2
exit 1
fi
@@ -102,9 +97,7 @@ print_bazel_test_log_tails() {
local -a bazel_info_args=(info)
if [[ -n "${BUILDBUDDY_API_KEY:-}" ]]; then
# `bazel info` needs the same CI config as the failed test invocation so
# platform-specific output roots match. On Windows, omitting `ci-windows`
# would point at `local_windows-fastbuild` even when the test ran with the
# MSVC host platform under `local_windows_msvc-fastbuild`.
# platform-specific output roots match.
bazel_info_args+=("--config=${ci_config}")
fi
@@ -113,7 +106,7 @@ print_bazel_test_log_tails() {
# mode can make `bazel info` fail, which would hide the real test log path.
for arg in "${post_config_bazel_args[@]}"; do
case "$arg" in
--host_platform=* | --repo_contents_cache=* | --repository_cache=*)
--host_platform=* | --platforms=* | --repo_contents_cache=* | --repository_cache=*)
bazel_info_args+=("$arg")
;;
esac
@@ -267,31 +260,7 @@ if [[ ${#bazel_args[@]} -eq 0 || ${#bazel_targets[@]} -eq 0 ]]; then
exit 1
fi
if [[ "${RUNNER_OS:-}" == "Windows" && $windows_cross_compile -eq 1 && -z "${BUILDBUDDY_API_KEY:-}" ]]; then
# Windows cross-compilation depends on authenticated RBE. Preserve the local
# Windows build shape when credentials are unavailable.
ci_config=ci-windows
windows_msvc_host_platform=1
fi
post_config_bazel_args=()
if [[ "${RUNNER_OS:-}" == "Windows" && $windows_msvc_host_platform -eq 1 ]]; then
has_host_platform_override=0
for arg in "${bazel_args[@]}"; do
if [[ "$arg" == --host_platform=* ]]; then
has_host_platform_override=1
break
fi
done
if [[ $has_host_platform_override -eq 0 ]]; then
# Use the MSVC Windows platform for jobs that need helper binaries like
# Rust test wrappers and V8 generators to resolve a compatible toolchain.
# Callers that need a different Windows target platform should pass an
# explicit `--platforms=...` flag.
post_config_bazel_args+=("--host_platform=//:local_windows_msvc")
fi
fi
if [[ $remote_download_toplevel -eq 1 ]]; then
# Override the CI config's remote_download_minimal setting when callers need
@@ -308,16 +277,22 @@ if [[ "${RUNNER_OS:-}" == "Windows" && -n "${BUILDBUDDY_API_KEY:-}" && ( $window
# `--enable_platform_specific_config` expands `common:windows` on Windows
# hosts after ordinary rc configs, which can override `ci-windows-cross`'s
# RBE host platform. Repeat it on the command line for cross builds. The
# Hybrid execution keeps its gnullvm host platform for local Rust actions.
# hybrid execution keeps its gnullvm host platform for local Rust actions.
post_config_bazel_args+=(--host_platform=//:rbe)
fi
fi
if [[ "${RUNNER_OS:-}" == "Windows" && $windows_cross_compile -eq 1 && -z "${BUILDBUDDY_API_KEY:-}" ]]; then
# The Windows cross-compile config depends on authenticated remote
# execution. When credentials are unavailable, keep the local build shape
# and its lower concurrency cap.
post_config_bazel_args+=(--jobs=8)
# execution. When credentials are unavailable, spell out the equivalent
# local gnullvm platforms and keep the lower concurrency cap.
post_config_bazel_args+=(
--host_platform=//:local_windows
--platforms=//:windows_x86_64_gnullvm
--extra_execution_platforms=//:windows_x86_64_gnullvm
--extra_toolchains=//:windows_gnullvm_tests_on_gnullvm_host_toolchain
--jobs=8
)
fi
if [[ -n "${BAZEL_REPO_CONTENTS_CACHE:-}" ]]; then
@@ -338,57 +313,28 @@ if [[ -n "${CODEX_BAZEL_EXECUTION_LOG_COMPACT_DIR:-}" ]]; then
fi
if [[ "${RUNNER_OS:-}" == "Windows" ]]; then
pass_windows_build_env=1
if [[ -n "${BUILDBUDDY_API_KEY:-}" && ( $windows_cross_compile -eq 1 || $windows_hybrid_execution -eq 1 ) ]]; then
# Generic build actions execute on Linux RBE workers. Passing the Windows
# runner's compiler environment there leaks VS/SDK paths and makes genrules
# try to execute tools that do not exist on the worker.
pass_windows_build_env=0
if [[ -z "${CODEX_BAZEL_WINDOWS_EXECUTION_PATH:-}" ]]; then
echo "CODEX_BAZEL_WINDOWS_EXECUTION_PATH must be set for Windows Bazel CI." >&2
exit 1
fi
if [[ $pass_windows_build_env -eq 1 ]]; then
windows_action_env_vars=(
INCLUDE
LIB
LIBPATH
UCRTVersion
UniversalCRTSdkDir
VCINSTALLDIR
VCToolsInstallDir
WindowsLibPath
WindowsSdkBinPath
WindowsSdkDir
WindowsSDKLibVersion
WindowsSDKVersion
)
for env_var in "${windows_action_env_vars[@]}"; do
if [[ -n "${!env_var:-}" ]]; then
post_config_bazel_args+=("--action_env=${env_var}" "--host_action_env=${env_var}")
fi
done
fi
if [[ -z "${CODEX_BAZEL_WINDOWS_PATH:-}" ]]; then
echo "CODEX_BAZEL_WINDOWS_PATH must be set for Windows Bazel CI." >&2
if [[ -z "${CODEX_BAZEL_WINDOWS_TEST_PATH:-}" ]]; then
echo "CODEX_BAZEL_WINDOWS_TEST_PATH must be set for Windows Bazel CI." >&2
exit 1
fi
if [[ $pass_windows_build_env -eq 1 ]]; then
post_config_bazel_args+=(
"--action_env=PATH=${CODEX_BAZEL_WINDOWS_PATH}"
"--host_action_env=PATH=${CODEX_BAZEL_WINDOWS_PATH}"
)
else
windows_execution_path="${CODEX_BAZEL_WINDOWS_EXECUTION_PATH}"
if [[ -n "${BUILDBUDDY_API_KEY:-}" && ( $windows_cross_compile -eq 1 || $windows_hybrid_execution -eq 1 ) ]]; then
# Remote build actions run on Linux RBE workers. Give their shell snippets
# a frozen Linux PATH while preserving CODEX_BAZEL_WINDOWS_PATH below only
# for local Windows test execution.
post_config_bazel_args+=(
"--action_env=PATH=/usr/bin:/bin"
"--host_action_env=PATH=/usr/bin:/bin"
)
# a frozen Linux execution-substrate path. Windows Rust and build-script
# actions receive the same value, which intentionally cannot discover
# runner-installed Windows compilers or SDK tools.
windows_execution_path="/usr/bin:/bin"
fi
post_config_bazel_args+=("--test_env=PATH=${CODEX_BAZEL_WINDOWS_PATH}")
post_config_bazel_args+=(
"--action_env=PATH=${windows_execution_path}"
"--host_action_env=PATH=${windows_execution_path}"
"--test_env=PATH=${CODEX_BAZEL_WINDOWS_TEST_PATH}"
)
fi
bazel_console_log="$(mktemp)"

164
.github/scripts/test_run_bazel_with_buildbuddy.py vendored
View File

@@ -12,6 +12,66 @@ import run_bazel_with_buildbuddy
class RunBazelWithBuildBuddyTest(unittest.TestCase):
def run_bazel_ci(
self,
temp_dir: str,
env_overrides: dict[str, str],
*args: str,
) -> list[str]:
fake_bazel_impl = Path(temp_dir) / "fake-bazel.py"
fake_bazel_impl.write_text(
"#!/usr/bin/env python3\n"
"import json\n"
"import sys\n"
"print(json.dumps(sys.argv[1:]))\n",
encoding="utf-8",
)
if os.name == "nt":
fake_bazel = Path(temp_dir) / "fake-bazel.cmd"
fake_bazel.write_text(
f'@"{sys.executable}" "{fake_bazel_impl}" %*\n',
encoding="utf-8",
)
else:
fake_bazel = fake_bazel_impl
fake_bazel.chmod(0o755)
env = os.environ.copy()
for name in (
"BUILDBUDDY_API_KEY",
"GITHUB_ACTIONS",
"GITHUB_EVENT_NAME",
"GITHUB_EVENT_PATH",
"GITHUB_REPOSITORY",
):
env.pop(name, None)
env.update(env_overrides)
env["CODEX_BAZEL_BIN"] = str(fake_bazel)
bash = "bash"
if os.name == "nt":
bash = str(Path(os.environ["ProgramFiles"]) / "Git" / "bin" / "bash.exe")
self.assertTrue(Path(bash).is_file(), bash)
result = subprocess.run(
[
bash,
str(Path(__file__).with_name("run-bazel-ci.sh")),
*args,
],
env=env,
check=False,
capture_output=True,
text=True,
)
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
return next(
json.loads(line)
for line in result.stdout.splitlines()
if line.startswith("[")
)
def github_env(
self,
temp_dir: str,
@@ -123,69 +183,21 @@ class RunBazelWithBuildBuddyTest(unittest.TestCase):
self,
) -> None:
with TemporaryDirectory() as temp_dir:
fake_bazel_impl = Path(temp_dir) / "fake-bazel.py"
fake_bazel_impl.write_text(
"#!/usr/bin/env python3\n"
"import json\n"
"import sys\n"
"print(json.dumps(sys.argv[1:]))\n",
encoding="utf-8",
)
if os.name == "nt":
fake_bazel = Path(temp_dir) / "fake-bazel.cmd"
fake_bazel.write_text(
f'@"{sys.executable}" "{fake_bazel_impl}" %*\n',
encoding="utf-8",
)
else:
fake_bazel = fake_bazel_impl
fake_bazel.chmod(0o755)
env = os.environ.copy()
for name in (
"GITHUB_ACTIONS",
"GITHUB_EVENT_NAME",
"GITHUB_EVENT_PATH",
"GITHUB_REPOSITORY",
):
env.pop(name, None)
env.update(
command = self.run_bazel_ci(
temp_dir,
{
"BUILDBUDDY_API_KEY": "token",
"CODEX_BAZEL_BIN": str(fake_bazel),
"CODEX_BAZEL_WINDOWS_PATH": r"C:\runtime\bin",
"CODEX_BAZEL_WINDOWS_EXECUTION_PATH": r"C:\substrate\bin",
"CODEX_BAZEL_WINDOWS_TEST_PATH": r"C:\runtime\bin",
"INCLUDE": r"C:\Visual Studio\include",
"RUNNER_OS": "Windows",
}
)
bash = "bash"
if os.name == "nt":
bash = str(Path(os.environ["ProgramFiles"]) / "Git" / "bin" / "bash.exe")
self.assertTrue(Path(bash).is_file(), bash)
result = subprocess.run(
[
bash,
str(Path(__file__).with_name("run-bazel-ci.sh")),
"--windows-hybrid-execution",
"--",
"build",
"--config=argument-comment-lint",
"--",
"//codex-rs/arg0:arg0",
],
env=env,
check=False,
capture_output=True,
text=True,
)
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
command = next(
json.loads(line)
for line in result.stdout.splitlines()
if line.startswith("[")
},
"--windows-hybrid-execution",
"--",
"build",
"--config=argument-comment-lint",
"--",
"//codex-rs/arg0:arg0",
)
self.assertIn("--config=ci-windows-hybrid", command)
self.assertIn("--shell_executable=/bin/bash", command)
@@ -203,6 +215,40 @@ class RunBazelWithBuildBuddyTest(unittest.TestCase):
)
)
def test_keyless_windows_cross_build_uses_gnullvm_substrate(self) -> None:
with TemporaryDirectory() as temp_dir:
command = self.run_bazel_ci(
temp_dir,
{
"CODEX_BAZEL_WINDOWS_EXECUTION_PATH": r"C:\substrate\bin",
"CODEX_BAZEL_WINDOWS_TEST_PATH": r"C:\runtime\bin",
"INCLUDE": r"C:\Visual Studio\include",
"RUNNER_OS": "Windows",
},
"--windows-cross-compile",
"--",
"build",
"--",
"//codex-rs/arg0:arg0",
)
self.assertIn("--jobs=8", command)
self.assertIn("--host_platform=//:local_windows", command)
self.assertIn("--platforms=//:windows_x86_64_gnullvm", command)
self.assertIn(
"--extra_execution_platforms=//:windows_x86_64_gnullvm", command
)
self.assertIn(
"--extra_toolchains=//:windows_gnullvm_tests_on_gnullvm_host_toolchain",
command,
)
self.assertIn(r"--action_env=PATH=C:\substrate\bin", command)
self.assertIn(r"--host_action_env=PATH=C:\substrate\bin", command)
self.assertIn(r"--test_env=PATH=C:\runtime\bin", command)
self.assertFalse(any("msvc" in arg.lower() for arg in command))
self.assertNotIn("--action_env=INCLUDE", command)
self.assertNotIn("--host_action_env=INCLUDE", command)
def test_query_remote_configuration_is_inserted_before_expression(self) -> None:
expression = 'kind("rust_library rule", //codex-rs/...)'
env = {"BUILDBUDDY_API_KEY": "fork-token"}