From 4e16a08e2867d45ccf983b1e37edab70b85db1ff Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Thu, 30 Apr 2026 23:59:27 -0700 Subject: [PATCH] ci: add Windows cargo-xwin release PoC --- .bazelrc | 14 ++ .../scripts/compute-bazel-windows-path.ps1 | 14 +- .github/scripts/run-bazel-ci.sh | 92 ++++++--- .github/workflows/bazel.yml | 11 +- ...rust-release-windows-cross-compile-poc.yml | 182 ++++++++++++++++++ BUILD.bazel | 34 ++++ .../src/unified_exec/tests.rs | 4 + defs.bzl | 76 +++++++- patches/v8_bazel_rules.patch | 3 +- patches/v8_source_portability.patch | 16 +- third_party/v8/BUILD.bazel | 24 +-- workspace_root_test_launcher.bat.tpl | 5 +- workspace_root_test_launcher.sh.tpl | 14 +- 13 files changed, 423 insertions(+), 66 deletions(-) create mode 100644 .github/workflows/rust-release-windows-cross-compile-poc.yml diff --git a/.bazelrc b/.bazelrc index 76f81ade40..d5fa794a10 100644 --- a/.bazelrc +++ b/.bazelrc @@ -153,6 +153,20 @@ common:ci-macos --config=remote common:ci-macos --strategy=remote common:ci-macos --strategy=TestRunner=darwin-sandbox,local +# On Windows, use Linux remote execution for build actions but keep test actions +# on the Windows runner so Bazel's normal test sharding and flaky-test retries +# still run against Windows binaries. +common:ci-windows-cross --config=ci-windows +common:ci-windows-cross --build_metadata=TAG_windows_cross_compile=true +common:ci-windows-cross --config=remote +common:ci-windows-cross --host_platform=//:rbe +common:ci-windows-cross --strategy=remote +common:ci-windows-cross --strategy=TestRunner=local +common:ci-windows-cross --local_test_jobs=8 +common:ci-windows-cross --platforms=//:windows_x86_64_gnullvm +common:ci-windows-cross --extra_execution_platforms=//:rbe,//:windows_x86_64_msvc +common:ci-windows-cross --extra_toolchains=//:windows_gnullvm_tests_on_msvc_host_toolchain + # Linux-only V8 CI config. common:ci-v8 --config=ci common:ci-v8 --build_metadata=TAG_workflow=v8 diff --git a/.github/scripts/compute-bazel-windows-path.ps1 b/.github/scripts/compute-bazel-windows-path.ps1 index 6b6bbe0462..81fd668c8b 100644 --- a/.github/scripts/compute-bazel-windows-path.ps1 +++ b/.github/scripts/compute-bazel-windows-path.ps1 @@ -5,9 +5,9 @@ tool entries, such as Maven, that can change independently of this repo and cause avoidable cache misses. This script derives a smaller, cache-stable PATH that keeps the Windows -toolchain entries Bazel-backed CI tasks need: MSVC and Windows SDK paths, Git, -PowerShell, Node, Python, DotSlash, and the standard Windows system -directories. +toolchain entries Bazel-backed CI tasks need: MSVC and Windows SDK paths, +MinGW runtime DLL paths for gnullvm-built tests, Git, PowerShell, Node, Python, +DotSlash, and the standard Windows system directories. `setup-bazel-ci` runs this after exporting the MSVC environment, and the script publishes the result via `GITHUB_ENV` as `CODEX_BAZEL_WINDOWS_PATH` so later steps can pass that explicit PATH to Bazel. @@ -49,6 +49,8 @@ foreach ($pathEntry in ($env:PATH -split ';')) { $pathEntry -like '*Microsoft Visual Studio*' -or $pathEntry -like '*Windows Kits*' -or $pathEntry -like '*Microsoft SDKs*' -or + $pathEntry -eq 'C:\mingw64\bin' -or + $pathEntry -like 'C:\msys64\*\bin' -or $pathEntry -like 'C:\Program Files\Git\*' -or $pathEntry -like 'C:\Program Files\PowerShell\*' -or $pathEntry -like 'C:\hostedtoolcache\windows\node\*' -or @@ -85,6 +87,12 @@ if ($pwshCommand) { Add-StablePathEntry (Split-Path $pwshCommand.Source -Parent) } +foreach ($mingwPath in @('C:\mingw64\bin', 'C:\msys64\mingw64\bin', 'C:\msys64\ucrt64\bin')) { + if (Test-Path $mingwPath) { + Add-StablePathEntry $mingwPath + } +} + if ($windowsAppsPath) { Add-StablePathEntry $windowsAppsPath } diff --git a/.github/scripts/run-bazel-ci.sh b/.github/scripts/run-bazel-ci.sh index b81e0a4d57..38dcec7bf8 100755 --- a/.github/scripts/run-bazel-ci.sh +++ b/.github/scripts/run-bazel-ci.sh @@ -6,6 +6,7 @@ print_failed_bazel_test_logs=0 print_failed_bazel_action_summary=0 remote_download_toplevel=0 windows_msvc_host_platform=0 +windows_cross_compile=0 while [[ $# -gt 0 ]]; do case "$1" in @@ -25,6 +26,10 @@ while [[ $# -gt 0 ]]; do windows_msvc_host_platform=1 shift ;; + --windows-cross-compile) + windows_cross_compile=1 + shift + ;; --) shift break @@ -37,7 +42,7 @@ while [[ $# -gt 0 ]]; do done if [[ $# -eq 0 ]]; then - echo "Usage: $0 [--print-failed-test-logs] [--print-failed-action-summary] [--remote-download-toplevel] [--windows-msvc-host-platform] -- -- " >&2 + echo "Usage: $0 [--print-failed-test-logs] [--print-failed-action-summary] [--remote-download-toplevel] [--windows-msvc-host-platform] [--windows-cross-compile] -- -- " >&2 exit 1 fi @@ -61,7 +66,11 @@ case "${RUNNER_OS:-}" in ci_config=ci-macos ;; Windows) - ci_config=ci-windows + if [[ $windows_cross_compile -eq 1 ]]; then + ci_config=ci-windows-cross + else + ci_config=ci-windows + fi ;; esac @@ -105,8 +114,8 @@ print_bazel_test_log_tails() { while IFS= read -r target; do failed_targets+=("$target") done < <( - grep -E '^FAIL: //' "$console_log" \ - | sed -E 's#^FAIL: (//[^ ]+).*#\1#' \ + grep -E '^(FAIL: //|ERROR: .* Testing //)' "$console_log" \ + | sed -E 's#^FAIL: (//[^ ]+).*#\1#; s#^ERROR: .* Testing (//[^ ]+) failed:.*#\1#' \ | sort -u ) @@ -269,6 +278,13 @@ if [[ $remote_download_toplevel -eq 1 ]]; then post_config_bazel_args+=(--remote_download_toplevel) fi +if [[ "${RUNNER_OS:-}" == "Windows" && $windows_cross_compile -eq 1 && -z "${BUILDBUDDY_API_KEY:-}" ]]; then + # The Windows cross-compile config depends on remote execution. Fork PRs do + # not receive the BuildBuddy secret, so fall back to the existing local build + # shape and keep its lower concurrency cap. + post_config_bazel_args+=(--jobs=8) +fi + if [[ -n "${BAZEL_REPO_CONTENTS_CACHE:-}" ]]; then # Windows self-hosted runners can run multiple Bazel jobs concurrently. Give # each job its own repo contents cache so they do not fight over the shared @@ -287,37 +303,57 @@ if [[ -n "${CODEX_BAZEL_EXECUTION_LOG_COMPACT_DIR:-}" ]]; then fi if [[ "${RUNNER_OS:-}" == "Windows" ]]; then - windows_action_env_vars=( - INCLUDE - LIB - LIBPATH - UCRTVersion - UniversalCRTSdkDir - VCINSTALLDIR - VCToolsInstallDir - WindowsLibPath - WindowsSdkBinPath - WindowsSdkDir - WindowsSDKLibVersion - WindowsSDKVersion - ) + pass_windows_build_env=1 + if [[ $windows_cross_compile -eq 1 && -n "${BUILDBUDDY_API_KEY:-}" ]]; then + # Remote build actions execute on Linux RBE workers. Passing the Windows + # runner's build environment there makes Bazel genrules try to execute + # C:\Program Files\Git\usr\bin\bash.exe on Linux. + pass_windows_build_env=0 + fi - for env_var in "${windows_action_env_vars[@]}"; do - if [[ -n "${!env_var:-}" ]]; then - post_config_bazel_args+=("--action_env=${env_var}" "--host_action_env=${env_var}") - fi - done + if [[ $pass_windows_build_env -eq 1 ]]; then + windows_action_env_vars=( + INCLUDE + LIB + LIBPATH + UCRTVersion + UniversalCRTSdkDir + VCINSTALLDIR + VCToolsInstallDir + WindowsLibPath + WindowsSdkBinPath + WindowsSdkDir + WindowsSDKLibVersion + WindowsSDKVersion + ) + + for env_var in "${windows_action_env_vars[@]}"; do + if [[ -n "${!env_var:-}" ]]; then + post_config_bazel_args+=("--action_env=${env_var}" "--host_action_env=${env_var}") + fi + done + fi if [[ -z "${CODEX_BAZEL_WINDOWS_PATH:-}" ]]; then echo "CODEX_BAZEL_WINDOWS_PATH must be set for Windows Bazel CI." >&2 exit 1 fi - post_config_bazel_args+=( - "--action_env=PATH=${CODEX_BAZEL_WINDOWS_PATH}" - "--host_action_env=PATH=${CODEX_BAZEL_WINDOWS_PATH}" - "--test_env=PATH=${CODEX_BAZEL_WINDOWS_PATH}" - ) + if [[ $pass_windows_build_env -eq 1 ]]; then + post_config_bazel_args+=( + "--action_env=PATH=${CODEX_BAZEL_WINDOWS_PATH}" + "--host_action_env=PATH=${CODEX_BAZEL_WINDOWS_PATH}" + ) + elif [[ $windows_cross_compile -eq 1 ]]; then + # Remote build actions run on Linux RBE workers. Give their shell snippets + # a Linux PATH while preserving CODEX_BAZEL_WINDOWS_PATH below for local + # Windows test execution. + post_config_bazel_args+=( + "--action_env=PATH=/usr/bin:/bin" + "--host_action_env=PATH=/usr/bin:/bin" + ) + fi + post_config_bazel_args+=("--test_env=PATH=${CODEX_BAZEL_WINDOWS_PATH}") fi bazel_console_log="$(mktemp)" diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index ef41330c46..f8724271f9 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -22,8 +22,6 @@ jobs: # true test failure in a rust_test() marked "flaky" that gets run 3x. # In that case, extra time generally does not give us more signal. # - # Ultimately we need true distributed builds (e.g., - # https://www.buildbuddy.io/docs/rbe-setup/) to speed things up. timeout-minutes: 30 strategy: fail-fast: false @@ -53,7 +51,7 @@ jobs: runs-on: ${{ matrix.os }} # Configure a human readable name for each job - name: Local Bazel build on ${{ matrix.os }} for ${{ matrix.target }} + name: Bazel test on ${{ matrix.os }} for ${{ matrix.target }} steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 @@ -91,6 +89,7 @@ jobs: ) bazel_wrapper_args=( + --print-failed-action-summary --print-failed-test-logs ) bazel_test_args=( @@ -100,8 +99,10 @@ jobs: --build_metadata=COMMIT_SHA=${GITHUB_SHA} ) if [[ "${RUNNER_OS}" == "Windows" ]]; then - bazel_wrapper_args+=(--windows-msvc-host-platform) - bazel_test_args+=(--jobs=8) + bazel_wrapper_args+=( + --windows-cross-compile + --remote-download-toplevel + ) fi ./.github/scripts/run-bazel-ci.sh \ diff --git a/.github/workflows/rust-release-windows-cross-compile-poc.yml b/.github/workflows/rust-release-windows-cross-compile-poc.yml new file mode 100644 index 0000000000..345cf23808 --- /dev/null +++ b/.github/workflows/rust-release-windows-cross-compile-poc.yml @@ -0,0 +1,182 @@ +name: rust-release-windows-cross-compile-poc + +on: + pull_request: + paths: + - ".github/workflows/rust-release-windows-cross-compile-poc.yml" + workflow_dispatch: + inputs: + release_lto: + description: Cargo release LTO mode + type: choice + default: thin + options: + - thin + - fat + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +env: + CARGO_PROFILE_RELEASE_LTO: ${{ github.event_name == 'workflow_dispatch' && inputs.release_lto || 'thin' }} + CARGO_TERM_COLOR: always + CARGO_XWIN_VERSION: 0.22.0 + WINDOWS_TARGET: x86_64-pc-windows-msvc + XWIN_ARCH: x86_64 + XWIN_CACHE_DIR: ${{ github.workspace }}/.xwin-cache + XWIN_VARIANT: desktop + +jobs: + build-windows-binaries: + name: Cross-compile Windows release binaries + runs-on: ubuntu-24.04 + timeout-minutes: 90 + defaults: + run: + working-directory: codex-rs + + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + + - name: Print runner specs + shell: bash + run: | + set -euo pipefail + cpu_model="$(lscpu | awk -F: '/Model name/ {gsub(/^[ \t]+/, "", $2); print $2; exit}')" + total_ram="$(awk '/MemTotal/ {printf "%.1f GiB\n", $2 / 1024 / 1024}' /proc/meminfo)" + echo "Runner: ${RUNNER_NAME:-unknown}" + echo "OS: $(uname -a)" + echo "CPU model: ${cpu_model}" + echo "Logical CPUs: $(nproc)" + echo "Total RAM: ${total_ram}" + echo "Disk usage:" + df -h . + + - name: Install cross-linker dependencies + shell: bash + run: | + set -euo pipefail + sudo apt-get update -y + sudo DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + clang \ + cmake \ + lld \ + llvm \ + nasm \ + ninja-build \ + pkg-config \ + strace + + - uses: dtolnay/rust-toolchain@a0b273b48ed29de4470960879e8381ff45632f26 # 1.93.0 + with: + targets: ${{ env.WINDOWS_TARGET }} + + - name: Compute cache key inputs + id: lockhash + shell: bash + run: | + set -euo pipefail + echo "lock_hash=$(sha256sum Cargo.lock | cut -d' ' -f1)" >> "$GITHUB_OUTPUT" + echo "toolchain_hash=$(sha256sum rust-toolchain.toml | cut -d' ' -f1)" >> "$GITHUB_OUTPUT" + + - name: Restore cargo-xwin cache + id: cache_cargo_xwin_restore + uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + ${{ env.XWIN_CACHE_DIR }} + key: cargo-xwin-${{ runner.os }}-${{ env.WINDOWS_TARGET }}-${{ env.CARGO_XWIN_VERSION }}-${{ steps.lockhash.outputs.lock_hash }}-${{ steps.lockhash.outputs.toolchain_hash }} + restore-keys: | + cargo-xwin-${{ runner.os }}-${{ env.WINDOWS_TARGET }}-${{ env.CARGO_XWIN_VERSION }}- + + - name: Install cargo-xwin + shell: bash + run: | + set -euo pipefail + if ! cargo xwin --version 2>/dev/null | grep -F "$CARGO_XWIN_VERSION" >/dev/null; then + cargo install cargo-xwin --locked --version "$CARGO_XWIN_VERSION" + fi + cargo xwin --version + + - name: Cache MSVC CRT and Windows SDK + shell: bash + run: cargo xwin cache xwin + + - name: Build codex-responses-api-proxy + shell: bash + run: | + set -euo pipefail + cargo xwin build \ + --release \ + --target "$WINDOWS_TARGET" \ + --timings \ + --bin codex-responses-api-proxy + + - name: Build codex-app-server + shell: bash + run: | + set -euo pipefail + cargo xwin build \ + --release \ + --target "$WINDOWS_TARGET" \ + --timings \ + --bin codex-app-server + + - name: Build codex + shell: bash + run: | + set -euo pipefail + cargo xwin build \ + --release \ + --target "$WINDOWS_TARGET" \ + --timings \ + --bin codex + + - name: Verify Windows artifacts + shell: bash + run: | + set -euo pipefail + for binary in codex-responses-api-proxy codex-app-server codex; do + path="target/${WINDOWS_TARGET}/release/${binary}.exe" + ls -lh "$path" + file "$path" + done + + - name: Upload Cargo timings + if: always() + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7 + with: + name: cargo-timings-rust-release-windows-cross-compile-poc-${{ env.WINDOWS_TARGET }}-primary-app-server + path: codex-rs/target/**/cargo-timings/cargo-timing.html + if-no-files-found: warn + + - name: Upload Windows binaries + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7 + with: + name: windows-cross-compile-poc-${{ env.WINDOWS_TARGET }}-primary-app-server + path: | + codex-rs/target/${{ env.WINDOWS_TARGET }}/release/codex-responses-api-proxy.exe + codex-rs/target/${{ env.WINDOWS_TARGET }}/release/codex-app-server.exe + codex-rs/target/${{ env.WINDOWS_TARGET }}/release/codex.exe + if-no-files-found: error + + - name: Save cargo-xwin cache + if: always() && !cancelled() && steps.cache_cargo_xwin_restore.outputs.cache-hit != 'true' + continue-on-error: true + uses: actions/cache/save@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + ${{ env.XWIN_CACHE_DIR }} + key: cargo-xwin-${{ runner.os }}-${{ env.WINDOWS_TARGET }}-${{ env.CARGO_XWIN_VERSION }}-${{ steps.lockhash.outputs.lock_hash }}-${{ steps.lockhash.outputs.toolchain_hash }} diff --git a/BUILD.bazel b/BUILD.bazel index 3f59ff1160..a82126e6f1 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -30,6 +30,40 @@ platform( parents = ["@platforms//host"], ) +platform( + name = "windows_x86_64_gnullvm", + constraint_values = [ + "@platforms//cpu:x86_64", + "@platforms//os:windows", + "@rules_rs//rs/experimental/platforms/constraints:windows_gnullvm", + ], +) + +platform( + name = "windows_x86_64_msvc", + constraint_values = [ + "@platforms//cpu:x86_64", + "@platforms//os:windows", + "@rules_rs//rs/experimental/platforms/constraints:windows_msvc", + ], +) + +toolchain( + name = "windows_gnullvm_tests_on_msvc_host_toolchain", + exec_compatible_with = [ + "@platforms//cpu:x86_64", + "@platforms//os:windows", + "@rules_rs//rs/experimental/platforms/constraints:windows_msvc", + ], + target_compatible_with = [ + "@platforms//cpu:x86_64", + "@platforms//os:windows", + "@rules_rs//rs/experimental/platforms/constraints:windows_gnullvm", + ], + toolchain = "@bazel_tools//tools/test:empty_toolchain", + toolchain_type = "@bazel_tools//tools/test:default_test_toolchain_type", +) + alias( name = "rbe", actual = "@rbe_platform", diff --git a/codex-rs/windows-sandbox-rs/src/unified_exec/tests.rs b/codex-rs/windows-sandbox-rs/src/unified_exec/tests.rs index b0530a4fb4..66f21807ba 100644 --- a/codex-rs/windows-sandbox-rs/src/unified_exec/tests.rs +++ b/codex-rs/windows-sandbox-rs/src/unified_exec/tests.rs @@ -50,6 +50,10 @@ fn pwsh_path() -> Option { } fn sandbox_cwd() -> PathBuf { + if let Ok(workspace_root) = std::env::var("INSTA_WORKSPACE_ROOT") { + return PathBuf::from(workspace_root); + } + PathBuf::from(env!("CARGO_MANIFEST_DIR")) .parent() .expect("repo root") diff --git a/defs.bzl b/defs.bzl index 71a3138418..275a2358e9 100644 --- a/defs.bzl +++ b/defs.bzl @@ -1,8 +1,8 @@ load("@crates//:data.bzl", "DEP_DATA") load("@crates//:defs.bzl", "all_crate_deps") load("@rules_platform//platform_data:defs.bzl", "platform_data") -load("@rules_rust//rust:defs.bzl", "rust_binary", "rust_library", "rust_proc_macro", "rust_test") load("@rules_rust//cargo/private:cargo_build_script_wrapper.bzl", "cargo_build_script") +load("@rules_rust//rust:defs.bzl", "rust_binary", "rust_library", "rust_proc_macro", "rust_test") PLATFORMS = [ "linux_arm64_musl", @@ -64,12 +64,16 @@ def _workspace_root_test_impl(ctx): test_bin = ctx.executable.test_bin workspace_root_marker = ctx.file.workspace_root_marker launcher_template = ctx.file._windows_launcher_template if is_windows else ctx.file._bash_launcher_template + runfile_env_exports = _windows_runfile_env_exports(ctx) if is_windows else _bash_runfile_env_exports(ctx) + workspace_root_setup = _windows_workspace_root_setup(ctx) if is_windows else _bash_workspace_root_setup(ctx) ctx.actions.expand_template( template = launcher_template, output = launcher, is_executable = True, substitutions = { + "__RUNFILE_ENV_EXPORTS__": runfile_env_exports, "__TEST_BIN__": test_bin.short_path, + "__WORKSPACE_ROOT_SETUP__": workspace_root_setup, "__WORKSPACE_ROOT_MARKER__": workspace_root_marker.short_path, }, ) @@ -78,6 +82,17 @@ def _workspace_root_test_impl(ctx): for data_dep in ctx.attr.data: runfiles = runfiles.merge(ctx.runfiles(files = data_dep[DefaultInfo].files.to_list())) runfiles = runfiles.merge(data_dep[DefaultInfo].default_runfiles) + for runfile_dep in ctx.attr.runfile_env: + executable = runfile_dep[DefaultInfo].files_to_run.executable + if executable == None: + fail("{} does not provide an executable for runfile_env".format(runfile_dep.label)) + runfiles = runfiles.merge(ctx.runfiles(files = [executable])) + + location_targets = ctx.attr.data + [ctx.attr.test_bin, ctx.attr.workspace_root_marker] + env = { + key: ctx.expand_location(value, targets = location_targets) + for key, value in ctx.attr.env.items() + } return [ DefaultInfo( @@ -86,18 +101,55 @@ def _workspace_root_test_impl(ctx): runfiles = runfiles, ), RunEnvironmentInfo( - environment = ctx.attr.env, + environment = env, ), ] +def _bash_runfile_env_exports(ctx): + lines = [] + for runfile_dep, env_var in ctx.attr.runfile_env.items(): + executable = runfile_dep[DefaultInfo].files_to_run.executable + if executable == None: + fail("{} does not provide an executable for runfile_env".format(runfile_dep.label)) + lines.append('RUNFILE_ENV_ARGS+=("{}=$(resolve_runfile "{}")")'.format(env_var, executable.short_path)) + return "\n".join(lines) + +def _windows_runfile_env_exports(ctx): + lines = [] + for runfile_dep, env_var in ctx.attr.runfile_env.items(): + executable = runfile_dep[DefaultInfo].files_to_run.executable + if executable == None: + fail("{} does not provide an executable for runfile_env".format(runfile_dep.label)) + lines.append('call :resolve_runfile {} "{}"'.format(env_var, executable.short_path)) + lines.append("if errorlevel 1 exit /b 1") + return "\n".join(lines) + +def _bash_workspace_root_setup(ctx): + if not ctx.attr.chdir_workspace_root: + return "" + return 'export INSTA_WORKSPACE_ROOT="${workspace_root}"\ncd "${workspace_root}"' + +def _windows_workspace_root_setup(ctx): + if not ctx.attr.chdir_workspace_root: + return "" + return """set "INSTA_WORKSPACE_ROOT=%workspace_root%" +cd /d "%workspace_root%" || exit /b 1""" + workspace_root_test = rule( implementation = _workspace_root_test_impl, test = True, + toolchains = ["@bazel_tools//tools/test:default_test_toolchain_type"], attrs = { + "chdir_workspace_root": attr.bool( + default = True, + ), "data": attr.label_list( allow_files = True, ), "env": attr.string_dict(), + "runfile_env": attr.label_keyed_string_dict( + cfg = "target", + ), "test_bin": attr.label( cfg = "target", executable = True, @@ -255,6 +307,7 @@ def codex_rust_crate( unit_test_name = name + "-unit-tests" unit_test_binary = name + "-unit-tests-bin" unit_test_shard_count = _test_shard_count(test_shard_counts, unit_test_name) + # Shard at the workspace_root_test layer. rules_rust's sharding wrapper # expects to run from its own runfiles cwd, while workspace_root_test # deliberately changes cwd so Insta sees Cargo-like snapshot paths. @@ -298,9 +351,11 @@ def codex_rust_crate( sanitized_binaries = [] cargo_env = {} + cargo_env_runfiles = {} for binary, main in binaries.items(): #binary = binary.replace("-", "_") sanitized_binaries.append(binary) + cargo_env_runfiles[":" + binary] = "CARGO_BIN_EXE_" + binary cargo_env["CARGO_BIN_EXE_" + binary] = "$(rlocationpath :%s)" % binary rust_binary( @@ -317,6 +372,7 @@ def codex_rust_crate( for binary_label in extra_binaries: sanitized_binaries.append(binary_label) binary = Label(binary_label).name + cargo_env_runfiles[binary_label] = "CARGO_BIN_EXE_" + binary cargo_env["CARGO_BIN_EXE_" + binary] = "$(rlocationpath %s)" % binary_label integration_test_kwargs = {} @@ -331,17 +387,17 @@ def codex_rust_crate( test_name = name + "-" + test_file_stem.replace("/", "-") if not test_name.endswith("-test"): test_name += "-test" + test_binary = test_name + "-bin" test_kwargs = {} test_kwargs.update(integration_test_kwargs) test_shard_count = _test_shard_count(test_shard_counts, test_name) if test_shard_count: - test_kwargs["experimental_enable_sharding"] = True test_kwargs["shard_count"] = test_shard_count test_kwargs["flaky"] = True rust_test( - name = test_name, + name = test_binary, crate_name = test_crate_name, crate_root = test, srcs = [test], @@ -356,10 +412,16 @@ def codex_rust_crate( "--remap-path-prefix=codex-rs=", ], rustc_env = rustc_env, - # Important: do not merge `test_env` here. Its unit-test-only - # `INSTA_WORKSPACE_ROOT="codex-rs"` is tuned for unit tests that - # execute from the repo root and can misplace integration snapshots. env = cargo_env, + tags = test_tags + ["manual"], + ) + + workspace_root_test( + name = test_name, + chdir_workspace_root = False, + runfile_env = cargo_env_runfiles, + test_bin = ":" + test_binary, + workspace_root_marker = "//codex-rs/utils/cargo-bin:repo_root.marker", tags = test_tags, **test_kwargs ) diff --git a/patches/v8_bazel_rules.patch b/patches/v8_bazel_rules.patch index 10e1a57679..df845939d0 100644 --- a/patches/v8_bazel_rules.patch +++ b/patches/v8_bazel_rules.patch @@ -133,7 +133,7 @@ index 85f31b7..7314584 100644 ], outs = [ "include/inspector/Debugger.h", -@@ -4426,15 +4426,18 @@ genrule( +@@ -4426,15 +4426,19 @@ genrule( "src/inspector/protocol/Schema.cpp", "src/inspector/protocol/Schema.h", ], @@ -145,6 +145,7 @@ index 85f31b7..7314584 100644 + --inspector_protocol_dir $$INSPECTOR_PROTOCOL_DIR \ --config $(location :src/inspector/inspector_protocol_config.json) \ --config_value protocol.path=$(location :include/js_protocol.pdl) \ ++ --config_value crdtp.dir=third_party/inspector_protocol/crdtp \ --output_base $(@D)/src/inspector", - local = 1, message = "Generating inspector files", diff --git a/patches/v8_source_portability.patch b/patches/v8_source_portability.patch index 4f5f46005f..d480e11c1a 100644 --- a/patches/v8_source_portability.patch +++ b/patches/v8_source_portability.patch @@ -83,9 +83,21 @@ index 420df0b..6f47969 100644 return __libc_stack_end; } diff --git a/orig/v8-14.6.202.11/src/base/platform/platform-win32.cc b/mod/v8-14.6.202.11/src/base/platform/platform-win32.cc -index f5d9ddc..542ea1a 100644 +index f5d9ddc..1c08b0f 100644 --- a/orig/v8-14.6.202.11/src/base/platform/platform-win32.cc +++ b/mod/v8-14.6.202.11/src/base/platform/platform-win32.cc +@@ -20,7 +20,11 @@ + #include + + // This has to come after windows.h. ++#ifdef __MINGW32__ ++#include ++#else + #include ++#endif + #include // For SymLoadModule64 and al. + #include // For _msize() + #include // For timeGetTime(). @@ -69,9 +69,7 @@ static_assert(offsetof(V8_CRITICAL_SECTION, SpinCount) == // Extra functions for MinGW. Most of these are the _s functions which are in // the Microsoft Visual Studio C++ CRT. @@ -171,7 +183,7 @@ diff --git a/orig/v8-14.6.202.11/src/heap/base/asm/x64/push_registers_masm.asm b index d0d0563..72e230b 100644 --- a/orig/v8-14.6.202.11/src/heap/base/asm/x64/push_registers_masm.asm +++ b/mod/v8-14.6.202.11/src/heap/base/asm/x64/push_registers_masm.asm -@@ -1,70 +1,30 @@ +@@ -1,70 +1,47 @@ -;; Copyright 2020 the V8 project authors. All rights reserved. -;; Use of this source code is governed by a BSD-style license that can be -;; found in the LICENSE file. diff --git a/third_party/v8/BUILD.bazel b/third_party/v8/BUILD.bazel index 27e9fa3ffe..563ed18d80 100644 --- a/third_party/v8/BUILD.bazel +++ b/third_party/v8/BUILD.bazel @@ -174,18 +174,18 @@ genrule( name = "binding_cc", srcs = ["@v8_crate_146_4_0//:binding_cc"], outs = ["binding.cc"], - # Keep this as a literal shell snippet. The string-concatenated form looked - # cleaner in Starlark but produced a broken `sed` invocation in CI. - cmd = """ - sed \ - -e '/#include "v8\\/src\\/flags\\/flags.h"/d' \ - -e 's|"v8/src/libplatform/default-platform.h"|"src/libplatform/default-platform.h"|' \ - -e 's| namespace i = v8::internal;| (void)usage;|' \ - -e '/using HelpOptions = i::FlagList::HelpOptions;/d' \ - -e '/HelpOptions help_options = HelpOptions(HelpOptions::kExit, usage);/d' \ - -e 's| i::FlagList::SetFlagsFromCommandLine(argc, argv, true, help_options);| v8::V8::SetFlagsFromCommandLine(argc, argv, true);|' \ - $(location @v8_crate_146_4_0//:binding_cc) > "$@" - """, + cmd = " ".join([ + "sed", + "-e '/#include \"v8\\/src\\/flags\\/flags.h\"/d'", + "-e 's|\"v8/src/libplatform/default-platform.h\"|\"src/libplatform/default-platform.h\"|'", + "-e 's| namespace i = v8::internal;| (void)usage;|'", + "-e '/using HelpOptions = i::FlagList::HelpOptions;/d'", + "-e '/HelpOptions help_options = HelpOptions(HelpOptions::kExit, usage);/d'", + "-e 's| i::FlagList::SetFlagsFromCommandLine(argc, argv, true, help_options);| v8::V8::SetFlagsFromCommandLine(argc, argv, true);|'", + "$(location @v8_crate_146_4_0//:binding_cc)", + ">", + '"$@"', + ]), ) copy_file( diff --git a/workspace_root_test_launcher.bat.tpl b/workspace_root_test_launcher.bat.tpl index af82e5ecff..5411b91629 100644 --- a/workspace_root_test_launcher.bat.tpl +++ b/workspace_root_test_launcher.bat.tpl @@ -10,8 +10,9 @@ for %%I in ("%workspace_root_marker_dir%..\..") do set "workspace_root=%%~fI" call :resolve_runfile test_bin "__TEST_BIN__" if errorlevel 1 exit /b 1 -set "INSTA_WORKSPACE_ROOT=%workspace_root%" -cd /d "%workspace_root%" || exit /b 1 +__RUNFILE_ENV_EXPORTS__ + +__WORKSPACE_ROOT_SETUP__ set "TOTAL_SHARDS=%RULES_RUST_TEST_TOTAL_SHARDS%" if not defined TOTAL_SHARDS set "TOTAL_SHARDS=%TEST_TOTAL_SHARDS%" diff --git a/workspace_root_test_launcher.sh.tpl b/workspace_root_test_launcher.sh.tpl index 1ba752506b..efbcfe1d4e 100644 --- a/workspace_root_test_launcher.sh.tpl +++ b/workspace_root_test_launcher.sh.tpl @@ -47,6 +47,9 @@ resolve_runfile() { workspace_root_marker="$(resolve_runfile "__WORKSPACE_ROOT_MARKER__")" workspace_root="$(dirname "$(dirname "$(dirname "${workspace_root_marker}")")")" test_bin="$(resolve_runfile "__TEST_BIN__")" +RUNFILE_ENV_ARGS=() + +__RUNFILE_ENV_EXPORTS__ test_shard_index() { local test_name="$1" @@ -75,7 +78,7 @@ run_sharded_libtest() { # Extra libtest args are usually ad-hoc local filters. Preserve those exactly # rather than combining them with generated exact filters. if [[ $# -gt 0 ]]; then - exec "${test_bin}" "$@" + exec env "${RUNFILE_ENV_ARGS[@]}" "${test_bin}" "$@" fi if [[ -z "${SHARD_INDEX}" ]]; then @@ -85,7 +88,7 @@ run_sharded_libtest() { local list_output local test_list - list_output="$("${test_bin}" --list --format terse)" + list_output="$(env "${RUNFILE_ENV_ARGS[@]}" "${test_bin}" --list --format terse)" test_list="$(printf '%s\n' "${list_output}" | grep ': test$' | sed 's/: test$//' | LC_ALL=C sort || true)" if [[ -z "${test_list}" ]]; then @@ -104,11 +107,10 @@ run_sharded_libtest() { exit 0 fi - exec "${test_bin}" "${shard_tests[@]}" --exact + exec env "${RUNFILE_ENV_ARGS[@]}" "${test_bin}" "${shard_tests[@]}" --exact } -export INSTA_WORKSPACE_ROOT="${workspace_root}" -cd "${workspace_root}" +__WORKSPACE_ROOT_SETUP__ TOTAL_SHARDS="${RULES_RUST_TEST_TOTAL_SHARDS:-${TEST_TOTAL_SHARDS:-}}" SHARD_INDEX="${RULES_RUST_TEST_SHARD_INDEX:-${TEST_SHARD_INDEX:-}}" @@ -116,4 +118,4 @@ if [[ -n "${TOTAL_SHARDS}" && "${TOTAL_SHARDS}" != "0" ]]; then run_sharded_libtest "$@" fi -exec "${test_bin}" "$@" +exec env "${RUNFILE_ENV_ARGS[@]}" "${test_bin}" "$@"