ci: route build IO through Dev Drives

2026-09-09 15:58:47 +00:00 · 2026-07-07 04:25:23 +00:00
parent bd1894e5a3
commit 81fbbac4ff
8 changed files with 162 additions and 73 deletions
--- a/.github/actions/setup-bazel-ci/action.yml
+++ b/.github/actions/setup-bazel-ci/action.yml
@@ -25,25 +25,7 @@ runs:
      id: configure_bazel_repository_cache
      shell: pwsh
      run: |
-        # Keep the repository cache under HOME on all runners. Windows `D:\a`
-        # cache paths match `.bazelrc`, but `actions/cache/restore` currently
-        # returns HTTP 400 for that path in the Windows clippy job.
-        $repositoryCachePath = Join-Path $HOME '.cache/bazel-repo-cache'
-        "repository-cache-path=$repositoryCachePath" | Out-File -FilePath $env:GITHUB_OUTPUT -Encoding utf8 -Append
-        "BAZEL_REPOSITORY_CACHE=$repositoryCachePath" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
-
-    - name: Configure Bazel output root (Windows)
-      if: runner.os == 'Windows'
-      shell: pwsh
-      run: |
-        # Use the shortest available drive to reduce argv/path length issues,
-        # but avoid the drive root because some Windows test launchers mis-handle
-        # MANIFEST paths there.
-        $hasDDrive = Test-Path 'D:\'
-        $bazelOutputUserRoot = if ($hasDDrive) { 'D:\b' } else { 'C:\b' }
-        $repoContentsCache = Join-Path $env:RUNNER_TEMP "bazel-repo-contents-cache-$env:GITHUB_RUN_ID-$env:GITHUB_JOB"
-        "BAZEL_OUTPUT_USER_ROOT=$bazelOutputUserRoot" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
-        "BAZEL_REPO_CONTENTS_CACHE=$repoContentsCache" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
+        "repository-cache-path=$env:BAZEL_REPOSITORY_CACHE" | Out-File -FilePath $env:GITHUB_OUTPUT -Encoding utf8 -Append

    - name: Expose MSVC SDK environment (Windows)
      if: runner.os == 'Windows'
--- a/.github/actions/setup-ci/action.yml
+++ b/.github/actions/setup-ci/action.yml
@@ -4,11 +4,49 @@ description: Prepare common tools and environment shared by CI jobs.
 runs:
  using: composite
  steps:
-    # TODO(anp): Move this under CI_BUILD_ROOT once all consumers use the
-    # shared build-path contract.
-    - name: Configure Cargo target directory
+    # setup-ci expects either this step or the Unix fallback below to define
+    # CI_BUILD_ROOT. Windows puts it on a Dev Drive because Cargo and Bazel
+    # spend significant time reading and writing build/cache trees.
+    - name: Configure Dev Drive (Windows)
+      if: runner.os == 'Windows'
+      shell: pwsh
+      run: ./.github/scripts/setup-dev-drive.ps1
+
+    - name: Configure CI build root (Unix)
+      if: runner.os != 'Windows'
      shell: bash
-      run: echo "CARGO_TARGET_DIR=${CARGO_TARGET_DIR:-$GITHUB_WORKSPACE/codex-rs/target}" >> "$GITHUB_ENV"
+      run: echo "CI_BUILD_ROOT=$HOME/.cache/codex-ci" >> "$GITHUB_ENV"
+
+    - name: Configure CI build paths
+      shell: bash
+      run: |
+        set -euo pipefail
+        # Keep this directory name tiny on every platform so the shared path
+        # contract preserves Windows' short Bazel output root. Long Windows
+        # Bazel paths can overflow argv and confuse test MANIFEST handling.
+        bazel_output_user_root="$CI_BUILD_ROOT/b"
+        bazel_repository_cache="$CI_BUILD_ROOT/bazel-repository-cache"
+        bazel_repo_contents_cache="$CI_BUILD_ROOT/bazel-repo-contents-cache-$GITHUB_RUN_ID-$GITHUB_JOB"
+        cargo_target_dir="$CI_BUILD_ROOT/cargo-target"
+        tmp="$CI_BUILD_ROOT/tmp"
+
+        build_dirs=(
+          "$bazel_output_user_root"
+          "$bazel_repository_cache"
+          "$bazel_repo_contents_cache"
+          "$cargo_target_dir"
+          "$tmp"
+        )
+        mkdir -p "${build_dirs[@]}"
+
+        {
+          echo "BAZEL_OUTPUT_USER_ROOT=$bazel_output_user_root"
+          echo "BAZEL_REPOSITORY_CACHE=$bazel_repository_cache"
+          echo "BAZEL_REPO_CONTENTS_CACHE=$bazel_repo_contents_cache"
+          echo "CARGO_TARGET_DIR=$cargo_target_dir"
+          echo "TMP=$tmp"
+          echo "TEMP=$tmp"
+        } >> "$GITHUB_ENV"

    - name: Prefer the Git CLI for Cargo git dependencies
      shell: bash
--- a/.github/scripts/run_bazel_with_buildbuddy.py
+++ b/.github/scripts/run_bazel_with_buildbuddy.py
@@ -131,25 +131,54 @@ def bazel_args_without_remote_execution(args: Sequence[str]) -> list[str]:
 def bazel_args_with_remote_config(
    args: Sequence[str], env: Mapping[str, str]
 ) -> list[str]:
+    command_idx = next(
+        (idx for idx, arg in enumerate(args) if not arg.startswith("-")),
+        None,
+    )
+    if command_idx is None:
+        raise ValueError("expected a Bazel command")
+
    config = remote_config(args, env)
    if config is None:
-        return bazel_args_without_remote_execution(args)
+        configured_args = bazel_args_without_remote_execution(args)
+    else:
+        # `remote_config()` returns a configuration only when this key is present.
+        api_key = env["BUILDBUDDY_API_KEY"]
+        remote_args = [
+            f"--config={config}",
+            f"--remote_header=x-buildbuddy-api-key={api_key}",
+        ]

-    # `remote_config()` returns a configuration only when this key is present.
-    api_key = env["BUILDBUDDY_API_KEY"]
-    remote_args = [
-        f"--config={config}",
-        f"--remote_header=x-buildbuddy-api-key={api_key}",
+        # Insert immediately after the Bazel command. This keeps wrapper-added
+        # options out of positional payloads and lets later CI configs override
+        # shared RBE defaults such as the Windows cross-compilation exec platforms.
+        configured_args = [
+            *args[: command_idx + 1],
+            *remote_args,
+            *args[command_idx + 1 :],
+        ]
+
+    try:
+        separator_idx = configured_args.index("--")
+    except ValueError:
+        separator_idx = len(configured_args)
+
+    cache_args = [
+        f"{option_prefix}{env[env_name]}"
+        for env_name, option_prefix in (
+            ("BAZEL_REPO_CONTENTS_CACHE", "--repo_contents_cache="),
+            ("BAZEL_REPOSITORY_CACHE", "--repository_cache="),
+        )
+        if env.get(env_name)
+        and not any(
+            arg.startswith(option_prefix) for arg in configured_args[:separator_idx]
+        )
+    ]
+    return [
+        *configured_args[:separator_idx],
+        *cache_args,
+        *configured_args[separator_idx:],
    ]
-
-    # Insert immediately after the Bazel command. This keeps wrapper-added
-    # options out of positional payloads and lets later CI configs override
-    # shared RBE defaults such as the Windows cross-compilation exec platforms.
-    insertion_idx = next(
-        (idx + 1 for idx, arg in enumerate(args) if not arg.startswith("-")),
-        len(args),
-    )
-    return [*args[:insertion_idx], *remote_args, *args[insertion_idx:]]


 def bazel_command(*args: str, env: Mapping[str, str] | None = None) -> list[str]:
--- a/.github/scripts/setup-dev-drive.ps1
+++ b/.github/scripts/setup-dev-drive.ps1
@@ -1,14 +1,15 @@
 # Configure a fast drive for Windows CI jobs.
 #
 # GitHub-hosted Windows runners do not always expose a secondary D: volume. When
-# they do not, try to create a Dev Drive VHD and fall back to C: if the runner
-# image does not allow that provisioning path.
+# they do not, create a Dev Drive VHD. CI depends on this path for its
+# build directories where CI spends significant time doing I/O, so fail the
+# job if no real Dev Drive is available.

-function Use-FallbackDrive {
-    param([string]$Reason)
+function Test-DevDrive {
+    param([string]$Drive)

-    Write-Warning "$Reason Falling back to C:"
-    return "C:"
+    & fsutil devdrv query $Drive *> $null
+    return $LASTEXITCODE -eq 0
 }

 function Invoke-BestEffort {
@@ -21,10 +22,14 @@ function Invoke-BestEffort {
    }
 }

-if (Test-Path "D:\") {
-    Write-Output "Using existing drive at D:"
+if ((Test-Path "D:\") -and (Test-DevDrive "D:")) {
+    Write-Output "Using existing Dev Drive at D:"
    $Drive = "D:"
 } else {
+    if (Test-Path "D:\") {
+        Write-Output "Existing D: volume is not a Dev Drive; provisioning a new Dev Drive VHD."
+    }
+
    try {
        $VhdPath = Join-Path $env:RUNNER_TEMP "codex-dev-drive.vhdx"
        $SizeBytes = 64GB
@@ -42,21 +47,17 @@ if (Test-Path "D:\") {

        $Drive = "$($Volume.DriveLetter):"

+        if (-not (Test-DevDrive $Drive)) {
+            throw "Provisioned volume at $Drive did not pass Dev Drive verification."
+        }
+
        Invoke-BestEffort { fsutil devdrv trust $Drive } "Trusting Dev Drive $Drive"
        Invoke-BestEffort { fsutil devdrv enable /disallowAv } "Disabling AV filter attachment for Dev Drives"
-        Invoke-BestEffort { fsutil devdrv query $Drive } "Querying Dev Drive $Drive"

        Write-Output "Using Dev Drive at $Drive"
    } catch {
-        $Drive = Use-FallbackDrive "Failed to create Dev Drive: $($_.Exception.Message)"
+        throw "Failed to create Dev Drive: $($_.Exception.Message)"
    }
 }

-$Tmp = "$Drive\codex-tmp"
-New-Item -Path $Tmp -ItemType Directory -Force | Out-Null
-
-@(
-    "DEV_DRIVE=$Drive"
-    "TMP=$Tmp"
-    "TEMP=$Tmp"
-) | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
+"CI_BUILD_ROOT=$Drive" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
--- a/.github/scripts/test_run_bazel_with_buildbuddy.py
+++ b/.github/scripts/test_run_bazel_with_buildbuddy.py
@@ -214,6 +214,48 @@ class RunBazelWithBuildBuddyTest(unittest.TestCase):
            ],
        )

+    def test_bazel_command_uses_configured_local_caches(self) -> None:
+        env = {
+            "BAZEL_REPO_CONTENTS_CACHE": "/tmp/bazel-repo-contents",
+            "BAZEL_REPOSITORY_CACHE": "/tmp/bazel-repository",
+        }
+
+        self.assertEqual(
+            run_bazel_with_buildbuddy.bazel_command(
+                "build",
+                "--config=local",
+                "//codex-rs/...",
+                env=env,
+            ),
+            [
+                "bazel",
+                "build",
+                "--config=local",
+                "//codex-rs/...",
+                "--repo_contents_cache=/tmp/bazel-repo-contents",
+                "--repository_cache=/tmp/bazel-repository",
+            ],
+        )
+
+    def test_bazel_command_adds_local_caches_before_separator(self) -> None:
+        self.assertEqual(
+            run_bazel_with_buildbuddy.bazel_command(
+                "build",
+                "//codex-rs/...",
+                "--",
+                "--program-arg",
+                env={"BAZEL_REPOSITORY_CACHE": "/tmp/bazel-repository"},
+            ),
+            [
+                "bazel",
+                "build",
+                "//codex-rs/...",
+                "--repository_cache=/tmp/bazel-repository",
+                "--",
+                "--program-arg",
+            ],
+        )
+
    def test_main_preserves_spaced_argument_and_child_exit_status(self) -> None:
        spaced_arg = (
            r"--test_env=PATH=C:\Program Files\PowerShell\7;C:\Program Files\Git\bin"
--- a/.github/scripts/test_v8_canary_changes.py
+++ b/.github/scripts/test_v8_canary_changes.py
@@ -47,13 +47,17 @@ version = "149.2.0"
            )
        )

-    def test_setup_ci_change_requires_canary_and_source_build(self) -> None:
-        changed_files = {".github/actions/setup-ci/action.yml"}
-
-        self.assertTrue(canary_required(changed_files, "149.2.0", "149.2.0"))
-        self.assertTrue(
-            windows_source_required(changed_files, "149.2.0", "149.2.0")
-        )
+    def test_shared_ci_setup_changes_require_canary_and_source_build(self) -> None:
+        for path in (
+            ".github/actions/setup-ci/action.yml",
+            ".github/scripts/setup-dev-drive.ps1",
+        ):
+            with self.subTest(path=path):
+                changed_files = {path}
+                self.assertTrue(canary_required(changed_files, "149.2.0", "149.2.0"))
+                self.assertTrue(
+                    windows_source_required(changed_files, "149.2.0", "149.2.0")
+                )

    def test_manual_dispatch_requires_source_build(self) -> None:
        self.assertTrue(
--- a/.github/scripts/v8_canary_changes.py
+++ b/.github/scripts/v8_canary_changes.py
@@ -25,6 +25,7 @@ CANARY_PATH_PATTERNS = {
    ".github/scripts/run_bazel_with_buildbuddy.py",
    ".github/scripts/rusty_v8_bazel.py",
    ".github/scripts/rusty_v8_module_bazel.py",
+    ".github/scripts/setup-dev-drive.ps1",
    ".github/scripts/v8_canary_changes.py",
    ".github/workflows/postmerge-ci.yml",
    ".github/workflows/rusty-v8-release.yml",
@@ -44,6 +45,7 @@ WINDOWS_SOURCE_BUILD_PATHS = {
    ".github/actions/setup-ci/**",
    ".github/scripts/rusty_v8_bazel.py",
    ".github/scripts/rusty_v8_module_bazel.py",
+    ".github/scripts/setup-dev-drive.ps1",
    ".github/scripts/v8_canary_changes.py",
    ".github/workflows/rusty-v8-release.yml",
    ".github/workflows/v8-canary.yml",
--- a/.github/workflows/rust-ci-full-nextest-platform.yml
+++ b/.github/workflows/rust-ci-full-nextest-platform.yml
@@ -71,11 +71,6 @@ jobs:
        with:
          persist-credentials: false

-      - name: Configure Dev Drive (Windows)
-        if: ${{ runner.os == 'Windows' }}
-        shell: pwsh
-        run: ../.github/scripts/setup-dev-drive.ps1
-
      - uses: ./.github/actions/setup-ci

      - name: Install Linux build dependencies
@@ -136,11 +131,7 @@ jobs:
            echo "Using sccache GitHub backend"
          else
            echo "SCCACHE_GHA_ENABLED=false" >> "$GITHUB_ENV"
-            if [[ -n "${DEV_DRIVE:-}" ]]; then
-              echo "SCCACHE_DIR=${DEV_DRIVE}\\.sccache" >> "$GITHUB_ENV"
-            else
-              echo "SCCACHE_DIR=${{ github.workspace }}/.sccache" >> "$GITHUB_ENV"
-            fi
+            echo "SCCACHE_DIR=${CI_BUILD_ROOT}/sccache" >> "$GITHUB_ENV"
            echo "Using sccache local disk + actions/cache fallback"
          fi