ci: route build IO through Dev Drives

.github/scripts/run_bazel_with_buildbuddy.py

@@ -131,25 +131,54 @@ def bazel_args_without_remote_execution(args: Sequence[str]) -> list[str]:
 def bazel_args_with_remote_config(
     args: Sequence[str], env: Mapping[str, str]
 ) -> list[str]:
+    command_idx = next(
+        (idx for idx, arg in enumerate(args) if not arg.startswith("-")),
+        None,
+    )
+    if command_idx is None:
+        raise ValueError("expected a Bazel command")
+
     config = remote_config(args, env)
     if config is None:
-        return bazel_args_without_remote_execution(args)
+        configured_args = bazel_args_without_remote_execution(args)
+    else:
+        # `remote_config()` returns a configuration only when this key is present.
+        api_key = env["BUILDBUDDY_API_KEY"]
+        remote_args = [
+            f"--config={config}",
+            f"--remote_header=x-buildbuddy-api-key={api_key}",
+        ]
 
-    # `remote_config()` returns a configuration only when this key is present.
-    api_key = env["BUILDBUDDY_API_KEY"]
-    remote_args = [
-        f"--config={config}",
-        f"--remote_header=x-buildbuddy-api-key={api_key}",
+        # Insert immediately after the Bazel command. This keeps wrapper-added
+        # options out of positional payloads and lets later CI configs override
+        # shared RBE defaults such as the Windows cross-compilation exec platforms.
+        configured_args = [
+            *args[: command_idx + 1],
+            *remote_args,
+            *args[command_idx + 1 :],
+        ]
+
+    try:
+        separator_idx = configured_args.index("--")
+    except ValueError:
+        separator_idx = len(configured_args)
+
+    cache_args = [
+        f"{option_prefix}{env[env_name]}"
+        for env_name, option_prefix in (
+            ("BAZEL_REPO_CONTENTS_CACHE", "--repo_contents_cache="),
+            ("BAZEL_REPOSITORY_CACHE", "--repository_cache="),
+        )
+        if env.get(env_name)
+        and not any(
+            arg.startswith(option_prefix) for arg in configured_args[:separator_idx]
+        )
+    ]
+    return [
+        *configured_args[:separator_idx],
+        *cache_args,
+        *configured_args[separator_idx:],
     ]
-
-    # Insert immediately after the Bazel command. This keeps wrapper-added
-    # options out of positional payloads and lets later CI configs override
-    # shared RBE defaults such as the Windows cross-compilation exec platforms.
-    insertion_idx = next(
-        (idx + 1 for idx, arg in enumerate(args) if not arg.startswith("-")),
-        len(args),
-    )
-    return [*args[:insertion_idx], *remote_args, *args[insertion_idx:]]
 
 
 def bazel_command(*args: str, env: Mapping[str, str] | None = None) -> list[str]:

.github/scripts/setup-dev-drive.ps1

@@ -1,14 +1,15 @@
 # Configure a fast drive for Windows CI jobs.
 #
 # GitHub-hosted Windows runners do not always expose a secondary D: volume. When
-# they do not, try to create a Dev Drive VHD and fall back to C: if the runner
-# image does not allow that provisioning path.
+# they do not, create a Dev Drive VHD. CI depends on this path for its
+# build directories where CI spends significant time doing I/O, so fail the
+# job if no real Dev Drive is available.
 
-function Use-FallbackDrive {
-    param([string]$Reason)
+function Test-DevDrive {
+    param([string]$Drive)
 
-    Write-Warning "$Reason Falling back to C:"
-    return "C:"
+    & fsutil devdrv query $Drive *> $null
+    return $LASTEXITCODE -eq 0
 }
 
 function Invoke-BestEffort {
@@ -21,10 +22,14 @@ function Invoke-BestEffort {
     }
 }
 
-if (Test-Path "D:\") {
-    Write-Output "Using existing drive at D:"
+if ((Test-Path "D:\") -and (Test-DevDrive "D:")) {
+    Write-Output "Using existing Dev Drive at D:"
     $Drive = "D:"
 } else {
+    if (Test-Path "D:\") {
+        Write-Output "Existing D: volume is not a Dev Drive; provisioning a new Dev Drive VHD."
+    }
+
     try {
         $VhdPath = Join-Path $env:RUNNER_TEMP "codex-dev-drive.vhdx"
         $SizeBytes = 64GB
@@ -42,21 +47,17 @@ if (Test-Path "D:\") {
 
         $Drive = "$($Volume.DriveLetter):"
 
+        if (-not (Test-DevDrive $Drive)) {
+            throw "Provisioned volume at $Drive did not pass Dev Drive verification."
+        }
+
         Invoke-BestEffort { fsutil devdrv trust $Drive } "Trusting Dev Drive $Drive"
         Invoke-BestEffort { fsutil devdrv enable /disallowAv } "Disabling AV filter attachment for Dev Drives"
-        Invoke-BestEffort { fsutil devdrv query $Drive } "Querying Dev Drive $Drive"
 
         Write-Output "Using Dev Drive at $Drive"
     } catch {
-        $Drive = Use-FallbackDrive "Failed to create Dev Drive: $($_.Exception.Message)"
+        throw "Failed to create Dev Drive: $($_.Exception.Message)"
     }
 }
 
-$Tmp = "$Drive\codex-tmp"
-New-Item -Path $Tmp -ItemType Directory -Force | Out-Null
-
-@(
-    "DEV_DRIVE=$Drive"
-    "TMP=$Tmp"
-    "TEMP=$Tmp"
-) | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
+"CI_BUILD_ROOT=$Drive" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append

.github/scripts/test_run_bazel_with_buildbuddy.py

@@ -214,6 +214,48 @@ class RunBazelWithBuildBuddyTest(unittest.TestCase):
             ],
         )
 
+    def test_bazel_command_uses_configured_local_caches(self) -> None:
+        env = {
+            "BAZEL_REPO_CONTENTS_CACHE": "/tmp/bazel-repo-contents",
+            "BAZEL_REPOSITORY_CACHE": "/tmp/bazel-repository",
+        }
+
+        self.assertEqual(
+            run_bazel_with_buildbuddy.bazel_command(
+                "build",
+                "--config=local",
+                "//codex-rs/...",
+                env=env,
+            ),
+            [
+                "bazel",
+                "build",
+                "--config=local",
+                "//codex-rs/...",
+                "--repo_contents_cache=/tmp/bazel-repo-contents",
+                "--repository_cache=/tmp/bazel-repository",
+            ],
+        )
+
+    def test_bazel_command_adds_local_caches_before_separator(self) -> None:
+        self.assertEqual(
+            run_bazel_with_buildbuddy.bazel_command(
+                "build",
+                "//codex-rs/...",
+                "--",
+                "--program-arg",
+                env={"BAZEL_REPOSITORY_CACHE": "/tmp/bazel-repository"},
+            ),
+            [
+                "bazel",
+                "build",
+                "//codex-rs/...",
+                "--repository_cache=/tmp/bazel-repository",
+                "--",
+                "--program-arg",
+            ],
+        )
+
     def test_main_preserves_spaced_argument_and_child_exit_status(self) -> None:
         spaced_arg = (
             r"--test_env=PATH=C:\Program Files\PowerShell\7;C:\Program Files\Git\bin"

.github/scripts/test_v8_canary_changes.py

@@ -47,13 +47,17 @@ version = "149.2.0"
             )
         )
 
-    def test_setup_ci_change_requires_canary_and_source_build(self) -> None:
-        changed_files = {".github/actions/setup-ci/action.yml"}
-
-        self.assertTrue(canary_required(changed_files, "149.2.0", "149.2.0"))
-        self.assertTrue(
-            windows_source_required(changed_files, "149.2.0", "149.2.0")
-        )
+    def test_shared_ci_setup_changes_require_canary_and_source_build(self) -> None:
+        for path in (
+            ".github/actions/setup-ci/action.yml",
+            ".github/scripts/setup-dev-drive.ps1",
+        ):
+            with self.subTest(path=path):
+                changed_files = {path}
+                self.assertTrue(canary_required(changed_files, "149.2.0", "149.2.0"))
+                self.assertTrue(
+                    windows_source_required(changed_files, "149.2.0", "149.2.0")
+                )
 
     def test_manual_dispatch_requires_source_build(self) -> None:
         self.assertTrue(

.github/scripts/v8_canary_changes.py

@@ -25,6 +25,7 @@ CANARY_PATH_PATTERNS = {
     ".github/scripts/run_bazel_with_buildbuddy.py",
     ".github/scripts/rusty_v8_bazel.py",
     ".github/scripts/rusty_v8_module_bazel.py",
+    ".github/scripts/setup-dev-drive.ps1",
     ".github/scripts/v8_canary_changes.py",
     ".github/workflows/postmerge-ci.yml",
     ".github/workflows/rusty-v8-release.yml",
@@ -44,6 +45,7 @@ WINDOWS_SOURCE_BUILD_PATHS = {
     ".github/actions/setup-ci/**",
     ".github/scripts/rusty_v8_bazel.py",
     ".github/scripts/rusty_v8_module_bazel.py",
+    ".github/scripts/setup-dev-drive.ps1",
     ".github/scripts/v8_canary_changes.py",
     ".github/workflows/rusty-v8-release.yml",
     ".github/workflows/v8-canary.yml",