diff --git a/.codex/skills/bazel-hot-cache/SKILL.md b/.codex/skills/bazel-hot-cache/SKILL.md index 37b77ee50d..be9bc5846d 100644 --- a/.codex/skills/bazel-hot-cache/SKILL.md +++ b/.codex/skills/bazel-hot-cache/SKILL.md @@ -27,6 +27,13 @@ The helper is read-only from a developer machine or devbox: Do not change this to upload laptop or devbox artifacts unless the user explicitly asks for a cache-warming writer flow. +The helper also bounds the known remote-cache long tail: one Bazel attempt may +stall after remote-cache progress stops even though the cache keyspace is hot. +The checked-in helper times out that stalled attempt after `120s`, shuts down +that attempt's Bazel server, and retries once. Override only for diagnostics +with `CODEX_BAZEL_HOT_CACHE_TIMEOUT_SECONDS` or +`CODEX_BAZEL_HOT_CACHE_MAX_ATTEMPTS`. + ## Local Flow From a Codex checkout with `gh` auth and `BUILDBUDDY_API_KEY` in the command @@ -90,6 +97,7 @@ The helper sets: - `--config=ci-macos` on macOS or `--config=ci-linux` on Linux - `--remote_download_toplevel` - `--noremote_upload_local_results` +- bounded retry defaults: `120s` timeout, `2` max attempts ## Reading Results @@ -107,6 +115,7 @@ bookkeeping, not cache misses. - Build mode supports macOS and Linux only. - `--print-latest-hot-main-commit` requires `gh` auth. - Build mode requires `BUILDBUDDY_API_KEY`. +- Build mode requires `python3` for bounded retry handling. - This is command/config specific. A successful `main` Bazel run is a good hot default for this helper's `verify-release-build` shape, not proof that every other Bazel command is hot. diff --git a/codex-rs/docs/bazel.md b/codex-rs/docs/bazel.md index 3bd72bb9c6..ac3b4e6d2e 100644 --- a/codex-rs/docs/bazel.md +++ b/codex-rs/docs/bazel.md @@ -156,6 +156,12 @@ Linux `verify-release-build` cache keyspace instead of the macOS one. Pass Bazel target patterns after the script name to build something other than the default `//codex-rs/cli:codex`. +The helper bounds the known remote-cache long tail: if a Bazel attempt stalls +after remote-cache progress stops, it times out that attempt after `120s`, +shuts down that attempt's Bazel server, and retries once. Override those +diagnostic knobs only when needed with `CODEX_BAZEL_HOT_CACHE_TIMEOUT_SECONDS` +or `CODEX_BAZEL_HOT_CACHE_MAX_ATTEMPTS`. + For routine developer use, keep Bazel's normal persistent user root and repository cache. A fresh worktree already gives Bazel a fresh workspace/output base while preserving the shared local Bazel state that makes this workflow diff --git a/scripts/run-bazel-hot-cache-build.sh b/scripts/run-bazel-hot-cache-build.sh index 234668396c..507765b70e 100755 --- a/scripts/run-bazel-hot-cache-build.sh +++ b/scripts/run-bazel-hot-cache-build.sh @@ -18,6 +18,7 @@ provided, builds //codex-rs/cli:codex. Requires: - macOS or Linux for build mode - BUILDBUDDY_API_KEY in the environment for build mode + - python3 for bounded retry handling in build mode - gh auth for --print-latest-hot-main-commit EOF } @@ -99,25 +100,94 @@ bazel_startup_args=() if [[ -n "${BAZEL_OUTPUT_USER_ROOT:-}" ]]; then bazel_startup_args+=("--output_user_root=${BAZEL_OUTPUT_USER_ROOT}") fi +bazel_timeout_seconds="${CODEX_BAZEL_HOT_CACHE_TIMEOUT_SECONDS:-120}" +bazel_max_attempts="${CODEX_BAZEL_HOT_CACHE_MAX_ATTEMPTS:-2}" +if ! [[ "${bazel_timeout_seconds}" =~ ^[1-9][0-9]*$ ]]; then + echo "CODEX_BAZEL_HOT_CACHE_TIMEOUT_SECONDS must be a positive integer." >&2 + exit 1 +fi +if ! [[ "${bazel_max_attempts}" =~ ^[1-9][0-9]*$ ]]; then + echo "CODEX_BAZEL_HOT_CACHE_MAX_ATTEMPTS must be a positive integer." >&2 + exit 1 +fi +if ! command -v python3 >/dev/null 2>&1; then + echo "python3 is required to bound Bazel hot-cache retries." >&2 + exit 1 +fi # Keep the explicit Rust debug-assertion flags before the platform CI config. # That matches the verify-release-build CI action key ordering that warms this # cache. -exec "${bazel_bin}" \ - "${bazel_startup_args[@]}" \ - --noexperimental_remote_repo_contents_cache \ - build \ - --config=buildbuddy-openai-rbe \ - "--remote_header=x-buildbuddy-api-key=${BUILDBUDDY_API_KEY}" \ - --compilation_mode=fastbuild \ - --@rules_rust//rust/settings:extra_rustc_flag=-Cdebug-assertions=no \ - --@rules_rust//rust/settings:extra_exec_rustc_flag=-Cdebug-assertions=no \ - "--build_metadata=COMMIT_SHA=${commit_sha}" \ - --build_metadata=TAG_job=verify-release-build \ - --build_metadata=TAG_rust_debug_assertions=off \ - "--config=${bazel_ci_config}" \ - --remote_download_toplevel \ - "--repository_cache=${repository_cache}" \ - --noremote_upload_local_results \ - -- \ +bazel_command=( + "${bazel_bin}" + "${bazel_startup_args[@]}" + --noexperimental_remote_repo_contents_cache + build + --config=buildbuddy-openai-rbe + "--remote_header=x-buildbuddy-api-key=${BUILDBUDDY_API_KEY}" + --compilation_mode=fastbuild + --@rules_rust//rust/settings:extra_rustc_flag=-Cdebug-assertions=no + --@rules_rust//rust/settings:extra_exec_rustc_flag=-Cdebug-assertions=no + "--build_metadata=COMMIT_SHA=${commit_sha}" + --build_metadata=TAG_job=verify-release-build + --build_metadata=TAG_rust_debug_assertions=off + "--config=${bazel_ci_config}" + --remote_download_toplevel + "--repository_cache=${repository_cache}" + --noremote_upload_local_results + -- "${targets[@]}" +) + +run_command_with_timeout() { + local timeout_seconds="$1" + shift + python3 - "${timeout_seconds}" "$@" <<'PY' +import os +import signal +import subprocess +import sys + +timeout_seconds = int(sys.argv[1]) +command = sys.argv[2:] +process = subprocess.Popen(command, start_new_session=True) +try: + raise SystemExit(process.wait(timeout=timeout_seconds)) +except subprocess.TimeoutExpired: + try: + os.killpg(process.pid, signal.SIGTERM) + except ProcessLookupError: + pass + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + process.wait() + raise SystemExit(124) +PY +} + +shutdown_bazel_server() { + run_command_with_timeout 10 \ + "${bazel_bin}" \ + "${bazel_startup_args[@]}" \ + shutdown \ + >/dev/null 2>&1 || true +} + +for ((attempt = 1; attempt <= bazel_max_attempts; attempt++)); do + if run_command_with_timeout "${bazel_timeout_seconds}" "${bazel_command[@]}"; then + exit 0 + else + status=$? + fi + if [[ ${status} -ne 124 || ${attempt} -eq ${bazel_max_attempts} ]]; then + exit "${status}" + fi + + echo "Bazel hot-cache attempt ${attempt}/${bazel_max_attempts} timed out after ${bazel_timeout_seconds}s; retrying." >&2 + shutdown_bazel_server +done