diff --git a/.github/scripts/run-argument-comment-lint-bazel.sh b/.github/scripts/run-argument-comment-lint-bazel.sh index fddca4cadb..4a7d34a80c 100755 --- a/.github/scripts/run-argument-comment-lint-bazel.sh +++ b/.github/scripts/run-argument-comment-lint-bazel.sh @@ -41,7 +41,7 @@ read_query_labels() { query_stdout="$(mktemp)" query_stderr="$(mktemp)" - if ! ./.github/scripts/run-bazel-query-ci.sh \ + if ! ./.github/scripts/run_bazel_query_ci.py \ --keep_going \ --output=label \ -- "$query" >"$query_stdout" 2>"$query_stderr"; then @@ -72,7 +72,7 @@ if [[ "${RUNNER_OS:-}" == "Windows" ]]; then fi fi -./.github/scripts/run-bazel-ci.sh \ +./.github/scripts/run_bazel_ci.py \ -- \ build \ "${bazel_lint_args[@]}" \ diff --git a/.github/scripts/run-bazel-ci.sh b/.github/scripts/run-bazel-ci.sh index 89f937a998..ec859f443e 100755 --- a/.github/scripts/run-bazel-ci.sh +++ b/.github/scripts/run-bazel-ci.sh @@ -2,416 +2,4 @@ set -euo pipefail -print_failed_bazel_test_logs=0 -print_failed_bazel_action_summary=0 -remote_download_toplevel=0 -windows_msvc_host_platform=0 -windows_cross_compile=0 - -while [[ $# -gt 0 ]]; do - case "$1" in - --print-failed-test-logs) - print_failed_bazel_test_logs=1 - shift - ;; - --print-failed-action-summary) - print_failed_bazel_action_summary=1 - shift - ;; - --remote-download-toplevel) - remote_download_toplevel=1 - shift - ;; - --windows-msvc-host-platform) - windows_msvc_host_platform=1 - shift - ;; - --windows-cross-compile) - windows_cross_compile=1 - shift - ;; - --) - shift - break - ;; - *) - echo "Unknown option: $1" >&2 - exit 1 - ;; - esac -done - -if [[ $# -eq 0 ]]; then - echo "Usage: $0 [--print-failed-test-logs] [--print-failed-action-summary] [--remote-download-toplevel] [--windows-msvc-host-platform] [--windows-cross-compile] -- -- " >&2 - exit 1 -fi - -bazel_startup_args=() -if [[ -n "${BAZEL_OUTPUT_USER_ROOT:-}" ]]; then - bazel_startup_args+=("--output_user_root=${BAZEL_OUTPUT_USER_ROOT}") -fi - -run_bazel() { - if [[ "${RUNNER_OS:-}" == "Windows" ]]; then - MSYS2_ARG_CONV_EXCL='*' "$(dirname "${BASH_SOURCE[0]}")/run_bazel_with_buildbuddy.py" "$@" - return - fi - - "$(dirname "${BASH_SOURCE[0]}")/run_bazel_with_buildbuddy.py" "$@" -} - -run_bazel_with_startup_args() { - if (( ${#bazel_startup_args[@]} > 0 )); then - run_bazel "${bazel_startup_args[@]}" "$@" - return - fi - - run_bazel "$@" -} - -ci_config=ci-linux -case "${RUNNER_OS:-}" in - macOS) - ci_config=ci-macos - ;; - Windows) - if [[ $windows_cross_compile -eq 1 ]]; then - ci_config=ci-windows-cross - else - ci_config=ci-windows - fi - ;; -esac - -print_bazel_test_log_tails() { - local console_log="$1" - local testlogs_dir - - local -a bazel_info_args=(info) - if [[ -n "${BUILDBUDDY_API_KEY:-}" ]]; then - # `bazel info` needs the same CI config as the failed test invocation so - # platform-specific output roots match. On Windows, omitting `ci-windows` - # would point at `local_windows-fastbuild` even when the test ran with the - # MSVC host platform under `local_windows_msvc-fastbuild`. - bazel_info_args+=("--config=${ci_config}") - fi - - # Only pass flags that affect Bazel's output-root selection or repository - # lookup. Test/build-only flags such as execution logs or remote download - # mode can make `bazel info` fail, which would hide the real test log path. - for arg in "${post_config_bazel_args[@]}"; do - case "$arg" in - --host_platform=* | --repo_contents_cache=* | --repository_cache=*) - bazel_info_args+=("$arg") - ;; - esac - done - - testlogs_dir="$(run_bazel_with_startup_args \ - --noexperimental_remote_repo_contents_cache \ - "${bazel_info_args[@]}" \ - bazel-testlogs 2>/dev/null || echo bazel-testlogs)" - - local failed_targets=() - while IFS= read -r target; do - failed_targets+=("$target") - done < <( - grep -E '^(FAIL: //|ERROR: .* Testing //)' "$console_log" \ - | sed -E 's#^FAIL: (//[^ ]+).*#\1#; s#^ERROR: .* Testing (//[^ ]+) failed:.*#\1#' \ - | sort -u - ) - - if [[ ${#failed_targets[@]} -eq 0 ]]; then - echo "No failed Bazel test targets were found in console output." - return - fi - - for target in "${failed_targets[@]}"; do - local rel_path="${target#//}" - rel_path="${rel_path/://}" - local test_log="${testlogs_dir}/${rel_path}/test.log" - local reported_test_log - reported_test_log="$(grep -F "FAIL: ${target} " "$console_log" | sed -nE 's#.* \(see (.*[\\/]test\.log)\).*#\1#p' | head -n 1 || true)" - if [[ -n "$reported_test_log" ]]; then - reported_test_log="${reported_test_log//\\//}" - test_log="$reported_test_log" - fi - - echo "::group::Bazel test log tail for ${target}" - if [[ -f "$test_log" ]]; then - tail -n 200 "$test_log" - else - echo "Missing test log: $test_log" - fi - echo "::endgroup::" - done -} - -print_bazel_action_failure_summary() { - local console_log="$1" - local escaped_summary - local summary - - summary="$( - awk ' - function clean(line) { - gsub(sprintf("%c", 27) "\\[[0-9;]*m", "", line) - sub(/^.*\t[^\t]*\t[0-9TZ:._-]+ /, "", line) - return line - } - - function is_diagnostic(line) { - return line ~ /^(error(\[[^]]+\])?:|warning:|note:|help:)/ || - line ~ /^[[:space:]]+-->/ || - line ~ /^[[:space:]]*[0-9]+[[:space:]]+\|/ || - line ~ /^[[:space:]]*\|/ || - line ~ /^[[:space:]]+= (note|help):/ || - line ~ /^[[:space:]]*\^[[:space:]^~-]*$/ || - line ~ /^For more information/ || - line ~ /^error: aborting/ - } - - { - line = clean($0) - } - - line ~ /^ERROR: .* failed:/ { - if (printed) { - print "" - } - print line - in_failure = 1 - seen_diagnostic = 0 - printed = 1 - next - } - - in_failure && is_diagnostic(line) { - print line - seen_diagnostic = 1 - next - } - - in_failure && seen_diagnostic && line == "" { - print "" - next - } - - in_failure && seen_diagnostic { - in_failure = 0 - seen_diagnostic = 0 - next - } - ' "$console_log" - )" - - if [[ -z "$summary" ]]; then - summary="$(grep -E '^ERROR: |^FAILED: ' "$console_log" | tail -n 50 || true)" - fi - - if [[ -z "$summary" ]]; then - echo "No Bazel action failures were found in the captured console output." - return - fi - - if [[ "${GITHUB_ACTIONS:-}" == "true" ]]; then - escaped_summary="$( - printf '%s' "$summary" \ - | awk 'BEGIN { ORS = "" } { - gsub(/%/, "%25") - gsub(/\r/, "%0D") - print sep $0 - sep = "%0A" - }' - )" - echo "::error title=Bazel failed action diagnostics::${escaped_summary}" - fi - - echo - echo "Bazel failed action diagnostics:" - echo "--------------------------------" - printf '%s\n' "$summary" - echo "--------------------------------" -} - -bazel_args=() -bazel_targets=() -found_target_separator=0 -for arg in "$@"; do - if [[ "$arg" == "--" && $found_target_separator -eq 0 ]]; then - found_target_separator=1 - continue - fi - - if [[ $found_target_separator -eq 0 ]]; then - bazel_args+=("$arg") - else - bazel_targets+=("$arg") - fi -done - -if [[ ${#bazel_args[@]} -eq 0 || ${#bazel_targets[@]} -eq 0 ]]; then - echo "Expected Bazel args and targets separated by --" >&2 - exit 1 -fi - -if [[ "${RUNNER_OS:-}" == "Windows" && $windows_cross_compile -eq 1 && -z "${BUILDBUDDY_API_KEY:-}" ]]; then - # Windows cross-compilation depends on authenticated RBE. Preserve the local - # Windows build shape when credentials are unavailable. - ci_config=ci-windows - windows_msvc_host_platform=1 -fi - -post_config_bazel_args=() -if [[ "${RUNNER_OS:-}" == "Windows" && $windows_msvc_host_platform -eq 1 ]]; then - has_host_platform_override=0 - for arg in "${bazel_args[@]}"; do - if [[ "$arg" == --host_platform=* ]]; then - has_host_platform_override=1 - break - fi - done - - if [[ $has_host_platform_override -eq 0 ]]; then - # Use the MSVC Windows platform for jobs that need helper binaries like - # Rust test wrappers and V8 generators to resolve a compatible toolchain. - # Callers that need a different Windows target platform should pass an - # explicit `--platforms=...` flag. - post_config_bazel_args+=("--host_platform=//:local_windows_msvc") - fi -fi - -if [[ $remote_download_toplevel -eq 1 ]]; then - # Override the CI config's remote_download_minimal setting when callers need - # the built artifact to exist on disk after the command completes. - post_config_bazel_args+=(--remote_download_toplevel) -fi - -if [[ "${RUNNER_OS:-}" == "Windows" && $windows_cross_compile -eq 1 && -n "${BUILDBUDDY_API_KEY:-}" ]]; then - # `--enable_platform_specific_config` expands `common:windows` on Windows - # hosts after ordinary rc configs, which can override `ci-windows-cross`'s - # RBE host platform. Repeat the host platform on the command line so V8 and - # other genrules execute on Linux RBE workers instead of Git Bash locally. - # - # Bazel also derives the default genrule shell from the client host. Without - # an explicit shell executable, remote Linux actions can be asked to run - # `C:\Program Files\Git\usr\bin\bash.exe`. - post_config_bazel_args+=(--host_platform=//:rbe --shell_executable=/bin/bash) -fi - -if [[ "${RUNNER_OS:-}" == "Windows" && $windows_cross_compile -eq 1 && -z "${BUILDBUDDY_API_KEY:-}" ]]; then - # The Windows cross-compile config depends on authenticated remote - # execution. When credentials are unavailable, keep the local build shape - # and its lower concurrency cap. - post_config_bazel_args+=(--jobs=8) -fi - -if [[ -n "${BAZEL_REPO_CONTENTS_CACHE:-}" ]]; then - # Windows self-hosted runners can run multiple Bazel jobs concurrently. Give - # each job its own repo contents cache so they do not fight over the shared - # path configured in `ci-windows`. - post_config_bazel_args+=("--repo_contents_cache=${BAZEL_REPO_CONTENTS_CACHE}") -fi - -if [[ -n "${BAZEL_REPOSITORY_CACHE:-}" ]]; then - post_config_bazel_args+=("--repository_cache=${BAZEL_REPOSITORY_CACHE}") -fi - -if [[ -n "${CODEX_BAZEL_EXECUTION_LOG_COMPACT_DIR:-}" ]]; then - post_config_bazel_args+=( - "--execution_log_compact_file=${CODEX_BAZEL_EXECUTION_LOG_COMPACT_DIR}/execution-log-${bazel_args[0]}-${GITHUB_JOB:-local}-$$.zst" - ) -fi - -if [[ "${RUNNER_OS:-}" == "Windows" ]]; then - pass_windows_build_env=1 - if [[ $windows_cross_compile -eq 1 && -n "${BUILDBUDDY_API_KEY:-}" ]]; then - # Remote build actions execute on Linux RBE workers. Passing the Windows - # runner's build environment there makes Bazel genrules try to execute - # C:\Program Files\Git\usr\bin\bash.exe on Linux. - pass_windows_build_env=0 - fi - - if [[ $pass_windows_build_env -eq 1 ]]; then - windows_action_env_vars=( - INCLUDE - LIB - LIBPATH - UCRTVersion - UniversalCRTSdkDir - VCINSTALLDIR - VCToolsInstallDir - WindowsLibPath - WindowsSdkBinPath - WindowsSdkDir - WindowsSDKLibVersion - WindowsSDKVersion - ) - - for env_var in "${windows_action_env_vars[@]}"; do - if [[ -n "${!env_var:-}" ]]; then - post_config_bazel_args+=("--action_env=${env_var}" "--host_action_env=${env_var}") - fi - done - fi - - if [[ -z "${CODEX_BAZEL_WINDOWS_PATH:-}" ]]; then - echo "CODEX_BAZEL_WINDOWS_PATH must be set for Windows Bazel CI." >&2 - exit 1 - fi - - if [[ $pass_windows_build_env -eq 1 ]]; then - post_config_bazel_args+=( - "--action_env=PATH=${CODEX_BAZEL_WINDOWS_PATH}" - "--host_action_env=PATH=${CODEX_BAZEL_WINDOWS_PATH}" - ) - elif [[ $windows_cross_compile -eq 1 ]]; then - # Remote build actions run on Linux RBE workers. Give their shell snippets - # a Linux PATH while preserving CODEX_BAZEL_WINDOWS_PATH below for local - # Windows test execution. - post_config_bazel_args+=( - "--action_env=PATH=/usr/bin:/bin" - "--host_action_env=PATH=/usr/bin:/bin" - ) - fi - post_config_bazel_args+=("--test_env=PATH=${CODEX_BAZEL_WINDOWS_PATH}") -fi - -bazel_console_log="$(mktemp)" -trap 'rm -f "$bazel_console_log"' EXIT - -bazel_run_args=( - "${bazel_args[@]}" -) -if [[ -n "${BUILDBUDDY_API_KEY:-}" ]]; then - echo "BuildBuddy API key is available; using remote Bazel configuration." - bazel_run_args+=("--config=${ci_config}") -else - echo "BuildBuddy API key is not available; using local Bazel configuration." -fi -if (( ${#post_config_bazel_args[@]} > 0 )); then - bazel_run_args+=("${post_config_bazel_args[@]}") -fi -set +e -# Work around Bazel 9 remote repo contents cache / overlay materialization -# failures seen in CI (for example "is not a symlink" or permission errors -# while materializing external repos such as rules_perl). This only disables -# the startup-level repo contents cache; keyed runs still use BuildBuddy. -run_bazel_with_startup_args \ - --noexperimental_remote_repo_contents_cache \ - "${bazel_run_args[@]}" \ - -- \ - "${bazel_targets[@]}" \ - 2>&1 | tee "$bazel_console_log" -bazel_status=${PIPESTATUS[0]} -set -e - -if [[ ${bazel_status:-0} -ne 0 ]]; then - if [[ $print_failed_bazel_action_summary -eq 1 ]]; then - print_bazel_action_failure_summary "$bazel_console_log" - fi - if [[ $print_failed_bazel_test_logs -eq 1 ]]; then - print_bazel_test_log_tails "$bazel_console_log" - fi - exit "$bazel_status" -fi +exec "$(dirname "${BASH_SOURCE[0]}")/run_bazel_ci.py" "$@" diff --git a/.github/scripts/run-bazel-query-ci.sh b/.github/scripts/run-bazel-query-ci.sh deleted file mode 100755 index 39fa04c47d..0000000000 --- a/.github/scripts/run-bazel-query-ci.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -# Run target-discovery queries with the same startup settings as the main -# build/test invocation so they can reuse the same Bazel server. Queries only -# enumerate labels, so they intentionally do not select a CI build/test config -# or remote execution. - -if [[ $# -lt 2 || "${@: -2:1}" != "--" ]]; then - echo "Usage: $0 [...] -- " >&2 - exit 1 -fi - -query_args=("${@:1:$#-2}") -query_expression="${@: -1}" - -run_bazel() { - if [[ "${RUNNER_OS:-}" == "Windows" ]]; then - MSYS2_ARG_CONV_EXCL='*' "$(dirname "${BASH_SOURCE[0]}")/run_bazel_with_buildbuddy.py" "$@" - return - fi - - "$(dirname "${BASH_SOURCE[0]}")/run_bazel_with_buildbuddy.py" "$@" -} - -bazel_query_args=(query) - -if [[ -n "${BAZEL_REPO_CONTENTS_CACHE:-}" ]]; then - bazel_query_args+=("--repo_contents_cache=${BAZEL_REPO_CONTENTS_CACHE}") -fi - -if [[ -n "${BAZEL_REPOSITORY_CACHE:-}" ]]; then - bazel_query_args+=("--repository_cache=${BAZEL_REPOSITORY_CACHE}") -fi - -if (( ${#query_args[@]} > 0 )); then - bazel_query_args+=("${query_args[@]}") -fi -bazel_query_args+=("$query_expression") - -run_bazel "${bazel_query_args[@]}" diff --git a/.github/scripts/run_bazel_ci.py b/.github/scripts/run_bazel_ci.py new file mode 100755 index 0000000000..21b756ae25 --- /dev/null +++ b/.github/scripts/run_bazel_ci.py @@ -0,0 +1,386 @@ +#!/usr/bin/env python3 + +"""Run Bazel in CI with platform configuration and useful failure output.""" + +from __future__ import annotations + +import os +import re +import subprocess +import sys +from collections.abc import Mapping +from collections.abc import Sequence +from dataclasses import dataclass +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from run_bazel_with_buildbuddy import bazel_command + + +USAGE = ( + "Usage: run_bazel_ci.py [--print-failed-test-logs] " + "[--print-failed-action-summary] [--remote-download-toplevel] " + "[--windows-msvc-host-platform] [--windows-cross-compile] " + "-- -- " +) +WINDOWS_ACTION_ENV_VARS = ( + "INCLUDE", + "LIB", + "LIBPATH", + "UCRTVersion", + "UniversalCRTSdkDir", + "VCINSTALLDIR", + "VCToolsInstallDir", + "WindowsLibPath", + "WindowsSdkBinPath", + "WindowsSdkDir", + "WindowsSDKLibVersion", + "WindowsSDKVersion", +) +INFO_POST_CONFIG_PREFIXES = ( + "--host_platform=", + "--repo_contents_cache=", + "--repository_cache=", +) +ANSI_ESCAPE_RE = re.compile(r"\x1b\[[0-9;]*m") +LOG_PREFIX_RE = re.compile(r"^.*\t[^\t]*\t[0-9TZ:._-]+ ") +FAILED_TARGET_RES = ( + re.compile(r"^FAIL: (//[^ ]+)"), + re.compile(r"^ERROR: .* Testing (//[^ ]+) failed:"), +) +REPORTED_TEST_LOG_RE = re.compile(r" \(see (.*[\\/]test\.log)\)") + + +@dataclass(frozen=True) +class Options: + print_failed_test_logs: bool = False + print_failed_action_summary: bool = False + remote_download_toplevel: bool = False + windows_msvc_host_platform: bool = False + windows_cross_compile: bool = False + + +@dataclass(frozen=True) +class Invocation: + command: list[str] + child_env: dict[str, str] + ci_config: str + post_config_args: list[str] + remote_enabled: bool + + +def parse_args(args: Sequence[str]) -> tuple[Options, list[str], list[str]]: + option_names = { + "--print-failed-test-logs": "print_failed_test_logs", + "--print-failed-action-summary": "print_failed_action_summary", + "--remote-download-toplevel": "remote_download_toplevel", + "--windows-msvc-host-platform": "windows_msvc_host_platform", + "--windows-cross-compile": "windows_cross_compile", + } + enabled = {name: False for name in option_names.values()} + index = 0 + while index < len(args) and args[index] != "--": + arg = args[index] + if arg not in option_names: + raise ValueError(f"Unknown option: {arg}") + enabled[option_names[arg]] = True + index += 1 + if index == len(args): + raise ValueError(USAGE) + + remaining = list(args[index + 1 :]) + try: + separator = remaining.index("--") + except ValueError as exc: + raise ValueError("Expected Bazel args and targets separated by --") from exc + bazel_args = remaining[:separator] + targets = remaining[separator + 1 :] + if not bazel_args or not targets: + raise ValueError("Expected Bazel args and targets separated by --") + return Options(**enabled), bazel_args, targets + + +def ci_config(options: Options, env: Mapping[str, str]) -> str: + if env.get("RUNNER_OS") == "macOS": + return "ci-macos" + if env.get("RUNNER_OS") == "Windows": + if options.windows_cross_compile and env.get("BUILDBUDDY_API_KEY"): + return "ci-windows-cross" + return "ci-windows" + return "ci-linux" + + +def post_config_args( + options: Options, + bazel_args: Sequence[str], + env: Mapping[str, str], + *, + pid: int, +) -> list[str]: + runner_os = env.get("RUNNER_OS") + is_windows = runner_os == "Windows" + has_buildbuddy_key = bool(env.get("BUILDBUDDY_API_KEY")) + args: list[str] = [] + + use_msvc_host = options.windows_msvc_host_platform or ( + options.windows_cross_compile and not has_buildbuddy_key + ) + if is_windows and use_msvc_host and not any( + arg.startswith("--host_platform=") for arg in bazel_args + ): + args.append("--host_platform=//:local_windows_msvc") + if options.remote_download_toplevel: + args.append("--remote_download_toplevel") + if is_windows and options.windows_cross_compile and has_buildbuddy_key: + args.extend(("--host_platform=//:rbe", "--shell_executable=/bin/bash")) + if is_windows and options.windows_cross_compile and not has_buildbuddy_key: + args.append("--jobs=8") + if repo_contents_cache := env.get("BAZEL_REPO_CONTENTS_CACHE"): + args.append(f"--repo_contents_cache={repo_contents_cache}") + if repository_cache := env.get("BAZEL_REPOSITORY_CACHE"): + args.append(f"--repository_cache={repository_cache}") + if execution_log_dir := env.get("CODEX_BAZEL_EXECUTION_LOG_COMPACT_DIR"): + job = env.get("GITHUB_JOB", "local") + args.append( + f"--execution_log_compact_file={execution_log_dir}/" + f"execution-log-{bazel_args[0]}-{job}-{pid}.zst" + ) + + if not is_windows: + return args + + windows_path = env.get("CODEX_BAZEL_WINDOWS_PATH") + if not windows_path: + raise ValueError("CODEX_BAZEL_WINDOWS_PATH must be set for Windows Bazel CI.") + + pass_windows_build_env = not ( + options.windows_cross_compile and has_buildbuddy_key + ) + if pass_windows_build_env: + for name in WINDOWS_ACTION_ENV_VARS: + if value := env.get(name): + args.extend((f"--action_env={name}", f"--host_action_env={name}")) + args.extend( + ( + f"--action_env=PATH={windows_path}", + f"--host_action_env=PATH={windows_path}", + ) + ) + else: + args.extend( + ( + "--action_env=PATH=/usr/bin:/bin", + "--host_action_env=PATH=/usr/bin:/bin", + ) + ) + args.append(f"--test_env=PATH={windows_path}") + return args + + +def build_invocation( + options: Options, + bazel_args: Sequence[str], + targets: Sequence[str], + env: Mapping[str, str], + *, + pid: int, +) -> Invocation: + config = ci_config(options, env) + post_args = post_config_args(options, bazel_args, env, pid=pid) + run_args = list(bazel_args) + remote_enabled = bool(env.get("BUILDBUDDY_API_KEY")) + if remote_enabled: + run_args.append(f"--config={config}") + run_args.extend(post_args) + + startup_args = [] + if output_user_root := env.get("BAZEL_OUTPUT_USER_ROOT"): + startup_args.append(f"--output_user_root={output_user_root}") + command = bazel_command( + *startup_args, + "--noexperimental_remote_repo_contents_cache", + *run_args, + "--", + *targets, + env=env, + ) + child_env = dict(env) + if env.get("RUNNER_OS") == "Windows": + child_env["MSYS2_ARG_CONV_EXCL"] = "*" + return Invocation(command, child_env, config, post_args, remote_enabled) + + +def clean_log_line(line: str) -> str: + return LOG_PREFIX_RE.sub("", ANSI_ESCAPE_RE.sub("", line)) + + +def is_diagnostic(line: str) -> bool: + stripped = line.lstrip() + return ( + bool(re.match(r"^(error(\[[^]]+\])?:|warning:|note:|help:)", line)) + or stripped.startswith("-->") + or bool(re.match(r"^[0-9]+[ ]+\|", stripped)) + or stripped.startswith("|") + or stripped.startswith("= note:") + or stripped.startswith("= help:") + or bool(re.match(r"^\^[\s^~-]*$", stripped)) + or line.startswith("For more information") + or line.startswith("error: aborting") + ) + + +def action_failure_summary(console_output: str) -> str | None: + summary: list[str] = [] + in_failure = False + seen_diagnostic = False + for raw_line in console_output.splitlines(): + line = clean_log_line(raw_line) + if line.startswith("ERROR: ") and " failed:" in line: + if summary: + summary.append("") + summary.append(line) + in_failure = True + seen_diagnostic = False + elif in_failure and is_diagnostic(line): + summary.append(line) + seen_diagnostic = True + elif in_failure and seen_diagnostic and not line: + summary.append("") + elif in_failure and seen_diagnostic: + in_failure = False + seen_diagnostic = False + + if not summary: + summary = [ + line + for line in console_output.splitlines() + if line.startswith(("ERROR: ", "FAILED: ")) + ][-50:] + result = "\n".join(summary).rstrip() + return result or None + + +def failed_test_targets(console_output: str) -> list[str]: + targets = set() + for line in console_output.splitlines(): + for pattern in FAILED_TARGET_RES: + if match := pattern.match(line): + targets.add(match.group(1)) + break + return sorted(targets) + + +def reported_test_log(console_output: str, target: str) -> Path | None: + prefix = f"FAIL: {target} " + for line in console_output.splitlines(): + if line.startswith(prefix) and (match := REPORTED_TEST_LOG_RE.search(line)): + return Path(match.group(1).replace("\\", "/")) + return None + + +def test_log_path(console_output: str, testlogs_dir: Path, target: str) -> Path: + if path := reported_test_log(console_output, target): + return path + relative = target.removeprefix("//").replace(":", "/", 1) + return testlogs_dir / relative / "test.log" + + +def bazel_testlogs_dir(invocation: Invocation) -> Path: + info_args = ["info"] + if invocation.remote_enabled: + info_args.append(f"--config={invocation.ci_config}") + info_args.extend( + arg + for arg in invocation.post_config_args + if arg.startswith(INFO_POST_CONFIG_PREFIXES) + ) + command = bazel_command( + *( + [arg for arg in invocation.command[1:] if arg.startswith("--output_user_root=")] + ), + "--noexperimental_remote_repo_contents_cache", + *info_args, + "bazel-testlogs", + env=invocation.child_env, + ) + result = subprocess.run( + command, + env=invocation.child_env, + check=False, + capture_output=True, + text=True, + ) + return Path(result.stdout.strip()) if result.returncode == 0 else Path("bazel-testlogs") + + +def print_action_failure_summary(console_output: str) -> None: + summary = action_failure_summary(console_output) + if summary is None: + print("No Bazel action failures were found in the captured console output.") + return + if os.environ.get("GITHUB_ACTIONS") == "true": + escaped = summary.replace("%", "%25").replace("\r", "%0D").replace("\n", "%0A") + print(f"::error title=Bazel failed action diagnostics::{escaped}") + print("\nBazel failed action diagnostics:") + print("--------------------------------") + print(summary) + print("--------------------------------") + + +def print_test_log_tails(console_output: str, invocation: Invocation) -> None: + targets = failed_test_targets(console_output) + if not targets: + print("No failed Bazel test targets were found in console output.") + return + testlogs_dir = bazel_testlogs_dir(invocation) + for target in targets: + path = test_log_path(console_output, testlogs_dir, target) + print(f"::group::Bazel test log tail for {target}") + if path.is_file(): + print("".join(path.read_text(errors="replace").splitlines(keepends=True)[-200:]), end="") + else: + print(f"Missing test log: {path}") + print("::endgroup::") + + +def run_and_tee(invocation: Invocation) -> tuple[int, str]: + process = subprocess.Popen( + invocation.command, + env=invocation.child_env, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + ) + output = [] + assert process.stdout is not None + for line in process.stdout: + print(line, end="") + output.append(line) + return process.wait(), "".join(output) + + +def main(argv: Sequence[str] | None = None, env: Mapping[str, str] | None = None) -> int: + argv = sys.argv[1:] if argv is None else argv + env = os.environ if env is None else env + try: + options, bazel_args, targets = parse_args(argv) + invocation = build_invocation(options, bazel_args, targets, env, pid=os.getpid()) + except ValueError as exc: + print(exc, file=sys.stderr) + return 1 + + if invocation.remote_enabled: + print("BuildBuddy API key is available; using remote Bazel configuration.") + else: + print("BuildBuddy API key is not available; using local Bazel configuration.") + status, console_output = run_and_tee(invocation) + if status != 0: + if options.print_failed_action_summary: + print_action_failure_summary(console_output) + if options.print_failed_test_logs: + print_test_log_tails(console_output, invocation) + return status + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/scripts/run_bazel_query_ci.py b/.github/scripts/run_bazel_query_ci.py new file mode 100755 index 0000000000..f70aec8c92 --- /dev/null +++ b/.github/scripts/run_bazel_query_ci.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python3 + +"""Run target-discovery Bazel queries with CI cache and server settings.""" + +from __future__ import annotations + +import os +import subprocess +import sys +from collections.abc import Mapping +from collections.abc import Sequence +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from run_bazel_with_buildbuddy import bazel_command + + +USAGE = "Usage: run_bazel_query_ci.py [...] -- " + + +def query_command(args: Sequence[str], env: Mapping[str, str]) -> list[str]: + if len(args) < 2 or args[-2] != "--": + raise ValueError(USAGE) + + query_args = ["query"] + if repo_contents_cache := env.get("BAZEL_REPO_CONTENTS_CACHE"): + query_args.append(f"--repo_contents_cache={repo_contents_cache}") + if repository_cache := env.get("BAZEL_REPOSITORY_CACHE"): + query_args.append(f"--repository_cache={repository_cache}") + query_args.extend(args[:-2]) + query_args.append(args[-1]) + + return bazel_command( + *query_args, + env=env, + enable_remote_config=False, + ) + + +def main(argv: Sequence[str] | None = None, env: Mapping[str, str] | None = None) -> int: + argv = sys.argv[1:] if argv is None else argv + env = os.environ if env is None else env + try: + command = query_command(argv, env) + except ValueError as exc: + print(exc, file=sys.stderr) + return 1 + + child_env = dict(env) + if env.get("RUNNER_OS") == "Windows": + child_env["MSYS2_ARG_CONV_EXCL"] = "*" + return subprocess.run(command, env=child_env, check=False).returncode + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/scripts/run_bazel_with_buildbuddy.py b/.github/scripts/run_bazel_with_buildbuddy.py index 4503b4fda3..281e414200 100755 --- a/.github/scripts/run_bazel_with_buildbuddy.py +++ b/.github/scripts/run_bazel_with_buildbuddy.py @@ -152,10 +152,15 @@ def bazel_args_with_remote_config( return [*args[:insertion_idx], *remote_args, *args[insertion_idx:]] -def bazel_command(*args: str, env: Mapping[str, str] | None = None) -> list[str]: +def bazel_command( + *args: str, + env: Mapping[str, str] | None = None, + enable_remote_config: bool = True, +) -> list[str]: env = os.environ if env is None else env bazel = env.get("CODEX_BAZEL_BIN", "bazel") - return [bazel, *startup_args(args, env), *bazel_args_with_remote_config(args, env)] + bazel_args = bazel_args_with_remote_config(args, env) if enable_remote_config else args + return [bazel, *startup_args(args, env), *bazel_args] def main() -> None: diff --git a/.github/scripts/test_run_bazel_ci.py b/.github/scripts/test_run_bazel_ci.py new file mode 100644 index 0000000000..3588ac4cfc --- /dev/null +++ b/.github/scripts/test_run_bazel_ci.py @@ -0,0 +1,212 @@ +#!/usr/bin/env python3 + +import json +import os +import stat +import subprocess +import sys +import unittest +from pathlib import Path +from tempfile import TemporaryDirectory + +import run_bazel_ci + + +class RunBazelCiTest(unittest.TestCase): + def test_parse_args_preserves_bazel_args_and_targets(self) -> None: + options, bazel_args, targets = run_bazel_ci.parse_args( + [ + "--print-failed-test-logs", + "--remote-download-toplevel", + "--", + "test", + "--keep_going", + "--", + "//codex-rs/...", + "-//third_party/v8:all", + ] + ) + + self.assertEqual( + options, + run_bazel_ci.Options( + print_failed_test_logs=True, + remote_download_toplevel=True, + ), + ) + self.assertEqual(bazel_args, ["test", "--keep_going"]) + self.assertEqual(targets, ["//codex-rs/...", "-//third_party/v8:all"]) + + def test_linux_remote_invocation_uses_shared_buildbuddy_wrapper(self) -> None: + invocation = run_bazel_ci.build_invocation( + run_bazel_ci.Options(remote_download_toplevel=True), + ["test", "--keep_going"], + ["//codex-rs/..."], + { + "BAZEL_OUTPUT_USER_ROOT": "/tmp/output", + "BUILDBUDDY_API_KEY": "token", + "CODEX_BAZEL_BIN": "fake-bazel", + }, + pid=123, + ) + + self.assertEqual( + invocation.command, + [ + "fake-bazel", + "--output_user_root=/tmp/output", + "--noexperimental_remote_repo_contents_cache", + "test", + "--config=buildbuddy-generic-rbe", + "--remote_header=x-buildbuddy-api-key=token", + "--keep_going", + "--config=ci-linux", + "--remote_download_toplevel", + "--", + "//codex-rs/...", + ], + ) + + def test_keyless_windows_cross_compile_falls_back_to_local_msvc(self) -> None: + invocation = run_bazel_ci.build_invocation( + run_bazel_ci.Options(windows_cross_compile=True), + ["build"], + ["//codex-rs/cli:codex"], + { + "CODEX_BAZEL_BIN": "fake-bazel", + "CODEX_BAZEL_WINDOWS_PATH": r"C:\Windows", + "RUNNER_OS": "Windows", + }, + pid=123, + ) + + self.assertEqual(invocation.ci_config, "ci-windows") + self.assertIn("--host_platform=//:local_windows_msvc", invocation.command) + self.assertIn("--jobs=8", invocation.command) + self.assertIn(r"--test_env=PATH=C:\Windows", invocation.command) + self.assertEqual(invocation.child_env["MSYS2_ARG_CONV_EXCL"], "*") + + def test_remote_windows_cross_compile_uses_linux_build_environment(self) -> None: + invocation = run_bazel_ci.build_invocation( + run_bazel_ci.Options(windows_cross_compile=True), + ["test"], + ["//codex-rs/..."], + { + "BUILDBUDDY_API_KEY": "token", + "CODEX_BAZEL_BIN": "fake-bazel", + "CODEX_BAZEL_WINDOWS_PATH": r"C:\Windows", + "RUNNER_OS": "Windows", + }, + pid=123, + ) + + self.assertEqual(invocation.ci_config, "ci-windows-cross") + self.assertIn("--host_platform=//:rbe", invocation.command) + self.assertIn("--action_env=PATH=/usr/bin:/bin", invocation.command) + self.assertNotIn(r"--action_env=PATH=C:\Windows", invocation.command) + + def test_action_failure_summary_keeps_rust_diagnostics(self) -> None: + output = "\n".join( + [ + "INFO: building", + "ERROR: /tmp/BUILD:1:1 Rustc failed:", + "error[E0308]: mismatched types", + " --> src/main.rs:1:1", + " |", + " 1 | nope", + " | ^^^^", + "note: expected unit", + "Target //other:thing failed to build", + ] + ) + + self.assertEqual( + run_bazel_ci.action_failure_summary(output), + "\n".join( + [ + "ERROR: /tmp/BUILD:1:1 Rustc failed:", + "error[E0308]: mismatched types", + " --> src/main.rs:1:1", + " |", + " 1 | nope", + " | ^^^^", + "note: expected unit", + ] + ), + ) + + def test_failed_test_log_resolution_prefers_reported_path(self) -> None: + console = ( + "FAIL: //codex-rs/core:core-tests (see " + r"C:\tmp\core-tests\test.log)" + ) + + self.assertEqual( + run_bazel_ci.failed_test_targets(console), + ["//codex-rs/core:core-tests"], + ) + self.assertEqual( + run_bazel_ci.test_log_path( + console, + Path("bazel-testlogs"), + "//codex-rs/core:core-tests", + ), + Path("C:/tmp/core-tests/test.log"), + ) + + def test_main_runs_bazel_and_preserves_exit_status(self) -> None: + with TemporaryDirectory() as temp_dir: + temp = Path(temp_dir) + args_path = temp / "args.json" + fake_bazel = temp / "fake-bazel" + fake_bazel.write_text( + "#!/usr/bin/env python3\n" + "import json, os, sys\n" + "open(os.environ['ARGS_PATH'], 'w').write(json.dumps(sys.argv[1:]))\n" + "print('FAILED: fake failure')\n" + "sys.exit(37)\n", + encoding="utf-8", + ) + fake_bazel.chmod(fake_bazel.stat().st_mode | stat.S_IXUSR) + env = os.environ.copy() + env.update( + { + "ARGS_PATH": str(args_path), + "CODEX_BAZEL_BIN": str(fake_bazel), + } + ) + env.pop("BUILDBUDDY_API_KEY", None) + + result = subprocess.run( + [ + sys.executable, + str(Path(run_bazel_ci.__file__)), + "--print-failed-action-summary", + "--", + "build", + "--keep_going", + "--", + "//codex-rs/cli:codex", + ], + env=env, + check=False, + capture_output=True, + text=True, + ) + + self.assertEqual(result.returncode, 37, result.stderr) + self.assertIn("Bazel failed action diagnostics:", result.stdout) + self.assertEqual( + json.loads(args_path.read_text()), + [ + "--noexperimental_remote_repo_contents_cache", + "build", + "--keep_going", + "--", + "//codex-rs/cli:codex", + ], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/scripts/test_run_bazel_query_ci.py b/.github/scripts/test_run_bazel_query_ci.py new file mode 100644 index 0000000000..a1b465e27a --- /dev/null +++ b/.github/scripts/test_run_bazel_query_ci.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 + +import json +import os +import stat +import subprocess +import sys +import unittest +from pathlib import Path +from tempfile import TemporaryDirectory + +import run_bazel_query_ci + + +class RunBazelQueryCiTest(unittest.TestCase): + def test_query_command_reuses_startup_and_repository_cache_settings(self) -> None: + expression = 'kind("rust_library rule", //codex-rs/...)' + + self.assertEqual( + run_bazel_query_ci.query_command( + ["--keep_going", "--output=label", "--", expression], + { + "BAZEL_OUTPUT_USER_ROOT": "/tmp/output", + "BAZEL_REPO_CONTENTS_CACHE": "/tmp/contents", + "BAZEL_REPOSITORY_CACHE": "/tmp/repository", + "BUILDBUDDY_API_KEY": "token", + "CODEX_BAZEL_BIN": "fake-bazel", + "GITHUB_ACTIONS": "true", + }, + ), + [ + "fake-bazel", + "--output_user_root=/tmp/output", + "--noexperimental_remote_repo_contents_cache", + "query", + "--repo_contents_cache=/tmp/contents", + "--repository_cache=/tmp/repository", + "--keep_going", + "--output=label", + expression, + ], + ) + + def test_query_command_requires_final_separator_and_expression(self) -> None: + for args in ([], ["--", "expression", "extra"], ["expression"]): + with self.subTest(args=args): + with self.assertRaisesRegex(ValueError, "Usage:"): + run_bazel_query_ci.query_command(args, {}) + + def test_main_runs_query_without_buildbuddy_configuration(self) -> None: + with TemporaryDirectory() as temp_dir: + temp = Path(temp_dir) + args_path = temp / "args.json" + fake_bazel = temp / "fake-bazel" + fake_bazel.write_text( + "#!/usr/bin/env python3\n" + "import json, os, sys\n" + "open(os.environ['ARGS_PATH'], 'w').write(json.dumps(sys.argv[1:]))\n" + "print('//codex-rs/cli:codex')\n", + encoding="utf-8", + ) + fake_bazel.chmod(fake_bazel.stat().st_mode | stat.S_IXUSR) + env = os.environ.copy() + env.update( + { + "ARGS_PATH": str(args_path), + "BUILDBUDDY_API_KEY": "token", + "CODEX_BAZEL_BIN": str(fake_bazel), + "GITHUB_ACTIONS": "true", + } + ) + + result = subprocess.run( + [ + sys.executable, + str(Path(run_bazel_query_ci.__file__)), + "--output=label", + "--", + "//codex-rs/...", + ], + env=env, + check=False, + capture_output=True, + text=True, + ) + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout, "//codex-rs/cli:codex\n") + self.assertEqual( + json.loads(args_path.read_text()), + [ + "--noexperimental_remote_repo_contents_cache", + "query", + "--output=label", + "//codex-rs/...", + ], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/scripts/test_run_bazel_with_buildbuddy.py b/.github/scripts/test_run_bazel_with_buildbuddy.py index f06e34b895..0c5d7b11b7 100644 --- a/.github/scripts/test_run_bazel_with_buildbuddy.py +++ b/.github/scripts/test_run_bazel_with_buildbuddy.py @@ -214,6 +214,20 @@ class RunBazelWithBuildBuddyTest(unittest.TestCase): ], ) + def test_bazel_command_can_explicitly_disable_remote_configuration(self) -> None: + self.assertEqual( + run_bazel_with_buildbuddy.bazel_command( + "query", + "//codex-rs/...", + env={ + "BUILDBUDDY_API_KEY": "token", + "CODEX_BAZEL_BIN": "fake-bazel", + }, + enable_remote_config=False, + ), + ["fake-bazel", "query", "//codex-rs/..."], + ) + def test_main_preserves_spaced_argument_and_child_exit_status(self) -> None: spaced_arg = ( r"--test_env=PATH=C:\Program Files\PowerShell\7;C:\Program Files\Git\bin" diff --git a/codex-rs/docs/bazel.md b/codex-rs/docs/bazel.md index f12eeefd70..b3c44a090b 100644 --- a/codex-rs/docs/bazel.md +++ b/codex-rs/docs/bazel.md @@ -78,7 +78,7 @@ configurations. GitHub Actions routes Bazel build and output-resolution commands through `.github/scripts/run_bazel_with_buildbuddy.py`. Higher-level helpers such as -`.github/scripts/run-bazel-ci.sh` and `.github/scripts/rusty_v8_bazel.py` +`.github/scripts/run_bazel_ci.py` and `.github/scripts/rusty_v8_bazel.py` delegate remote configuration selection to that wrapper. The wrapper reads the GitHub Actions repository and event payload rather than relying on workflow files to duplicate tenant-selection logic. It also normalizes GitHub Actions diff --git a/scripts/list-bazel-clippy-targets.sh b/scripts/list-bazel-clippy-targets.sh index d12a256d00..a384973f04 100755 --- a/scripts/list-bazel-clippy-targets.sh +++ b/scripts/list-bazel-clippy-targets.sh @@ -23,7 +23,7 @@ done # continue with a partial list if `bazel query` fails. Target discovery is # local on all platforms. manual_rust_test_targets="$( - ./.github/scripts/run-bazel-query-ci.sh \ + ./.github/scripts/run_bazel_query_ci.py \ --output=label \ -- 'kind("rust_test rule", attr(tags, "manual", //codex-rs/... except //codex-rs/v8-poc/...))' )" diff --git a/tools/argument-comment-lint/list-bazel-targets.sh b/tools/argument-comment-lint/list-bazel-targets.sh index f8cb4f5e20..ccf2d06fb5 100755 --- a/tools/argument-comment-lint/list-bazel-targets.sh +++ b/tools/argument-comment-lint/list-bazel-targets.sh @@ -10,7 +10,7 @@ cd "${repo_root}" # Add only those manual rust_test targets explicitly so inline `#[cfg(test)]` # call sites are linted without pulling in unrelated manual release targets. manual_rust_test_targets="$( - ./.github/scripts/run-bazel-query-ci.sh \ + ./.github/scripts/run_bazel_query_ci.py \ --output=label \ -- 'kind("rust_test rule", attr(tags, "manual", //codex-rs/...))' )"