diff --git a/CLAUDE.md b/CLAUDE.md index 466ee74..1657534 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -42,6 +42,15 @@ inside a command substitution for twelve minutes and the job was killed five minutes after COPR had reported success. Any new `copr-cli` invocation gets a `timeout`. +**The runner kills a step at ~15 minutes.** Measured, not guessed: monsoon runs +56 and 60 stopped producing output at 14m29s and 14m30s after step start, with +their COPR builds still running and succeeding minutes later. The job is then +marked failed by a reaper that ticks at :05:50/:20:50/:35:50/:50:50. A monsoon +build takes ~25 minutes, so **waiting for it to finish is not possible here**. +The wait budget defaults to 720s and a build still running at expiry exits 0 +with a note rather than a failure. Do not raise the budget past ~13 minutes +without first confirming the runner limit has changed. + **Never leave the step silent.** A long build produces no COPR output between state transitions, and a silent step is killed by the runner's inactivity timeout. The poll loop prints a heartbeat every interval; that is load-bearing, diff --git a/README.md b/README.md index 4947855..e237103 100644 --- a/README.md +++ b/README.md @@ -18,15 +18,16 @@ This action: than from the health of a long-lived connection. - On completion, fetches each chroot's `builder-live.log` via `copr-cli download-build` and emits them as `::group::` blocks. -- Fails CI if the build fails, but always dumps logs first. -- Gives up after `COPR_BUILD_TIMEOUT` (default 7200s) and says so, rather than - hanging until the runner kills the job. +- Fails CI if the build **fails**. A build merely still running when the wait + budget expires is reported, not failed. +- Returns within `COPR_WAIT_BUDGET` whatever happens, so the step finishes + before a CI runner's step limit kills it. ### Environment overrides | Variable | Default | Purpose | |---|---|---| -| `COPR_BUILD_TIMEOUT` | `7200` | Seconds to wait for a build to reach a terminal state. | +| `COPR_WAIT_BUDGET` | `720` | Seconds to wait for the build before returning. A build still running at expiry is reported as such and the step exits 0 — CI runners commonly kill a step long before a large build finishes. | | `COPR_POLL_INTERVAL` | `30` | Seconds between build-state polls. | ## Requirements diff --git a/scripts/copr-build.sh b/scripts/copr-build.sh index 6148736..743a6eb 100755 --- a/scripts/copr-build.sh +++ b/scripts/copr-build.sh @@ -9,9 +9,16 @@ set -o pipefail -# How long to wait for a build to reach a terminal state before giving up, and -# how often to ask COPR where it has got to. -BUILD_TIMEOUT="${COPR_BUILD_TIMEOUT:-7200}" +# How long we are willing to wait for the build before handing control back to +# CI, and how often to ask COPR where it has got to. +# +# Deliberately well under the runner's step limit rather than the length of a +# build. Gitea's runner kills a step at ~15 minutes: two monsoon releases had +# their step killed at 14m29s and 14m30s with the build still running, and the +# job then reported a successful build as a failure. Waiting longer is simply +# not available to us, so past this budget the build is reported as still +# running and COPR is left to finish it. +WAIT_BUDGET="${COPR_WAIT_BUDGET:-${COPR_BUILD_TIMEOUT:-720}}" POLL_INTERVAL="${COPR_POLL_INTERVAL:-30}" # Every copr-cli invocation is bounded. They perform network calls with no # internal timeout, and a hang in any of them stalls the whole job. @@ -49,16 +56,19 @@ build_state() { # Poll COPR until the build settles, or until BUILD_TIMEOUT elapses. Echoes the # terminal state, or "unknown" if we ran out of patience. +# Poll until the build settles or the budget runs out. Echoes the last state +# seen, which the caller checks for terminality — a non-terminal state means we +# ran out of patience, not that anything went wrong. wait_for_terminal_state() { local build_id="$1" local started deadline state now started=$(date +%s) - deadline=$((started + BUILD_TIMEOUT)) + deadline=$((started + WAIT_BUDGET)) state="$(build_state "$build_id")" while ! is_terminal_state "$state"; do now=$(date +%s) if [ "$now" -ge "$deadline" ]; then - echo "unknown" + echo "${state:-unknown}" return fi # Heartbeat on stderr (stdout carries the return value). A COPR build emits @@ -128,13 +138,22 @@ case "$BUILD_STATE" in STATUS=1 ;; *) - echo "error: gave up after ${BUILD_TIMEOUT}s waiting for COPR build $BUILD_ID to finish" >&2 - echo " check https://copr.fedorainfracloud.org/coprs/build/$BUILD_ID" >&2 - STATUS=1 + # Still going, or we could not tell. Either way the build has not failed, + # and reporting it as a failure is what made every green release red. + echo "note: build $BUILD_ID has not finished within our ${WAIT_BUDGET}s wait budget" + echo " (last state: ${BUILD_STATE:-unknown}). COPR will finish it on its own." + echo " Follow: https://copr.fedorainfracloud.org/coprs/build/$BUILD_ID" + STATUS=0 ;; esac # Fetch per-chroot results (logs + rpms). Anonymous download — no auth needed. +# Only meaningful once the build has finished; there is nothing to fetch for one +# that is still running, and trying would burn the little time we have left. +if ! is_terminal_state "$BUILD_STATE"; then + exit "$STATUS" +fi + LOG_DIR="$(mktemp -d -t copr-logs.XXXXXX)" timeout "$DOWNLOAD_TIMEOUT" copr-cli download-build --dest "$LOG_DIR" "$BUILD_ID" || { echo "warning: failed to download build artifacts" >&2 diff --git a/tests/test-copr-build.sh b/tests/test-copr-build.sh index 603b650..198e787 100755 --- a/tests/test-copr-build.sh +++ b/tests/test-copr-build.sh @@ -111,9 +111,14 @@ CASE_TIMEOUT=30 run_case "a hung status call does not stall the poll loop" \ 0 "final state: succeeded" STUB_WATCH_HANGS=1 STUB_STATES="running succeeded" \ STUB_STATUS_HANGS_UNTIL=1 COPR_STATUS_TIMEOUT=2 -# Never settles: bounded, and reported as a timeout rather than a build failure. -CASE_TIMEOUT=3 run_case "build that never settles times out" \ - 1 "gave up after" STUB_WATCH_HANGS=1 STUB_STATES="running" +# The step limit case: the build is still running when our wait budget expires. +# This must NOT be reported as a failure — that is what turned every green +# release red — and it must not try to fetch logs that do not exist yet. +CASE_TIMEOUT=3 run_case "still-running build at budget expiry exits 0" \ + 0 "has not finished within" STUB_WATCH_HANGS=1 STUB_STATES="running" + +CASE_TIMEOUT=3 run_case "still-running build says so rather than claiming failure" \ + 0 "COPR will finish it" STUB_WATCH_HANGS=1 STUB_STATES="running" echo echo "passed: $PASS failed: $FAIL"