diff --git a/scripts/copr-build.sh b/scripts/copr-build.sh index 7d15fea..db8542c 100755 --- a/scripts/copr-build.sh +++ b/scripts/copr-build.sh @@ -39,14 +39,24 @@ build_state() { # terminal state, or "unknown" if we ran out of patience. wait_for_terminal_state() { local build_id="$1" - local deadline=$(($(date +%s) + BUILD_TIMEOUT)) - local state + local started deadline state now + started=$(date +%s) + deadline=$((started + BUILD_TIMEOUT)) state="$(build_state "$build_id")" while ! is_terminal_state "$state"; do - if [ "$(date +%s)" -ge "$deadline" ]; then + now=$(date +%s) + if [ "$now" -ge "$deadline" ]; then echo "unknown" return fi + # Heartbeat on stderr (stdout carries the return value). A COPR build emits + # nothing between state transitions, so a long build leaves the step silent + # for its whole duration — and a silent step gets killed by the runner's + # inactivity timeout before it can finish. Two builds died this way after + # ~24-27 minutes of no output, one of them 66 seconds before COPR reported + # success. Printing every poll keeps the step alive and shows progress. + printf ' [%3dm %3ds] build %s: %s\n' \ + $(((now - started) / 60)) $(((now - started) % 60)) "$build_id" "${state:-unknown}" >&2 sleep "$POLL_INTERVAL" state="$(build_state "$build_id")" done diff --git a/tests/test-copr-build.sh b/tests/test-copr-build.sh index a364fd5..f2dfdb8 100755 --- a/tests/test-copr-build.sh +++ b/tests/test-copr-build.sh @@ -92,6 +92,11 @@ run_case "canceled build exits 1" \ run_case "skipped build exits 0" \ 0 "was skipped" STUB_WATCH_HANGS=0 STUB_STATES="skipped" +# A build that takes a while must keep producing output, or the runner's +# inactivity timeout kills the step before COPR finishes. +CASE_TIMEOUT=6 run_case "slow build emits a heartbeat while waiting" \ + 0 "build 12345: running" STUB_WATCH_HANGS=1 STUB_STATES="running running running succeeded" + # Never settles: bounded, and reported as a timeout rather than a build failure. CASE_TIMEOUT=3 run_case "build that never settles times out" \ 1 "gave up after" STUB_WATCH_HANGS=1 STUB_STATES="running"