rob thijssen
95a84e4ad8
Two things. A build-level "succeeded" says the build finished, not that every chroot produced an RPM. The job could go green while fedora-43, fedora-44 or rawhide had failed or was still building. Poll build-chroot/list alongside the build state, keep waiting while any chroot is unfinished, and fail if any finished in anything other than succeeded or skipped. Per-chroot results are printed at the end and included in the heartbeat, so a slow chroot is visible while it runs. copr-cli has no per-chroot status subcommand, so this reads COPR's public API directly with python3 — already a hard dependency of copr-cli, so nothing new is required. COPR_API_BASE overrides the instance. Separately, renaming BUILD_TIMEOUT to WAIT_BUDGET in v1.0.5 missed the watch-build invocation, leaving `timeout "" copr-cli watch-build`. That failed immediately, so the progress stream has not actually run since v1.0.5 — harmless only because the verdict comes from polling, and caught here because the new tests surfaced the error text. The unfinished-versus-failed distinction is deliberate and load-bearing: reporting a still-running chroot as a failure would be the same mistake this action has now made in three different guises.
256 lines
9.9 KiB
Bash
Executable File
256 lines
9.9 KiB
Bash
Executable File
#!/bin/bash
|
|
# Submit an SRPM to COPR, watch the build, and dump per-chroot build logs
|
|
# to stdout so they are captured in CI output.
|
|
#
|
|
# Usage: copr-build.sh <project> <srpm> [srpm...]
|
|
# Example: copr-build.sh helexa/cortex ./cortex-0.1.2-1.fc43.src.rpm
|
|
#
|
|
# Requires: copr-cli on PATH, a valid ~/.config/copr.
|
|
|
|
set -o pipefail
|
|
|
|
# How long we are willing to wait for the build before handing control back to
|
|
# CI, and how often to ask COPR where it has got to.
|
|
#
|
|
# Long enough to actually see a build through — a Rust release build runs ~25
|
|
# minutes — but under the runner's absolute wall-clock cap so we always exit
|
|
# with a verdict of our own rather than being killed mid-wait.
|
|
#
|
|
# This was briefly 720s, when a runner-side reaper sampled the container log
|
|
# for signs of life. Step output never reaches that log, so any step lasting
|
|
# over ~15 minutes was killed however much it printed, and successful builds
|
|
# were reported as failures. That reaper now derives progress from the Gitea
|
|
# job log, which does see our heartbeat, so waiting properly is possible again.
|
|
# The heartbeat below is what keeps the job alive: do not remove it.
|
|
WAIT_BUDGET="${COPR_WAIT_BUDGET:-${COPR_BUILD_TIMEOUT:-2700}}"
|
|
POLL_INTERVAL="${COPR_POLL_INTERVAL:-30}"
|
|
# Every copr-cli invocation is bounded. They perform network calls with no
|
|
# internal timeout, and a hang in any of them stalls the whole job.
|
|
STATUS_TIMEOUT="${COPR_STATUS_TIMEOUT:-60}"
|
|
SUBMIT_TIMEOUT="${COPR_SUBMIT_TIMEOUT:-600}"
|
|
DOWNLOAD_TIMEOUT="${COPR_DOWNLOAD_TIMEOUT:-600}"
|
|
|
|
PROJECT="$1"
|
|
shift
|
|
|
|
if [ -z "$PROJECT" ] || [ "$#" -eq 0 ]; then
|
|
echo "usage: $0 <project> <srpm> [srpm...]" >&2
|
|
exit 2
|
|
fi
|
|
|
|
# COPR states that mean the build has finished, one way or another. Anything
|
|
# else (pending, starting, running, importing, waiting) is still in progress.
|
|
is_terminal_state() {
|
|
case "$1" in
|
|
succeeded | failed | canceled | skipped) return 0 ;;
|
|
*) return 1 ;;
|
|
esac
|
|
}
|
|
|
|
# Ask COPR for a build's state. Empty output means "could not tell", which the
|
|
# caller treats as non-terminal and retries.
|
|
#
|
|
# Bounded, because copr-cli's network calls can block indefinitely. One release
|
|
# hung here for over twelve minutes: the poll loop sat inside this command
|
|
# substitution, heartbeats stopped, and the job was killed five minutes after
|
|
# COPR had already reported success.
|
|
build_state() {
|
|
timeout "$STATUS_TIMEOUT" copr-cli status "$1" 2>/dev/null | tail -n1 | tr -d '[:space:]'
|
|
}
|
|
|
|
# Per-chroot states, one "<name> <state>" per line.
|
|
#
|
|
# The build-level state is not enough on its own: it describes the build as a
|
|
# whole, and a green tick there is not the same as "every chroot produced an
|
|
# RPM". copr-cli has no per-chroot status subcommand, so this uses COPR's public
|
|
# API. python3 is a hard dependency of copr-cli itself, so it adds no new
|
|
# requirement, and the endpoint needs no authentication.
|
|
chroot_states() {
|
|
timeout "$STATUS_TIMEOUT" python3 -c '
|
|
import json, os, sys, urllib.request
|
|
base = os.environ.get("COPR_API_BASE", "https://copr.fedorainfracloud.org")
|
|
url = f"{base}/api_3/build-chroot/list?build_id={sys.argv[1]}"
|
|
with urllib.request.urlopen(url, timeout=20) as r:
|
|
for c in json.load(r).get("items", []):
|
|
print(c.get("name", "?"), c.get("state", "unknown"))
|
|
' "$1" 2>/dev/null
|
|
}
|
|
|
|
# True when every chroot has finished. Empty or unreadable output is treated as
|
|
# "not yet", so a transient API failure retries rather than passing the build.
|
|
all_chroots_finished() {
|
|
local out name state
|
|
out="$(chroot_states "$1")"
|
|
[ -n "$out" ] || return 1
|
|
while read -r name state; do
|
|
[ -n "$name" ] || continue
|
|
is_terminal_state "$state" || return 1
|
|
done <<< "$out"
|
|
return 0
|
|
}
|
|
|
|
# Poll COPR until the build settles, or until WAIT_BUDGET elapses. Echoes the
|
|
# terminal state, or "unknown" if we ran out of patience.
|
|
# Poll until the build settles or the budget runs out. Echoes the last state
|
|
# seen, which the caller checks for terminality — a non-terminal state means we
|
|
# ran out of patience, not that anything went wrong.
|
|
wait_for_terminal_state() {
|
|
local build_id="$1"
|
|
local started deadline state now
|
|
started=$(date +%s)
|
|
deadline=$((started + WAIT_BUDGET))
|
|
state="$(build_state "$build_id")"
|
|
while ! is_terminal_state "$state" || ! all_chroots_finished "$build_id"; do
|
|
now=$(date +%s)
|
|
if [ "$now" -ge "$deadline" ]; then
|
|
echo "${state:-unknown}"
|
|
return
|
|
fi
|
|
# Heartbeat on stderr (stdout carries the return value). A COPR build emits
|
|
# nothing between state transitions, so a long build leaves the step silent
|
|
# for its whole duration — and a silent step gets killed by the runner's
|
|
# inactivity timeout before it can finish. Two builds died this way after
|
|
# ~24-27 minutes of no output, one of them 66 seconds before COPR reported
|
|
# success. Printing every poll keeps the step alive and shows progress.
|
|
printf ' [%3dm %3ds] build %s: %s | %s\n' \
|
|
$(((now - started) / 60)) $(((now - started) % 60)) "$build_id" "${state:-unknown}" \
|
|
"$(chroot_states "$build_id" | awk '{printf "%s=%s ", $1, $2}')" >&2
|
|
sleep "$POLL_INTERVAL"
|
|
state="$(build_state "$build_id")"
|
|
done
|
|
echo "$state"
|
|
}
|
|
|
|
# Submit without waiting; capture the build ID from stdout.
|
|
SUBMIT_OUT=$(timeout "$SUBMIT_TIMEOUT" copr-cli build --nowait "$PROJECT" "$@")
|
|
echo "$SUBMIT_OUT"
|
|
BUILD_ID=$(echo "$SUBMIT_OUT" | grep -oP 'Created builds: \K[0-9]+' | head -n1)
|
|
|
|
if [ -z "$BUILD_ID" ]; then
|
|
echo "error: could not parse build ID from copr-cli output" >&2
|
|
exit 1
|
|
fi
|
|
|
|
echo
|
|
echo "Build $BUILD_ID submitted to $PROJECT"
|
|
echo "Follow live: https://copr.fedorainfracloud.org/coprs/build/$BUILD_ID"
|
|
echo
|
|
|
|
# Stream status transitions for the operator's benefit, in the background and
|
|
# bounded so it cannot hang the job.
|
|
#
|
|
# Its exit status is deliberately ignored, and it is NOT what decides the
|
|
# verdict. `watch-build` holds a long-lived connection to COPR, and it has been
|
|
# seen to stop responding while the build carries on and finishes normally: in
|
|
# one case the build succeeded at 08:30:30 while the watcher sat silent until
|
|
# the runner killed the step at 08:35:50, which the job then reported as a
|
|
# build failure. Conflating "the watcher lost its connection" with "the build
|
|
# failed" turns a green release into a red one and blocks any dependent job.
|
|
timeout "$WAIT_BUDGET" copr-cli watch-build "$BUILD_ID" &
|
|
WATCH_PID=$!
|
|
|
|
# The verdict comes from COPR itself. Polling also means we stop as soon as the
|
|
# build settles, rather than waiting on the watcher to notice.
|
|
BUILD_STATE="$(wait_for_terminal_state "$BUILD_ID")"
|
|
|
|
kill "$WATCH_PID" 2>/dev/null || true
|
|
wait "$WATCH_PID" 2>/dev/null || true
|
|
|
|
echo
|
|
echo "COPR build $BUILD_ID final state: $BUILD_STATE"
|
|
|
|
CHROOTS="$(chroot_states "$BUILD_ID")"
|
|
if [ -n "$CHROOTS" ]; then
|
|
echo "Per-chroot results:"
|
|
while read -r cname cstate; do
|
|
[ -n "$cname" ] || continue
|
|
echo " $cname: $cstate"
|
|
done <<< "$CHROOTS"
|
|
fi
|
|
|
|
# A chroot that failed while the build as a whole reads succeeded still means no
|
|
# RPM for that release, so the job must not go green on it. A chroot that simply
|
|
# has not finished is a different thing and must not be reported as a failure —
|
|
# that is the mistake this action has made in every other guise.
|
|
FAILED_CHROOTS="$(
|
|
while read -r cname cstate; do
|
|
[ -n "$cname" ] || continue
|
|
case "$cstate" in
|
|
succeeded | skipped | importing | pending | starting | running | waiting) ;;
|
|
*) printf '%s(%s) ' "$cname" "$cstate" ;;
|
|
esac
|
|
done <<< "$CHROOTS"
|
|
)"
|
|
UNFINISHED_CHROOTS="$(
|
|
while read -r cname cstate; do
|
|
[ -n "$cname" ] || continue
|
|
is_terminal_state "$cstate" || printf '%s(%s) ' "$cname" "$cstate"
|
|
done <<< "$CHROOTS"
|
|
)"
|
|
|
|
case "$BUILD_STATE" in
|
|
succeeded)
|
|
if [ -n "$FAILED_CHROOTS" ]; then
|
|
echo "error: build $BUILD_ID reports succeeded but these chroots did not: $FAILED_CHROOTS" >&2
|
|
STATUS=1
|
|
elif [ -n "$UNFINISHED_CHROOTS" ]; then
|
|
echo "note: build $BUILD_ID has not finished within our ${WAIT_BUDGET}s wait budget"
|
|
echo " (chroots still going: $UNFINISHED_CHROOTS). COPR will finish it on its own."
|
|
echo " Follow: https://copr.fedorainfracloud.org/coprs/build/$BUILD_ID"
|
|
STATUS=0
|
|
else
|
|
STATUS=0
|
|
fi
|
|
;;
|
|
skipped)
|
|
# COPR already had this exact build; nothing was rebuilt, but nothing is
|
|
# wrong either.
|
|
echo "note: build $BUILD_ID was skipped (already built)"
|
|
STATUS=0
|
|
;;
|
|
failed | canceled)
|
|
echo "error: COPR build $BUILD_ID finished with state '$BUILD_STATE'" >&2
|
|
STATUS=1
|
|
;;
|
|
*)
|
|
# Still going, or we could not tell. Either way the build has not failed,
|
|
# and reporting it as a failure is what made every green release red.
|
|
echo "note: build $BUILD_ID has not finished within our ${WAIT_BUDGET}s wait budget"
|
|
echo " (last state: ${BUILD_STATE:-unknown}). COPR will finish it on its own."
|
|
echo " Follow: https://copr.fedorainfracloud.org/coprs/build/$BUILD_ID"
|
|
STATUS=0
|
|
;;
|
|
esac
|
|
|
|
# Fetch per-chroot results (logs + rpms). Anonymous download — no auth needed.
|
|
# Only meaningful once the build has finished; there is nothing to fetch for one
|
|
# that is still running, and trying would burn the little time we have left.
|
|
if ! is_terminal_state "$BUILD_STATE"; then
|
|
exit "$STATUS"
|
|
fi
|
|
|
|
LOG_DIR="$(mktemp -d -t copr-logs.XXXXXX)"
|
|
timeout "$DOWNLOAD_TIMEOUT" copr-cli download-build --dest "$LOG_DIR" "$BUILD_ID" || {
|
|
echo "warning: failed to download build artifacts" >&2
|
|
}
|
|
|
|
# Dump each chroot's builder-live.log as a collapsible group. COPR stores
|
|
# the log gzipped on the mirror once the build has finished; fall back to
|
|
# the plain file in case a backend version ever serves it uncompressed.
|
|
for chroot_dir in "$LOG_DIR"/*/; do
|
|
[ -d "$chroot_dir" ] || continue
|
|
chroot=$(basename "$chroot_dir")
|
|
echo
|
|
echo "::group::${chroot} builder-live.log"
|
|
if [ -f "${chroot_dir}builder-live.log.gz" ]; then
|
|
zcat "${chroot_dir}builder-live.log.gz"
|
|
elif [ -f "${chroot_dir}builder-live.log" ]; then
|
|
cat "${chroot_dir}builder-live.log"
|
|
else
|
|
echo "(no builder-live.log found for ${chroot})"
|
|
fi
|
|
echo "::endgroup::"
|
|
done
|
|
|
|
exit "$STATUS"
|