rob thijssen
95973e731b
The verdict logic was right; the waiting was impossible. Gitea's runner kills a step at ~15 minutes. monsoon runs 56 and 60 stopped producing output at 14m29s and 14m30s after step start — within one second of each other, with perfectly regular 31s heartbeats right up to the cut — while their COPR builds carried on and succeeded at 10:15:20 and 11:34:55. The job was then marked failed by a reaper ticking at :05:50/:20:50/:35:50/:50:50. So the step was never going to survive a ~25 minute build, and every fix so far addressed a real defect that was not this one. Default the wait budget to 720s, comfortably inside the limit, and change what expiry means: a build still running is reported with its URL and exits 0, because it has not failed. A build that reaches failed or canceled inside the budget still fails the job, which catches the early failures that make up most build breakage. Skip the artifact download entirely when the build has not finished, since there is nothing to fetch and no time to spend. The trade is explicit: a build that fails after the budget will not be caught by CI. That is strictly better than the previous behaviour of failing every successful build and blocking dependent jobs.
181 lines
6.9 KiB
Bash
Executable File
181 lines
6.9 KiB
Bash
Executable File
#!/bin/bash
|
|
# Submit an SRPM to COPR, watch the build, and dump per-chroot build logs
|
|
# to stdout so they are captured in CI output.
|
|
#
|
|
# Usage: copr-build.sh <project> <srpm> [srpm...]
|
|
# Example: copr-build.sh helexa/cortex ./cortex-0.1.2-1.fc43.src.rpm
|
|
#
|
|
# Requires: copr-cli on PATH, a valid ~/.config/copr.
|
|
|
|
set -o pipefail
|
|
|
|
# How long we are willing to wait for the build before handing control back to
|
|
# CI, and how often to ask COPR where it has got to.
|
|
#
|
|
# Deliberately well under the runner's step limit rather than the length of a
|
|
# build. Gitea's runner kills a step at ~15 minutes: two monsoon releases had
|
|
# their step killed at 14m29s and 14m30s with the build still running, and the
|
|
# job then reported a successful build as a failure. Waiting longer is simply
|
|
# not available to us, so past this budget the build is reported as still
|
|
# running and COPR is left to finish it.
|
|
WAIT_BUDGET="${COPR_WAIT_BUDGET:-${COPR_BUILD_TIMEOUT:-720}}"
|
|
POLL_INTERVAL="${COPR_POLL_INTERVAL:-30}"
|
|
# Every copr-cli invocation is bounded. They perform network calls with no
|
|
# internal timeout, and a hang in any of them stalls the whole job.
|
|
STATUS_TIMEOUT="${COPR_STATUS_TIMEOUT:-60}"
|
|
SUBMIT_TIMEOUT="${COPR_SUBMIT_TIMEOUT:-600}"
|
|
DOWNLOAD_TIMEOUT="${COPR_DOWNLOAD_TIMEOUT:-600}"
|
|
|
|
PROJECT="$1"
|
|
shift
|
|
|
|
if [ -z "$PROJECT" ] || [ "$#" -eq 0 ]; then
|
|
echo "usage: $0 <project> <srpm> [srpm...]" >&2
|
|
exit 2
|
|
fi
|
|
|
|
# COPR states that mean the build has finished, one way or another. Anything
|
|
# else (pending, starting, running, importing, waiting) is still in progress.
|
|
is_terminal_state() {
|
|
case "$1" in
|
|
succeeded | failed | canceled | skipped) return 0 ;;
|
|
*) return 1 ;;
|
|
esac
|
|
}
|
|
|
|
# Ask COPR for a build's state. Empty output means "could not tell", which the
|
|
# caller treats as non-terminal and retries.
|
|
#
|
|
# Bounded, because copr-cli's network calls can block indefinitely. One release
|
|
# hung here for over twelve minutes: the poll loop sat inside this command
|
|
# substitution, heartbeats stopped, and the job was killed five minutes after
|
|
# COPR had already reported success.
|
|
build_state() {
|
|
timeout "$STATUS_TIMEOUT" copr-cli status "$1" 2>/dev/null | tail -n1 | tr -d '[:space:]'
|
|
}
|
|
|
|
# Poll COPR until the build settles, or until BUILD_TIMEOUT elapses. Echoes the
|
|
# terminal state, or "unknown" if we ran out of patience.
|
|
# Poll until the build settles or the budget runs out. Echoes the last state
|
|
# seen, which the caller checks for terminality — a non-terminal state means we
|
|
# ran out of patience, not that anything went wrong.
|
|
wait_for_terminal_state() {
|
|
local build_id="$1"
|
|
local started deadline state now
|
|
started=$(date +%s)
|
|
deadline=$((started + WAIT_BUDGET))
|
|
state="$(build_state "$build_id")"
|
|
while ! is_terminal_state "$state"; do
|
|
now=$(date +%s)
|
|
if [ "$now" -ge "$deadline" ]; then
|
|
echo "${state:-unknown}"
|
|
return
|
|
fi
|
|
# Heartbeat on stderr (stdout carries the return value). A COPR build emits
|
|
# nothing between state transitions, so a long build leaves the step silent
|
|
# for its whole duration — and a silent step gets killed by the runner's
|
|
# inactivity timeout before it can finish. Two builds died this way after
|
|
# ~24-27 minutes of no output, one of them 66 seconds before COPR reported
|
|
# success. Printing every poll keeps the step alive and shows progress.
|
|
printf ' [%3dm %3ds] build %s: %s\n' \
|
|
$(((now - started) / 60)) $(((now - started) % 60)) "$build_id" "${state:-unknown}" >&2
|
|
sleep "$POLL_INTERVAL"
|
|
state="$(build_state "$build_id")"
|
|
done
|
|
echo "$state"
|
|
}
|
|
|
|
# Submit without waiting; capture the build ID from stdout.
|
|
SUBMIT_OUT=$(timeout "$SUBMIT_TIMEOUT" copr-cli build --nowait "$PROJECT" "$@")
|
|
echo "$SUBMIT_OUT"
|
|
BUILD_ID=$(echo "$SUBMIT_OUT" | grep -oP 'Created builds: \K[0-9]+' | head -n1)
|
|
|
|
if [ -z "$BUILD_ID" ]; then
|
|
echo "error: could not parse build ID from copr-cli output" >&2
|
|
exit 1
|
|
fi
|
|
|
|
echo
|
|
echo "Build $BUILD_ID submitted to $PROJECT"
|
|
echo "Follow live: https://copr.fedorainfracloud.org/coprs/build/$BUILD_ID"
|
|
echo
|
|
|
|
# Stream status transitions for the operator's benefit, in the background and
|
|
# bounded so it cannot hang the job.
|
|
#
|
|
# Its exit status is deliberately ignored, and it is NOT what decides the
|
|
# verdict. `watch-build` holds a long-lived connection to COPR, and it has been
|
|
# seen to stop responding while the build carries on and finishes normally: in
|
|
# one case the build succeeded at 08:30:30 while the watcher sat silent until
|
|
# the runner killed the step at 08:35:50, which the job then reported as a
|
|
# build failure. Conflating "the watcher lost its connection" with "the build
|
|
# failed" turns a green release into a red one and blocks any dependent job.
|
|
timeout "$BUILD_TIMEOUT" copr-cli watch-build "$BUILD_ID" &
|
|
WATCH_PID=$!
|
|
|
|
# The verdict comes from COPR itself. Polling also means we stop as soon as the
|
|
# build settles, rather than waiting on the watcher to notice.
|
|
BUILD_STATE="$(wait_for_terminal_state "$BUILD_ID")"
|
|
|
|
kill "$WATCH_PID" 2>/dev/null || true
|
|
wait "$WATCH_PID" 2>/dev/null || true
|
|
|
|
echo
|
|
echo "COPR build $BUILD_ID final state: $BUILD_STATE"
|
|
|
|
case "$BUILD_STATE" in
|
|
succeeded)
|
|
STATUS=0
|
|
;;
|
|
skipped)
|
|
# COPR already had this exact build; nothing was rebuilt, but nothing is
|
|
# wrong either.
|
|
echo "note: build $BUILD_ID was skipped (already built)"
|
|
STATUS=0
|
|
;;
|
|
failed | canceled)
|
|
echo "error: COPR build $BUILD_ID finished with state '$BUILD_STATE'" >&2
|
|
STATUS=1
|
|
;;
|
|
*)
|
|
# Still going, or we could not tell. Either way the build has not failed,
|
|
# and reporting it as a failure is what made every green release red.
|
|
echo "note: build $BUILD_ID has not finished within our ${WAIT_BUDGET}s wait budget"
|
|
echo " (last state: ${BUILD_STATE:-unknown}). COPR will finish it on its own."
|
|
echo " Follow: https://copr.fedorainfracloud.org/coprs/build/$BUILD_ID"
|
|
STATUS=0
|
|
;;
|
|
esac
|
|
|
|
# Fetch per-chroot results (logs + rpms). Anonymous download — no auth needed.
|
|
# Only meaningful once the build has finished; there is nothing to fetch for one
|
|
# that is still running, and trying would burn the little time we have left.
|
|
if ! is_terminal_state "$BUILD_STATE"; then
|
|
exit "$STATUS"
|
|
fi
|
|
|
|
LOG_DIR="$(mktemp -d -t copr-logs.XXXXXX)"
|
|
timeout "$DOWNLOAD_TIMEOUT" copr-cli download-build --dest "$LOG_DIR" "$BUILD_ID" || {
|
|
echo "warning: failed to download build artifacts" >&2
|
|
}
|
|
|
|
# Dump each chroot's builder-live.log as a collapsible group. COPR stores
|
|
# the log gzipped on the mirror once the build has finished; fall back to
|
|
# the plain file in case a backend version ever serves it uncompressed.
|
|
for chroot_dir in "$LOG_DIR"/*/; do
|
|
[ -d "$chroot_dir" ] || continue
|
|
chroot=$(basename "$chroot_dir")
|
|
echo
|
|
echo "::group::${chroot} builder-live.log"
|
|
if [ -f "${chroot_dir}builder-live.log.gz" ]; then
|
|
zcat "${chroot_dir}builder-live.log.gz"
|
|
elif [ -f "${chroot_dir}builder-live.log" ]; then
|
|
cat "${chroot_dir}builder-live.log"
|
|
else
|
|
echo "(no builder-live.log found for ${chroot})"
|
|
fi
|
|
echo "::endgroup::"
|
|
done
|
|
|
|
exit "$STATUS"
|