diff --git a/.gitea/workflows/deploy.yaml b/.gitea/workflows/deploy.yaml index 4fc2391..0651d53 100644 --- a/.gitea/workflows/deploy.yaml +++ b/.gitea/workflows/deploy.yaml @@ -286,30 +286,46 @@ jobs: echo "--- hashing ---" # The counter is the only honest evidence this process is doing work # rather than idling on a failed connection. After a restart the miner - # must reconnect and receive a job, so poll rather than sample twice. + # exports miner_gpu_devices only once it has connected to the node and + # miner_hashes_total only once it has hashed, so first wait for the + # gauge to appear (readiness), then require the counter to advance. # Here-strings, not pipes: a pipe whose reader exits early SIGPIPEs the # writer and pipefail turns that into exit 141. - deadline=$((SECONDS + 120)) - m=$(run "curl -fsS http://127.0.0.1:${{ env.MINER_METRICS_PORT }}/metrics") - h1=$(awk '/^miner_hashes_total/{print $2; exit}' <<<"$m") - devs=$(awk '/^miner_gpu_devices/{print $2; exit}' <<<"$m") - if [ "${devs%.*}" != "${{ matrix.gpu_devices }}" ]; then - echo "miner_gpu_devices is ${devs:-?}, matrix says ${{ matrix.gpu_devices }}" >&2; fail=1 - else - echo " ok miner_gpu_devices ${devs%.*}" - fi - ok=0 + scrape() { ssh $SSHOPTS gitea_ci@"${{ matrix.host }}" "curl -fsS http://127.0.0.1:${{ env.MINER_METRICS_PORT }}/metrics" || true; } + deadline=$((SECONDS + 90)) + devs="" while [ $SECONDS -lt $deadline ]; do - sleep 10 - m=$(run "curl -fsS http://127.0.0.1:${{ env.MINER_METRICS_PORT }}/metrics") - h2=$(awk '/^miner_hashes_total/{print $2; exit}' <<<"$m") - if [ -n "${h1:-}" ] && [ -n "${h2:-}" ] && [ "${h2%.*}" -gt "${h1%.*}" ]; then - echo " ok miner_hashes_total ${h1%.*} -> ${h2%.*}" - ok=1; break - fi + m=$(scrape) + devs=$(awk '/^miner_gpu_devices /{print $2; exit}' <<<"$m") + [ -n "$devs" ] && break + sleep 5 done - if [ "$ok" != 1 ]; then - echo " miner_hashes_total did not advance within 120s (${h1:-?} -> ${h2:-?})" >&2 + if [ -z "$devs" ]; then + echo " miner did not connect to ${{ matrix.node }} within 90s (no miner_gpu_devices exported)" >&2; fail=1 + elif [ "${devs%.*}" != "${{ matrix.gpu_devices }}" ]; then + echo " miner_gpu_devices is ${devs}, matrix says ${{ matrix.gpu_devices }}" >&2; fail=1 + else + echo " ok connected; miner_gpu_devices ${devs%.*}" + fi + + deadline=$((SECONDS + 120)) + h1=""; h2=""; ok=0 + while [ $SECONDS -lt $deadline ]; do + m=$(scrape) + h=$(awk '/^miner_hashes_total /{print $2; exit}' <<<"$m") + if [ -n "$h" ]; then + if [ -z "$h1" ]; then + h1="$h" + elif [ "${h%.*}" -gt "${h1%.*}" ]; then + h2="$h"; ok=1; break + fi + fi + sleep 10 + done + if [ "$ok" = 1 ]; then + echo " ok miner_hashes_total ${h1%.*} -> ${h2%.*}" + else + echo " miner_hashes_total did not advance within 120s (${h1:-none} -> ${h2:-none})" >&2 fail=1 fi @@ -324,7 +340,10 @@ jobs: run() { ssh $SSHOPTS gitea_ci@"${{ matrix.host }}" "$@"; } if run test -x /usr/local/bin/quantus-miner.prev; then echo "validate failed after a restart — restoring the previous binary" - run sudo cp -p /usr/local/bin/quantus-miner.prev /usr/local/bin/quantus-miner + # install, not cp: cp writes in place and fails with "Text file busy" + # on a running binary; install unlinks the destination first (the + # same reason rsync's temp-file-and-rename works for the push). + run sudo install -m 0755 /usr/local/bin/quantus-miner.prev /usr/local/bin/quantus-miner run sudo restorecon -R /usr/local/bin/quantus-miner /etc/quantus-miner /var/lib/quantus-miner run sudo systemctl restart quantus-miner.service echo "restored: $(run /usr/local/bin/quantus-miner --version)" diff --git a/deploy/infra-setup.sh b/deploy/infra-setup.sh index da55c0d..45e9281 100755 --- a/deploy/infra-setup.sh +++ b/deploy/infra-setup.sh @@ -110,11 +110,12 @@ gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl restart quantus-miner.service # mining for a measurement window and resume it afterwards. gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl stop quantus-miner.service gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl start quantus-miner.service -# Rollback: the deploy keeps the previous binary as .prev and restores it when -# validate fails. Deploying from main means a bad commit reaches production; +# Rollback: the deploy keeps the previous binary as .prev (cp) and restores it +# with install when validate fails; cp back would hit "Text file busy" on the +# running binary, install unlinks the destination first. Deploying from main means a bad commit reaches production; # this is what makes that survivable. gitea_ci ALL=(root) NOPASSWD: /usr/bin/cp -p /usr/local/bin/quantus-miner /usr/local/bin/quantus-miner.prev -gitea_ci ALL=(root) NOPASSWD: /usr/bin/cp -p /usr/local/bin/quantus-miner.prev /usr/local/bin/quantus-miner +gitea_ci ALL=(root) NOPASSWD: /usr/bin/install -m 0755 /usr/local/bin/quantus-miner.prev /usr/local/bin/quantus-miner SUDO chmod 0440 "$tmp" visudo -cf "$tmp"