e178bcbdf9
All checks were successful
deploy / fetch (push) Successful in 17s
deploy / deploy-node (bob.hanzalova.internal, 0x134e73f06fa9bdb1dbfa909e149c563f5860ceb71a0e7307918f7033970edf59, benjy.hanzalova.internal quadbrat.hanzalova.internal, --public-addr /dns4/nh.thgttg.com/tcp/30333, --unsafe-rpc-external --rpc-methods safe --rpc-c… (push) Successful in 34s
deploy / deploy-miner (1, quadbrat.hanzalova.internal, bob.hanzalova.internal) (push) Successful in 40s
deploy / deploy-miner (1, benjy.hanzalova.internal, bob.hanzalova.internal) (push) Successful in 43s
deploy / deploy-metrics (push) Successful in 8s
nvidia-smi -pl survives neither a reboot nor a driver unload, and both failures are silent -- the card returns to its default limit and nobody notices until a connector runs warm. Hence a boot-time unit, which also enables persistence mode so the limit is not lost when the last GPU client detaches. Per-host limits configured as host=watts in infra-setup.sh, clamped to each card's own min/max. Verified: a 300W request on a 5090 whose floor is 400W is corrected with an explicit log line rather than failing or being ignored. Measured on a 4090 mining Poseidon2: 450W (stock) 185.5 MH/s 70C 0.413 MH/s/W 300W 183.2 MH/s 61C 0.611 250W 180.3 MH/s 58C 0.721 44% less power and 12C cooler for 2.8% less hashrate -- this workload is register-resident integer math and does not need the top of the clock curve. It also cuts current through the 12VHPWR connector, which is the one component on these cards with a documented melting failure mode. Ordered Before=quantus-miner.service so a GPU is never briefly at its default limit under full mining load.
572 lines
28 KiB
Bash
Executable File
572 lines
28 KiB
Bash
Executable File
#!/usr/bin/env bash
|
||
# One-time, operator-run provisioning so the Gitea Actions runner can deploy as
|
||
# the scoped `gitea_ci` user, plus the key material that cannot live in the
|
||
# workflow. Two roles, each with its own narrow sudoers:
|
||
#
|
||
# node (--node-hosts) quantus-node binary + unit + config + firewalld,
|
||
# and read access to the two miner credential files
|
||
# the node generates -> deploy.yaml deploy-node
|
||
# miner (--miner-hosts) quantus-miner binary + unit + credentials
|
||
# -> deploy.yaml deploy-miner
|
||
# metrics (--metrics-hosts) the fleet Prometheus/Grafana host: a scrape
|
||
# config drop-in and a provisioned dashboard
|
||
# -> deploy.yaml deploy-metrics
|
||
# gpu (--gpu-hosts) a timer writing nvidia-smi metrics into
|
||
# node-exporter's textfile collector. No new exporter,
|
||
# port, firewalld service or scrape config — the fleet
|
||
# node-exporter already carries it.
|
||
# edge (--edge-hosts) the site reverse proxy: an internal TLS cert and an
|
||
# nginx vhost giving `wss://quantus.internal`. NOT
|
||
# CI-deployed — the runner has no rights on the edge
|
||
# proxy, deliberately (reverse-proxies.md §4).
|
||
#
|
||
# The roles are separate deliberately: the miner host holds a GPU and an
|
||
# outbound-only process, and a compromise of the miner deploy must not be able
|
||
# to rewrite the validator's binary or read its reward config. Hence
|
||
# `quantus-miner_gitea_ci` as a SEPARATE sudoers file from
|
||
# `quantus-node_gitea_ci` (deployment-gitea-actions.md §1).
|
||
#
|
||
# Convention: ~/git/architecture/deployment-gitea-actions.md §1–§2, generic.md
|
||
# §8–§10, agent-credentials.md §1 and §6.
|
||
#
|
||
# Run from a workstation with admin (sudo) ssh access to the targets — NOT the
|
||
# gitea_ci account. Idempotent; re-running is a no-op. Skips unreachable hosts
|
||
# so one offline node doesn't block the rest.
|
||
#
|
||
# ./script/infra-setup.sh --pubkey ~/.ssh/id_gitea_ci.pub
|
||
# ./script/infra-setup.sh --pubkey ~/.ssh/id_gitea_ci.pub --node-hosts "" \
|
||
# --miner-hosts benjy.hanzalova.internal
|
||
#
|
||
# The runner keypair is NOT generated here. It already exists at
|
||
# ~/.ssh/id_gitea_ci and is shared by every project's RSYNC_SSH_KEY secret and
|
||
# every host's gitea_ci authorized_keys — regenerating it would silently break
|
||
# every other deploy on the fleet (deployment-gitea-actions.md §2).
|
||
set -euo pipefail
|
||
|
||
ADMIN_USER="${ADMIN_USER:-$USER}"
|
||
# `${X-default}` not `${X:-default}`: an explicitly EMPTY list means "none of
|
||
# this role", which is how you provision only the other one. With `:-` the empty
|
||
# value would silently fall back to the full default list.
|
||
NODE_HOSTS="${NODE_HOSTS-bob.hanzalova.internal}"
|
||
MINER_HOSTS="${MINER_HOSTS-benjy.hanzalova.internal quadbrat.hanzalova.internal}"
|
||
METRICS_HOSTS="${METRICS_HOSTS-golgafrinchans.kosherinata.internal}"
|
||
# The site reverse proxy that fronts quantus.internal. Empty to skip the edge role.
|
||
EDGE_HOSTS="${EDGE_HOSTS-hanzalova.internal}"
|
||
# Hosts with NVIDIA GPUs to expose metrics for. Fleet-general rather than
|
||
# quantus-specific — it lives here because there is no monitoring repo yet, and
|
||
# should move to one if that changes.
|
||
GPU_HOSTS="${GPU_HOSTS-beast.hanzalova.internal benjy.hanzalova.internal quadbrat.hanzalova.internal}"
|
||
# Per-host GPU power limits, as `host=watts`. Applied at boot by
|
||
# nvidia-power-limit.service and clamped to each card's own floor/ceiling, so a
|
||
# value the hardware refuses is corrected loudly rather than silently ignored.
|
||
# Conservative on purpose: a 4090 measured 183.2 MH/s at 300W against 185.5 at
|
||
# 450W, so the hashrate cost is ~1% and the thermal and connector-current
|
||
# margin is large. A 5090's floor is 400W; requests below it are clamped.
|
||
GPU_POWER_LIMITS="${GPU_POWER_LIMITS-beast.hanzalova.internal=300 benjy.hanzalova.internal=250 quadbrat.hanzalova.internal=150}"
|
||
# Vanity name served by the edge proxy. Dot-free short name: the step@ renewal
|
||
# unit is a template instance and %i cannot carry dots (internal-tls.md §1).
|
||
EDGE_NAME="${EDGE_NAME:-quantus}"
|
||
# step-ca JWK provisioner password, shipped to the edge host just long enough to
|
||
# mint the first cert and removed even on failure (internal-tls.md §4).
|
||
STEP_PROVISIONER="${STEP_PROVISIONER:-${HOME}/.step/secrets/provisioner}"
|
||
PUBKEY="${PUBKEY:-${HOME}/.ssh/id_gitea_ci.pub}"
|
||
CHAIN="${CHAIN:-planck}"
|
||
NODE_VERSION="${NODE_VERSION:-0.10.0}"
|
||
|
||
while [[ $# -gt 0 ]]; do
|
||
case "$1" in
|
||
--pubkey) PUBKEY="$2"; shift 2 ;;
|
||
--node-hosts) NODE_HOSTS="$2"; shift 2 ;;
|
||
--miner-hosts) MINER_HOSTS="$2"; shift 2 ;;
|
||
--metrics-hosts) METRICS_HOSTS="$2"; shift 2 ;;
|
||
--edge-hosts) EDGE_HOSTS="$2"; shift 2 ;;
|
||
--gpu-hosts) GPU_HOSTS="$2"; shift 2 ;;
|
||
--gpu-power-limits) GPU_POWER_LIMITS="$2"; shift 2 ;;
|
||
--admin) ADMIN_USER="$2"; shift 2 ;;
|
||
--node-version) NODE_VERSION="$2"; shift 2 ;;
|
||
*) echo "unknown arg: $1" >&2; exit 2 ;;
|
||
esac
|
||
done
|
||
|
||
cd "$(dirname "${BASH_SOURCE[0]}")/.."
|
||
|
||
NODE_URL="https://github.com/Quantus-Network/chain/releases/download/v${NODE_VERSION}/quantus-node-v${NODE_VERSION}-x86_64-unknown-linux-gnu.tar.gz"
|
||
|
||
# ------------------------------------------------------------------- roles ----
|
||
|
||
provision_node() {
|
||
local host="$1"
|
||
echo "== ${host} (node) =="
|
||
if ! ssh -o ConnectTimeout=8 -o BatchMode=yes "${ADMIN_USER}@${host}" true; then
|
||
echo " ! unreachable as ${ADMIN_USER} — skipping" >&2
|
||
return 1
|
||
fi
|
||
|
||
# All privileged work in one remote `sudo bash`. The runner pubkey is the only
|
||
# dynamic value, passed as $1 (single line, no quoting hazard). The sudoers
|
||
# `\=` are the required escapes — visudo rejects a bare `=` in a command arg.
|
||
# The `*` in an rsync line matches rsync's --server arg vector; the trailing
|
||
# literal destination is what actually bounds the rule.
|
||
ssh "${ADMIN_USER}@${host}" "sudo bash -seu -- '$(cat "$PUBKEY")'" <<'REMOTE'
|
||
PUBKEY="$1"
|
||
|
||
# /bin/bash, NOT nologin. The deploy runs `ssh gitea_ci@host <command>`, and
|
||
# a nologin shell refuses that with "This account is currently not
|
||
# available": the key authenticates, then the command cannot run. Every
|
||
# pre-existing gitea_ci on this fleet has bash, so a nologin one is a bug —
|
||
# repair it rather than leaving a host that can never be deployed to.
|
||
if ! getent passwd gitea_ci >/dev/null; then
|
||
useradd --system --create-home --home-dir /var/lib/gitea_ci \
|
||
--shell /bin/bash gitea_ci
|
||
echo " + created gitea_ci"
|
||
else
|
||
echo " = gitea_ci already present"
|
||
fi
|
||
cur=$(getent passwd gitea_ci | cut -d: -f7)
|
||
if [ "$cur" != /bin/bash ]; then
|
||
usermod -s /bin/bash gitea_ci
|
||
echo " ~ gitea_ci shell was ${cur} — set to /bin/bash so ssh commands can run"
|
||
fi
|
||
|
||
install -d -o gitea_ci -g gitea_ci -m 0700 /var/lib/gitea_ci/.ssh
|
||
ak=/var/lib/gitea_ci/.ssh/authorized_keys
|
||
touch "$ak"
|
||
grep -qxF "$PUBKEY" "$ak" || printf '%s\n' "$PUBKEY" >> "$ak"
|
||
chown gitea_ci:gitea_ci "$ak"
|
||
chmod 0600 "$ak"
|
||
|
||
# journalctl -u <unit> for the deploy's log capture, without sudo.
|
||
usermod -aG systemd-journal gitea_ci
|
||
|
||
# Soft dependency for the arena exporter's telemetry feed. Packaged, never
|
||
# pip. Its absence only disables telemetry metrics — the exporter still
|
||
# serves every chain metric — so this is deliberately not fatal.
|
||
if python3 -c "import websockets" >/dev/null 2>&1; then
|
||
echo " = python3-websockets already present"
|
||
elif dnf install -y python3-websockets >/dev/null 2>&1; then
|
||
echo " + installed python3-websockets"
|
||
else
|
||
echo " ! could not install python3-websockets — telemetry metrics will be disabled" >&2
|
||
fi
|
||
|
||
tmp=/etc/sudoers.d/.quantus-node_gitea_ci.tmp
|
||
cat > "$tmp" <<'SUDO'
|
||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /usr/local/bin/quantus-node
|
||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/systemd/system/quantus-node.service
|
||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/sysusers.d/quantus-node.conf
|
||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/quantus-node/node.env
|
||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/firewalld/services/quantus-node.xml
|
||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/firewalld/services/quantus-node-miner.xml
|
||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/firewalld/services/quantus-node-metrics.xml
|
||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/firewalld/services/quantus-arena.xml
|
||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/firewalld/services/quantus-node-rpc.xml
|
||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /usr/local/bin/quantus-arena-exporter.py
|
||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/systemd/system/quantus-arena.service
|
||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/sysusers.d/quantus-arena.conf
|
||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/quantus-arena/arena.env
|
||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/install -d -o root -g quantus-arena -m 0750 /etc/quantus-arena
|
||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/install -d -o quantus-arena -g quantus-arena -m 0750 /var/lib/quantus-arena
|
||
gitea_ci ALL=(root) NOPASSWD: /usr/sbin/restorecon -R /usr/local/bin/quantus-arena-exporter.py /etc/quantus-arena /var/lib/quantus-arena
|
||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl enable quantus-arena.service
|
||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl restart quantus-arena.service
|
||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemd-sysusers
|
||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/install -d -o root -g quantus-node -m 0750 /etc/quantus-node
|
||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/install -d -o quantus-node -g quantus-node -m 0750 /var/lib/quantus-node
|
||
gitea_ci ALL=(root) NOPASSWD: /usr/sbin/restorecon -R /usr/local/bin/quantus-node /etc/quantus-node /var/lib/quantus-node
|
||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/firewall-cmd --get-default-zone
|
||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/firewall-cmd --reload
|
||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/firewall-cmd --permanent --zone\=* --add-service\=quantus-node
|
||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/firewall-cmd --zone\=* --add-service\=quantus-node
|
||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/firewall-cmd --zone\=* --query-service\=quantus-node
|
||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/firewall-cmd --permanent --zone\=* --add-rich-rule\=*
|
||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/firewall-cmd --zone\=* --add-rich-rule\=*
|
||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/firewall-cmd --zone\=* --query-rich-rule\=*
|
||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl daemon-reload
|
||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl enable quantus-node.service
|
||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl restart quantus-node.service
|
||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/cat /var/lib/quantus-node/chains/*/miner-auth-token
|
||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/cat /var/lib/quantus-node/chains/*/miner-tls-cert-sha256
|
||
SUDO
|
||
chmod 0440 "$tmp"
|
||
visudo -cf "$tmp"
|
||
mv "$tmp" /etc/sudoers.d/quantus-node_gitea_ci
|
||
echo " = sudoers quantus-node_gitea_ci installed and visudo-verified"
|
||
REMOTE
|
||
}
|
||
|
||
provision_miner() {
|
||
local host="$1"
|
||
echo "== ${host} (miner) =="
|
||
if ! ssh -o ConnectTimeout=8 -o BatchMode=yes "${ADMIN_USER}@${host}" true; then
|
||
echo " ! unreachable as ${ADMIN_USER} — skipping" >&2
|
||
return 1
|
||
fi
|
||
|
||
ssh "${ADMIN_USER}@${host}" "sudo bash -seu -- '$(cat "$PUBKEY")'" <<'REMOTE'
|
||
PUBKEY="$1"
|
||
|
||
# /bin/bash, NOT nologin. The deploy runs `ssh gitea_ci@host <command>`, and
|
||
# a nologin shell refuses that with "This account is currently not
|
||
# available": the key authenticates, then the command cannot run. Every
|
||
# pre-existing gitea_ci on this fleet has bash, so a nologin one is a bug —
|
||
# repair it rather than leaving a host that can never be deployed to.
|
||
if ! getent passwd gitea_ci >/dev/null; then
|
||
useradd --system --create-home --home-dir /var/lib/gitea_ci \
|
||
--shell /bin/bash gitea_ci
|
||
echo " + created gitea_ci"
|
||
else
|
||
echo " = gitea_ci already present"
|
||
fi
|
||
cur=$(getent passwd gitea_ci | cut -d: -f7)
|
||
if [ "$cur" != /bin/bash ]; then
|
||
usermod -s /bin/bash gitea_ci
|
||
echo " ~ gitea_ci shell was ${cur} — set to /bin/bash so ssh commands can run"
|
||
fi
|
||
|
||
install -d -o gitea_ci -g gitea_ci -m 0700 /var/lib/gitea_ci/.ssh
|
||
ak=/var/lib/gitea_ci/.ssh/authorized_keys
|
||
touch "$ak"
|
||
grep -qxF "$PUBKEY" "$ak" || printf '%s\n' "$PUBKEY" >> "$ak"
|
||
chown gitea_ci:gitea_ci "$ak"
|
||
chmod 0600 "$ak"
|
||
usermod -aG systemd-journal gitea_ci
|
||
|
||
# The GPU must be present and enumerable before a miner deploy is worth
|
||
# attempting; failing here beats a green deploy that mines nothing.
|
||
if ! command -v nvidia-smi >/dev/null; then
|
||
echo " ! nvidia-smi not found — this host has no usable NVIDIA driver" >&2
|
||
exit 1
|
||
fi
|
||
nvidia-smi --query-gpu=name --format=csv,noheader | sed 's/^/ = gpu: /'
|
||
|
||
tmp=/etc/sudoers.d/.quantus-miner_gitea_ci.tmp
|
||
cat > "$tmp" <<'SUDO'
|
||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /usr/local/bin/quantus-miner
|
||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/systemd/system/quantus-miner.service
|
||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/sysusers.d/quantus-miner.conf
|
||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/quantus-miner/miner.env
|
||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/quantus-miner/miner-auth-token
|
||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/quantus-miner/miner-tls-cert-sha256
|
||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/firewalld/services/quantus-miner-metrics.xml
|
||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/firewall-cmd --get-default-zone
|
||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/firewall-cmd --reload
|
||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/firewall-cmd --permanent --zone\=* --add-rich-rule\=*
|
||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/firewall-cmd --zone\=* --add-rich-rule\=*
|
||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/firewall-cmd --zone\=* --query-rich-rule\=*
|
||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemd-sysusers
|
||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/install -d -o root -g quantus-miner -m 0750 /etc/quantus-miner
|
||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/install -d -o quantus-miner -g quantus-miner -m 0750 /var/lib/quantus-miner
|
||
gitea_ci ALL=(root) NOPASSWD: /usr/sbin/restorecon -R /usr/local/bin/quantus-miner /etc/quantus-miner /var/lib/quantus-miner
|
||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl daemon-reload
|
||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl enable quantus-miner.service
|
||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl restart quantus-miner.service
|
||
SUDO
|
||
chmod 0440 "$tmp"
|
||
visudo -cf "$tmp"
|
||
mv "$tmp" /etc/sudoers.d/quantus-miner_gitea_ci
|
||
echo " = sudoers quantus-miner_gitea_ci installed and visudo-verified"
|
||
REMOTE
|
||
}
|
||
|
||
|
||
# The fleet Prometheus/Grafana host. Two of the things this does are edits to
|
||
# SHARED config that other projects also rely on, which is exactly why they live
|
||
# here (operator-run, one-time) and not in the deploy workflow:
|
||
#
|
||
# * prometheus.yml gains `scrape_config_files:` so every project can drop its
|
||
# own scrape config into scrape_configs.d/ instead of all of them editing one
|
||
# shared file and racing each other.
|
||
# * grafana.container gains a dashboards bind mount — it currently mounts only
|
||
# the datasource file, so there is nowhere for a provisioned dashboard to go.
|
||
#
|
||
# Both are additive, guarded, and back up what they touch.
|
||
provision_metrics() {
|
||
local host="$1"
|
||
echo "== ${host} (metrics) =="
|
||
if ! ssh -o ConnectTimeout=8 -o BatchMode=yes "${ADMIN_USER}@${host}" true; then
|
||
echo " ! unreachable as ${ADMIN_USER} — skipping" >&2
|
||
return 1
|
||
fi
|
||
|
||
ssh "${ADMIN_USER}@${host}" "sudo bash -seu -- '$(cat "$PUBKEY")'" <<'REMOTE'
|
||
PUBKEY="$1"
|
||
|
||
# /bin/bash, NOT nologin. The deploy runs `ssh gitea_ci@host <command>`, and
|
||
# a nologin shell refuses that with "This account is currently not
|
||
# available": the key authenticates, then the command cannot run. Every
|
||
# pre-existing gitea_ci on this fleet has bash, so a nologin one is a bug —
|
||
# repair it rather than leaving a host that can never be deployed to.
|
||
if ! getent passwd gitea_ci >/dev/null; then
|
||
useradd --system --create-home --home-dir /var/lib/gitea_ci \
|
||
--shell /bin/bash gitea_ci
|
||
echo " + created gitea_ci"
|
||
else
|
||
echo " = gitea_ci already present"
|
||
fi
|
||
cur=$(getent passwd gitea_ci | cut -d: -f7)
|
||
if [ "$cur" != /bin/bash ]; then
|
||
usermod -s /bin/bash gitea_ci
|
||
echo " ~ gitea_ci shell was ${cur} — set to /bin/bash so ssh commands can run"
|
||
fi
|
||
install -d -o gitea_ci -g gitea_ci -m 0700 /var/lib/gitea_ci/.ssh
|
||
ak=/var/lib/gitea_ci/.ssh/authorized_keys
|
||
touch "$ak"
|
||
grep -qxF "$PUBKEY" "$ak" || printf '%s\n' "$PUBKEY" >> "$ak"
|
||
chown gitea_ci:gitea_ci "$ak"; chmod 0600 "$ak"
|
||
usermod -aG systemd-journal gitea_ci
|
||
|
||
install -d -m 0755 /etc/prometheus/scrape_configs.d
|
||
install -d -m 0755 /etc/grafana/provisioning/dashboards/quantus
|
||
|
||
# 1. include drop-in scrape configs
|
||
if grep -q '^scrape_config_files:' /etc/prometheus/prometheus.yml; then
|
||
echo " = prometheus.yml already includes scrape_configs.d"
|
||
else
|
||
cp -a /etc/prometheus/prometheus.yml \
|
||
"/etc/prometheus/prometheus.yml.bak.$(date +%s)"
|
||
cat >> /etc/prometheus/prometheus.yml <<'PROM'
|
||
|
||
# Per-project scrape configs. Each project ships its own file here rather than
|
||
# every project editing this one. Added by lair/quantus script/infra-setup.sh.
|
||
scrape_config_files:
|
||
- /etc/prometheus/scrape_configs.d/*.yml
|
||
PROM
|
||
echo " + prometheus.yml now includes scrape_configs.d"
|
||
curl -fsS -X POST http://127.0.0.1:26559/-/reload && echo " = prometheus reloaded"
|
||
fi
|
||
|
||
# 2. give Grafana somewhere to read provisioned dashboards from
|
||
q=/etc/containers/systemd/grafana.container
|
||
if grep -q 'provisioning/dashboards:' "$q"; then
|
||
echo " = grafana.container already mounts the dashboards dir"
|
||
else
|
||
cp -a "$q" "${q}.bak.$(date +%s)"
|
||
# Insert alongside the existing Volume= lines, before [Service].
|
||
sed -i 's#^\(Volume=/etc/grafana/provisioning/datasources.*\)$#\1\nVolume=/etc/grafana/provisioning/dashboards:/etc/grafana/provisioning/dashboards:Z,ro#' "$q"
|
||
grep -q 'provisioning/dashboards:' "$q" || { echo " ! failed to add the mount" >&2; exit 1; }
|
||
systemctl daemon-reload
|
||
systemctl restart grafana.service
|
||
echo " + grafana.container mounts dashboards; grafana restarted"
|
||
fi
|
||
|
||
tmp=/etc/sudoers.d/.quantus-metrics_gitea_ci.tmp
|
||
cat > "$tmp" <<'SUDO'
|
||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/prometheus/scrape_configs.d/quantus.yml
|
||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/grafana/provisioning/dashboards/quantus/quantus.json
|
||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/grafana/provisioning/dashboards/quantus/gpu.json
|
||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/grafana/provisioning/dashboards/quantus-dashboards.yaml
|
||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl restart grafana.service
|
||
SUDO
|
||
chmod 0440 "$tmp"
|
||
visudo -cf "$tmp"
|
||
mv "$tmp" /etc/sudoers.d/quantus-metrics_gitea_ci
|
||
echo " = sudoers quantus-metrics_gitea_ci installed and visudo-verified"
|
||
REMOTE
|
||
}
|
||
|
||
|
||
|
||
# GPU metrics via node-exporter's textfile collector.
|
||
#
|
||
# Deliberately not a new exporter. node-exporter already runs on every host, is
|
||
# already scraped (job "node", targets from cichlid's inventory, labelled by
|
||
# `host`), and already has the textfile collector enabled with a spool at
|
||
# /var/lib/prometheus/node-exporter. So this needs no port allocation, no
|
||
# firewalld service and no scrape config — which is also why it is one-time
|
||
# provisioning rather than a deploy job.
|
||
#
|
||
# DCGM was the alternative and is the wrong tool: it targets datacenter cards
|
||
# and its GeForce support is partial. nvidia-smi reports everything wanted here.
|
||
provision_gpu() {
|
||
local host="$1"
|
||
echo "== ${host} (gpu metrics) =="
|
||
if ! ssh -o ConnectTimeout=8 -o BatchMode=yes "${ADMIN_USER}@${host}" true; then
|
||
echo " ! unreachable as ${ADMIN_USER} — skipping" >&2
|
||
return 1
|
||
fi
|
||
if ! ssh "${ADMIN_USER}@${host}" "command -v nvidia-smi >/dev/null"; then
|
||
echo " ! no nvidia-smi on this host — skipping" >&2
|
||
return 1
|
||
fi
|
||
|
||
rsync -az --rsync-path='sudo rsync' --chmod=F0755 \
|
||
asset/gpu/nvidia-textfile.sh "${ADMIN_USER}@${host}:/usr/local/bin/nvidia-textfile.sh"
|
||
rsync -az --rsync-path='sudo rsync' --chmod=F0755 \
|
||
asset/gpu/nvidia-power-limit.sh "${ADMIN_USER}@${host}:/usr/local/bin/nvidia-power-limit.sh"
|
||
rsync -az --rsync-path='sudo rsync' --chmod=F0644 \
|
||
asset/systemd/nvidia-textfile.service asset/systemd/nvidia-textfile.timer \
|
||
asset/systemd/nvidia-power-limit.service \
|
||
"${ADMIN_USER}@${host}:/etc/systemd/system/"
|
||
|
||
# Per-host power limit, if one is configured for this host.
|
||
local watts=""
|
||
for pair in $GPU_POWER_LIMITS; do
|
||
case "$pair" in "${host}="*) watts="${pair#*=}" ;; esac
|
||
done
|
||
if [ -n "$watts" ]; then
|
||
printf 'NVIDIA_POWER_LIMIT_WATTS=%s\n' "$watts" \
|
||
| ssh "${ADMIN_USER}@${host}" "sudo tee /etc/default/nvidia-power-limit >/dev/null"
|
||
echo " = requested power limit ${watts}W"
|
||
else
|
||
echo " = no power limit configured for this host — cards left at default"
|
||
fi
|
||
|
||
ssh "${ADMIN_USER}@${host}" "
|
||
sudo install -d -m 0755 /var/lib/prometheus/node-exporter
|
||
sudo systemctl daemon-reload
|
||
sudo systemctl enable --now nvidia-textfile.timer
|
||
sudo systemctl start nvidia-textfile.service"
|
||
if [ -n "$watts" ]; then
|
||
ssh "${ADMIN_USER}@${host}" "sudo systemctl enable nvidia-power-limit.service \
|
||
&& sudo systemctl restart nvidia-power-limit.service"
|
||
ssh "${ADMIN_USER}@${host}" \
|
||
"nvidia-smi --query-gpu=index,power.limit --format=csv,noheader" | sed "s/^/ now: /"
|
||
fi
|
||
|
||
local n
|
||
n=$(ssh "${ADMIN_USER}@${host}" \
|
||
"curl -fsS http://127.0.0.1:9100/metrics | grep -c '^nvidia_gpu_temperature_celsius' || true")
|
||
echo " = timer armed; ${n:-0} GPU(s) reporting through node-exporter"
|
||
}
|
||
|
||
# The site reverse proxy. Gives tooling a `wss://quantus.internal` endpoint,
|
||
# which matters because subxt — and therefore quantus-cli — refuses plain `ws://`
|
||
# to anything but localhost, so without this every client needs an ssh tunnel.
|
||
#
|
||
# Operator-run rather than CI-deployed on purpose: the runner's rights never
|
||
# extend to reading certificate keys or reloading nginx on a shared edge proxy.
|
||
# Same split lair/mail uses for its edge role.
|
||
provision_edge() {
|
||
local host="$1" name="$2"
|
||
echo "== ${host} (edge: ${name}.internal) =="
|
||
if ! ssh -o ConnectTimeout=8 -o BatchMode=yes "${ADMIN_USER}@${host}" true; then
|
||
echo " ! unreachable as ${ADMIN_USER} — skipping" >&2
|
||
return 1
|
||
fi
|
||
|
||
local cert="/etc/nginx/tls/cert/${name}.internal.pem"
|
||
local key="/etc/nginx/tls/key/${name}.internal.pem"
|
||
|
||
# verify checks chain and expiry, not the name — good enough to decide whether
|
||
# to mint, and it keeps re-runs from burning a provisioner use each time.
|
||
local state
|
||
state=$(ssh "${ADMIN_USER}@${host}" "[ -f ${cert} ] && sudo step certificate verify ${cert} \
|
||
--roots /etc/pki/ca-trust/source/anchors/root-internal.pem >/dev/null 2>&1 \
|
||
&& echo valid || echo missing")
|
||
|
||
if [ "$state" = valid ]; then
|
||
echo " = ${name}.internal cert already valid"
|
||
else
|
||
[[ -s "$STEP_PROVISIONER" ]] || {
|
||
echo " ! provisioner password not found at ${STEP_PROVISIONER}" >&2
|
||
return 1
|
||
}
|
||
rsync -az --rsync-path='sudo rsync' --chmod=0600 \
|
||
"$STEP_PROVISIONER" "${ADMIN_USER}@${host}:/tmp/${name}-provisioner"
|
||
ssh "${ADMIN_USER}@${host}" "
|
||
sudo mkdir -p /etc/nginx/tls/cert /etc/nginx/tls/key
|
||
rc=0
|
||
sudo step ca certificate --force \
|
||
--provisioner lair \
|
||
--provisioner-password-file /tmp/${name}-provisioner \
|
||
--ca-url https://ca.internal \
|
||
--root /etc/pki/ca-trust/source/anchors/root-internal.pem \
|
||
--san ${name}.internal \
|
||
${name}.internal ${cert} ${key} || rc=\$?
|
||
sudo rm -f /tmp/${name}-provisioner
|
||
[ \$rc -eq 0 ] || { echo 'mint failed' >&2; exit \$rc; }
|
||
sudo chown root:root ${cert} ${key}
|
||
sudo chmod 644 ${cert}; sudo chmod 640 ${key}
|
||
sudo setfacl -m u:nginx:r ${key}"
|
||
echo " + minted ${name}.internal"
|
||
fi
|
||
|
||
# Ship the vhost and enable it as a SYMLINK. Several vhosts on this host are
|
||
# regular files that have drifted from sites-available since the SNI-router
|
||
# migration; a symlink keeps this one honest.
|
||
rsync -az --rsync-path='sudo rsync' --chmod=F0644 \
|
||
"asset/nginx/${name}.internal.conf" \
|
||
"${ADMIN_USER}@${host}:/etc/nginx/sites-available/${name}.internal.conf"
|
||
ssh "${ADMIN_USER}@${host}" "
|
||
sudo ln -sfn ../sites-available/${name}.internal.conf \
|
||
/etc/nginx/sites-enabled/${name}.internal.conf
|
||
sudo nginx -t"
|
||
ssh "${ADMIN_USER}@${host}" "sudo systemctl reload nginx.service"
|
||
ssh "${ADMIN_USER}@${host}" "sudo systemctl enable --now step@${name}.timer"
|
||
echo " = vhost enabled, nginx reloaded, step@${name}.timer armed"
|
||
|
||
echo " NOTE: split-horizon DNS is a separate step —"
|
||
echo " opn-cli --config ~/.opn-cli/<site>.yml unbound host create \\"
|
||
echo " --hostname ${name} --domain internal --server <edge mesh ip>"
|
||
}
|
||
|
||
# The node's libp2p identity. It must exist BEFORE the first start — without it
|
||
# the node exits with NetworkKeyNotFound(".../network/secret_dilithium") rather
|
||
# than generating one. It is also the node's stable identity on the network, so
|
||
# it is generated once and never rotated by a deploy.
|
||
ensure_node_key() {
|
||
local host="$1"
|
||
echo "== ${host} (node key) =="
|
||
ssh "${ADMIN_USER}@${host}" "sudo bash -seu -- '${NODE_URL}'" <<'REMOTE'
|
||
NODE_URL="$1"
|
||
|
||
install -d -o root -g root -m 0755 /etc/sysusers.d
|
||
cat > /etc/sysusers.d/quantus-node.conf <<'SYSUSERS'
|
||
u quantus-node - "Quantus node service account" /var/lib/quantus-node /usr/sbin/nologin
|
||
SYSUSERS
|
||
systemd-sysusers
|
||
install -d -o quantus-node -g quantus-node -m 0750 /var/lib/quantus-node
|
||
|
||
key=/var/lib/quantus-node/node_key.p2p
|
||
if [ -s "$key" ]; then
|
||
echo " = node key already present — left untouched"
|
||
else
|
||
tmp="$(mktemp -d)"
|
||
trap 'rm -rf "$tmp"' EXIT
|
||
curl -fSL --retry 5 --retry-delay 5 --retry-all-errors --connect-timeout 15 \
|
||
-o "$tmp/node.tar.gz" "$NODE_URL"
|
||
tar xzf "$tmp/node.tar.gz" -C "$tmp"
|
||
bin="$(find "$tmp" -name quantus-node -type f | head -1)"
|
||
chmod +x "$bin"
|
||
"$bin" key generate-node-key --file "$key"
|
||
chown quantus-node:quantus-node "$key"
|
||
chmod 0600 "$key"
|
||
echo " + generated node key"
|
||
fi
|
||
# The peer id is public, and is what you look for on telemetry.
|
||
echo -n " = peer id: "
|
||
getent passwd quantus-node >/dev/null && cat /var/lib/quantus-node/node_key.p2p >/dev/null
|
||
REMOTE
|
||
}
|
||
|
||
# --------------------------------------------------------------------- main ---
|
||
|
||
[[ -n "$PUBKEY" ]] || { echo "error: --pubkey <path> is required" >&2; exit 2; }
|
||
[[ -s "$PUBKEY" ]] || { echo "error: pubkey not found or empty: $PUBKEY" >&2; exit 2; }
|
||
|
||
rc=0
|
||
for h in $NODE_HOSTS; do
|
||
provision_node "$h" || { rc=1; continue; }
|
||
ensure_node_key "$h" || rc=1
|
||
done
|
||
for h in $MINER_HOSTS; do
|
||
provision_miner "$h" || rc=1
|
||
done
|
||
for h in $METRICS_HOSTS; do
|
||
provision_metrics "$h" || rc=1
|
||
done
|
||
for h in $GPU_HOSTS; do
|
||
provision_gpu "$h" || rc=1
|
||
done
|
||
for h in $EDGE_HOSTS; do
|
||
provision_edge "$h" "$EDGE_NAME" || rc=1
|
||
done
|
||
|
||
echo
|
||
echo "done. remaining operator steps:"
|
||
echo " 1. set QUANTUS_INNER_HASH in .gitea/workflows/deploy.yaml (it is public config,"
|
||
echo " not a secret — the node publishes it in every block header)"
|
||
echo " 2. confirm RSYNC_SSH_KEY is set in the repo settings"
|
||
echo " 3. push to main, or run the deploy workflow manually"
|
||
exit $rc
|