#!/usr/bin/env bash
#
# One-time host provisioning for blackbeard.observer.
#
# Run by an OPERATOR from a workstation with full sudo ssh to the targets — not
# by CI. The runner deploys as a scoped `gitea_ci` user and deliberately has no
# rights to create accounts, read certificate keys, or reload nginx on a shared
# edge proxy.
#
#   ./script/infra-setup.sh --pubkey ~/.ssh/id_gitea_ci.pub
#
# Idempotent, and it skips past unreachable hosts so one offline node does not
# block the rest. Re-run it whenever the deploy gains a new file to ship: every
# deploy job preflights the target's sudoers against the grants below and fails
# up front naming what is missing, so the two cannot silently drift.
#
# Roles (all run by default; pass --role to narrow):
#
#   api       the host running blackbeard-api beside quantus-node
#   dns       the public apex CNAME on Cloudflare
#   cert      the Let's Encrypt certificate for the public name
#   edge      the site's nginx proxy: vhosts, webroot, internal cert
#   database  Postgres roles, database, and the pg_ident CN mapping
#
# `dns` and `cert` run BEFORE `edge`, and in that order: certbot uses a DNS-01
# challenge, and nginx -t fails on a missing ssl_certificate — which blocks
# every reload on the proxy, not just this vhost.
#
# Conventions: architecture/generic.md §8-§11, deployment-gitea-actions.md.

set -euo pipefail

# --- infra truth, matching .gitea/workflows/deploy.yaml -----------------------
API_HOST="${API_HOST:-bob.hanzalova.internal}"
API_PORT="${API_PORT:-25864}"
EDGE_HOST="${EDGE_HOST:-oolon.kosherinata.internal}"
PG_PRIMARY="${PG_PRIMARY:-magrathea.kosherinata.internal}"
# The standby needs the same ident mapping: pg_ident.conf contents are NOT
# replicated, and a failover to a server missing it locks the app out.
PG_STANDBY="${PG_STANDBY:-frankie.hanzalova.internal}"
PG_VERSION="${PG_VERSION:-18}"
WEBROOT="${WEBROOT:-/var/www/blackbeard.observer}"
PUBLIC_NAME="${PUBLIC_NAME:-blackbeard.observer}"
INTERNAL_NAME="${INTERNAL_NAME:-blackbeard.internal}"
DB_NAME="${DB_NAME:-blackbeard}"
DB_ROLE="${DB_ROLE:-blackbeard_rw}"
# The per-site indirection name the public record CNAMEs to. `bl` is the DC
# (oolon), `nh` the office (hanzalova) — architecture/public-dns.md §2. A vhost
# record must never carry a site address directly: the address changes and every
# hardcoded A record then has to be hunted down across zones.
SITE_INDIRECTION="${SITE_INDIRECTION:-bl.thgttg.com}"
CERT_EMAIL="${CERT_EMAIL:-ops@blackbeard.observer}"

PUBKEY=""
ROLES="api dns cert edge database"
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"

info() { printf '\033[36m==\033[0m %s\n' "$*"; }
warn() { printf '\033[33m!!\033[0m %s\n' "$*" >&2; }
die()  { printf '\033[31mXX\033[0m %s\n' "$*" >&2; exit 1; }

usage() {
    sed -n '3,30p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'
    exit "${1:-0}"
}

while [ $# -gt 0 ]; do
    case "$1" in
        --pubkey) PUBKEY="$2"; shift 2 ;;
        --role)   ROLES="$2"; shift 2 ;;
        -h|--help) usage 0 ;;
        *) die "unknown argument: $1 (try --help)" ;;
    esac
done

has_role() { [[ " $ROLES " == *" $1 "* ]]; }

# Reachability is checked once per host and the result reused, so an offline
# host produces one clear message rather than a failure per step.
reachable() {
    local host="$1"
    if ssh -o ConnectTimeout=8 -o BatchMode=yes "$host" true; then
        return 0
    fi
    warn "$host is unreachable — skipping its steps"
    return 1
}

# --- the scoped sudoers grants ------------------------------------------------
#
# These strings are the single source of truth for what CI may do on each host.
# The deploy workflow's preflight extracts the paths from THIS FILE and compares
# them against `sudo -n -l` on the target, so adding a file to the deploy means
# adding a line here and re-running this script — nothing else keeps them in
# step.
#
# `:` and `=` are reserved in sudoers and must be escaped inside command
# arguments, or visudo rejects the file.

api_sudoers() {
    cat <<'EOF'
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /usr/local/bin/blackbeard-api
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /usr/local/bin/blackbeard
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/blackbeard/config.toml
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/sysusers.d/blackbeard.conf
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/systemd/system/blackbeard-api.service
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/systemd/system/blackbeard-api-cert.path
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/systemd/system/blackbeard-api-cert-reload.service
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/firewalld/services/blackbeard-api.xml
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemd-sysusers
gitea_ci ALL=(root) NOPASSWD: /usr/bin/install -d -o root -g blackbeard -m 0750 /etc/blackbeard
gitea_ci ALL=(root) NOPASSWD: /usr/bin/setfacl -m u\:blackbeard\:r /etc/pki/tls/private/*
gitea_ci ALL=(root) NOPASSWD: /usr/sbin/restorecon -R /usr/local/bin/blackbeard-api /usr/local/bin/blackbeard /etc/blackbeard
gitea_ci ALL=(root) NOPASSWD: /usr/sbin/semanage port -l
gitea_ci ALL=(root) NOPASSWD: /usr/sbin/semanage port -a -t http_port_t -p tcp 25864
gitea_ci ALL=(root) NOPASSWD: /usr/bin/firewall-cmd --reload
gitea_ci ALL=(root) NOPASSWD: /usr/bin/firewall-cmd --get-default-zone
gitea_ci ALL=(root) NOPASSWD: /usr/bin/firewall-cmd --zone=* --query-service=blackbeard-api
gitea_ci ALL=(root) NOPASSWD: /usr/bin/firewall-cmd --permanent --zone=* --add-service=blackbeard-api
gitea_ci ALL=(root) NOPASSWD: /usr/bin/firewall-cmd --zone=* --add-service=blackbeard-api
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl daemon-reload
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl enable blackbeard-api.service
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl enable --now blackbeard-api-cert.path
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl restart blackbeard-api.service
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl is-active blackbeard-api.service
# Runas is (blackbeard), not (root): the deploy validates the config AS the
# service account, which is the only way to prove the account can actually read
# the config and the certificate key it names. A (root) grant would not permit
# `sudo -u blackbeard` at all, and a root-run check would pass on a config the
# service cannot read.
gitea_ci ALL=(blackbeard) NOPASSWD: /usr/local/bin/blackbeard-api --config /etc/blackbeard/config.toml --check
EOF
}

edge_sudoers() {
    cat <<'EOF'
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /var/www/blackbeard.observer/
gitea_ci ALL=(root) NOPASSWD: /usr/sbin/restorecon -R /var/www/blackbeard.observer
gitea_ci ALL=(root) NOPASSWD: /usr/sbin/nginx -t
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl reload nginx
EOF
}

# --- gitea_ci -----------------------------------------------------------------

provision_gitea_ci() {
    local host="$1" sudoers_body="$2"

    [ -n "$PUBKEY" ] || die "--pubkey is required to provision gitea_ci (the runner's public key)"
    [ -f "$PUBKEY" ] || die "$PUBKEY does not exist"

    info "$host: gitea_ci account"
    # A real shell, never nologin. The deploy runs `ssh gitea_ci@host <command>`;
    # a nologin shell authenticates the key and then refuses the command with
    # "This account is currently not available", which reads as an auth problem
    # rather than a shell one. Every pre-existing gitea_ci on the fleet has bash,
    # and one provisioned with nologin can never be deployed to — so repair it
    # rather than leaving it.
    ssh "$host" sudo bash -euo pipefail <<'REMOTE'
        if ! id gitea_ci >/dev/null 2>&1; then
            useradd --system --create-home --home-dir /var/lib/gitea_ci --shell /bin/bash gitea_ci
        fi
        current=$(getent passwd gitea_ci | cut -d: -f7)
        if [ "$current" != "/bin/bash" ]; then
            echo "repairing gitea_ci shell: $current -> /bin/bash"
            usermod --shell /bin/bash gitea_ci
        fi
        install -d -o gitea_ci -g gitea_ci -m 0700 /var/lib/gitea_ci/.ssh
        # Lets the deploy capture `journalctl -u <unit>` after a restart without
        # a sudoers entry for it.
        usermod -aG systemd-journal gitea_ci
REMOTE

    rsync -a --chown gitea_ci:gitea_ci --chmod 0600 --rsync-path 'sudo rsync' \
        "$PUBKEY" "$host:/var/lib/gitea_ci/.ssh/authorized_keys"

    info "$host: scoped sudoers"
    # Named <app>_gitea_ci, not bare gitea_ci, so several apps can drop their own
    # files on a shared host without clobbering each other.
    printf '%s\n' "$sudoers_body" | \
        ssh "$host" 'sudo tee /etc/sudoers.d/blackbeard_gitea_ci > /dev/null && sudo chmod 0440 /etc/sudoers.d/blackbeard_gitea_ci'
    # Verify before leaving: a syntax error in a sudoers drop-in can lock every
    # sudo on the host, not just this one.
    ssh "$host" sudo visudo -cf /etc/sudoers.d/blackbeard_gitea_ci
}

# --- roles --------------------------------------------------------------------

role_api() {
    reachable "$API_HOST" || return 0
    provision_gitea_ci "$API_HOST" "$(api_sudoers)"

    info "$API_HOST: service account and directories"
    rsync -a --rsync-path 'sudo rsync' \
        "$REPO_ROOT/asset/systemd/blackbeard.sysusers.conf" \
        "$API_HOST:/etc/sysusers.d/blackbeard.conf"
    ssh "$API_HOST" sudo bash -euo pipefail <<'REMOTE'
        systemd-sysusers
        install -d -o root -g blackbeard -m 0750 /etc/blackbeard
        install -d -o blackbeard -g blackbeard -m 0750 /var/lib/blackbeard
        # The mTLS credential for Postgres. The key is not world-readable; the
        # service account is granted read access here and again on every deploy,
        # because a certificate rotation replaces the file and drops the ACL.
        setfacl -m "u:blackbeard:r" "/etc/pki/tls/private/$(hostname -f).pem"
REMOTE

    info "$API_HOST: SELinux port label"
    ssh "$API_HOST" sudo bash -euo pipefail <<REMOTE
        if semanage port -l | grep -qE "^http_port_t.*\b$API_PORT\b"; then
            echo "port $API_PORT already labelled http_port_t"
        else
            semanage port -a -t http_port_t -p tcp "$API_PORT"
        fi
REMOTE
}

role_edge() {
    reachable "$EDGE_HOST" || return 0
    provision_gitea_ci "$EDGE_HOST" "$(edge_sudoers)"

    info "$EDGE_HOST: webroot"
    ssh "$EDGE_HOST" sudo bash -euo pipefail <<REMOTE
        install -d -o root -g root -m 0755 "$WEBROOT"
        # An unlabelled webroot makes nginx return 403 for every file, with
        # nothing in the nginx error log to explain it.
        restorecon -R "$WEBROOT"
REMOTE

    info "$EDGE_HOST: internal certificate for $INTERNAL_NAME"
    # Minted from the internal step-ca and renewed by a templated step@ unit
    # (architecture/internal-tls.md). Certs are issued with a 24-hour expiry, so
    # the timer is not optional.
    ssh "$EDGE_HOST" sudo bash -euo pipefail <<REMOTE
        install -d -m 0755 /etc/nginx/tls/cert
        install -d -m 0700 /etc/nginx/tls/key
        if [ -f "/etc/nginx/tls/cert/$INTERNAL_NAME.pem" ]; then
            echo "$INTERNAL_NAME certificate present"
            systemctl enable --now "step@blackbeard.timer" || \
                echo "step@ renewal timer not armed — renew $INTERNAL_NAME manually until it is"
        else
            # Not fatal. The mesh vhost is a convenience; the public site is the
            # deliverable, and nginx -t fails on a missing ssl_certificate —
            # which would block every reload on this proxy, not just ours.
            echo "NOTE: no internal certificate for $INTERNAL_NAME; the mesh vhost will be skipped."
            echo "      Mint it with the lair provisioner (architecture/internal-tls.md §4)"
            echo "      and re-run --role edge to enable it."
        fi
REMOTE

    info "$EDGE_HOST: nginx configuration"
    rsync -a --rsync-path 'sudo rsync' \
        "$REPO_ROOT/asset/nginx/blackbeard-upstream.conf" \
        "$EDGE_HOST:/etc/nginx/conf.d/blackbeard-upstream.conf"
    rsync -a --rsync-path 'sudo rsync' \
        "$REPO_ROOT/asset/nginx/$PUBLIC_NAME.conf" \
        "$REPO_ROOT/asset/nginx/$INTERNAL_NAME.conf" \
        "$EDGE_HOST:/etc/nginx/sites-available/"
    ssh "$EDGE_HOST" sudo bash -euo pipefail <<REMOTE
        # sites-enabled holds only symlinks, and relative ones.
        ln -sfn "../sites-available/$PUBLIC_NAME.conf" "/etc/nginx/sites-enabled/$PUBLIC_NAME.conf"
        if [ -f "/etc/nginx/tls/cert/$INTERNAL_NAME.pem" ]; then
            ln -sfn "../sites-available/$INTERNAL_NAME.conf" "/etc/nginx/sites-enabled/$INTERNAL_NAME.conf"
        else
            rm -f "/etc/nginx/sites-enabled/$INTERNAL_NAME.conf"
        fi

        # The vhosts use \$connection_upgrade for the WebSocket upgrade; without
        # the map the socket silently degrades to a hanging request and the page
        # shows "reconnecting" forever with nothing in any log.
        if ! grep -rqs 'connection_upgrade' /etc/nginx/conf.d/ /etc/nginx/nginx.conf; then
            cat > /etc/nginx/conf.d/websocket-upgrade.conf <<'MAP'
map \$http_upgrade \$connection_upgrade {
    default upgrade;
    ''      close;
}
MAP
            echo "installed the connection_upgrade map"
        else
            echo "connection_upgrade map already present"
        fi

        # nginx -t parses without binding, so it catches syntax and missing
        # certs but not a port owned across http{} and stream{}. Verify the
        # reload landed rather than trusting the test.
        nginx -t
        systemctl reload nginx
        sleep 1
        systemctl is-active --quiet nginx || { echo "nginx did not come back after reload" >&2; exit 1; }
REMOTE

    cat <<EOF

$EDGE_HOST is configured. One step is NOT automated here:

  Split-horizon DNS for $INTERNAL_NAME, on BOTH site routers. A record on
  only one router NXDOMAINs everywhere else:

       for site in hanzalova kosherinata; do
         opn-cli --config ~/.opn-cli/\$site.yml unbound host create \\
           --hostname blackbeard --domain internal --rr A --server <edge mesh ip>
       done

     \`create\` only saves; POST /api/unbound/service/reconfigure on each router
     to apply.

EOF
}

# The Cloudflare API token lives on the edge proxy and is never copied off it:
# it can rewrite DNS for every zone on the account, including MX records for
# domains whose mail we host (architecture/public-dns.md §1). Every call that
# needs it therefore runs ON the proxy.
role_dns() {
    reachable "$EDGE_HOST" || return 0
    info "$EDGE_HOST: public DNS for $PUBLIC_NAME"

    ssh "$EDGE_HOST" "PUBLIC_NAME='$PUBLIC_NAME' SITE='$SITE_INDIRECTION' bash -euo pipefail" <<'REMOTE'
        TOKEN=$(sudo grep -oP '(?<=dns_cloudflare_api_token\s=\s).*' /root/.certbot-internal | tr -d "\"' ")
        [ -n "$TOKEN" ] || { echo "no cloudflare token in /root/.certbot-internal" >&2; exit 1; }
        api() { curl -sS -H "Authorization: Bearer $TOKEN" "$@"; }

        zid=$(api "https://api.cloudflare.com/client/v4/zones?name=${PUBLIC_NAME}"               | python3 -c 'import sys,json; r=json.load(sys.stdin)["result"]; print(r[0]["id"] if r else "")')
        [ -n "$zid" ] || { echo "zone $PUBLIC_NAME is not on this Cloudflare account" >&2; exit 1; }

        # List the name's FULL record set first, all types. Filtering by the one
        # type you expect and finding nothing does not mean the name is free —
        # it may be a CNAME, and adding an A beside it is a conflict at best
        # (architecture/public-dns.md §4).
        existing=$(api "https://api.cloudflare.com/client/v4/zones/${zid}/dns_records?name=${PUBLIC_NAME}")
        echo "$existing" | python3 -c '
import sys, json
rs = json.load(sys.stdin)["result"]
print(f"existing records for the apex: {len(rs)}")
for r in rs:
    print("  %s %s -> %s proxied=%s" % (r["type"], r["name"], r["content"], r["proxied"]))
'
        current=$(echo "$existing" | python3 -c '
import sys, json
rs = json.load(sys.stdin)["result"]
cn = [r for r in rs if r["type"] == "CNAME"]
print(cn[0]["content"] if cn else "")
')
        if [ "$current" = "$SITE" ]; then
            echo "apex already CNAMEs to $SITE — nothing to do"
            exit 0
        fi
        if [ -n "$current" ]; then
            echo "apex already CNAMEs to $current, not $SITE." >&2
            echo "Refusing to repoint an existing record; change it deliberately." >&2
            exit 1
        fi
        if echo "$existing" | grep -q '"type"'; then
            # Records exist but none is a CNAME. A CNAME cannot coexist with
            # other types at a name, so this needs a human decision.
            echo "apex carries non-CNAME records; refusing to add a CNAME beside them." >&2
            exit 1
        fi

        # Cloudflare flattens an apex CNAME, answering with an A — which is what
        # makes this convention work at a zone apex at all. It is
        # Cloudflare-specific, not portable DNS.
        api -X POST "https://api.cloudflare.com/client/v4/zones/${zid}/dns_records"             -H 'Content-Type: application/json'             --data "$(python3 -c '
import json, os
print(json.dumps({"type":"CNAME","name":os.environ["PUBLIC_NAME"],"content":os.environ["SITE"],
                  "ttl":1,"proxied":False,"comment":"Quantus-Network/blackbeard.observer"}))')"         | python3 -c '
import sys, json
d = json.load(sys.stdin)
print("created %s -> %s" % (d["result"]["name"], d["result"]["content"]) if d["success"]
      else "FAILED %s" % d["errors"])
import sys as s
s.exit(0 if d["success"] else 1)
'
REMOTE
}

role_cert() {
    reachable "$EDGE_HOST" || return 0
    info "$EDGE_HOST: Let's Encrypt certificate for $PUBLIC_NAME"

    # DNS-01 via the same Cloudflare credential, ECDSA, per
    # architecture/external-tls.md §1. --keep-until-expiring makes this a no-op
    # when the cert is still valid, so the role is safe to run every time.
    #
    # `sudo test`, not bare `test`: /etc/letsencrypt/live is root-only 0700 and
    # an unprivileged check silently returns false, concluding the cert is
    # missing and re-issuing on every run.
    ssh "$EDGE_HOST" "PUBLIC_NAME='$PUBLIC_NAME' CERT_EMAIL='$CERT_EMAIL' bash -euo pipefail" <<'REMOTE'
        if sudo test -d "/etc/letsencrypt/live/${PUBLIC_NAME}"; then
            echo "certificate lineage ${PUBLIC_NAME} already exists"
        fi
        sudo certbot certonly             -m "$CERT_EMAIL" --agree-tos --no-eff-email --noninteractive             --cert-name "$PUBLIC_NAME"             --key-type ecdsa             --dns-cloudflare             --dns-cloudflare-credentials /root/.certbot-internal             --dns-cloudflare-propagation-seconds 60             --keep-until-expiring             -d "$PUBLIC_NAME"
        sudo test -f "/etc/letsencrypt/live/${PUBLIC_NAME}/fullchain.pem"             || { echo "certbot reported success but no fullchain.pem exists" >&2; exit 1; }

        # One deploy hook per host, not per cert. Never silence it: a reload is
        # only a request, and nginx exits 0 having kept its old certificates
        # whenever it cannot re-acquire a socket. On a 90-day public cert a
        # stuck reload takes months to become visible.
        hook=/etc/letsencrypt/renewal-hooks/deploy/reload-nginx.sh
        if ! sudo test -x "$hook"; then
            sudo install -d -m 0755 /etc/letsencrypt/renewal-hooks/deploy
            printf '%s\n' '#!/bin/sh'                 'systemctl reload nginx || logger -t reload-nginx -p daemon.err \'                 '    "nginx reload failed after certbot renewal; cert on disk is newer than the one served"'                 | sudo tee "$hook" > /dev/null
            sudo chmod +x "$hook"
            echo "installed the certbot deploy hook"
        else
            echo "certbot deploy hook already present"
        fi
        systemctl is-enabled certbot-renew.timer >/dev/null 2>&1             && echo "certbot-renew.timer is enabled"             || echo "WARNING: certbot-renew.timer is not enabled on this host"
REMOTE
}

role_database() {
    reachable "$PG_PRIMARY" || return 0

    info "$PG_PRIMARY: roles and database"
    # Roles and databases are created on the PRIMARY only — replication carries
    # them to the standby.
    ssh "$PG_PRIMARY" sudo -u postgres bash -euo pipefail <<REMOTE
        if psql -tAc "select 1 from pg_database where datname = '$DB_NAME'" | grep -q 1; then
            echo "database $DB_NAME already exists"
        else
            # createdb, not the bootstrap SQL: create database cannot run inside
            # a transaction or a DO block.
            psql -c "create role $DB_ROLE with login" || true
            createdb -O "$DB_ROLE" "$DB_NAME"
        fi
REMOTE
    rsync -a --rsync-path 'sudo rsync' "$REPO_ROOT/asset/sql/bootstrap.sql" \
        "$PG_PRIMARY:/tmp/blackbeard-bootstrap.sql"
    ssh "$PG_PRIMARY" "sudo -u postgres psql -v ON_ERROR_STOP=1 -f /tmp/blackbeard-bootstrap.sql && sudo rm -f /tmp/blackbeard-bootstrap.sql"

    # The CN → role mapping, on BOTH servers. pg_ident.conf contents are not
    # replicated, and a failover to a server missing this mapping locks the app
    # out entirely — with an authentication error that looks like a certificate
    # problem.
    local api_fqdn
    api_fqdn=$(ssh "$API_HOST" hostname -f 2>/dev/null || echo "$API_HOST")
    for server in "$PG_PRIMARY" "$PG_STANDBY"; do
        reachable "$server" || continue
        info "$server: pg_ident mapping for $api_fqdn -> $DB_ROLE"
        printf 'cert_cn %s %s\n' "$api_fqdn" "$DB_ROLE" | \
            ssh "$server" "sudo install -d -m 0700 -o postgres -g postgres /var/lib/pgsql/$PG_VERSION/data/pg_ident.conf.d && sudo tee /var/lib/pgsql/$PG_VERSION/data/pg_ident.conf.d/$api_fqdn.conf > /dev/null"
        ssh "$server" "sudo chown postgres:postgres /var/lib/pgsql/$PG_VERSION/data/pg_ident.conf.d/$api_fqdn.conf"
        # Reload, not restart: pg_ident is re-read on SIGHUP.
        ssh "$server" "sudo systemctl reload postgresql-$PG_VERSION"
    done

    info "verifying the mapping from $API_HOST"
    if reachable "$API_HOST"; then
        # `psql` is not installed on an app host and there is no reason for it
        # to be, so its absence must not be reported as an authentication
        # failure — that reads as a broken mapping and sends you looking in the
        # wrong place. When it is missing, the deploy's own health probe is the
        # verification: the daemon connects with exactly these credentials.
        if ssh "$API_HOST" "command -v psql >/dev/null"; then
            ssh "$API_HOST" "sudo -u blackbeard psql 'host=$PG_PRIMARY port=5432 dbname=$DB_NAME user=$DB_ROLE sslmode=verify-full sslrootcert=/etc/pki/ca-trust/source/anchors/root-internal.pem sslcert=/etc/pki/tls/misc/\$(hostname -f).pem sslkey=/etc/pki/tls/private/\$(hostname -f).pem' -tAc 'select current_user'" \
                || warn "the app host could not authenticate to $PG_PRIMARY — check the CN mapping and the key ACL"
        else
            info "psql absent on $API_HOST; the deploy's health probe verifies the connection instead"
        fi
    fi

    # Not a warning to be dismissed. pg_ident.conf contents are NOT replicated,
    # so a standby that never received this mapping will refuse the app the
    # moment it is promoted — with an authentication error that looks like a
    # certificate problem and arrives during an outage.
    if ! ssh -o ConnectTimeout=8 -o BatchMode=yes "$PG_STANDBY" true 2>/dev/null; then
        warn "$PG_STANDBY did not receive the pg_ident mapping."
        warn "Re-run: ./script/infra-setup.sh --role database   once it is reachable."
        warn "Until then a failover to it will lock blackbeard-api out of the database."
    fi
}

# --- run ----------------------------------------------------------------------

info "roles: $ROLES"
has_role api      && role_api
# dns before cert (DNS-01 needs the zone), cert before edge (nginx -t fails on
# a missing ssl_certificate and blocks every reload on the proxy).
has_role dns      && role_dns
has_role cert     && role_cert
has_role edge     && role_edge
has_role database && role_database
info "done"