rob thijssen
36a10ac691
`CorsLayer::allow_origin` replaces rather than appends, so folding over the configured list left only `blackbeard.internal` allowed and silently refused `blackbeard.observer` — the site's own public origin. It broke nothing, because the frontend is served same-origin and never consults CORS, which is precisely why it would have gone unnoticed until something else called the API. Verified on the deployed vhost: both configured origins are now echoed back and an unlisted one is refused. Also records the deployment gotchas this session turned up (exact-argument sudoers matching, the runas spec for the config check, the cross-site hop the loopback probe cannot see, and why a WebSocket upgrade test needs --http1.1). Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01MSDYiibCtELsrjQq6KXnoi
484 lines
24 KiB
Bash
Executable File
484 lines
24 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
#
|
|
# One-time host provisioning for blackbeard.observer.
|
|
#
|
|
# Run by an OPERATOR from a workstation with full sudo ssh to the targets — not
|
|
# by CI. The runner deploys as a scoped `gitea_ci` user and deliberately has no
|
|
# rights to create accounts, read certificate keys, or reload nginx on a shared
|
|
# edge proxy.
|
|
#
|
|
# ./script/infra-setup.sh --pubkey ~/.ssh/id_gitea_ci.pub
|
|
#
|
|
# Idempotent, and it skips past unreachable hosts so one offline node does not
|
|
# block the rest. Re-run it whenever the deploy gains a new file to ship: every
|
|
# deploy job preflights the target's sudoers against the grants below and fails
|
|
# up front naming what is missing, so the two cannot silently drift.
|
|
#
|
|
# Roles (all run by default; pass --role to narrow):
|
|
#
|
|
# api the host running blackbeard-api beside quantus-node
|
|
# dns the public apex CNAME on Cloudflare
|
|
# cert the Let's Encrypt certificate for the public name
|
|
# edge the site's nginx proxy: vhosts, webroot, internal cert
|
|
# database Postgres roles, database, and the pg_ident CN mapping
|
|
#
|
|
# `dns` and `cert` run BEFORE `edge`, and in that order: certbot uses a DNS-01
|
|
# challenge, and nginx -t fails on a missing ssl_certificate — which blocks
|
|
# every reload on the proxy, not just this vhost.
|
|
#
|
|
# Conventions: architecture/generic.md §8-§11, deployment-gitea-actions.md.
|
|
|
|
set -euo pipefail
|
|
|
|
# --- infra truth, matching .gitea/workflows/deploy.yaml -----------------------
|
|
API_HOST="${API_HOST:-bob.hanzalova.internal}"
|
|
API_PORT="${API_PORT:-25864}"
|
|
EDGE_HOST="${EDGE_HOST:-oolon.kosherinata.internal}"
|
|
PG_PRIMARY="${PG_PRIMARY:-magrathea.kosherinata.internal}"
|
|
# The standby needs the same ident mapping: pg_ident.conf contents are NOT
|
|
# replicated, and a failover to a server missing it locks the app out.
|
|
PG_STANDBY="${PG_STANDBY:-frankie.hanzalova.internal}"
|
|
PG_VERSION="${PG_VERSION:-18}"
|
|
WEBROOT="${WEBROOT:-/var/www/blackbeard.observer}"
|
|
PUBLIC_NAME="${PUBLIC_NAME:-blackbeard.observer}"
|
|
INTERNAL_NAME="${INTERNAL_NAME:-blackbeard.internal}"
|
|
DB_NAME="${DB_NAME:-blackbeard}"
|
|
DB_ROLE="${DB_ROLE:-blackbeard_rw}"
|
|
# The per-site indirection name the public record CNAMEs to. `bl` is the DC
|
|
# (oolon), `nh` the office (hanzalova) — architecture/public-dns.md §2. A vhost
|
|
# record must never carry a site address directly: the address changes and every
|
|
# hardcoded A record then has to be hunted down across zones.
|
|
SITE_INDIRECTION="${SITE_INDIRECTION:-bl.thgttg.com}"
|
|
CERT_EMAIL="${CERT_EMAIL:-ops@blackbeard.observer}"
|
|
|
|
PUBKEY=""
|
|
ROLES="api dns cert edge database"
|
|
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
|
|
info() { printf '\033[36m==\033[0m %s\n' "$*"; }
|
|
warn() { printf '\033[33m!!\033[0m %s\n' "$*" >&2; }
|
|
die() { printf '\033[31mXX\033[0m %s\n' "$*" >&2; exit 1; }
|
|
|
|
usage() {
|
|
sed -n '3,30p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'
|
|
exit "${1:-0}"
|
|
}
|
|
|
|
while [ $# -gt 0 ]; do
|
|
case "$1" in
|
|
--pubkey) PUBKEY="$2"; shift 2 ;;
|
|
--role) ROLES="$2"; shift 2 ;;
|
|
-h|--help) usage 0 ;;
|
|
*) die "unknown argument: $1 (try --help)" ;;
|
|
esac
|
|
done
|
|
|
|
has_role() { [[ " $ROLES " == *" $1 "* ]]; }
|
|
|
|
# Reachability is checked once per host and the result reused, so an offline
|
|
# host produces one clear message rather than a failure per step.
|
|
reachable() {
|
|
local host="$1"
|
|
if ssh -o ConnectTimeout=8 -o BatchMode=yes "$host" true; then
|
|
return 0
|
|
fi
|
|
warn "$host is unreachable — skipping its steps"
|
|
return 1
|
|
}
|
|
|
|
# --- the scoped sudoers grants ------------------------------------------------
|
|
#
|
|
# These strings are the single source of truth for what CI may do on each host.
|
|
# The deploy workflow's preflight extracts the paths from THIS FILE and compares
|
|
# them against `sudo -n -l` on the target, so adding a file to the deploy means
|
|
# adding a line here and re-running this script — nothing else keeps them in
|
|
# step.
|
|
#
|
|
# `:` and `=` are reserved in sudoers and must be escaped inside command
|
|
# arguments, or visudo rejects the file.
|
|
|
|
api_sudoers() {
|
|
cat <<'EOF'
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /usr/local/bin/blackbeard-api
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /usr/local/bin/blackbeard
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/blackbeard/config.toml
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/sysusers.d/blackbeard.conf
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/systemd/system/blackbeard-api.service
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/systemd/system/blackbeard-api-cert.path
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/systemd/system/blackbeard-api-cert-reload.service
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/firewalld/services/blackbeard-api.xml
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemd-sysusers
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/bin/install -d -o root -g blackbeard -m 0750 /etc/blackbeard
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/bin/setfacl -m u\:blackbeard\:r /etc/pki/tls/private/*
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/sbin/restorecon -R /usr/local/bin/blackbeard-api /usr/local/bin/blackbeard /etc/blackbeard
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/sbin/semanage port -l
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/sbin/semanage port -a -t http_port_t -p tcp 25864
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/bin/firewall-cmd --reload
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/bin/firewall-cmd --get-default-zone
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/bin/firewall-cmd --zone=* --query-service=blackbeard-api
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/bin/firewall-cmd --permanent --zone=* --add-service=blackbeard-api
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/bin/firewall-cmd --zone=* --add-service=blackbeard-api
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl daemon-reload
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl enable blackbeard-api.service
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl enable --now blackbeard-api-cert.path
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl restart blackbeard-api.service
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl is-active blackbeard-api.service
|
|
# Runas is (blackbeard), not (root): the deploy validates the config AS the
|
|
# service account, which is the only way to prove the account can actually read
|
|
# the config and the certificate key it names. A (root) grant would not permit
|
|
# `sudo -u blackbeard` at all, and a root-run check would pass on a config the
|
|
# service cannot read.
|
|
gitea_ci ALL=(blackbeard) NOPASSWD: /usr/local/bin/blackbeard-api --config /etc/blackbeard/config.toml --check
|
|
EOF
|
|
}
|
|
|
|
edge_sudoers() {
|
|
cat <<'EOF'
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /var/www/blackbeard.observer/
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/sbin/restorecon -R /var/www/blackbeard.observer
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/sbin/nginx -t
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl reload nginx
|
|
EOF
|
|
}
|
|
|
|
# --- gitea_ci -----------------------------------------------------------------
|
|
|
|
provision_gitea_ci() {
|
|
local host="$1" sudoers_body="$2"
|
|
|
|
[ -n "$PUBKEY" ] || die "--pubkey is required to provision gitea_ci (the runner's public key)"
|
|
[ -f "$PUBKEY" ] || die "$PUBKEY does not exist"
|
|
|
|
info "$host: gitea_ci account"
|
|
# A real shell, never nologin. The deploy runs `ssh gitea_ci@host <command>`;
|
|
# a nologin shell authenticates the key and then refuses the command with
|
|
# "This account is currently not available", which reads as an auth problem
|
|
# rather than a shell one. Every pre-existing gitea_ci on the fleet has bash,
|
|
# and one provisioned with nologin can never be deployed to — so repair it
|
|
# rather than leaving it.
|
|
ssh "$host" sudo bash -euo pipefail <<'REMOTE'
|
|
if ! id gitea_ci >/dev/null 2>&1; then
|
|
useradd --system --create-home --home-dir /var/lib/gitea_ci --shell /bin/bash gitea_ci
|
|
fi
|
|
current=$(getent passwd gitea_ci | cut -d: -f7)
|
|
if [ "$current" != "/bin/bash" ]; then
|
|
echo "repairing gitea_ci shell: $current -> /bin/bash"
|
|
usermod --shell /bin/bash gitea_ci
|
|
fi
|
|
install -d -o gitea_ci -g gitea_ci -m 0700 /var/lib/gitea_ci/.ssh
|
|
# Lets the deploy capture `journalctl -u <unit>` after a restart without
|
|
# a sudoers entry for it.
|
|
usermod -aG systemd-journal gitea_ci
|
|
REMOTE
|
|
|
|
rsync -a --chown gitea_ci:gitea_ci --chmod 0600 --rsync-path 'sudo rsync' \
|
|
"$PUBKEY" "$host:/var/lib/gitea_ci/.ssh/authorized_keys"
|
|
|
|
info "$host: scoped sudoers"
|
|
# Named <app>_gitea_ci, not bare gitea_ci, so several apps can drop their own
|
|
# files on a shared host without clobbering each other.
|
|
printf '%s\n' "$sudoers_body" | \
|
|
ssh "$host" 'sudo tee /etc/sudoers.d/blackbeard_gitea_ci > /dev/null && sudo chmod 0440 /etc/sudoers.d/blackbeard_gitea_ci'
|
|
# Verify before leaving: a syntax error in a sudoers drop-in can lock every
|
|
# sudo on the host, not just this one.
|
|
ssh "$host" sudo visudo -cf /etc/sudoers.d/blackbeard_gitea_ci
|
|
}
|
|
|
|
# --- roles --------------------------------------------------------------------
|
|
|
|
role_api() {
|
|
reachable "$API_HOST" || return 0
|
|
provision_gitea_ci "$API_HOST" "$(api_sudoers)"
|
|
|
|
info "$API_HOST: service account and directories"
|
|
rsync -a --rsync-path 'sudo rsync' \
|
|
"$REPO_ROOT/asset/systemd/blackbeard.sysusers.conf" \
|
|
"$API_HOST:/etc/sysusers.d/blackbeard.conf"
|
|
ssh "$API_HOST" sudo bash -euo pipefail <<'REMOTE'
|
|
systemd-sysusers
|
|
install -d -o root -g blackbeard -m 0750 /etc/blackbeard
|
|
install -d -o blackbeard -g blackbeard -m 0750 /var/lib/blackbeard
|
|
# The mTLS credential for Postgres. The key is not world-readable; the
|
|
# service account is granted read access here and again on every deploy,
|
|
# because a certificate rotation replaces the file and drops the ACL.
|
|
setfacl -m "u:blackbeard:r" "/etc/pki/tls/private/$(hostname -f).pem"
|
|
REMOTE
|
|
|
|
info "$API_HOST: SELinux port label"
|
|
ssh "$API_HOST" sudo bash -euo pipefail <<REMOTE
|
|
if semanage port -l | grep -qE "^http_port_t.*\b$API_PORT\b"; then
|
|
echo "port $API_PORT already labelled http_port_t"
|
|
else
|
|
semanage port -a -t http_port_t -p tcp "$API_PORT"
|
|
fi
|
|
REMOTE
|
|
}
|
|
|
|
role_edge() {
|
|
reachable "$EDGE_HOST" || return 0
|
|
provision_gitea_ci "$EDGE_HOST" "$(edge_sudoers)"
|
|
|
|
info "$EDGE_HOST: webroot"
|
|
ssh "$EDGE_HOST" sudo bash -euo pipefail <<REMOTE
|
|
install -d -o root -g root -m 0755 "$WEBROOT"
|
|
# An unlabelled webroot makes nginx return 403 for every file, with
|
|
# nothing in the nginx error log to explain it.
|
|
restorecon -R "$WEBROOT"
|
|
REMOTE
|
|
|
|
info "$EDGE_HOST: internal certificate for $INTERNAL_NAME"
|
|
# Minted from the internal step-ca and renewed by a templated step@ unit
|
|
# (architecture/internal-tls.md). Certs are issued with a 24-hour expiry, so
|
|
# the timer is not optional.
|
|
ssh "$EDGE_HOST" sudo bash -euo pipefail <<REMOTE
|
|
install -d -m 0755 /etc/nginx/tls/cert
|
|
install -d -m 0700 /etc/nginx/tls/key
|
|
if [ -f "/etc/nginx/tls/cert/$INTERNAL_NAME.pem" ]; then
|
|
echo "$INTERNAL_NAME certificate present"
|
|
systemctl enable --now "step@blackbeard.timer" || \
|
|
echo "step@ renewal timer not armed — renew $INTERNAL_NAME manually until it is"
|
|
else
|
|
# Not fatal. The mesh vhost is a convenience; the public site is the
|
|
# deliverable, and nginx -t fails on a missing ssl_certificate —
|
|
# which would block every reload on this proxy, not just ours.
|
|
echo "NOTE: no internal certificate for $INTERNAL_NAME; the mesh vhost will be skipped."
|
|
echo " Mint it with the lair provisioner (architecture/internal-tls.md §4)"
|
|
echo " and re-run --role edge to enable it."
|
|
fi
|
|
REMOTE
|
|
|
|
info "$EDGE_HOST: nginx configuration"
|
|
rsync -a --rsync-path 'sudo rsync' \
|
|
"$REPO_ROOT/asset/nginx/blackbeard-upstream.conf" \
|
|
"$EDGE_HOST:/etc/nginx/conf.d/blackbeard-upstream.conf"
|
|
rsync -a --rsync-path 'sudo rsync' \
|
|
"$REPO_ROOT/asset/nginx/$PUBLIC_NAME.conf" \
|
|
"$REPO_ROOT/asset/nginx/$INTERNAL_NAME.conf" \
|
|
"$EDGE_HOST:/etc/nginx/sites-available/"
|
|
ssh "$EDGE_HOST" sudo bash -euo pipefail <<REMOTE
|
|
# sites-enabled holds only symlinks, and relative ones.
|
|
ln -sfn "../sites-available/$PUBLIC_NAME.conf" "/etc/nginx/sites-enabled/$PUBLIC_NAME.conf"
|
|
if [ -f "/etc/nginx/tls/cert/$INTERNAL_NAME.pem" ]; then
|
|
ln -sfn "../sites-available/$INTERNAL_NAME.conf" "/etc/nginx/sites-enabled/$INTERNAL_NAME.conf"
|
|
else
|
|
rm -f "/etc/nginx/sites-enabled/$INTERNAL_NAME.conf"
|
|
fi
|
|
|
|
# The vhosts use \$connection_upgrade for the WebSocket upgrade; without
|
|
# the map the socket silently degrades to a hanging request and the page
|
|
# shows "reconnecting" forever with nothing in any log.
|
|
if ! grep -rqs 'connection_upgrade' /etc/nginx/conf.d/ /etc/nginx/nginx.conf; then
|
|
cat > /etc/nginx/conf.d/websocket-upgrade.conf <<'MAP'
|
|
map \$http_upgrade \$connection_upgrade {
|
|
default upgrade;
|
|
'' close;
|
|
}
|
|
MAP
|
|
echo "installed the connection_upgrade map"
|
|
else
|
|
echo "connection_upgrade map already present"
|
|
fi
|
|
|
|
# nginx -t parses without binding, so it catches syntax and missing
|
|
# certs but not a port owned across http{} and stream{}. Verify the
|
|
# reload landed rather than trusting the test.
|
|
nginx -t
|
|
systemctl reload nginx
|
|
sleep 1
|
|
systemctl is-active --quiet nginx || { echo "nginx did not come back after reload" >&2; exit 1; }
|
|
REMOTE
|
|
|
|
cat <<EOF
|
|
|
|
$EDGE_HOST is configured. One step is NOT automated here:
|
|
|
|
Split-horizon DNS for $INTERNAL_NAME, on BOTH site routers. A record on
|
|
only one router NXDOMAINs everywhere else:
|
|
|
|
for site in hanzalova kosherinata; do
|
|
opn-cli --config ~/.opn-cli/\$site.yml unbound host create \\
|
|
--hostname blackbeard --domain internal --rr A --server <edge mesh ip>
|
|
done
|
|
|
|
\`create\` only saves; POST /api/unbound/service/reconfigure on each router
|
|
to apply.
|
|
|
|
EOF
|
|
}
|
|
|
|
# The Cloudflare API token lives on the edge proxy and is never copied off it:
|
|
# it can rewrite DNS for every zone on the account, including MX records for
|
|
# domains whose mail we host (architecture/public-dns.md §1). Every call that
|
|
# needs it therefore runs ON the proxy.
|
|
role_dns() {
|
|
reachable "$EDGE_HOST" || return 0
|
|
info "$EDGE_HOST: public DNS for $PUBLIC_NAME"
|
|
|
|
ssh "$EDGE_HOST" "PUBLIC_NAME='$PUBLIC_NAME' SITE='$SITE_INDIRECTION' bash -euo pipefail" <<'REMOTE'
|
|
TOKEN=$(sudo grep -oP '(?<=dns_cloudflare_api_token\s=\s).*' /root/.certbot-internal | tr -d "\"' ")
|
|
[ -n "$TOKEN" ] || { echo "no cloudflare token in /root/.certbot-internal" >&2; exit 1; }
|
|
api() { curl -sS -H "Authorization: Bearer $TOKEN" "$@"; }
|
|
|
|
zid=$(api "https://api.cloudflare.com/client/v4/zones?name=${PUBLIC_NAME}" | python3 -c 'import sys,json; r=json.load(sys.stdin)["result"]; print(r[0]["id"] if r else "")')
|
|
[ -n "$zid" ] || { echo "zone $PUBLIC_NAME is not on this Cloudflare account" >&2; exit 1; }
|
|
|
|
# List the name's FULL record set first, all types. Filtering by the one
|
|
# type you expect and finding nothing does not mean the name is free —
|
|
# it may be a CNAME, and adding an A beside it is a conflict at best
|
|
# (architecture/public-dns.md §4).
|
|
existing=$(api "https://api.cloudflare.com/client/v4/zones/${zid}/dns_records?name=${PUBLIC_NAME}")
|
|
echo "$existing" | python3 -c '
|
|
import sys, json
|
|
rs = json.load(sys.stdin)["result"]
|
|
print(f"existing records for the apex: {len(rs)}")
|
|
for r in rs:
|
|
print(" %s %s -> %s proxied=%s" % (r["type"], r["name"], r["content"], r["proxied"]))
|
|
'
|
|
current=$(echo "$existing" | python3 -c '
|
|
import sys, json
|
|
rs = json.load(sys.stdin)["result"]
|
|
cn = [r for r in rs if r["type"] == "CNAME"]
|
|
print(cn[0]["content"] if cn else "")
|
|
')
|
|
if [ "$current" = "$SITE" ]; then
|
|
echo "apex already CNAMEs to $SITE — nothing to do"
|
|
exit 0
|
|
fi
|
|
if [ -n "$current" ]; then
|
|
echo "apex already CNAMEs to $current, not $SITE." >&2
|
|
echo "Refusing to repoint an existing record; change it deliberately." >&2
|
|
exit 1
|
|
fi
|
|
if echo "$existing" | grep -q '"type"'; then
|
|
# Records exist but none is a CNAME. A CNAME cannot coexist with
|
|
# other types at a name, so this needs a human decision.
|
|
echo "apex carries non-CNAME records; refusing to add a CNAME beside them." >&2
|
|
exit 1
|
|
fi
|
|
|
|
# Cloudflare flattens an apex CNAME, answering with an A — which is what
|
|
# makes this convention work at a zone apex at all. It is
|
|
# Cloudflare-specific, not portable DNS.
|
|
api -X POST "https://api.cloudflare.com/client/v4/zones/${zid}/dns_records" -H 'Content-Type: application/json' --data "$(python3 -c '
|
|
import json, os
|
|
print(json.dumps({"type":"CNAME","name":os.environ["PUBLIC_NAME"],"content":os.environ["SITE"],
|
|
"ttl":1,"proxied":False,"comment":"Quantus-Network/blackbeard.observer"}))')" | python3 -c '
|
|
import sys, json
|
|
d = json.load(sys.stdin)
|
|
print("created %s -> %s" % (d["result"]["name"], d["result"]["content"]) if d["success"]
|
|
else "FAILED %s" % d["errors"])
|
|
import sys as s
|
|
s.exit(0 if d["success"] else 1)
|
|
'
|
|
REMOTE
|
|
}
|
|
|
|
role_cert() {
|
|
reachable "$EDGE_HOST" || return 0
|
|
info "$EDGE_HOST: Let's Encrypt certificate for $PUBLIC_NAME"
|
|
|
|
# DNS-01 via the same Cloudflare credential, ECDSA, per
|
|
# architecture/external-tls.md §1. --keep-until-expiring makes this a no-op
|
|
# when the cert is still valid, so the role is safe to run every time.
|
|
#
|
|
# `sudo test`, not bare `test`: /etc/letsencrypt/live is root-only 0700 and
|
|
# an unprivileged check silently returns false, concluding the cert is
|
|
# missing and re-issuing on every run.
|
|
ssh "$EDGE_HOST" "PUBLIC_NAME='$PUBLIC_NAME' CERT_EMAIL='$CERT_EMAIL' bash -euo pipefail" <<'REMOTE'
|
|
if sudo test -d "/etc/letsencrypt/live/${PUBLIC_NAME}"; then
|
|
echo "certificate lineage ${PUBLIC_NAME} already exists"
|
|
fi
|
|
sudo certbot certonly -m "$CERT_EMAIL" --agree-tos --no-eff-email --noninteractive --cert-name "$PUBLIC_NAME" --key-type ecdsa --dns-cloudflare --dns-cloudflare-credentials /root/.certbot-internal --dns-cloudflare-propagation-seconds 60 --keep-until-expiring -d "$PUBLIC_NAME"
|
|
sudo test -f "/etc/letsencrypt/live/${PUBLIC_NAME}/fullchain.pem" || { echo "certbot reported success but no fullchain.pem exists" >&2; exit 1; }
|
|
|
|
# One deploy hook per host, not per cert. Never silence it: a reload is
|
|
# only a request, and nginx exits 0 having kept its old certificates
|
|
# whenever it cannot re-acquire a socket. On a 90-day public cert a
|
|
# stuck reload takes months to become visible.
|
|
hook=/etc/letsencrypt/renewal-hooks/deploy/reload-nginx.sh
|
|
if ! sudo test -x "$hook"; then
|
|
sudo install -d -m 0755 /etc/letsencrypt/renewal-hooks/deploy
|
|
printf '%s\n' '#!/bin/sh' 'systemctl reload nginx || logger -t reload-nginx -p daemon.err \' ' "nginx reload failed after certbot renewal; cert on disk is newer than the one served"' | sudo tee "$hook" > /dev/null
|
|
sudo chmod +x "$hook"
|
|
echo "installed the certbot deploy hook"
|
|
else
|
|
echo "certbot deploy hook already present"
|
|
fi
|
|
systemctl is-enabled certbot-renew.timer >/dev/null 2>&1 && echo "certbot-renew.timer is enabled" || echo "WARNING: certbot-renew.timer is not enabled on this host"
|
|
REMOTE
|
|
}
|
|
|
|
role_database() {
|
|
reachable "$PG_PRIMARY" || return 0
|
|
|
|
info "$PG_PRIMARY: roles and database"
|
|
# Roles and databases are created on the PRIMARY only — replication carries
|
|
# them to the standby.
|
|
ssh "$PG_PRIMARY" sudo -u postgres bash -euo pipefail <<REMOTE
|
|
if psql -tAc "select 1 from pg_database where datname = '$DB_NAME'" | grep -q 1; then
|
|
echo "database $DB_NAME already exists"
|
|
else
|
|
# createdb, not the bootstrap SQL: create database cannot run inside
|
|
# a transaction or a DO block.
|
|
psql -c "create role $DB_ROLE with login" || true
|
|
createdb -O "$DB_ROLE" "$DB_NAME"
|
|
fi
|
|
REMOTE
|
|
rsync -a --rsync-path 'sudo rsync' "$REPO_ROOT/asset/sql/bootstrap.sql" \
|
|
"$PG_PRIMARY:/tmp/blackbeard-bootstrap.sql"
|
|
ssh "$PG_PRIMARY" "sudo -u postgres psql -v ON_ERROR_STOP=1 -f /tmp/blackbeard-bootstrap.sql && sudo rm -f /tmp/blackbeard-bootstrap.sql"
|
|
|
|
# The CN → role mapping, on BOTH servers. pg_ident.conf contents are not
|
|
# replicated, and a failover to a server missing this mapping locks the app
|
|
# out entirely — with an authentication error that looks like a certificate
|
|
# problem.
|
|
local api_fqdn
|
|
api_fqdn=$(ssh "$API_HOST" hostname -f 2>/dev/null || echo "$API_HOST")
|
|
for server in "$PG_PRIMARY" "$PG_STANDBY"; do
|
|
reachable "$server" || continue
|
|
info "$server: pg_ident mapping for $api_fqdn -> $DB_ROLE"
|
|
printf 'cert_cn %s %s\n' "$api_fqdn" "$DB_ROLE" | \
|
|
ssh "$server" "sudo install -d -m 0700 -o postgres -g postgres /var/lib/pgsql/$PG_VERSION/data/pg_ident.conf.d && sudo tee /var/lib/pgsql/$PG_VERSION/data/pg_ident.conf.d/$api_fqdn.conf > /dev/null"
|
|
ssh "$server" "sudo chown postgres:postgres /var/lib/pgsql/$PG_VERSION/data/pg_ident.conf.d/$api_fqdn.conf"
|
|
# Reload, not restart: pg_ident is re-read on SIGHUP.
|
|
ssh "$server" "sudo systemctl reload postgresql-$PG_VERSION"
|
|
done
|
|
|
|
info "verifying the mapping from $API_HOST"
|
|
if reachable "$API_HOST"; then
|
|
# `psql` is not installed on an app host and there is no reason for it
|
|
# to be, so its absence must not be reported as an authentication
|
|
# failure — that reads as a broken mapping and sends you looking in the
|
|
# wrong place. When it is missing, the deploy's own health probe is the
|
|
# verification: the daemon connects with exactly these credentials.
|
|
if ssh "$API_HOST" "command -v psql >/dev/null"; then
|
|
ssh "$API_HOST" "sudo -u blackbeard psql 'host=$PG_PRIMARY port=5432 dbname=$DB_NAME user=$DB_ROLE sslmode=verify-full sslrootcert=/etc/pki/ca-trust/source/anchors/root-internal.pem sslcert=/etc/pki/tls/misc/\$(hostname -f).pem sslkey=/etc/pki/tls/private/\$(hostname -f).pem' -tAc 'select current_user'" \
|
|
|| warn "the app host could not authenticate to $PG_PRIMARY — check the CN mapping and the key ACL"
|
|
else
|
|
info "psql absent on $API_HOST; the deploy's health probe verifies the connection instead"
|
|
fi
|
|
fi
|
|
|
|
# Not a warning to be dismissed. pg_ident.conf contents are NOT replicated,
|
|
# so a standby that never received this mapping will refuse the app the
|
|
# moment it is promoted — with an authentication error that looks like a
|
|
# certificate problem and arrives during an outage.
|
|
if ! ssh -o ConnectTimeout=8 -o BatchMode=yes "$PG_STANDBY" true 2>/dev/null; then
|
|
warn "$PG_STANDBY did not receive the pg_ident mapping."
|
|
warn "Re-run: ./script/infra-setup.sh --role database once it is reachable."
|
|
warn "Until then a failover to it will lock blackbeard-api out of the database."
|
|
fi
|
|
}
|
|
|
|
# --- run ----------------------------------------------------------------------
|
|
|
|
info "roles: $ROLES"
|
|
has_role api && role_api
|
|
# dns before cert (DNS-01 needs the zone), cert before edge (nginx -t fails on
|
|
# a missing ssl_certificate and blocks every reload on the proxy).
|
|
has_role dns && role_dns
|
|
has_role cert && role_cert
|
|
has_role edge && role_edge
|
|
has_role database && role_database
|
|
info "done"
|