rob thijssen
110b523fd0
Wires up the prod deployment per architecture-doc conventions:
- api → nikola.kosherinata.internal, loopback bind 127.0.0.1:42424
(less-common port, registered with SELinux as http_port_t).
- worker → frootmig.kosherinata.internal, no listening port.
- web (static ui/dist + nginx server_name rob.tn) → nikola, with
/api/* reverse-proxied to the loopback API.
- db → existing magrathea cluster via mTLS, hostname-baked DATABASE_URL
rendered into /etc/moments/{api,worker}.env at deploy time.
Cert rotation: step-ca renews host certs every 24h; .path units watch
/etc/pki/tls/misc/<host>.pem and trigger systemctl restart of the
relevant service. Both binaries hold cert state in rustls and read
once at startup, so restart is the right reload semantics.
deploy.sh contract matches the architecture doc: positional env arg,
component list (or `all` / `default`), --dry-run support. Renders
config templates from `pass`, rsyncs over ssh+sudo, runs sysusers /
restorecon / semanage / systemctl / nginx -t idempotently.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
327 lines
10 KiB
Bash
Executable File
327 lines
10 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
#
|
|
# moments deployment script.
|
|
#
|
|
# ./script/deploy.sh <environment> [component...]
|
|
# ./script/deploy.sh prod api worker web
|
|
# ./script/deploy.sh prod all
|
|
#
|
|
# Builds artifacts locally, resolves secrets from `pass`, renders config
|
|
# templates, rsyncs everything to the target hosts, and reloads systemd /
|
|
# nginx / firewalld / SELinux state idempotently.
|
|
|
|
set -euo pipefail
|
|
shopt -s nullglob
|
|
|
|
repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
manifest="${repo_root}/asset/manifest.yml"
|
|
dry_run=0
|
|
|
|
usage() {
|
|
cat <<EOF >&2
|
|
usage: $(basename "$0") <environment> [component...] [--dry-run]
|
|
$(basename "$0") prod api worker web
|
|
$(basename "$0") prod all
|
|
$(basename "$0") prod default # api + web (worker isn't restarted unless asked)
|
|
EOF
|
|
exit 2
|
|
}
|
|
|
|
log() { printf '\033[1;34m[deploy]\033[0m %s\n' "$*" >&2; }
|
|
warn() { printf '\033[1;33m[deploy]\033[0m %s\n' "$*" >&2; }
|
|
die() { printf '\033[1;31m[deploy]\033[0m %s\n' "$*" >&2; exit 1; }
|
|
|
|
run() {
|
|
if (( dry_run )); then
|
|
printf '\033[2m[dry-run]\033[0m %s\n' "$*" >&2
|
|
else
|
|
"$@"
|
|
fi
|
|
}
|
|
|
|
ssh_run() {
|
|
local host="$1"; shift
|
|
if (( dry_run )); then
|
|
printf '\033[2m[dry-run]\033[0m ssh %s -- %s\n' "$host" "$*" >&2
|
|
else
|
|
ssh -o BatchMode=yes "$host" "$@"
|
|
fi
|
|
}
|
|
|
|
[[ $# -ge 1 ]] || usage
|
|
environment="$1"; shift
|
|
components=()
|
|
while [[ $# -gt 0 ]]; do
|
|
case "$1" in
|
|
--dry-run) dry_run=1 ;;
|
|
*) components+=("$1") ;;
|
|
esac
|
|
shift
|
|
done
|
|
|
|
[[ -f "$manifest" ]] || die "manifest not found: $manifest"
|
|
command -v yq >/dev/null 2>&1 || die "yq is required"
|
|
command -v pass >/dev/null 2>&1 || die "pass is required"
|
|
command -v rsync >/dev/null 2>&1 || die "rsync is required"
|
|
command -v cargo >/dev/null 2>&1 || die "cargo is required"
|
|
|
|
# Resolve component list ----------------------------------------------------
|
|
|
|
env_path=".environments.${environment}"
|
|
yq -e "${env_path}" "$manifest" >/dev/null \
|
|
|| die "environment '$environment' not found in manifest"
|
|
|
|
mapfile -t all_components < <(yq -r "${env_path}.components | keys | .[]" "$manifest")
|
|
|
|
if [[ ${#components[@]} -eq 0 ]]; then
|
|
usage
|
|
fi
|
|
case "${components[0]:-}" in
|
|
all) components=("${all_components[@]}") ;;
|
|
default) components=(api web) ;;
|
|
esac
|
|
|
|
# Build artifacts -----------------------------------------------------------
|
|
|
|
needs_rust=0
|
|
needs_web=0
|
|
for c in "${components[@]}"; do
|
|
case "$c" in
|
|
api|worker) needs_rust=1 ;;
|
|
web) needs_web=1 ;;
|
|
esac
|
|
done
|
|
|
|
if (( needs_rust )); then
|
|
log "cargo build --release (api, worker)"
|
|
run cargo build --release --bin moments-api --bin moments-worker --manifest-path "${repo_root}/Cargo.toml"
|
|
fi
|
|
|
|
if (( needs_web )); then
|
|
log "vite build (ui)"
|
|
run sh -c "cd '${repo_root}/ui' && pnpm install --frozen-lockfile && pnpm run build"
|
|
fi
|
|
|
|
# Per-component deploy ------------------------------------------------------
|
|
|
|
deploy_api() {
|
|
local host="$1"
|
|
log "api -> $host"
|
|
|
|
if (( dry_run )); then
|
|
printf '\033[2m[dry-run]\033[0m render api.env (HOSTNAME=%s) + units, rsync to %s:/, run sysusers/restorecon/semanage/systemctl on %s\n' \
|
|
"$host" "$host" "$host" >&2
|
|
return 0
|
|
fi
|
|
|
|
local fqdn
|
|
fqdn="$host"
|
|
|
|
local stage
|
|
stage="$(mktemp -d)"
|
|
trap "rm -rf '$stage'" RETURN
|
|
|
|
install -d "$stage/etc/moments" "$stage/etc/systemd/system" "$stage/etc/sysusers.d" "$stage/usr/local/bin"
|
|
|
|
# Render env file with hostname substitution.
|
|
sed "s|{{HOSTNAME}}|${fqdn}|g" "${repo_root}/asset/config/api.env.tmpl" \
|
|
> "$stage/etc/moments/api.env"
|
|
|
|
sed "s|{{HOSTNAME}}|${fqdn}|g" "${repo_root}/asset/systemd/moments-api-cert.path" \
|
|
> "$stage/etc/systemd/system/moments-api-cert.path"
|
|
|
|
install -m 0644 "${repo_root}/asset/systemd/moments-api.service" "$stage/etc/systemd/system/"
|
|
install -m 0644 "${repo_root}/asset/systemd/moments-api-cert-reload.service" "$stage/etc/systemd/system/"
|
|
install -m 0644 "${repo_root}/asset/systemd/moments.sysusers.conf" "$stage/etc/sysusers.d/moments.conf"
|
|
install -m 0755 "${repo_root}/target/release/moments-api" "$stage/usr/local/bin/moments-api"
|
|
|
|
# Permissions on the rendered env: root-owned, moments group readable.
|
|
chmod 0640 "$stage/etc/moments/api.env"
|
|
|
|
if (( dry_run )); then
|
|
printf '\033[2m[dry-run]\033[0m rsync staged -> %s:/\n' "$host" >&2
|
|
else
|
|
rsync -aHAX --rsync-path="sudo rsync" "$stage/" "${host}:/"
|
|
fi
|
|
|
|
ssh_run "$host" "sudo bash -s" <<'REMOTE_EOF'
|
|
set -euo pipefail
|
|
fqdn="$(hostname -f)"
|
|
|
|
systemd-sysusers /etc/sysusers.d/moments.conf
|
|
|
|
install -d -o root -g moments -m 0750 /etc/moments
|
|
install -d -o moments -g moments -m 0750 /var/lib/moments
|
|
chown root:moments /etc/moments/api.env
|
|
chmod 0640 /etc/moments/api.env
|
|
|
|
# Grant the moments user read access to the host private key — required for
|
|
# the postgres mTLS connection.
|
|
setfacl -m u:moments:r "/etc/pki/tls/private/${fqdn}.pem" || true
|
|
|
|
# Label loopback API port. Idempotent — the -m flag turns "already labelled"
|
|
# into a no-op.
|
|
if ! semanage port -l | awk '{print $1, $3}' | grep -qE "^http_port_t .*42424"; then
|
|
semanage port -a -t http_port_t -p tcp 42424 || \
|
|
semanage port -m -t http_port_t -p tcp 42424
|
|
fi
|
|
|
|
restorecon -Rv /usr/local/bin/moments-api /etc/moments /var/lib/moments
|
|
|
|
systemctl daemon-reload
|
|
systemctl enable --now moments-api-cert.path
|
|
systemctl enable --now moments-api.service
|
|
systemctl restart moments-api.service
|
|
|
|
# Health probe.
|
|
for i in 1 2 3 4 5 6 7 8 9 10; do
|
|
if curl -fsS http://127.0.0.1:42424/v1/healthz >/dev/null; then
|
|
echo "moments-api healthy"
|
|
exit 0
|
|
fi
|
|
sleep 1
|
|
done
|
|
echo "moments-api did not become healthy" >&2
|
|
journalctl -u moments-api.service -n 50 --no-pager >&2
|
|
exit 1
|
|
REMOTE_EOF
|
|
}
|
|
|
|
deploy_worker() {
|
|
local host="$1"
|
|
log "worker -> $host"
|
|
|
|
if (( dry_run )); then
|
|
printf '\033[2m[dry-run]\033[0m render worker.env (HOSTNAME=%s, GITHUB_TOKEN from pass) + units, rsync to %s:/, run sysusers/restorecon/systemctl on %s\n' \
|
|
"$host" "$host" "$host" >&2
|
|
return 0
|
|
fi
|
|
|
|
local fqdn
|
|
fqdn="$host"
|
|
|
|
local github_token=""
|
|
if pass show github.com/grenade/admin-token >/dev/null 2>&1; then
|
|
github_token="$(pass show github.com/grenade/admin-token)"
|
|
else
|
|
warn "no github admin-token in pass; worker will run without GITHUB_TOKEN"
|
|
fi
|
|
|
|
local stage
|
|
stage="$(mktemp -d)"
|
|
trap "rm -rf '$stage'" RETURN
|
|
|
|
install -d "$stage/etc/moments" "$stage/etc/systemd/system" "$stage/etc/sysusers.d" "$stage/usr/local/bin"
|
|
|
|
sed -e "s|{{HOSTNAME}}|${fqdn}|g" \
|
|
-e "s|{{GITHUB_TOKEN}}|${github_token}|g" \
|
|
"${repo_root}/asset/config/worker.env.tmpl" > "$stage/etc/moments/worker.env"
|
|
|
|
sed "s|{{HOSTNAME}}|${fqdn}|g" "${repo_root}/asset/systemd/moments-worker-cert.path" \
|
|
> "$stage/etc/systemd/system/moments-worker-cert.path"
|
|
|
|
install -m 0644 "${repo_root}/asset/systemd/moments-worker.service" "$stage/etc/systemd/system/"
|
|
install -m 0644 "${repo_root}/asset/systemd/moments-worker-cert-reload.service" "$stage/etc/systemd/system/"
|
|
install -m 0644 "${repo_root}/asset/systemd/moments.sysusers.conf" "$stage/etc/sysusers.d/moments.conf"
|
|
install -m 0755 "${repo_root}/target/release/moments-worker" "$stage/usr/local/bin/moments-worker"
|
|
|
|
chmod 0640 "$stage/etc/moments/worker.env"
|
|
|
|
if (( dry_run )); then
|
|
printf '\033[2m[dry-run]\033[0m rsync staged -> %s:/\n' "$host" >&2
|
|
else
|
|
rsync -aHAX --rsync-path="sudo rsync" "$stage/" "${host}:/"
|
|
fi
|
|
|
|
ssh_run "$host" "sudo bash -s" <<'REMOTE_EOF'
|
|
set -euo pipefail
|
|
fqdn="$(hostname -f)"
|
|
|
|
systemd-sysusers /etc/sysusers.d/moments.conf
|
|
|
|
install -d -o root -g moments -m 0750 /etc/moments
|
|
install -d -o moments -g moments -m 0750 /var/lib/moments
|
|
chown root:moments /etc/moments/worker.env
|
|
chmod 0640 /etc/moments/worker.env
|
|
|
|
setfacl -m u:moments:r "/etc/pki/tls/private/${fqdn}.pem" || true
|
|
|
|
restorecon -Rv /usr/local/bin/moments-worker /etc/moments /var/lib/moments
|
|
|
|
systemctl daemon-reload
|
|
systemctl enable --now moments-worker-cert.path
|
|
systemctl enable --now moments-worker.service
|
|
systemctl restart moments-worker.service
|
|
|
|
# Liveness probe — worker doesn't expose a port, so check is-active.
|
|
if ! systemctl is-active --quiet moments-worker.service; then
|
|
journalctl -u moments-worker.service -n 50 --no-pager >&2
|
|
exit 1
|
|
fi
|
|
echo "moments-worker active"
|
|
REMOTE_EOF
|
|
}
|
|
|
|
deploy_web() {
|
|
local host="$1"
|
|
log "web -> $host"
|
|
|
|
if (( dry_run )); then
|
|
printf '\033[2m[dry-run]\033[0m rsync ui/dist/ to %s:/var/www/moments/ + nginx config, run nginx -t/reload on %s\n' \
|
|
"$host" "$host" >&2
|
|
return 0
|
|
fi
|
|
|
|
local stage
|
|
stage="$(mktemp -d)"
|
|
trap "rm -rf '$stage'" RETURN
|
|
|
|
install -d "$stage/var/www/moments" "$stage/etc/nginx/conf.d"
|
|
|
|
rsync -a "${repo_root}/ui/dist/" "$stage/var/www/moments/"
|
|
install -m 0644 "${repo_root}/asset/nginx/rob.tn.conf" "$stage/etc/nginx/conf.d/rob.tn.conf"
|
|
|
|
if (( dry_run )); then
|
|
printf '\033[2m[dry-run]\033[0m rsync staged -> %s:/\n' "$host" >&2
|
|
else
|
|
rsync -aHAX --delete --rsync-path="sudo rsync" "$stage/var/www/moments/" "${host}:/var/www/moments/"
|
|
rsync -aHAX --rsync-path="sudo rsync" "$stage/etc/nginx/conf.d/rob.tn.conf" "${host}:/etc/nginx/conf.d/rob.tn.conf"
|
|
fi
|
|
|
|
ssh_run "$host" "sudo bash -s" <<'REMOTE_EOF'
|
|
set -euo pipefail
|
|
|
|
# Allow nginx to talk upstream to the loopback API socket.
|
|
setsebool -P httpd_can_network_connect on
|
|
|
|
restorecon -Rv /var/www/moments /etc/nginx/conf.d/rob.tn.conf
|
|
|
|
if ! nginx -t; then
|
|
echo "nginx config check failed" >&2
|
|
exit 1
|
|
fi
|
|
systemctl reload nginx
|
|
echo "nginx reloaded"
|
|
REMOTE_EOF
|
|
}
|
|
|
|
# Dispatch ------------------------------------------------------------------
|
|
|
|
failed=()
|
|
for component in "${components[@]}"; do
|
|
mapfile -t hosts < <(yq -r "${env_path}.components.${component}.hosts[]" "$manifest")
|
|
for host in "${hosts[@]}"; do
|
|
case "$component" in
|
|
api) deploy_api "$host" || failed+=("api@$host") ;;
|
|
worker) deploy_worker "$host" || failed+=("worker@$host") ;;
|
|
web) deploy_web "$host" || failed+=("web@$host") ;;
|
|
*) warn "unknown component: $component" ;;
|
|
esac
|
|
done
|
|
done
|
|
|
|
if [[ ${#failed[@]} -gt 0 ]]; then
|
|
die "failed: ${failed[*]}"
|
|
fi
|
|
log "deploy complete"
|