rob thijssen
3db1f586a8
Add newsfeed-modelwatch: a one-shot, systemd-timer-driven producer that watches
the Hugging Face Hub for open-weight releases and pushes filtered candidates to
the ingest endpoint. Answers "how do I automate the HF firehose into my feed" —
the RSS half already works on the pull rail; this is the JSON/API half.
How it works: each run polls the HF models API (a watchlist of orgs +
trendingScore), applies an admission predicate (license allowlist, total-params
cap, safetensors present, not gated, pipeline_tag), and POSTs the survivors to
POST /v1/ingest/candidates under a bearer token, tagged source="huggingface".
Two properties of the existing ingest rail shape the design:
- Ingest is idempotent on (user, external_id). Using the repo id as external_id
makes the producer STATELESS — no dedup table; re-runs/overlapping timers just
re-submit and the server drops repeats.
- HF `tags` are copied onto the candidate, so the user's per-interest weights do
the ranking. The predicate is only an ADMISSION filter (what's worth surfacing
at all) and stays subordinate to explicit weights, per the house rule.
Layout: newsfeed-modelwatch is a client of the API, not an internal component —
it holds no core/data deps. Pure logic (HF types, predicate, mapping to
CandidateSubmission, param formatting) lives in the lib with unit tests; the bin
does the HTTP polling/posting. CandidateSubmission gains Serialize so the
in-workspace producer can build and post one.
Ops:
- asset/systemd/newsfeed-modelwatch.{service,timer}: oneshot + 3-hourly timer.
The ingest token is a secret, kept in /etc/newsfeed/modelwatch.env
(NEWSFEED_TOKEN, mapped to `token` by figment) so a redeploy of the config
never clobbers it.
- asset/config/modelwatch.toml.tmpl: watchlist + predicate, no secret.
- infra-setup.sh installs the units, creates the env placeholder (never
overwritten), enables the timer, and prints the token step.
- deploy.yml builds + ships the binary and the (secret-free) config each deploy.
Verified: unit tests (gated union, license precedence, admit accept/reject,
mapping); dry-run against live HF; full loop against a local API — minted token,
submitted a real release, confirmed it landed in the feed with the huggingface
source linked and HF tags carried through. fmt/clippy/test all green.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_016fKZzDpvjiJ9eYbPGgJvUP
307 lines
14 KiB
Bash
Executable File
307 lines
14 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
#
|
|
# One-time host provisioning for newsfeed's CI-driven deploy (architecture
|
|
# deployment-gitea-actions.md §2). Run by the operator from a workstation with full sudo
|
|
# and mesh ssh access — NOT by the CI runner. Idempotent; skips unreachable hosts.
|
|
#
|
|
# It provisions, per host:
|
|
# * the `gitea_ci` deploy user + the runner's authorized_key + systemd-journal group
|
|
# * a scoped /etc/sudoers.d/newsfeed_gitea_ci drop-in (visudo-verified)
|
|
# and per role:
|
|
# * API host (slartibartfast): the `newsfeed` service account, /etc/newsfeed +
|
|
# /var/lib/newsfeed, SELinux port label for 22672, firewalld service
|
|
# * WEB host (oolon): /var/www/newsfeed webroot, the internal `newsfeed.internal` TLS
|
|
# cert (lair provisioner) + step@newsfeed renewal, and the nginx vhost
|
|
#
|
|
# Usage: ./script/infra-setup.sh [--dry-run]
|
|
set -euo pipefail
|
|
|
|
# --- infra truth (edit here if the topology changes) -------------------------------
|
|
API_HOST="slartibartfast.kosherinata.internal" # newsfeed-api + newsfeed-worker
|
|
WEB_HOST="oolon.kosherinata.internal" # nginx: SPA + API reverse proxy
|
|
API_PORT="22672"
|
|
CERT_NAME="newsfeed" # dot-free; serves ${CERT_NAME}.internal
|
|
PUBLIC_FQDN="rob.fyi" # WAN name (Let's Encrypt); "" to skip
|
|
CERT_EMAIL="ops@rob.fyi" # certbot registration email
|
|
CF_CREDS="/root/.certbot-internal" # Cloudflare API token on the web host
|
|
RUNNER_PUBKEY="${HOME}/.ssh/id_gitea_ci.pub" # runner key distributed to gitea_ci
|
|
PROVISIONER_PW="${HOME}/.step/secrets/provisioner"
|
|
ROOT_CA="/etc/pki/ca-trust/source/anchors/root-internal.pem"
|
|
CA_URL="https://ca.internal"
|
|
|
|
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
DRY_RUN=0
|
|
[[ "${1:-}" == "--dry-run" ]] && DRY_RUN=1
|
|
|
|
info() { printf '\033[1;34m==>\033[0m %s\n' "$*"; }
|
|
warn() { printf '\033[1;33mWARN\033[0m %s\n' "$*" >&2; }
|
|
|
|
run_remote() {
|
|
# run_remote <host> <script>
|
|
local host="$1" script="$2"
|
|
if [[ "$DRY_RUN" == 1 ]]; then
|
|
printf -- '--- would run on %s ---\n%s\n' "$host" "$script"
|
|
return 0
|
|
fi
|
|
ssh -o StrictHostKeyChecking=accept-new "$host" "sudo bash -euo pipefail -s" <<<"$script"
|
|
}
|
|
|
|
reachable() {
|
|
ssh -o StrictHostKeyChecking=accept-new -o ConnectTimeout=8 "$1" true 2>/dev/null
|
|
}
|
|
|
|
require_files() {
|
|
[[ -f "$RUNNER_PUBKEY" ]] || { warn "runner pubkey $RUNNER_PUBKEY missing (ssh-keygen -t ed25519 -f ~/.ssh/id_gitea_ci)"; exit 1; }
|
|
}
|
|
|
|
# --- gitea_ci user, key, journal group, sudoers (both hosts) ------------------------
|
|
provision_gitea_ci() {
|
|
local host="$1" sudoers="$2"
|
|
local pubkey; pubkey="$(cat "$RUNNER_PUBKEY")"
|
|
info "[$host] provisioning gitea_ci deploy user"
|
|
run_remote "$host" "
|
|
id gitea_ci >/dev/null 2>&1 || useradd --system --create-home --home-dir /var/lib/gitea_ci --shell /usr/sbin/nologin gitea_ci
|
|
usermod -aG systemd-journal gitea_ci
|
|
install -d -m 0700 -o gitea_ci -g gitea_ci /var/lib/gitea_ci/.ssh
|
|
grep -qF '${pubkey}' /var/lib/gitea_ci/.ssh/authorized_keys 2>/dev/null || echo '${pubkey}' >> /var/lib/gitea_ci/.ssh/authorized_keys
|
|
chown gitea_ci:gitea_ci /var/lib/gitea_ci/.ssh/authorized_keys
|
|
chmod 0600 /var/lib/gitea_ci/.ssh/authorized_keys
|
|
umask 022
|
|
cat > /etc/sudoers.d/newsfeed_gitea_ci <<'SUDOERS'
|
|
${sudoers}
|
|
SUDOERS
|
|
chmod 0440 /etc/sudoers.d/newsfeed_gitea_ci
|
|
visudo -cf /etc/sudoers.d/newsfeed_gitea_ci
|
|
"
|
|
}
|
|
|
|
API_SUDOERS='# Scoped deploy permissions for newsfeed api+worker on this host.
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /usr/local/bin/newsfeed-api
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /usr/local/bin/newsfeed-worker
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/newsfeed/config.toml
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/newsfeed/worker.toml
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl daemon-reload
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl restart newsfeed-api.service
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl restart newsfeed-worker.service
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl is-active newsfeed-api.service
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/sbin/restorecon -R /usr/local/bin/newsfeed-api /usr/local/bin/newsfeed-worker /etc/newsfeed /var/lib/newsfeed'
|
|
|
|
WEB_SUDOERS='# Scoped deploy permissions for the newsfeed SPA on the oolon proxy.
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /var/www/newsfeed/
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/sbin/restorecon -R /var/www/newsfeed
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/sbin/nginx -t
|
|
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl reload nginx.service'
|
|
|
|
# --- API host: service account, dirs, SELinux port, firewalld -----------------------
|
|
provision_api_host() {
|
|
local host="$API_HOST"
|
|
if ! reachable "$host"; then warn "[$host] unreachable, skipping"; return 0; fi
|
|
provision_gitea_ci "$host" "$API_SUDOERS"
|
|
|
|
info "[$host] service account, directories, SELinux, firewalld"
|
|
local sysusers firewalld
|
|
sysusers="$(cat "$REPO_ROOT/asset/systemd/newsfeed.sysusers.conf")"
|
|
firewalld="$(cat "$REPO_ROOT/asset/firewalld/newsfeed-api.xml")"
|
|
run_remote "$host" "
|
|
# service account
|
|
cat > /etc/sysusers.d/newsfeed.conf <<'SYSU'
|
|
${sysusers}
|
|
SYSU
|
|
systemd-sysusers /etc/sysusers.d/newsfeed.conf
|
|
|
|
# directories (config: root-owned, readable by service; state: service-owned)
|
|
install -d -m 0750 -o root -g newsfeed /etc/newsfeed
|
|
install -d -m 0750 -o newsfeed -g newsfeed /var/lib/newsfeed
|
|
install -d -m 0755 -o root -g root /usr/local/bin
|
|
|
|
# modelwatch ingest token holder — a secret, so it lives outside the deployable
|
|
# config and is created empty here, never clobbered by a redeploy.
|
|
if [ ! -f /etc/newsfeed/modelwatch.env ]; then
|
|
printf '# Ingest token for newsfeed-modelwatch. Mint one in the newsfeed UI\n# (Settings -> API tokens), paste below, then: systemctl start newsfeed-modelwatch\nNEWSFEED_TOKEN=\n' > /etc/newsfeed/modelwatch.env
|
|
chown root:newsfeed /etc/newsfeed/modelwatch.env
|
|
chmod 0640 /etc/newsfeed/modelwatch.env
|
|
fi
|
|
restorecon -R /etc/newsfeed /var/lib/newsfeed
|
|
|
|
# SELinux: label the API port so the daemon may bind it
|
|
semanage port -l | grep -qE '^http_port_t.*[[:space:]]${API_PORT}([,[:space:]]|\$)' || semanage port -a -t http_port_t -p tcp ${API_PORT}
|
|
|
|
# firewalld named service (mesh-reachable API port)
|
|
cat > /etc/firewalld/services/newsfeed-api.xml <<'FWD'
|
|
${firewalld}
|
|
FWD
|
|
firewall-cmd --reload
|
|
zone=\$(firewall-cmd --get-default-zone)
|
|
firewall-cmd --permanent --zone=\$zone --query-service=newsfeed-api >/dev/null 2>&1 || firewall-cmd --permanent --zone=\$zone --add-service=newsfeed-api
|
|
firewall-cmd --zone=\$zone --query-service=newsfeed-api >/dev/null 2>&1 || firewall-cmd --zone=\$zone --add-service=newsfeed-api
|
|
"
|
|
|
|
info "[$host] installing systemd units"
|
|
for unit in newsfeed-api.service newsfeed-worker.service \
|
|
newsfeed-modelwatch.service newsfeed-modelwatch.timer; do
|
|
local content; content="$(cat "$REPO_ROOT/asset/systemd/$unit")"
|
|
run_remote "$host" "cat > /etc/systemd/system/$unit <<'UNIT'
|
|
${content}
|
|
UNIT
|
|
"
|
|
done
|
|
run_remote "$host" "
|
|
systemctl daemon-reload
|
|
systemctl enable newsfeed-api.service newsfeed-worker.service
|
|
# modelwatch is a oneshot driven by its timer — enable the timer, not the service.
|
|
systemctl enable --now newsfeed-modelwatch.timer"
|
|
}
|
|
|
|
# --- WEB host (oolon): webroot, internal cert, nginx vhost --------------------------
|
|
provision_web_host() {
|
|
local host="$WEB_HOST"
|
|
if ! reachable "$host"; then warn "[$host] unreachable, skipping"; return 0; fi
|
|
provision_gitea_ci "$host" "$WEB_SUDOERS"
|
|
|
|
info "[$host] webroot"
|
|
run_remote "$host" "
|
|
install -d -m 0755 -o nginx -g nginx /var/www/newsfeed
|
|
# placeholder so nginx -t passes before the first SPA deploy
|
|
[ -f /var/www/newsfeed/index.html ] || echo '<!doctype html><title>newsfeed</title>' > /var/www/newsfeed/index.html
|
|
restorecon -R /var/www/newsfeed
|
|
"
|
|
|
|
mint_internal_cert "$host"
|
|
|
|
info "[$host] nginx vhost (newsfeed.internal)"
|
|
local vhost; vhost="$(cat "$REPO_ROOT/asset/nginx/newsfeed.internal.conf")"
|
|
run_remote "$host" "
|
|
install -d -m 0755 /etc/nginx/sites-available /etc/nginx/sites-enabled
|
|
cat > /etc/nginx/sites-available/newsfeed.internal.conf <<'VHOST'
|
|
${vhost}
|
|
VHOST
|
|
ln -sfn ../sites-available/newsfeed.internal.conf /etc/nginx/sites-enabled/newsfeed.internal.conf
|
|
nginx -t
|
|
systemctl reload nginx
|
|
"
|
|
|
|
provision_public_vhost "$host"
|
|
}
|
|
|
|
# Issue the public Let's Encrypt cert for $PUBLIC_FQDN (idempotent) and install the WAN
|
|
# vhost, gated on the cert existing (external-tls.md). Skips cleanly if PUBLIC_FQDN is
|
|
# unset or the Cloudflare credential isn't on the host.
|
|
provision_public_vhost() {
|
|
local host="$1"
|
|
[[ -n "$PUBLIC_FQDN" ]] || { info "[$host] PUBLIC_FQDN unset; skipping WAN vhost"; return 0; }
|
|
|
|
if [[ "$DRY_RUN" == 1 ]]; then info "[$host] would issue LE cert + install ${PUBLIC_FQDN} vhost"; return 0; fi
|
|
|
|
info "[$host] issuing Let's Encrypt cert for ${PUBLIC_FQDN} (certbot, Cloudflare DNS-01)"
|
|
# --keep-until-expiring makes this a no-op when the cert is still valid, so it is safe
|
|
# to run unconditionally. DNS-01 needs no inbound :80 and works even before the WAN A
|
|
# record / OPNsense forward exist. Requires the Cloudflare token at $CF_CREDS.
|
|
run_remote "$host" "
|
|
if [ ! -f '$CF_CREDS' ]; then
|
|
echo 'WARN: $CF_CREDS missing; cannot issue ${PUBLIC_FQDN} cert' >&2
|
|
exit 0
|
|
fi
|
|
command -v certbot >/dev/null || { echo 'WARN: certbot not installed' >&2; exit 0; }
|
|
certbot certonly \
|
|
-m '$CERT_EMAIL' --agree-tos --no-eff-email --noninteractive \
|
|
--cert-name '$PUBLIC_FQDN' \
|
|
--key-type ecdsa \
|
|
--dns-cloudflare \
|
|
--dns-cloudflare-credentials '$CF_CREDS' \
|
|
--dns-cloudflare-propagation-seconds 60 \
|
|
--keep-until-expiring \
|
|
-d '$PUBLIC_FQDN'
|
|
# nginx reload after future renewals (external-tls.md §3), once per host.
|
|
install -d -m 0755 /etc/letsencrypt/renewal-hooks/deploy
|
|
cat > /etc/letsencrypt/renewal-hooks/deploy/reload-nginx.sh <<'HOOK'
|
|
#!/bin/sh
|
|
systemctl reload nginx 2>/dev/null || true
|
|
HOOK
|
|
chmod +x /etc/letsencrypt/renewal-hooks/deploy/reload-nginx.sh
|
|
"
|
|
|
|
info "[$host] installing WAN vhost (${PUBLIC_FQDN})"
|
|
local vhost; vhost="$(cat "$REPO_ROOT/asset/nginx/newsfeed.public.conf")"
|
|
# Gate the vhost on the cert existing — nginx -t fails on a missing ssl_certificate and
|
|
# would block ALL of nginx from reloading (external-tls.md §4).
|
|
run_remote "$host" "
|
|
if ! test -d /etc/letsencrypt/live/${PUBLIC_FQDN}; then
|
|
echo 'NOTE: ${PUBLIC_FQDN} cert not present; skipping WAN vhost install' >&2
|
|
exit 0
|
|
fi
|
|
cat > /etc/nginx/sites-available/newsfeed.public.conf <<'VHOST'
|
|
${vhost}
|
|
VHOST
|
|
ln -sfn ../sites-available/newsfeed.public.conf /etc/nginx/sites-enabled/newsfeed.public.conf
|
|
nginx -t
|
|
systemctl reload nginx
|
|
"
|
|
info "[$host] ${PUBLIC_FQDN} served. Ensure the Cloudflare A record ${PUBLIC_FQDN} ->"
|
|
info " kosherinata WAN IP exists (unproxied) and OPNsense forwards :443 -> oolon"
|
|
info " (reverse-proxies.md §2)."
|
|
}
|
|
|
|
# Mint the first internal cert (idempotent) and enable renewal (internal-tls.md §4).
|
|
mint_internal_cert() {
|
|
local host="$1"
|
|
local cert="/etc/nginx/tls/cert/${CERT_NAME}.internal.pem"
|
|
local key="/etc/nginx/tls/key/${CERT_NAME}.internal.pem"
|
|
|
|
if [[ "$DRY_RUN" == 1 ]]; then info "[$host] would mint ${CERT_NAME}.internal cert"; return 0; fi
|
|
[[ -f "$PROVISIONER_PW" ]] || { warn "[$host] provisioner password $PROVISIONER_PW missing; skipping cert mint"; return 0; }
|
|
|
|
local state
|
|
state="$(ssh "$host" "[ -f $cert ] && step certificate verify $cert --roots $ROOT_CA >/dev/null 2>&1 && echo valid || echo missing")"
|
|
if [[ "$state" != valid ]]; then
|
|
info "[$host] minting ${CERT_NAME}.internal cert via lair provisioner"
|
|
rsync -az --rsync-path='sudo rsync' --chmod=0600 "$PROVISIONER_PW" "$host:/tmp/${CERT_NAME}-provisioner"
|
|
run_remote "$host" "
|
|
mkdir -p /etc/nginx/tls/cert /etc/nginx/tls/key
|
|
rc=0
|
|
step ca certificate --force \
|
|
--provisioner lair \
|
|
--provisioner-password-file /tmp/${CERT_NAME}-provisioner \
|
|
--ca-url $CA_URL --root $ROOT_CA \
|
|
--san ${CERT_NAME}.internal \
|
|
${CERT_NAME}.internal $cert $key || rc=\$?
|
|
rm -f /tmp/${CERT_NAME}-provisioner
|
|
[ \$rc -eq 0 ] || { echo 'cert mint failed' >&2; exit \$rc; }
|
|
chown root:root $cert $key
|
|
chmod 644 $cert; chmod 640 $key
|
|
setfacl -m u:nginx:r $key
|
|
"
|
|
else
|
|
info "[$host] ${CERT_NAME}.internal cert already valid"
|
|
fi
|
|
run_remote "$host" "systemctl enable --now step@${CERT_NAME}.timer"
|
|
}
|
|
|
|
# DNS is not managed by this script, but the deployment doesn't work without two records.
|
|
# Print them explicitly — a missing or wrong split-horizon record is the classic reason a
|
|
# mesh client gets the wrong site/cert (reverse-proxies.md §2, external-tls.md §5).
|
|
dns_reminders() {
|
|
local web_ip; web_ip="$(getent hosts "$WEB_HOST" | awk '{print $1}' | head -1)"
|
|
web_ip="${web_ip:-<oolon mesh IP>}"
|
|
info "DNS (manage separately — not done by this script):"
|
|
info " • add split-horizon ${CERT_NAME}.internal -> ${web_ip} (mesh access path)"
|
|
if [[ -n "$PUBLIC_FQDN" ]]; then
|
|
info " • ${PUBLIC_FQDN}: public A -> kosherinata WAN only. Do NOT add an internal"
|
|
info " override for it — an internal record pointing elsewhere (e.g. the office"
|
|
info " proxy) makes mesh clients hit the wrong vhost. Mesh users use ${CERT_NAME}.internal."
|
|
fi
|
|
}
|
|
|
|
main() {
|
|
require_files
|
|
info "newsfeed infra-setup (api=$API_HOST web=$WEB_HOST)$([[ $DRY_RUN == 1 ]] && echo ' [dry-run]')"
|
|
provision_api_host
|
|
provision_web_host
|
|
dns_reminders
|
|
info "modelwatch: mint an ingest token in the newsfeed UI (Settings -> API tokens),"
|
|
info " put it in ${API_HOST}:/etc/newsfeed/modelwatch.env (NEWSFEED_TOKEN=...), then"
|
|
info " 'systemctl start newsfeed-modelwatch' to seed immediately (the timer runs 3-hourly)."
|
|
info "done"
|
|
}
|
|
|
|
main "$@"
|