rob thijssen
a27da00b8f
The journal capture runs `if: always()` so a failed service start still leaves a usable record. When the job failed *before* authenticating -- the missing-secrets preflight, for instance -- it ran anyway, failed with "Host key verification failed", and that became the last error in the log. The actual cause was twenty lines up and easy to miss. Conditions it on the authenticate step having succeeded, so it still captures every case it was added for and stays quiet for the ones it cannot help with. Observed on run 6: the preflight correctly reported all three missing secrets, then the journal step overwrote that with an SSH error. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01XZG2i4AmfSqE97EJGBVb64
212 lines
8.3 KiB
YAML
212 lines
8.3 KiB
YAML
name: deploy
|
|
|
|
on:
|
|
push: { branches: [main] }
|
|
workflow_dispatch:
|
|
|
|
concurrency:
|
|
group: deploy
|
|
cancel-in-progress: false
|
|
|
|
env:
|
|
# --- infra truth: hosts, ports and paths live here, not in a manifest ---
|
|
SERVICE_HOST: bob.hanzalova.internal
|
|
# Bind the mesh address rather than a wildcard: the reverse proxy is on a
|
|
# different host, so loopback will not do, and a wildcard bind on a host that
|
|
# may later gain another interface would publish the registry there too.
|
|
LISTEN_ADDR: 10.6.0.193:20482
|
|
APP_PORT: "20482"
|
|
S3_ENDPOINT: http://caveman.kosherinata.internal:9000
|
|
S3_BUCKET: rustingface
|
|
UPSTREAM_ENABLED: "true"
|
|
|
|
jobs:
|
|
build:
|
|
runs-on: rust
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
|
|
# The gate runs before anything is built for deployment, so a broken
|
|
# commit on main never reaches a host.
|
|
- name: format
|
|
run: cargo fmt --all -- --check
|
|
|
|
- name: clippy
|
|
run: cargo clippy --workspace --all-targets --all-features -- -D warnings
|
|
|
|
- name: test
|
|
run: cargo test --workspace --all-features
|
|
|
|
# Static musl: the runner is Fedora 44 and the target is Fedora 43, so a
|
|
# dynamically linked binary could reference a newer glibc than bob has.
|
|
- name: build
|
|
run: |
|
|
rustup target add x86_64-unknown-linux-musl
|
|
cargo build --release --target x86_64-unknown-linux-musl --bin rustingface
|
|
|
|
- uses: actions/upload-artifact@v3
|
|
with:
|
|
name: rustingface
|
|
path: target/x86_64-unknown-linux-musl/release/rustingface
|
|
|
|
deploy:
|
|
needs: build
|
|
runs-on: fedora-43
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- uses: actions/download-artifact@v3
|
|
with:
|
|
name: rustingface
|
|
path: dist
|
|
|
|
# Without this, a missing secret first surfaces as ssh failing to parse an
|
|
# empty key, which names neither the secret nor the fix.
|
|
- name: check the required secrets are set
|
|
env:
|
|
RSYNC_SSH_KEY: ${{ secrets.RSYNC_SSH_KEY }}
|
|
S3_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY_ID }}
|
|
S3_SECRET_ACCESS_KEY: ${{ secrets.S3_SECRET_ACCESS_KEY }}
|
|
run: |
|
|
missing=""
|
|
for name in RSYNC_SSH_KEY S3_ACCESS_KEY_ID S3_SECRET_ACCESS_KEY; do
|
|
[ -n "${!name:-}" ] || missing="$missing $name"
|
|
done
|
|
if [ -n "$missing" ]; then
|
|
echo "missing repo secret(s):$missing" >&2
|
|
echo "Set them in Settings > Actions > Secrets. HF_TOKEN is optional and" >&2
|
|
echo "only needed for gated repositories." >&2
|
|
exit 1
|
|
fi
|
|
echo "all required secrets are present"
|
|
|
|
- name: authenticate to the target
|
|
id: auth
|
|
run: |
|
|
install -d -m 0700 ~/.ssh
|
|
printf '%s\n' "${{ secrets.RSYNC_SSH_KEY }}" > ~/.ssh/id_gitea_ci
|
|
chmod 600 ~/.ssh/id_gitea_ci
|
|
cat > ~/.ssh/config <<CFG
|
|
Host $SERVICE_HOST
|
|
User gitea_ci
|
|
IdentityFile ~/.ssh/id_gitea_ci
|
|
IdentitiesOnly yes
|
|
StrictHostKeyChecking accept-new
|
|
CFG
|
|
ssh "$SERVICE_HOST" hostname -f
|
|
|
|
- name: render config
|
|
env:
|
|
S3_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY_ID }}
|
|
S3_SECRET_ACCESS_KEY: ${{ secrets.S3_SECRET_ACCESS_KEY }}
|
|
HF_TOKEN: ${{ secrets.HF_TOKEN }}
|
|
run: |
|
|
# Literal substitution, not a shell expansion: a secret containing
|
|
# $, ` or \ must survive intact.
|
|
python3 - <<'PY'
|
|
import os, pathlib
|
|
template = pathlib.Path("asset/config/config.toml.tmpl").read_text()
|
|
for key in ("LISTEN_ADDR", "S3_ENDPOINT", "S3_BUCKET", "UPSTREAM_ENABLED"):
|
|
template = template.replace("{{%s}}" % key, os.environ[key])
|
|
pathlib.Path("dist/config.toml").write_text(template)
|
|
for name, var in (("s3-access-key", "S3_ACCESS_KEY_ID"),
|
|
("s3-secret-key", "S3_SECRET_ACCESS_KEY"),
|
|
("hf-token", "HF_TOKEN")):
|
|
pathlib.Path("dist", name).write_text(os.environ.get(var, ""))
|
|
PY
|
|
# Fail loudly rather than shipping a config with an unrendered
|
|
# placeholder that would only surface as a runtime parse error.
|
|
if grep -n '{{' dist/config.toml; then
|
|
echo "unrendered placeholder in the config" >&2
|
|
exit 1
|
|
fi
|
|
if [ ! -s dist/s3-access-key ] || [ ! -s dist/s3-secret-key ]; then
|
|
echo "S3 credentials are empty; set the S3_ACCESS_KEY_ID and S3_SECRET_ACCESS_KEY secrets" >&2
|
|
exit 1
|
|
fi
|
|
|
|
- name: create the service account and its config directory
|
|
run: |
|
|
rsync -az --rsync-path='sudo rsync' --mkpath \
|
|
asset/systemd/rustingface.sysusers.conf \
|
|
"$SERVICE_HOST:/etc/sysusers.d/rustingface.conf"
|
|
ssh "$SERVICE_HOST" '
|
|
set -euo pipefail
|
|
sudo systemd-sysusers
|
|
sudo install -d -o root -g rustingface -m 0750 /etc/rustingface'
|
|
|
|
- name: ship the binary, config and credentials
|
|
run: |
|
|
set -euo pipefail
|
|
rsync -az --rsync-path='sudo rsync' \
|
|
dist/rustingface "$SERVICE_HOST:/usr/local/bin/rustingface"
|
|
for f in config.toml s3-access-key s3-secret-key hf-token; do
|
|
rsync -az --rsync-path='sudo rsync' --mkpath \
|
|
"dist/$f" "$SERVICE_HOST:/etc/rustingface/$f"
|
|
done
|
|
rsync -az --rsync-path='sudo rsync' --mkpath \
|
|
asset/systemd/rustingface.service \
|
|
"$SERVICE_HOST:/etc/systemd/system/rustingface.service"
|
|
ssh "$SERVICE_HOST" '
|
|
set -euo pipefail
|
|
sudo chmod 0755 /usr/local/bin/rustingface
|
|
sudo chown -R root:rustingface /etc/rustingface
|
|
sudo chmod 0640 /etc/rustingface/config.toml /etc/rustingface/s3-access-key /etc/rustingface/s3-secret-key /etc/rustingface/hf-token
|
|
sudo restorecon -R /usr/local/bin/rustingface /etc/rustingface'
|
|
|
|
- name: firewalld
|
|
run: |
|
|
set -euo pipefail
|
|
# rsync the XML first: firewalld only learns a freshly-shipped custom
|
|
# service after --reload, and querying it before that fails.
|
|
rsync -az --rsync-path='sudo rsync' --mkpath \
|
|
asset/firewalld/rustingface.xml \
|
|
"$SERVICE_HOST:/etc/firewalld/services/rustingface.xml"
|
|
ssh "$SERVICE_HOST" '
|
|
set -euo pipefail
|
|
sudo firewall-cmd --reload
|
|
zone=$(sudo firewall-cmd --get-default-zone)
|
|
if sudo firewall-cmd --zone="$zone" --query-service=rustingface; then
|
|
echo "rustingface already enabled in $zone"
|
|
else
|
|
sudo firewall-cmd --permanent --zone="$zone" --add-service=rustingface
|
|
sudo firewall-cmd --zone="$zone" --add-service=rustingface
|
|
fi'
|
|
|
|
- name: restart
|
|
run: |
|
|
ssh "$SERVICE_HOST" '
|
|
set -euo pipefail
|
|
sudo systemctl daemon-reload
|
|
sudo systemctl enable rustingface.service
|
|
sudo systemctl restart rustingface.service'
|
|
|
|
- name: health check
|
|
run: |
|
|
set -euo pipefail
|
|
ssh "$SERVICE_HOST" 'sudo systemctl is-active rustingface.service'
|
|
for attempt in $(seq 1 20); do
|
|
if ssh "$SERVICE_HOST" "curl -fsS http://$LISTEN_ADDR/healthz"; then
|
|
echo "healthy after $attempt attempt(s)"
|
|
exit 0
|
|
fi
|
|
sleep 2
|
|
done
|
|
echo "rustingface did not answer /healthz" >&2
|
|
exit 1
|
|
|
|
- name: doctor
|
|
run: |
|
|
# Proves the deployment can actually reach and write to the bucket,
|
|
# which a liveness probe deliberately does not.
|
|
ssh "$SERVICE_HOST" \
|
|
'sudo -u rustingface /usr/local/bin/rustingface --config /etc/rustingface/config.toml doctor'
|
|
|
|
# Capture the unit's startup journal even when a later step failed --
|
|
# that is the record worth having. Conditioned on the target actually
|
|
# being reachable: without it, a job that failed before authenticating
|
|
# ends with "Host key verification failed", which buries the real cause
|
|
# under an error about something else entirely.
|
|
- name: journal
|
|
if: always() && steps.auth.outcome == 'success'
|
|
run: ssh "$SERVICE_HOST" 'journalctl -u rustingface.service -n 200 --no-pager'
|