From bb2f5b1f9b3e4cfa004def0a9cd5d894fabc5330 Mon Sep 17 00:00:00 2001 From: rob thijssen Date: Tue, 21 Jul 2026 11:42:39 +0300 Subject: [PATCH] fix(web): stop binding 443 behind the edge SNI router; verify the reload oolon's TCP 443 belongs to the stream SNI router, which ssl_prereads the handshake and forwards to the local https tier on 127.0.0.1:14443 with PROXY protocol. site.conf.tmpl predates that and still bound 443 itself, so every deploy and every daily refresh rsynced a vhost that collides with the router. Nothing in the pipeline caught it. `nginx -t` only detects duplicate listeners within a context, not across http{} and stream{}, and `systemctl reload` merely sends SIGHUP, so it exits 0 while nginx logs "bind() to 0.0.0.0:443 failed (98: Address already in use) ... still could not bind()", aborts the reconfiguration and keeps its old cycle. The deploy went green while oolon's running config was frozen. It stayed frozen for a day, stranding every cert the step@ timers renewed on disk until eleven internal vhosts were serving expired certs. A cold start would have failed outright, taking the whole public edge down. Template the listen line from WEB_LISTEN (manifest web.config.listen for script/deploy.sh, which renders the same template), and assert that the reload landed by requiring a fresh worker generation, dumping the nginx error log when it did not. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_0182wzZE8DguMPWhxD21gfP2 --- .gitea/workflows/deploy.yml | 30 +++++++++++++++++++++++++++++- .gitea/workflows/refresh.yml | 16 +++++++++++++++- asset/manifest.yml | 5 +++++ asset/nginx/site.conf.tmpl | 5 ++++- asset/sudoers.d/web-host.conf | 4 ++++ script/deploy.sh | 10 +++++++++- 6 files changed, 66 insertions(+), 4 deletions(-) diff --git a/.gitea/workflows/deploy.yml b/.gitea/workflows/deploy.yml index 49eb4d0..19d234c 100644 --- a/.gitea/workflows/deploy.yml +++ b/.gitea/workflows/deploy.yml @@ -45,6 +45,16 @@ env: API_PORT: "42424" SERVER_NAME: rob.tn WEB_ROOT: /var/www/rob.tn + # TCP 443 on the oolon edge is owned by the stream SNI router + # (/etc/nginx/streams-enabled/sni-router.conf), which ssl_prereads the + # handshake and forwards to the local https tier on 127.0.0.1:14443 with + # PROXY protocol. An http vhost that binds 443 itself collides with the + # router: `nginx -t` still passes (nginx only detects duplicate listeners + # within a context, not across http{} and stream{}), but every subsequent + # reload aborts with "bind() to 0.0.0.0:443 failed (98: Address already in + # use)" while `systemctl reload` still exits 0. That silently froze oolon's + # running config for a day and stranded every renewed cert on disk. + WEB_LISTEN: 127.0.0.1:14443 ssl proxy_protocol API_UPSTREAM_SCHEME: http API_UPSTREAM_ADDR: nikola.kosherinata.internal:42424 MUSL_TARGET: x86_64-unknown-linux-musl @@ -336,6 +346,7 @@ jobs: subs = { "SERVER_NAME": os.environ["SERVER_NAME"], "DOCROOT": os.environ["WEB_ROOT"], + "WEB_LISTEN": os.environ["WEB_LISTEN"], "API_UPSTREAM_SCHEME": os.environ["API_UPSTREAM_SCHEME"], "API_UPSTREAM_ADDR": os.environ["API_UPSTREAM_ADDR"], } @@ -366,4 +377,21 @@ jobs: fi sudo /usr/sbin/restorecon -R /etc/nginx/conf.d/'"${SERVER_NAME}"'.conf sudo /usr/sbin/nginx -t - sudo /usr/bin/systemctl reload nginx' + # `nginx -t` cannot see a listener that collides across http{} and + # stream{}, and `systemctl reload` is just SIGHUP — it exits 0 even + # when nginx aborts the reconfiguration and keeps the old cycle. A + # reload that really landed always spawns a fresh worker generation, + # so assert that rather than trusting the exit code. + master=$(cat /run/nginx.pid) + before=$(pgrep -P "$master" | sort | tr "\n" " ") + sudo /usr/bin/systemctl reload nginx + for _ in $(seq 1 10); do + sleep 1 + after=$(pgrep -P "$master" | sort | tr "\n" " ") + [ "$before" = "$after" ] || break + done + if [ "$before" = "$after" ]; then + echo "nginx reload did not take effect: worker generation unchanged" >&2 + sudo /usr/bin/tail -n 50 /var/log/nginx/error.log >&2 || true + exit 1 + fi' diff --git a/.gitea/workflows/refresh.yml b/.gitea/workflows/refresh.yml index 6e34f45..b9063b7 100644 --- a/.gitea/workflows/refresh.yml +++ b/.gitea/workflows/refresh.yml @@ -99,4 +99,18 @@ jobs: fi sudo /usr/sbin/restorecon -R /etc/nginx/conf.d/'"${SERVER_NAME}"'.conf sudo /usr/sbin/nginx -t - sudo /usr/bin/systemctl reload nginx' + # Verify the reload actually landed — see deploy.yml for why neither + # `nginx -t` nor `systemctl reload`s exit code is enough. + master=$(cat /run/nginx.pid) + before=$(pgrep -P "$master" | sort | tr "\n" " ") + sudo /usr/bin/systemctl reload nginx + for _ in $(seq 1 10); do + sleep 1 + after=$(pgrep -P "$master" | sort | tr "\n" " ") + [ "$before" = "$after" ] || break + done + if [ "$before" = "$after" ]; then + echo "nginx reload did not take effect: worker generation unchanged" >&2 + sudo /usr/bin/tail -n 50 /var/log/nginx/error.log >&2 || true + exit 1 + fi' diff --git a/asset/manifest.yml b/asset/manifest.yml index 07232a1..0828a33 100644 --- a/asset/manifest.yml +++ b/asset/manifest.yml @@ -34,3 +34,8 @@ environments: server_name: rob.tn root: /var/www/rob.tn api_upstream: http://nikola.kosherinata.internal:42424 + # oolon's stream SNI router owns TCP 443 and forwards to the local + # https tier on 14443 with PROXY protocol. Do not bind 443 here: + # `nginx -t` will not catch the collision (it is across http{} and + # stream{}) and every subsequent reload silently aborts. + listen: 127.0.0.1:14443 ssl proxy_protocol diff --git a/asset/nginx/site.conf.tmpl b/asset/nginx/site.conf.tmpl index cceb223..1d62c1f 100644 --- a/asset/nginx/site.conf.tmpl +++ b/asset/nginx/site.conf.tmpl @@ -5,7 +5,10 @@ upstream moments_api { server { server_name {{SERVER_NAME}}; - listen 443 ssl; + # Behind the edge's stream SNI router — see WEB_LISTEN in deploy.yml for why + # this must not bind 443 directly. real_ip recovery lives in the edge's + # conf.d/proxy-protocol.conf. + listen {{WEB_LISTEN}}; http2 on; ssl_certificate /etc/letsencrypt/live/{{SERVER_NAME}}/fullchain.pem; diff --git a/asset/sudoers.d/web-host.conf b/asset/sudoers.d/web-host.conf index e13a797..c772777 100644 --- a/asset/sudoers.d/web-host.conf +++ b/asset/sudoers.d/web-host.conf @@ -18,3 +18,7 @@ gitea_ci ALL=(root) NOPASSWD: /usr/sbin/semanage port -l gitea_ci ALL=(root) NOPASSWD: /usr/sbin/semanage port -a -t http_port_t -p tcp 42424 gitea_ci ALL=(root) NOPASSWD: /usr/sbin/nginx -t gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl reload nginx +# Read-only, so a reload that silently failed can report why (the error log is +# 0640 nginx:root). The deploy asserts the worker generation changed and dumps +# this on mismatch. +gitea_ci ALL=(root) NOPASSWD: /usr/bin/tail -n 50 /var/log/nginx/error.log diff --git a/script/deploy.sh b/script/deploy.sh index f52a91e..1479933 100755 --- a/script/deploy.sh +++ b/script/deploy.sh @@ -433,10 +433,17 @@ deploy_web() { local host="$1" log "web -> $host" - local server_name web_root api_upstream + local server_name web_root api_upstream web_listen server_name="$(yq --raw-output "${env_path}.components.web.config.server_name" "$manifest")" web_root="$(yq --raw-output "${env_path}.components.web.config.root" "$manifest")" api_upstream="$(yq --raw-output "${env_path}.components.web.config.api_upstream" "$manifest")" + # The edge's stream SNI router owns TCP 443; vhosts sit behind it on 14443. + # Binding 443 here collides with the router — `nginx -t` passes, but every + # later reload aborts while systemctl still reports success. See the + # web.config.listen note in asset/manifest.yml. + web_listen="$(yq --raw-output "${env_path}.components.web.config.listen" "$manifest")" + [[ -n "$web_listen" && "$web_listen" != "null" ]] \ + || web_listen='127.0.0.1:14443 ssl proxy_protocol' [[ -n "$server_name" && "$server_name" != "null" ]] || die "web.config.server_name missing in manifest" [[ -n "$web_root" && "$web_root" != "null" ]] || die "web.config.root missing in manifest" [[ -n "$api_upstream" && "$api_upstream" != "null" ]] || die "web.config.api_upstream missing in manifest" @@ -475,6 +482,7 @@ deploy_web() { local rendered rendered="$(<"${repo_root}/asset/nginx/site.conf.tmpl")" rendered=${rendered//'{{SERVER_NAME}}'/$server_name} + rendered=${rendered//'{{WEB_LISTEN}}'/$web_listen} rendered=${rendered//'{{DOCROOT}}'/$web_root} rendered=${rendered//'{{API_UPSTREAM_SCHEME}}'/$api_upstream_scheme} rendered=${rendered//'{{API_UPSTREAM_ADDR}}'/$api_upstream_addr}