diff --git a/.gitea/workflows/deploy.yml b/.gitea/workflows/deploy.yml index 49eb4d0..19d234c 100644 --- a/.gitea/workflows/deploy.yml +++ b/.gitea/workflows/deploy.yml @@ -45,6 +45,16 @@ env: API_PORT: "42424" SERVER_NAME: rob.tn WEB_ROOT: /var/www/rob.tn + # TCP 443 on the oolon edge is owned by the stream SNI router + # (/etc/nginx/streams-enabled/sni-router.conf), which ssl_prereads the + # handshake and forwards to the local https tier on 127.0.0.1:14443 with + # PROXY protocol. An http vhost that binds 443 itself collides with the + # router: `nginx -t` still passes (nginx only detects duplicate listeners + # within a context, not across http{} and stream{}), but every subsequent + # reload aborts with "bind() to 0.0.0.0:443 failed (98: Address already in + # use)" while `systemctl reload` still exits 0. That silently froze oolon's + # running config for a day and stranded every renewed cert on disk. + WEB_LISTEN: 127.0.0.1:14443 ssl proxy_protocol API_UPSTREAM_SCHEME: http API_UPSTREAM_ADDR: nikola.kosherinata.internal:42424 MUSL_TARGET: x86_64-unknown-linux-musl @@ -336,6 +346,7 @@ jobs: subs = { "SERVER_NAME": os.environ["SERVER_NAME"], "DOCROOT": os.environ["WEB_ROOT"], + "WEB_LISTEN": os.environ["WEB_LISTEN"], "API_UPSTREAM_SCHEME": os.environ["API_UPSTREAM_SCHEME"], "API_UPSTREAM_ADDR": os.environ["API_UPSTREAM_ADDR"], } @@ -366,4 +377,21 @@ jobs: fi sudo /usr/sbin/restorecon -R /etc/nginx/conf.d/'"${SERVER_NAME}"'.conf sudo /usr/sbin/nginx -t - sudo /usr/bin/systemctl reload nginx' + # `nginx -t` cannot see a listener that collides across http{} and + # stream{}, and `systemctl reload` is just SIGHUP — it exits 0 even + # when nginx aborts the reconfiguration and keeps the old cycle. A + # reload that really landed always spawns a fresh worker generation, + # so assert that rather than trusting the exit code. + master=$(cat /run/nginx.pid) + before=$(pgrep -P "$master" | sort | tr "\n" " ") + sudo /usr/bin/systemctl reload nginx + for _ in $(seq 1 10); do + sleep 1 + after=$(pgrep -P "$master" | sort | tr "\n" " ") + [ "$before" = "$after" ] || break + done + if [ "$before" = "$after" ]; then + echo "nginx reload did not take effect: worker generation unchanged" >&2 + sudo /usr/bin/tail -n 50 /var/log/nginx/error.log >&2 || true + exit 1 + fi' diff --git a/.gitea/workflows/refresh.yml b/.gitea/workflows/refresh.yml index 6e34f45..b9063b7 100644 --- a/.gitea/workflows/refresh.yml +++ b/.gitea/workflows/refresh.yml @@ -99,4 +99,18 @@ jobs: fi sudo /usr/sbin/restorecon -R /etc/nginx/conf.d/'"${SERVER_NAME}"'.conf sudo /usr/sbin/nginx -t - sudo /usr/bin/systemctl reload nginx' + # Verify the reload actually landed — see deploy.yml for why neither + # `nginx -t` nor `systemctl reload`s exit code is enough. + master=$(cat /run/nginx.pid) + before=$(pgrep -P "$master" | sort | tr "\n" " ") + sudo /usr/bin/systemctl reload nginx + for _ in $(seq 1 10); do + sleep 1 + after=$(pgrep -P "$master" | sort | tr "\n" " ") + [ "$before" = "$after" ] || break + done + if [ "$before" = "$after" ]; then + echo "nginx reload did not take effect: worker generation unchanged" >&2 + sudo /usr/bin/tail -n 50 /var/log/nginx/error.log >&2 || true + exit 1 + fi' diff --git a/asset/manifest.yml b/asset/manifest.yml index 07232a1..0828a33 100644 --- a/asset/manifest.yml +++ b/asset/manifest.yml @@ -34,3 +34,8 @@ environments: server_name: rob.tn root: /var/www/rob.tn api_upstream: http://nikola.kosherinata.internal:42424 + # oolon's stream SNI router owns TCP 443 and forwards to the local + # https tier on 14443 with PROXY protocol. Do not bind 443 here: + # `nginx -t` will not catch the collision (it is across http{} and + # stream{}) and every subsequent reload silently aborts. + listen: 127.0.0.1:14443 ssl proxy_protocol diff --git a/asset/nginx/site.conf.tmpl b/asset/nginx/site.conf.tmpl index cceb223..1d62c1f 100644 --- a/asset/nginx/site.conf.tmpl +++ b/asset/nginx/site.conf.tmpl @@ -5,7 +5,10 @@ upstream moments_api { server { server_name {{SERVER_NAME}}; - listen 443 ssl; + # Behind the edge's stream SNI router — see WEB_LISTEN in deploy.yml for why + # this must not bind 443 directly. real_ip recovery lives in the edge's + # conf.d/proxy-protocol.conf. + listen {{WEB_LISTEN}}; http2 on; ssl_certificate /etc/letsencrypt/live/{{SERVER_NAME}}/fullchain.pem; diff --git a/asset/sudoers.d/web-host.conf b/asset/sudoers.d/web-host.conf index e13a797..c772777 100644 --- a/asset/sudoers.d/web-host.conf +++ b/asset/sudoers.d/web-host.conf @@ -18,3 +18,7 @@ gitea_ci ALL=(root) NOPASSWD: /usr/sbin/semanage port -l gitea_ci ALL=(root) NOPASSWD: /usr/sbin/semanage port -a -t http_port_t -p tcp 42424 gitea_ci ALL=(root) NOPASSWD: /usr/sbin/nginx -t gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl reload nginx +# Read-only, so a reload that silently failed can report why (the error log is +# 0640 nginx:root). The deploy asserts the worker generation changed and dumps +# this on mismatch. +gitea_ci ALL=(root) NOPASSWD: /usr/bin/tail -n 50 /var/log/nginx/error.log diff --git a/script/deploy.sh b/script/deploy.sh index f52a91e..1479933 100755 --- a/script/deploy.sh +++ b/script/deploy.sh @@ -433,10 +433,17 @@ deploy_web() { local host="$1" log "web -> $host" - local server_name web_root api_upstream + local server_name web_root api_upstream web_listen server_name="$(yq --raw-output "${env_path}.components.web.config.server_name" "$manifest")" web_root="$(yq --raw-output "${env_path}.components.web.config.root" "$manifest")" api_upstream="$(yq --raw-output "${env_path}.components.web.config.api_upstream" "$manifest")" + # The edge's stream SNI router owns TCP 443; vhosts sit behind it on 14443. + # Binding 443 here collides with the router — `nginx -t` passes, but every + # later reload aborts while systemctl still reports success. See the + # web.config.listen note in asset/manifest.yml. + web_listen="$(yq --raw-output "${env_path}.components.web.config.listen" "$manifest")" + [[ -n "$web_listen" && "$web_listen" != "null" ]] \ + || web_listen='127.0.0.1:14443 ssl proxy_protocol' [[ -n "$server_name" && "$server_name" != "null" ]] || die "web.config.server_name missing in manifest" [[ -n "$web_root" && "$web_root" != "null" ]] || die "web.config.root missing in manifest" [[ -n "$api_upstream" && "$api_upstream" != "null" ]] || die "web.config.api_upstream missing in manifest" @@ -475,6 +482,7 @@ deploy_web() { local rendered rendered="$(<"${repo_root}/asset/nginx/site.conf.tmpl")" rendered=${rendered//'{{SERVER_NAME}}'/$server_name} + rendered=${rendered//'{{WEB_LISTEN}}'/$web_listen} rendered=${rendered//'{{DOCROOT}}'/$web_root} rendered=${rendered//'{{API_UPSTREAM_SCHEME}}'/$api_upstream_scheme} rendered=${rendered//'{{API_UPSTREAM_ADDR}}'/$api_upstream_addr}