fix(core): the per-key manifest lock did not actually exclude

The lock added alongside compare-and-set removed its map entry inside
the critical section. A queued waiter still held that mutex, so the next
arrival found no entry, created a fresh one, and the two then ran
concurrently -- defeating the lock exactly when there was contention,
which is the only time it does anything.

I previously reported this fix as working on the strength of ten clean
release-mode runs. That was wrong: in debug, where the timing differs,
the guarding test fails 3 times in 12. Retiring the entry only after
dropping our own reference, under the same shard lock `entry` takes,
gives 0 failures in 25 -- and restoring the old behaviour reproduces 3
in 12, so the test really does catch it.

The consequence was the one the test names: a file served to a client
but never recorded, and so invisible the moment the instance is sealed.
Real against the local filesystem store, which has no conditional put
and relies on the verify-after-write retry; MinIO's compare-and-set
covers the same race, which is why the deployed path never showed it.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01XZG2i4AmfSqE97EJGBVb64

script/infra-setup.sh

@@ -13,8 +13,14 @@
 # What it does:
 #   1. bob      — gitea_ci user, runner key, scoped sudoers, SELinux port label
 #   2. caveman  — the rustingface bucket and a scoped MinIO service account
-#   3. hanzalova— rf.internal cert, nginx vhost, webroot, step@ renewal timer,
-#                 and the gitea_ci account CI uses to ship the built frontend
+#   3. hanzalova— rf.internal cert (internal CA) and rustingface.com cert
+#                 (Let's Encrypt), both vhosts, the webroot, the renewal
+#                 timers, and the gitea_ci account CI uses to ship the frontend
+#
+# Public DNS is NOT touched here. The Cloudflare token can rewrite DNS for
+# every zone on the account, including MX records for domains whose mail we
+# host (public-dns.md §1), so publishing the name is a separate, deliberate
+# operator step. This script prints the exact call and verifies the result.
 #
 # Application config is NOT shipped here. The deploy workflow renders it from
 # Gitea secrets on every deploy.
@@ -30,6 +36,9 @@ readonly MINIO_URL=http://caveman.kosherinata.internal:9000
 readonly BUCKET=rustingface
 readonly VHOST=rf
 readonly WEBROOT=/var/www/rustingface
+readonly PUBLIC_NAME=rustingface.com
+readonly SITE_INDIRECTION=nh.thgttg.com          # hanzalova's WAN address
+readonly CERTBOT_CREDENTIALS=/root/.certbot-internal
 readonly RUNNER_PUBKEY="${RUNNER_PUBKEY:-$HOME/.ssh/id_gitea_ci.pub}"
 readonly PROVISIONER_PASSWORD="${PROVISIONER_PASSWORD:-$HOME/.step/secrets/provisioner}"
 readonly ROOT_CA=/etc/pki/ca-trust/source/anchors/root-internal.pem
@@ -107,13 +116,14 @@ gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/${APP}/config.toml
 gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/${APP}/s3-access-key
 gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/${APP}/s3-secret-key
 gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/${APP}/hf-token
+gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/${APP}/client-tokens
 gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/systemd/system/${APP}.service
 gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/sysusers.d/${APP}.conf
 gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/firewalld/services/${APP}.xml
 gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemd-sysusers
 gitea_ci ALL=(root) NOPASSWD: /usr/bin/install -d -o root -g ${APP} -m 0750 /etc/${APP}
 gitea_ci ALL=(root) NOPASSWD: /usr/bin/chown -R root\:${APP} /etc/${APP}
-gitea_ci ALL=(root) NOPASSWD: /usr/bin/chmod 0640 /etc/${APP}/config.toml /etc/${APP}/s3-access-key /etc/${APP}/s3-secret-key /etc/${APP}/hf-token
+gitea_ci ALL=(root) NOPASSWD: /usr/bin/chmod 0640 /etc/${APP}/config.toml /etc/${APP}/s3-access-key /etc/${APP}/s3-secret-key /etc/${APP}/hf-token /etc/${APP}/client-tokens
 gitea_ci ALL=(root) NOPASSWD: /usr/bin/chmod 0755 /usr/local/bin/${APP}
 gitea_ci ALL=(root) NOPASSWD: /usr/sbin/restorecon -R /usr/local/bin/${APP} /etc/${APP}
 gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl daemon-reload
@@ -242,19 +252,61 @@ provision_proxy() {
     ssh "$PROXY_HOST" "sudo systemctl enable --now step@${VHOST}.timer"
     ok "renewal timer enabled"
 
-    # The vhost references the cert paths, so it is only installed after the
-    # cert exists: nginx -t fails on a missing ssl_certificate and that blocks
-    # every unrelated reload on the host.
+    # The public cert. Idempotent: --keep-until-expiring makes a repeat run a
+    # no-op while the lineage is still valid. DNS-01 means this works before
+    # the name resolves anywhere, so the cert can exist before the record does.
+    local have_public
+    have_public=$(ssh "$PROXY_HOST" \
+        "sudo test -d /etc/letsencrypt/live/${PUBLIC_NAME} && echo yes || echo no" | tail -1)
+    if [ "$have_public" = no ]; then
+        if ! ssh "$PROXY_HOST" "sudo test -f ${CERTBOT_CREDENTIALS}"; then
+            die "${CERTBOT_CREDENTIALS} is missing on ${PROXY_HOST}; cannot issue ${PUBLIC_NAME}"
+        fi
+        ssh "$PROXY_HOST" "sudo certbot certonly \
+            -m ops@${PUBLIC_NAME} --agree-tos --no-eff-email --noninteractive \
+            --cert-name ${PUBLIC_NAME} \
+            --key-type ecdsa \
+            --dns-cloudflare \
+            --dns-cloudflare-credentials ${CERTBOT_CREDENTIALS} \
+            --dns-cloudflare-propagation-seconds 60 \
+            --keep-until-expiring \
+            -d ${PUBLIC_NAME} -d www.${PUBLIC_NAME}"
+        ok "issued ${PUBLIC_NAME}"
+    else
+        ok "${PUBLIC_NAME} cert is present"
+    fi
+
+    # certbot renews on its own timer, but a reload is only a request and a
+    # failed one leaves the old cert on the wire for up to 90 days. The hook
+    # must therefore leave a trace rather than swallow the failure.
+    ssh "$PROXY_HOST" 'sudo bash -s' <<'REMOTE'
+set -euo pipefail
+install -d -m 0755 /etc/letsencrypt/renewal-hooks/deploy
+cat > /etc/letsencrypt/renewal-hooks/deploy/reload-nginx.sh <<'HOOK'
+#!/bin/sh
+systemctl reload nginx || logger -t reload-nginx -p daemon.err \
+    "nginx reload failed after certbot renewal; cert on disk is newer than the one served"
+HOOK
+chmod +x /etc/letsencrypt/renewal-hooks/deploy/reload-nginx.sh
+REMOTE
+    ok "certbot deploy hook installed"
+
+    # The vhosts reference cert paths, so they are only installed once both
+    # certs exist: nginx -t fails on a missing ssl_certificate, and that
+    # blocks every unrelated reload on this shared proxy.
     rsync -az --rsync-path='sudo rsync' --mkpath \
-        asset/nginx/${VHOST}.internal.conf \
-        "$PROXY_HOST:/etc/nginx/sites-available/${VHOST}.internal.conf"
+        asset/nginx/rustingface-upstream.conf \
+        "$PROXY_HOST:/etc/nginx/conf.d/rustingface-upstream.conf"
+    for conf in ${VHOST}.internal.conf ${PUBLIC_NAME}.conf; do
+        rsync -az --rsync-path='sudo rsync' --mkpath \
+            "asset/nginx/${conf}" "$PROXY_HOST:/etc/nginx/sites-available/${conf}"
+        ssh "$PROXY_HOST" "sudo ln -sfn ../sites-available/${conf} /etc/nginx/sites-enabled/${conf}"
+    done
     ssh "$PROXY_HOST" "
         set -euo pipefail
-        sudo ln -sfn ../sites-available/${VHOST}.internal.conf \
-            /etc/nginx/sites-enabled/${VHOST}.internal.conf
         sudo nginx -t
         sudo systemctl reload nginx"
-    ok "vhost installed and nginx reloaded"
+    ok "both vhosts installed and nginx reloaded"
 
     # A reload is a request, not a guarantee: nginx keeps its old cycle if it
     # cannot rebind, and then serves the certificate it loaded then. Compare
@@ -320,6 +372,10 @@ gitea_ci ALL=(root) NOPASSWD: /usr/bin/install -d -o root -g root -m 0755 ${WEBR
 gitea_ci ALL=(root) NOPASSWD: /usr/sbin/restorecon -R ${WEBROOT}
 gitea_ci ALL=(root) NOPASSWD: /usr/sbin/nginx -t
 gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl reload nginx
+# The deploy's health check asks whether the public certificate exists
+# before probing the public vhost; /etc/letsencrypt/live is root-only, so
+# an unprivileged test silently returns false and would skip the check.
+gitea_ci ALL=(root) NOPASSWD: /usr/bin/test -d /etc/letsencrypt/live/rustingface.com
 SUDOERS
 chmod 0440 /etc/sudoers.d/.${APP}_gitea_ci.new
 visudo -cf /etc/sudoers.d/.${APP}_gitea_ci.new
@@ -327,7 +383,31 @@ mv /etc/sudoers.d/.${APP}_gitea_ci.new /etc/sudoers.d/${APP}_gitea_ci
 REMOTE
     ok "webroot ${WEBROOT} and scoped sudoers on the proxy"
 
-    info "DNS"
+    info "public DNS (${PUBLIC_NAME})"
+    if getent hosts "${PUBLIC_NAME}" >/dev/null; then
+        ok "${PUBLIC_NAME} resolves to $(getent hosts "${PUBLIC_NAME}" | awk '{print $1}' | head -1)"
+    else
+        echo "    ${PUBLIC_NAME} does not resolve yet. It is a zone apex, so it takes an"
+        echo "    A record rather than a CNAME (public-dns.md §3). From ${PROXY_HOST}:"
+        echo
+        echo "      TOKEN=\$(sudo grep -oP '(?<=dns_cloudflare_api_token\s=\s).*' \\"
+        echo "          ${CERTBOT_CREDENTIALS} | tr -d \"'\" \")"
+        echo "      zid=\$(curl -sS -H \"Authorization: Bearer \$TOKEN\" \\"
+        echo "          'https://api.cloudflare.com/client/v4/zones?name=${PUBLIC_NAME}' | jq -r '.result[0].id')"
+        echo "      # apex: A record to this site's WAN address"
+        echo "      curl -sS -X POST \"https://api.cloudflare.com/client/v4/zones/\$zid/dns_records\" \\"
+        echo "          -H \"Authorization: Bearer \$TOKEN\" -H 'Content-Type: application/json' \\"
+        echo "          --data '{\"type\":\"A\",\"name\":\"${PUBLIC_NAME}\",\"content\":\"$(dig +short ${SITE_INDIRECTION} A | head -1)\",\"ttl\":1,\"proxied\":false,\"comment\":\"rustingface\"}'"
+        echo "      # www: CNAME to the site indirection, so a re-homed site is one record"
+        echo "      curl -sS -X POST \"https://api.cloudflare.com/client/v4/zones/\$zid/dns_records\" \\"
+        echo "          -H \"Authorization: Bearer \$TOKEN\" -H 'Content-Type: application/json' \\"
+        echo "          --data '{\"type\":\"CNAME\",\"name\":\"www.${PUBLIC_NAME}\",\"content\":\"${SITE_INDIRECTION}\",\"ttl\":1,\"proxied\":false,\"comment\":\"rustingface\"}'"
+        echo
+        echo "    Run these from ${PROXY_HOST}: the token is already there, and copying it"
+        echo "    off the host spreads a credential that can edit every zone we own."
+    fi
+
+    info "mesh DNS"
     if getent hosts "${VHOST}.internal" >/dev/null; then
         ok "${VHOST}.internal resolves"
         return 0