From 6d8c3fe3ea9d45393a63107d543ffcf8a895f2b2 Mon Sep 17 00:00:00 2001 From: rob thijssen Date: Tue, 22 Sep 2026 15:03:12 +0300 Subject: [PATCH] infra: a dedicated access log for the public vhost, and deny hosting networks The proxy's shared access.log mixes every site on oolon and records neither Host nor Origin. Origin is the only field that separates our UI's WebSocket from another site's, because browsers send no Referer on the upgrade. The public vhost now writes its own JSON-lines log carrying both. Ten days of the shared log showed no third-party site using the API, and two headless pollers: an OVH host polling summary once a minute (88% of all no-referer REST) and a Hetzner host identifying as `stardust-devops`. The public vhost now refuses the hosting networks of both providers, from the prefixes RIPEstat reports as announced by AS24940, AS213230, AS212317 and AS16276, generated by script/deny-networks.py and committed so the list is reviewed. AS35540 is OVH's residential broadband and is left out. Against the same ten days, the list matches the two pollers and no visitor. Refusals are a plain-text 403 and are still logged with "denied":1, so a VPN user caught by it shows up rather than disappearing. Closes #24 Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01E4Ldjix2ofMdQ7dzqgnYA8 --- asset/nginx/blackbeard-deny.conf | 735 +++++++++++++++++++++++++++ asset/nginx/blackbeard-upstream.conf | 22 + asset/nginx/blackbeard.observer.conf | 31 +- script/deny-networks.py | 75 +++ script/infra-setup.sh | 8 +- 5 files changed, 869 insertions(+), 2 deletions(-) create mode 100644 asset/nginx/blackbeard-deny.conf create mode 100755 script/deny-networks.py diff --git a/asset/nginx/blackbeard-deny.conf b/asset/nginx/blackbeard-deny.conf new file mode 100644 index 0000000..c30ee50 --- /dev/null +++ b/asset/nginx/blackbeard-deny.conf @@ -0,0 +1,735 @@ +# GENERATED by script/deny-networks.py — do not edit by hand; regenerate. +# Announced prefixes per RIPEstat, 2026-09-22. +# +# Installed to /etc/nginx/conf.d/blackbeard-deny.conf. `$blackbeard_denied` +# is 1 for a client inside any network below; the public vhost refuses it. +# `geo` reads $remote_addr, which conf.d/proxy-protocol.conf has already +# restored to the client's address — without that every request would come +# from 127.0.0.1 and nothing would ever match. + +geo $blackbeard_denied { + default 0; + + # AS24940 Hetzner Online (dedicated): 84 networks + 2.28.0.0/15 1; + 5.9.0.0/16 1; + 5.75.128.0/17 1; + 23.88.0.0/17 1; + 31.58.88.0/24 1; + 37.27.0.0/16 1; + 45.15.99.0/24 1; + 45.145.227.0/24 1; + 46.4.0.0/16 1; + 46.62.128.0/17 1; + 46.224.0.0/15 1; + 49.12.0.0/15 1; + 62.238.0.0/17 1; + 65.21.0.0/16 1; + 65.108.0.0/15 1; + 77.42.0.0/17 1; + 78.46.0.0/15 1; + 78.138.62.0/24 1; + 85.10.192.0/18 1; + 85.93.20.0/24 1; + 88.99.0.0/16 1; + 88.198.0.0/16 1; + 89.167.0.0/17 1; + 91.98.0.0/15 1; + 91.107.128.0/17 1; + 91.190.240.0/21 1; + 94.130.0.0/16 1; + 95.216.0.0/15 1; + 116.202.0.0/15 1; + 128.140.0.0/17 1; + 135.181.0.0/16 1; + 136.243.0.0/16 1; + 138.199.128.0/17 1; + 138.201.0.0/16 1; + 142.132.128.0/17 1; + 144.76.0.0/16 1; + 148.251.0.0/16 1; + 157.90.0.0/16 1; + 157.180.0.0/17 1; + 159.69.0.0/16 1; + 162.55.0.0/16 1; + 167.233.0.0/16 1; + 167.235.0.0/16 1; + 168.119.0.0/16 1; + 171.25.225.0/24 1; + 176.9.0.0/16 1; + 178.63.0.0/16 1; + 178.104.0.0/15 1; + 178.212.75.0/24 1; + 185.12.65.0/24 1; + 185.50.120.0/23 1; + 185.107.52.0/22 1; + 185.126.28.0/22 1; + 185.157.83.0/24 1; + 185.157.176.0/22 1; + 185.171.224.0/22 1; + 185.189.228.0/22 1; + 185.213.45.0/24 1; + 185.216.237.0/24 1; + 185.226.99.0/24 1; + 185.228.8.0/23 1; + 188.34.128.0/17 1; + 188.40.0.0/16 1; + 188.245.0.0/16 1; + 193.105.82.0/24 1; + 193.110.6.0/23 1; + 193.163.198.0/24 1; + 194.42.180.0/22 1; + 194.42.184.0/22 1; + 194.62.106.0/24 1; + 195.60.226.0/24 1; + 195.201.0.0/16 1; + 195.248.224.0/24 1; + 197.242.84.0/22 1; + 201.131.3.0/24 1; + 204.29.146.0/24 1; + 204.168.128.0/17 1; + 213.133.96.0/19 1; + 213.239.192.0/18 1; + 216.55.108.0/22 1; + 2a01:4f8::/31 1; + 2a01:4ff:ff01::/48 1; + 2a06:be80::/29 1; + 2a12:2240::/29 1; + + # AS213230 Hetzner Cloud: 4 networks + 5.161.0.0/16 1; + 87.99.128.0/17 1; + 178.156.128.0/17 1; + 2a01:4ff::/40 1; + + # AS212317 Hetzner Cloud (US, Singapore): 2 networks + 5.78.0.0/16 1; + 2a01:4ff:100::/40 1; + + # AS16276 OVH SAS (hosting): 625 networks + 2.26.132.0/24 1; + 2.26.153.0/24 1; + 2.27.97.0/24 1; + 2.27.164.0/24 1; + 2.58.172.0/24 1; + 5.39.0.0/17 1; + 5.135.0.0/16 1; + 5.172.36.0/24 1; + 5.175.162.0/24 1; + 5.175.168.0/24 1; + 5.175.195.0/24 1; + 5.178.106.0/24 1; + 5.178.110.0/24 1; + 5.182.19.0/24 1; + 5.196.0.0/16 1; + 5.230.153.0/24 1; + 5.230.196.0/24 1; + 5.231.207.0/24 1; + 5.231.255.0/24 1; + 8.7.244.0/24 1; + 8.18.128.0/24 1; + 8.18.172.0/24 1; + 8.20.110.0/24 1; + 8.21.41.0/24 1; + 8.24.8.0/21 1; + 8.26.94.0/24 1; + 8.29.224.0/24 1; + 8.30.208.0/21 1; + 8.33.96.0/21 1; + 8.33.128.0/21 1; + 8.33.136.0/23 1; + 15.204.0.0/16 1; + 15.235.0.0/16 1; + 16.5.198.0/24 1; + 16.216.27.0/24 1; + 16.217.96.0/24 1; + 23.92.224.0/19 1; + 23.137.200.0/24 1; + 23.151.184.0/24 1; + 23.156.24.0/23 1; + 23.230.14.0/24 1; + 23.230.93.0/24 1; + 31.6.62.0/24 1; + 31.24.81.0/24 1; + 31.41.37.0/24 1; + 31.56.219.0/24 1; + 31.57.161.0/24 1; + 31.57.199.0/24 1; + 31.59.68.0/24 1; + 31.77.120.0/24 1; + 31.99.205.0/24 1; + 37.59.0.0/16 1; + 37.60.48.0/20 1; + 37.187.0.0/16 1; + 40.27.29.0/24 1; + 40.160.0.0/17 1; + 40.160.128.0/19 1; + 40.160.224.0/22 1; + 40.160.228.0/24 1; + 40.160.230.0/23 1; + 40.160.232.0/21 1; + 40.160.240.0/23 1; + 40.160.242.0/24 1; + 40.160.244.0/24 1; + 40.160.246.0/24 1; + 40.160.248.0/24 1; + 40.160.250.0/24 1; + 40.160.252.0/24 1; + 40.160.254.0/24 1; + 43.226.0.0/23 1; + 45.39.79.0/24 1; + 45.43.142.0/24 1; + 45.66.82.0/23 1; + 45.92.60.0/22 1; + 45.94.49.0/24 1; + 45.112.195.0/24 1; + 45.128.77.0/24 1; + 45.149.185.0/24 1; + 45.149.243.0/24 1; + 45.152.164.0/24 1; + 45.154.157.0/24 1; + 45.158.9.0/24 1; + 45.177.236.0/22 1; + 46.17.217.0/24 1; + 46.28.236.0/24 1; + 46.105.0.0/16 1; + 46.202.232.0/22 1; + 46.202.240.0/22 1; + 46.203.108.0/22 1; + 46.203.116.0/22 1; + 46.203.128.0/22 1; + 46.203.140.0/22 1; + 46.236.211.0/24 1; + 46.244.32.0/20 1; + 50.114.91.0/24 1; + 51.38.0.0/16 1; + 51.68.0.0/16 1; + 51.75.0.0/16 1; + 51.77.0.0/16 1; + 51.79.0.0/16 1; + 51.81.0.0/16 1; + 51.83.0.0/16 1; + 51.89.0.0/16 1; + 51.91.0.0/16 1; + 51.161.0.0/16 1; + 51.178.0.0/16 1; + 51.195.0.0/16 1; + 51.210.0.0/16 1; + 51.222.0.0/16 1; + 51.241.1.0/24 1; + 51.254.0.0/15 1; + 54.36.0.0/14 1; + 57.128.0.0/15 1; + 57.130.0.0/16 1; + 57.131.0.0/17 1; + 57.131.128.0/18 1; + 62.122.126.0/24 1; + 63.251.117.0/24 1; + 64.94.92.0/23 1; + 64.95.150.0/23 1; + 64.112.98.0/24 1; + 64.204.11.0/24 1; + 64.225.244.0/23 1; + 66.70.128.0/17 1; + 66.92.11.0/24 1; + 66.92.25.0/24 1; + 66.92.161.0/24 1; + 66.92.168.0/24 1; + 66.179.218.0/23 1; + 68.65.214.0/24 1; + 68.232.108.0/24 1; + 69.72.31.0/24 1; + 72.11.139.0/24 1; + 72.244.230.0/24 1; + 72.251.0.0/17 1; + 74.0.7.0/24 1; + 76.9.104.0/23 1; + 77.67.28.0/24 1; + 77.73.34.0/24 1; + 77.74.120.0/23 1; + 77.74.122.0/24 1; + 77.74.230.0/24 1; + 77.81.138.0/24 1; + 77.83.244.0/24 1; + 77.87.123.0/24 1; + 77.107.88.0/24 1; + 78.105.156.0/24 1; + 78.111.138.0/24 1; + 79.110.61.0/24 1; + 79.137.0.0/17 1; + 79.172.216.0/24 1; + 79.175.105.0/24 1; + 80.86.211.0/24 1; + 80.87.206.0/24 1; + 80.240.94.0/24 1; + 81.168.32.0/24 1; + 81.168.80.0/24 1; + 81.201.183.0/24 1; + 82.21.139.0/24 1; + 82.22.15.0/24 1; + 82.22.25.0/24 1; + 82.22.118.0/24 1; + 82.24.96.0/22 1; + 82.24.190.0/24 1; + 82.25.14.0/24 1; + 82.25.44.0/24 1; + 82.25.146.0/23 1; + 82.26.81.0/24 1; + 82.26.176.0/20 1; + 82.29.93.0/24 1; + 82.29.126.0/24 1; + 82.38.82.0/24 1; + 82.38.124.0/22 1; + 82.38.140.0/22 1; + 82.38.152.0/21 1; + 82.38.160.0/21 1; + 82.38.219.0/24 1; + 82.38.224.0/19 1; + 82.39.108.0/24 1; + 82.39.119.0/24 1; + 82.39.156.0/22 1; + 82.39.176.0/21 1; + 82.39.216.0/22 1; + 82.39.224.0/22 1; + 82.40.0.0/20 1; + 82.40.16.0/22 1; + 82.41.44.0/22 1; + 82.41.52.0/22 1; + 82.41.68.0/22 1; + 82.41.80.0/22 1; + 82.41.88.0/22 1; + 82.41.100.0/22 1; + 82.41.104.0/22 1; + 82.41.124.0/22 1; + 82.41.134.0/24 1; + 82.41.148.0/22 1; + 82.41.152.0/22 1; + 82.41.160.0/21 1; + 82.41.172.0/22 1; + 82.41.176.0/22 1; + 82.41.184.0/21 1; + 82.41.204.0/22 1; + 82.41.212.0/22 1; + 82.41.224.0/21 1; + 82.47.63.0/24 1; + 82.47.98.0/24 1; + 82.47.180.0/24 1; + 82.117.230.0/23 1; + 82.117.245.0/24 1; + 82.129.0.0/22 1; + 82.152.8.0/24 1; + 82.152.57.0/24 1; + 82.152.58.0/24 1; + 82.152.73.0/24 1; + 82.152.75.0/24 1; + 82.152.98.0/24 1; + 82.152.109.0/24 1; + 82.152.197.0/24 1; + 82.152.200.0/24 1; + 82.152.211.0/24 1; + 82.152.226.0/24 1; + 82.152.240.0/24 1; + 82.152.243.0/24 1; + 82.153.66.0/24 1; + 82.153.205.0/24 1; + 83.136.214.0/23 1; + 83.143.16.0/21 1; + 83.175.167.0/24 1; + 83.175.173.0/24 1; + 84.32.10.0/24 1; + 84.75.33.0/24 1; + 85.93.21.0/24 1; + 85.118.164.0/24 1; + 85.217.144.0/23 1; + 86.54.26.0/24 1; + 87.76.137.0/24 1; + 87.76.140.0/24 1; + 87.83.66.0/24 1; + 87.86.181.0/24 1; + 87.86.252.0/24 1; + 87.98.128.0/17 1; + 87.229.40.0/24 1; + 87.229.51.0/24 1; + 88.216.221.0/24 1; + 88.218.34.0/24 1; + 89.19.44.0/24 1; + 89.40.83.0/24 1; + 89.125.172.0/24 1; + 89.126.233.0/24 1; + 89.149.17.0/24 1; + 89.213.67.0/24 1; + 91.121.0.0/16 1; + 91.134.0.0/16 1; + 91.198.19.0/24 1; + 91.199.32.0/24 1; + 91.199.83.0/24 1; + 91.213.192.0/24 1; + 91.224.117.0/24 1; + 91.246.38.0/24 1; + 92.62.117.0/24 1; + 92.62.241.0/24 1; + 92.113.13.0/24 1; + 92.118.168.0/24 1; + 92.222.0.0/16 1; + 92.246.224.0/19 1; + 93.114.69.0/24 1; + 93.174.111.0/24 1; + 94.23.0.0/16 1; + 94.158.184.0/24 1; + 95.81.72.0/22 1; + 95.135.166.0/23 1; + 96.62.105.0/24 1; + 103.5.12.0/22 1; + 103.166.228.0/24 1; + 103.167.178.0/23 1; + 103.168.196.0/23 1; + 103.170.116.0/23 1; + 103.189.191.0/24 1; + 103.199.80.0/24 1; + 103.206.156.0/23 1; + 103.239.50.0/24 1; + 103.244.227.0/24 1; + 104.153.82.0/24 1; + 104.164.137.0/24 1; + 104.167.16.0/24 1; + 104.222.182.0/24 1; + 104.225.253.0/24 1; + 104.234.50.0/24 1; + 104.234.60.0/24 1; + 104.234.94.0/23 1; + 104.234.135.0/24 1; + 104.234.168.0/24 1; + 104.234.198.0/24 1; + 104.239.79.0/24 1; + 107.189.64.0/18 1; + 108.174.65.0/24 1; + 109.105.195.0/24 1; + 109.110.184.0/24 1; + 109.122.15.0/24 1; + 109.122.20.0/24 1; + 109.122.58.0/24 1; + 109.176.40.0/21 1; + 109.176.48.0/21 1; + 109.176.153.0/24 1; + 109.232.89.0/24 1; + 114.129.44.0/24 1; + 117.18.104.0/24 1; + 123.100.227.0/24 1; + 128.0.118.0/24 1; + 132.243.197.0/24 1; + 134.195.148.0/23 1; + 135.125.0.0/16 1; + 135.148.0.0/16 1; + 136.0.95.0/24 1; + 136.143.200.0/24 1; + 137.74.0.0/16 1; + 137.83.50.0/24 1; + 139.99.0.0/16 1; + 140.233.177.0/24 1; + 141.8.242.0/24 1; + 141.11.1.0/24 1; + 141.11.18.0/24 1; + 141.11.21.0/24 1; + 141.11.32.0/24 1; + 141.11.39.0/24 1; + 141.11.40.0/24 1; + 141.11.45.0/24 1; + 141.11.49.0/24 1; + 141.11.74.0/23 1; + 141.11.96.0/24 1; + 141.11.107.0/24 1; + 141.11.187.0/24 1; + 141.11.250.0/24 1; + 141.94.0.0/15 1; + 141.227.128.0/21 1; + 141.227.136.0/22 1; + 141.227.140.0/24 1; + 141.227.142.0/24 1; + 141.227.148.0/22 1; + 141.227.152.0/24 1; + 141.227.154.0/24 1; + 141.227.156.0/24 1; + 141.227.158.0/24 1; + 141.227.160.0/24 1; + 141.227.162.0/24 1; + 141.227.164.0/23 1; + 141.227.166.0/24 1; + 141.227.168.0/24 1; + 141.227.170.0/24 1; + 141.227.172.0/24 1; + 141.227.174.0/24 1; + 141.227.176.0/24 1; + 141.227.178.0/24 1; + 141.227.180.0/24 1; + 141.227.186.0/24 1; + 141.227.188.0/23 1; + 141.227.190.0/24 1; + 142.4.192.0/19 1; + 142.44.128.0/17 1; + 142.111.85.0/24 1; + 142.249.92.0/24 1; + 142.252.51.0/24 1; + 142.252.115.0/24 1; + 142.252.127.0/24 1; + 143.14.59.0/24 1; + 143.14.231.0/24 1; + 143.20.60.0/24 1; + 143.20.66.0/24 1; + 143.20.195.0/24 1; + 143.20.215.0/24 1; + 143.109.54.0/24 1; + 144.2.32.0/19 1; + 144.31.149.0/24 1; + 144.217.0.0/16 1; + 144.225.52.0/24 1; + 144.225.123.0/24 1; + 144.225.127.0/24 1; + 144.225.162.0/24 1; + 144.225.178.0/24 1; + 145.79.151.0/24 1; + 145.239.0.0/16 1; + 146.59.0.0/16 1; + 146.103.10.0/24 1; + 147.135.0.0/16 1; + 148.113.0.0/18 1; + 148.113.70.0/24 1; + 148.113.72.0/24 1; + 148.113.128.0/17 1; + 148.135.192.0/24 1; + 148.222.40.0/22 1; + 148.222.120.0/22 1; + 149.5.230.0/24 1; + 149.18.101.0/24 1; + 149.56.0.0/16 1; + 149.202.0.0/16 1; + 151.80.0.0/16 1; + 151.240.1.0/24 1; + 151.241.68.0/24 1; + 151.242.76.0/24 1; + 151.242.103.0/24 1; + 151.242.139.0/24 1; + 151.243.160.0/22 1; + 151.243.236.0/24 1; + 151.245.54.0/24 1; + 151.245.111.0/24 1; + 151.245.238.0/24 1; + 151.246.177.0/24 1; + 151.246.190.0/24 1; + 151.247.37.0/24 1; + 151.247.192.0/24 1; + 151.247.205.0/24 1; + 152.228.128.0/17 1; + 153.52.116.0/24 1; + 155.117.5.0/24 1; + 155.117.6.0/24 1; + 155.117.11.0/24 1; + 155.117.13.0/24 1; + 155.117.127.0/24 1; + 155.117.197.0/24 1; + 155.117.233.0/24 1; + 155.117.234.0/24 1; + 155.117.248.0/24 1; + 157.254.30.0/24 1; + 158.62.199.0/24 1; + 158.69.0.0/16 1; + 158.94.170.0/24 1; + 158.173.221.0/24 1; + 159.173.128.0/17 1; + 160.20.158.0/23 1; + 160.236.145.0/24 1; + 162.19.0.0/16 1; + 162.141.71.0/24 1; + 162.212.35.0/24 1; + 162.223.195.0/24 1; + 163.5.62.0/24 1; + 163.5.132.0/24 1; + 163.5.149.0/24 1; + 163.5.187.0/24 1; + 163.223.88.0/24 1; + 164.37.39.0/24 1; + 164.132.0.0/16 1; + 166.0.112.0/24 1; + 166.1.85.0/24 1; + 166.1.88.0/24 1; + 166.1.90.0/24 1; + 167.114.0.0/16 1; + 167.148.125.0/24 1; + 167.148.193.0/24 1; + 167.234.38.0/24 1; + 167.253.62.0/24 1; + 168.222.43.0/24 1; + 168.222.49.0/24 1; + 168.222.183.0/24 1; + 168.222.243.0/24 1; + 168.245.185.0/24 1; + 169.40.32.0/24 1; + 172.82.188.0/24 1; + 172.83.201.0/24 1; + 176.31.0.0/16 1; + 176.105.231.0/24 1; + 177.111.0.0/19 1; + 177.111.111.0/24 1; + 178.32.0.0/15 1; + 178.83.45.0/24 1; + 178.83.46.0/24 1; + 178.83.166.0/24 1; + 178.92.111.0/24 1; + 178.92.120.0/24 1; + 178.93.247.0/24 1; + 178.94.22.0/24 1; + 178.94.251.0/24 1; + 180.131.145.0/24 1; + 181.41.194.0/24 1; + 184.174.96.0/23 1; + 185.3.73.0/24 1; + 185.5.39.0/24 1; + 185.6.28.0/24 1; + 185.10.200.0/22 1; + 185.12.32.0/23 1; + 185.30.212.0/23 1; + 185.45.160.0/22 1; + 185.68.137.0/24 1; + 185.95.157.0/24 1; + 185.101.104.0/24 1; + 185.113.138.0/24 1; + 185.113.249.0/24 1; + 185.127.28.0/24 1; + 185.129.220.0/24 1; + 185.129.222.0/24 1; + 185.133.74.0/24 1; + 185.135.188.0/24 1; + 185.137.181.0/24 1; + 185.146.195.0/24 1; + 185.155.218.0/24 1; + 185.162.233.0/24 1; + 185.170.155.0/24 1; + 185.196.221.0/24 1; + 185.207.132.0/24 1; + 185.207.134.0/24 1; + 185.216.126.0/24 1; + 185.220.196.0/24 1; + 185.225.74.0/23 1; + 185.226.181.0/24 1; + 185.228.207.0/24 1; + 185.240.238.0/24 1; + 185.241.50.0/23 1; + 185.250.41.0/24 1; + 185.251.234.0/24 1; + 185.255.28.0/24 1; + 188.68.164.0/22 1; + 188.165.0.0/16 1; + 188.220.22.0/24 1; + 188.220.38.0/24 1; + 188.220.150.0/24 1; + 188.220.202.0/24 1; + 188.255.193.0/24 1; + 191.44.99.0/24 1; + 191.96.140.0/23 1; + 191.101.150.0/24 1; + 191.101.210.0/24 1; + 191.101.218.0/24 1; + 192.6.107.0/24 1; + 192.30.124.0/24 1; + 192.65.20.0/22 1; + 192.70.246.0/23 1; + 192.95.0.0/18 1; + 192.99.0.0/16 1; + 192.109.11.0/24 1; + 192.124.170.0/24 1; + 192.152.126.0/24 1; + 192.177.90.0/24 1; + 192.207.105.0/24 1; + 192.240.152.0/21 1; + 193.17.223.0/24 1; + 193.19.76.0/23 1; + 193.33.176.0/23 1; + 193.70.0.0/17 1; + 193.93.52.0/24 1; + 193.93.254.0/24 1; + 193.149.28.0/22 1; + 193.151.166.0/24 1; + 193.221.202.0/24 1; + 193.243.147.0/24 1; + 194.61.44.0/23 1; + 194.76.36.0/23 1; + 194.76.173.0/24 1; + 194.77.220.0/24 1; + 194.87.205.0/24 1; + 194.88.232.0/24 1; + 194.104.138.0/24 1; + 194.110.171.0/24 1; + 194.147.159.0/24 1; + 194.156.227.0/24 1; + 194.164.230.0/24 1; + 195.20.146.0/24 1; + 195.38.19.0/24 1; + 195.62.72.0/23 1; + 195.66.30.0/23 1; + 195.123.191.0/24 1; + 195.144.5.0/24 1; + 195.252.178.0/24 1; + 198.17.79.0/24 1; + 198.27.64.0/18 1; + 198.49.103.0/24 1; + 198.50.128.0/17 1; + 198.100.144.0/20 1; + 198.101.27.0/24 1; + 198.244.128.0/17 1; + 198.245.48.0/20 1; + 199.7.141.0/24 1; + 199.48.178.0/24 1; + 199.168.192.0/22 1; + 199.193.138.0/24 1; + 199.195.140.0/23 1; + 199.204.24.0/24 1; + 200.141.186.0/24 1; + 201.4.67.0/24 1; + 202.2.60.0/22 1; + 202.92.214.0/23 1; + 202.181.0.0/24 1; + 203.5.184.0/24 1; + 203.27.201.0/24 1; + 205.234.176.0/24 1; + 206.168.95.0/24 1; + 206.168.174.0/23 1; + 206.206.126.0/24 1; + 207.166.205.0/24 1; + 207.166.206.0/24 1; + 207.180.13.0/24 1; + 207.230.127.0/24 1; + 207.244.209.0/24 1; + 208.64.136.0/21 1; + 208.122.8.0/24 1; + 209.71.36.0/24 1; + 209.112.80.0/22 1; + 209.126.71.0/24 1; + 209.151.124.0/24 1; + 209.251.24.0/24 1; + 212.69.8.0/24 1; + 212.74.39.0/24 1; + 212.100.171.0/24 1; + 212.134.90.0/24 1; + 212.192.253.0/24 1; + 213.32.0.0/17 1; + 213.130.130.0/24 1; + 213.130.149.0/24 1; + 213.177.172.0/24 1; + 213.186.32.0/19 1; + 213.218.214.0/24 1; + 213.251.128.0/18 1; + 216.122.123.0/24 1; + 216.183.120.0/24 1; + 216.203.15.0/24 1; + 216.211.218.0/24 1; + 216.247.96.0/24 1; + 217.11.174.0/24 1; + 217.60.196.0/24 1; + 217.65.73.0/24 1; + 217.182.0.0/16 1; + 217.217.26.0/24 1; + 222.167.237.0/24 1; + 2001:41d0::/32 1; + 2402:1f00::/32 1; + 2604:2dc0::/32 1; + 2607:5300::/32 1; +} diff --git a/asset/nginx/blackbeard-upstream.conf b/asset/nginx/blackbeard-upstream.conf index 155b6a7..614f3d7 100644 --- a/asset/nginx/blackbeard-upstream.conf +++ b/asset/nginx/blackbeard-upstream.conf @@ -12,3 +12,25 @@ upstream blackbeard_api { # without keepalive every one pays a fresh TCP handshake to the node host. keepalive 16; } + +# The public vhost's own access log, declared here because `log_format` is only +# valid at http{} level (blackbeard/observer#24). The proxy's shared access.log +# mixes every site on oolon and records neither Host nor Origin, and Origin is +# the only field that separates our UI's WebSocket from another site's: browsers +# send no Referer on the upgrade handshake, so every /v1/ws looks the same in +# the shared log. JSON lines so a question about it is a `jq` one-liner rather +# than a quote-counting awk script. +log_format blackbeard_json escape=json + '{"time":"$time_iso8601",' + '"ip":"$remote_addr",' + '"host":"$host",' + '"method":"$request_method",' + '"uri":"$request_uri",' + '"proto":"$server_protocol",' + '"status":$status,' + '"bytes":$body_bytes_sent,' + '"request_time":$request_time,' + '"origin":"$http_origin",' + '"referer":"$http_referer",' + '"ua":"$http_user_agent",' + '"denied":$blackbeard_denied}'; diff --git a/asset/nginx/blackbeard.observer.conf b/asset/nginx/blackbeard.observer.conf index 0d3195d..ef74749 100644 --- a/asset/nginx/blackbeard.observer.conf +++ b/asset/nginx/blackbeard.observer.conf @@ -43,6 +43,35 @@ server { # without a trailing slash (reverse-proxies.md §4). absolute_redirect off; + # Replaces the proxy's shared access.log for this name, rather than adding + # to it: the format is declared in conf.d/blackbeard-upstream.conf. + access_log /var/log/nginx/blackbeard.observer.access.log blackbeard_json; + + # Hetzner and OVH hosting networks, from conf.d/blackbeard-deny.conf + # (script/deny-networks.py). Over 2026-09-12..22 this matched exactly two + # headless pollers and no visitor (blackbeard/observer#24). Whole vhost, not + # just /v1/: the page is useless without the API, and a plain 403 says why + # where a half-loaded site would not. Denials still reach the log above with + # "denied":1, so a VPN user caught by it is visible rather than silent. + # + # A speed bump rather than a gate: anything refused here can move to + # another provider. It does not replace an Origin check or credentials. + # + # Rewritten into a location rather than `return 403 "..."` here, because + # default_type cannot be set inside a server-level `if`, and the proxy's + # default of application/octet-stream makes a browser download the refusal + # instead of showing it. Not `error_page 403`, which would also answer a + # mislabelled webroot with this message and send the diagnosis elsewhere. + if ($blackbeard_denied) { + rewrite ^ /.denied last; + } + + location = /.denied { + internal; + default_type text/plain; + return 403 "blackbeard.observer does not serve requests from hosting-provider networks.\n"; + } + root /var/www/blackbeard.observer; index index.html; @@ -90,7 +119,7 @@ server { # Public, read-only, entirely derived from public block headers. No auth, # nothing to leak; documented here so the absence is deliberate rather than - # an oversight. + # an oversight. The network deny list above is about load, not secrecy. add_header X-Content-Type-Options nosniff always; add_header Referrer-Policy same-origin always; } diff --git a/script/deny-networks.py b/script/deny-networks.py new file mode 100755 index 0000000..1b33eeb --- /dev/null +++ b/script/deny-networks.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +# +# Regenerate asset/nginx/blackbeard-deny.conf: the networks the public vhost +# refuses, as an nginx `geo` map from the prefixes each ASN announces. +# +# ./script/deny-networks.py > asset/nginx/blackbeard-deny.conf +# +# Then review the diff and install it with `infra-setup.sh --role edge`. The +# output is committed rather than fetched at install time so that what the edge +# refuses is something a reviewer has read, and so a RIPEstat outage cannot turn +# an install into an empty list. +# +# Hosting networks only. AS35540 (OVH Telecom) is left out on purpose: it is +# OVH's residential broadband, and the point is headless pollers, not people. +# See blackbeard/observer#24 for the log analysis that justified each entry. + +import datetime +import ipaddress +import json +import sys +import urllib.request + +ASNS = { + 24940: "Hetzner Online (dedicated)", + 213230: "Hetzner Cloud", + 212317: "Hetzner Cloud (US, Singapore)", + 16276: "OVH SAS (hosting)", +} + +SOURCE = "https://stat.ripe.net/data/announced-prefixes/data.json?resource=AS{}" + + +def announced(asn): + with urllib.request.urlopen(SOURCE.format(asn), timeout=60) as r: + body = json.load(r) + if body.get("status") != "ok": + # A failed query must not produce a file that quietly stops denying a + # provider — the diff would show a deletion nobody asked for. + sys.exit(f"RIPEstat answered {body.get('status')!r} for AS{asn}") + nets = [ipaddress.ip_network(p["prefix"]) for p in body["data"]["prefixes"]] + if not nets: + sys.exit(f"AS{asn} announces nothing; refusing to write an empty entry") + # Collapsing removes duplicates and more-specifics; nginx's geo warns on a + # duplicate network and the file is smaller for it. + v4 = ipaddress.collapse_addresses(n for n in nets if n.version == 4) + v6 = ipaddress.collapse_addresses(n for n in nets if n.version == 6) + return list(v4) + list(v6) + + +def main(): + today = datetime.date.today().isoformat() + out = [ + "# GENERATED by script/deny-networks.py — do not edit by hand; regenerate.", + f"# Announced prefixes per RIPEstat, {today}.", + "#", + "# Installed to /etc/nginx/conf.d/blackbeard-deny.conf. `$blackbeard_denied`", + "# is 1 for a client inside any network below; the public vhost refuses it.", + "# `geo` reads $remote_addr, which conf.d/proxy-protocol.conf has already", + "# restored to the client's address — without that every request would come", + "# from 127.0.0.1 and nothing would ever match.", + "", + "geo $blackbeard_denied {", + " default 0;", + ] + for asn, name in ASNS.items(): + nets = announced(asn) + out.append("") + out.append(f" # AS{asn} {name}: {len(nets)} networks") + out.extend(f" {n} 1;" for n in nets) + out.append("}") + print("\n".join(out)) + + +if __name__ == "__main__": + main() diff --git a/script/infra-setup.sh b/script/infra-setup.sh index f091032..1081e77 100755 --- a/script/infra-setup.sh +++ b/script/infra-setup.sh @@ -255,9 +255,15 @@ REMOTE REMOTE info "$EDGE_HOST: nginx configuration" + # Shipped together: the log format in the upstream file reads + # $blackbeard_denied, which only blackbeard-deny.conf defines, and nginx -t + # rejects a config naming a variable nothing defines. The deny list is + # regenerated here by script/deny-networks.py and never edited on the proxy + # (blackbeard/observer#24). rsync -a --rsync-path 'sudo rsync' \ + "$REPO_ROOT/asset/nginx/blackbeard-deny.conf" \ "$REPO_ROOT/asset/nginx/blackbeard-upstream.conf" \ - "$EDGE_HOST:/etc/nginx/conf.d/blackbeard-upstream.conf" + "$EDGE_HOST:/etc/nginx/conf.d/" rsync -a --rsync-path 'sudo rsync' \ "$REPO_ROOT/asset/nginx/$PUBLIC_NAME.conf" \ "$REPO_ROOT/asset/nginx/$INTERNAL_NAME.conf" \