fix(core): the per-key manifest lock did not actually exclude
The lock added alongside compare-and-set removed its map entry inside the critical section. A queued waiter still held that mutex, so the next arrival found no entry, created a fresh one, and the two then ran concurrently -- defeating the lock exactly when there was contention, which is the only time it does anything. I previously reported this fix as working on the strength of ten clean release-mode runs. That was wrong: in debug, where the timing differs, the guarding test fails 3 times in 12. Retiring the entry only after dropping our own reference, under the same shard lock `entry` takes, gives 0 failures in 25 -- and restoring the old behaviour reproduces 3 in 12, so the test really does catch it. The consequence was the one the test names: a file served to a client but never recorded, and so invisible the moment the instance is sealed. Real against the local filesystem store, which has no conditional put and relies on the verify-after-write retry; MinIO's compare-and-set covers the same race, which is why the deployed path never showed it. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01XZG2i4AmfSqE97EJGBVb64
This commit is contained in:
@@ -19,6 +19,9 @@ env:
|
||||
S3_ENDPOINT: http://caveman.kosherinata.internal:9000
|
||||
S3_BUCKET: rustingface
|
||||
UPSTREAM_ENABLED: "true"
|
||||
AUTH_MODE: bearer
|
||||
AUTH_ANONYMOUS: catalog
|
||||
ALLOW_NEW_REPOS: "false"
|
||||
PROXY_HOST: hanzalova.internal
|
||||
WEBROOT: /var/www/rustingface
|
||||
|
||||
@@ -93,9 +96,10 @@ jobs:
|
||||
RSYNC_SSH_KEY: ${{ secrets.RSYNC_SSH_KEY }}
|
||||
S3_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY_ID }}
|
||||
S3_SECRET_ACCESS_KEY: ${{ secrets.S3_SECRET_ACCESS_KEY }}
|
||||
CLIENT_TOKENS: ${{ secrets.CLIENT_TOKENS }}
|
||||
run: |
|
||||
missing=""
|
||||
for name in RSYNC_SSH_KEY S3_ACCESS_KEY_ID S3_SECRET_ACCESS_KEY; do
|
||||
for name in RSYNC_SSH_KEY S3_ACCESS_KEY_ID S3_SECRET_ACCESS_KEY CLIENT_TOKENS; do
|
||||
[ -n "${!name:-}" ] || missing="$missing $name"
|
||||
done
|
||||
if [ -n "$missing" ]; then
|
||||
@@ -126,18 +130,21 @@ jobs:
|
||||
S3_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY_ID }}
|
||||
S3_SECRET_ACCESS_KEY: ${{ secrets.S3_SECRET_ACCESS_KEY }}
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN }}
|
||||
CLIENT_TOKENS: ${{ secrets.CLIENT_TOKENS }}
|
||||
run: |
|
||||
# Literal substitution, not a shell expansion: a secret containing
|
||||
# $, ` or \ must survive intact.
|
||||
python3 - <<'PY'
|
||||
import os, pathlib
|
||||
template = pathlib.Path("asset/config/config.toml.tmpl").read_text()
|
||||
for key in ("LISTEN_ADDR", "S3_ENDPOINT", "S3_BUCKET", "UPSTREAM_ENABLED"):
|
||||
for key in ("LISTEN_ADDR", "S3_ENDPOINT", "S3_BUCKET", "UPSTREAM_ENABLED",
|
||||
"AUTH_MODE", "AUTH_ANONYMOUS", "ALLOW_NEW_REPOS"):
|
||||
template = template.replace("{{%s}}" % key, os.environ[key])
|
||||
pathlib.Path("dist/config.toml").write_text(template)
|
||||
for name, var in (("s3-access-key", "S3_ACCESS_KEY_ID"),
|
||||
("s3-secret-key", "S3_SECRET_ACCESS_KEY"),
|
||||
("hf-token", "HF_TOKEN")):
|
||||
("hf-token", "HF_TOKEN"),
|
||||
("client-tokens", "CLIENT_TOKENS")):
|
||||
pathlib.Path("dist", name).write_text(os.environ.get(var, ""))
|
||||
PY
|
||||
# Fail loudly rather than shipping a config with an unrendered
|
||||
@@ -152,6 +159,13 @@ jobs:
|
||||
echo "S3 credentials are empty; set the S3_ACCESS_KEY_ID and S3_SECRET_ACCESS_KEY secrets" >&2
|
||||
exit 1
|
||||
fi
|
||||
# Bearer auth with no tokens refuses to start, so catch it here
|
||||
# rather than as a service that restart-loops after the files land.
|
||||
if [ "$AUTH_MODE" = bearer ] && [ ! -s dist/client-tokens ]; then
|
||||
echo "AUTH_MODE is bearer but the CLIENT_TOKENS secret is empty." >&2
|
||||
echo "Set it to one accepted token per line; '#' comments are allowed." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: create the service account and its config directory
|
||||
run: |
|
||||
@@ -168,7 +182,7 @@ jobs:
|
||||
set -euo pipefail
|
||||
rsync -az --rsync-path='sudo rsync' \
|
||||
dist/rustingface "$SERVICE_HOST:/usr/local/bin/rustingface"
|
||||
for f in config.toml s3-access-key s3-secret-key hf-token; do
|
||||
for f in config.toml s3-access-key s3-secret-key hf-token client-tokens; do
|
||||
rsync -az --rsync-path='sudo rsync' --mkpath \
|
||||
"dist/$f" "$SERVICE_HOST:/etc/rustingface/$f"
|
||||
done
|
||||
@@ -179,7 +193,7 @@ jobs:
|
||||
set -euo pipefail
|
||||
sudo chmod 0755 /usr/local/bin/rustingface
|
||||
sudo chown -R root:rustingface /etc/rustingface
|
||||
sudo chmod 0640 /etc/rustingface/config.toml /etc/rustingface/s3-access-key /etc/rustingface/s3-secret-key /etc/rustingface/hf-token
|
||||
sudo chmod 0640 /etc/rustingface/config.toml /etc/rustingface/s3-access-key /etc/rustingface/s3-secret-key /etc/rustingface/hf-token /etc/rustingface/client-tokens
|
||||
sudo restorecon -R /usr/local/bin/rustingface /etc/rustingface'
|
||||
|
||||
- name: firewalld
|
||||
@@ -310,8 +324,28 @@ jobs:
|
||||
curl -fsS https://rf.internal/healthz | grep -q ok
|
||||
# A client route must fall back to the app rather than 404.
|
||||
curl -fsS https://rf.internal/models | grep -q "id=.root."
|
||||
# And an API path must still reach the service, not the app.
|
||||
curl -fsS https://rf.internal/api/whoami-v2 | grep -q rustingface'
|
||||
# An API path must still reach the service, not the app. With
|
||||
# bearer auth on this is a 401, which is itself the proof that the
|
||||
# request reached the service rather than the static root.
|
||||
code=$(curl -s -o /dev/null -w "%{http_code}" https://rf.internal/api/whoami-v2)
|
||||
case "$code" in 200|401) ;; *) echo "api path returned $code" >&2; exit 1;; esac
|
||||
# The public vhost, if its certificate exists yet. Checked over
|
||||
# loopback with the SNI name, because the public name may not
|
||||
# resolve from inside the mesh until the DNS record is published.
|
||||
if sudo test -d /etc/letsencrypt/live/rustingface.com; then
|
||||
curl -fsS --resolve rustingface.com:443:127.0.0.1 \
|
||||
https://rustingface.com/ | grep -q "id=.root."
|
||||
# The catalogue is the public face and must answer anonymously.
|
||||
curl -fsS --resolve rustingface.com:443:127.0.0.1 \
|
||||
https://rustingface.com/v1/status | grep -q bucket
|
||||
# /metrics must not be public.
|
||||
m=$(curl -s -o /dev/null -w "%{http_code}" --resolve rustingface.com:443:127.0.0.1 \
|
||||
https://rustingface.com/metrics)
|
||||
[ "$m" = 404 ] || { echo "/metrics is exposed publicly ($m)" >&2; exit 1; }
|
||||
echo "public vhost answering, /metrics closed"
|
||||
else
|
||||
echo "rustingface.com has no certificate yet; skipping the public checks"
|
||||
fi'
|
||||
echo "frontend and API both answering through rf.internal"
|
||||
|
||||
- name: nginx log
|
||||
|
||||
@@ -60,6 +60,27 @@ statically from the nginx host and reading the `/v1/` API.
|
||||
`v1_repo_detail` does — and test it against `/resolve/` in a *directory*
|
||||
name, not just in a repo name.
|
||||
|
||||
## Auth, and what actually bounds the bucket
|
||||
|
||||
Two credentials that are easy to confuse, pointing in opposite directions:
|
||||
|
||||
- `upstream.token_file` (`hf-token`) authenticates **rustingface to the Hub**.
|
||||
Outbound only, never echoed downstream, never logged. Needed for gated repos.
|
||||
- `auth.token_file` (`client-tokens`) authenticates **clients to rustingface**.
|
||||
|
||||
A client's token is never forwarded upstream, and rustingface's token is never
|
||||
shown to a client. Do not let these merge.
|
||||
|
||||
`auth.mode = "bearer"` controls *who*, not *what* — a token holder can still
|
||||
cause any repository to be fetched. **`policy.allow_new_repos = false` is what
|
||||
bounds the bucket**, and the deployed configuration relies on it.
|
||||
|
||||
`auth.anonymous = "catalog"` opens `/v1/` to unauthenticated requests. That is
|
||||
only safe while every handler under `/v1/` reads the bucket and never consults
|
||||
upstream. **If you add a route there that can fetch, you have made the public
|
||||
face able to fill the bucket** — there is a test asserting the whole surface
|
||||
causes zero upstream requests, and it should stay that way.
|
||||
|
||||
## Where things live
|
||||
|
||||
Types in `entities`, logic in `core`, I/O in `data`, HTTP in `api`, wiring in
|
||||
|
||||
@@ -45,14 +45,26 @@ disable_xet = true
|
||||
# freeze: the first resolution of a mutable ref is recorded and reused
|
||||
# indefinitely. Moving to a newer revision is `rustingface refresh`.
|
||||
ref_resolution = "freeze"
|
||||
allow_new_repos = true
|
||||
# false: only repositories already in the bucket may be served, so no client
|
||||
# -- authenticated or not -- can cause an arbitrary model to be fetched and
|
||||
# stored. Growing the registry is an operator action: `rustingface fetch`.
|
||||
# This, not auth, is what bounds how large the bucket gets.
|
||||
allow_new_repos = {{ALLOW_NEW_REPOS}}
|
||||
allowlist = []
|
||||
|
||||
[auth]
|
||||
# The vhost is mesh-only and there is no per-client identity to enforce yet.
|
||||
# Switch to "bearer" and populate /etc/rustingface/client-tokens to change that.
|
||||
mode = "none"
|
||||
# This registry is published publicly at rustingface.com as well as on the
|
||||
# mesh at rf.internal, so downloads carry a credential.
|
||||
mode = "{{AUTH_MODE}}"
|
||||
token_file = "/etc/rustingface/client-tokens"
|
||||
# What a request with no valid token may reach:
|
||||
# nothing — 401 everywhere
|
||||
# catalog — the /v1/ inventory and model cards, and nothing else
|
||||
#
|
||||
# "catalog" is the public shop window: an operator evaluating rustingface can
|
||||
# browse what is stored and read the cards, but cannot download a weight and
|
||||
# cannot cause a fetch, because that surface only ever reads the bucket.
|
||||
anonymous = "{{AUTH_ANONYMOUS}}"
|
||||
|
||||
[observability]
|
||||
metrics = true
|
||||
|
||||
@@ -13,11 +13,6 @@
|
||||
# `/resolve/`, plus the API prefixes, goes to the service; everything else is
|
||||
# the single-page app.
|
||||
|
||||
upstream rustingface {
|
||||
server bob.hanzalova.internal:20482;
|
||||
keepalive 8;
|
||||
}
|
||||
|
||||
server {
|
||||
listen 127.0.0.1:14443 ssl proxy_protocol;
|
||||
server_name rf.internal;
|
||||
|
||||
@@ -0,0 +1,11 @@
|
||||
# The rustingface service, shared by both vhosts.
|
||||
#
|
||||
# Its own file because nginx rejects a duplicate `upstream` block, and both
|
||||
# rf.internal (internal CA, mesh) and rustingface.com (Let's Encrypt, public)
|
||||
# proxy to the same service. Installed into conf.d/ so it loads once, at
|
||||
# http{} scope, before either vhost.
|
||||
|
||||
upstream rustingface {
|
||||
server bob.hanzalova.internal:20482;
|
||||
keepalive 8;
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
# rustingface.com — the public face.
|
||||
#
|
||||
# A shop window for operators evaluating whether to run their own instance:
|
||||
# the catalogue, the model cards, and the app are readable by anyone. Weights
|
||||
# are not. That split is enforced by the service (`auth.mode = "bearer"` with
|
||||
# `auth.anonymous = "catalog"`), not by this file — nginx here is defence in
|
||||
# depth, and the two must not be allowed to disagree.
|
||||
#
|
||||
# Installed on hanzalova, whose WAN address is nh.thgttg.com. TCP :443 belongs
|
||||
# to the stream SNI router, so this listens on the https tier behind it.
|
||||
#
|
||||
# Public name => Let's Encrypt (external-tls.md). The mesh name rf.internal is
|
||||
# a separate vhost with an internal-CA cert, sharing this webroot and upstream
|
||||
# (reverse-proxies.md §2).
|
||||
|
||||
server {
|
||||
listen 127.0.0.1:14443 ssl proxy_protocol;
|
||||
http2 on;
|
||||
server_name rustingface.com www.rustingface.com;
|
||||
|
||||
ssl_certificate /etc/letsencrypt/live/rustingface.com/fullchain.pem;
|
||||
ssl_certificate_key /etc/letsencrypt/live/rustingface.com/privkey.pem;
|
||||
ssl_protocols TLSv1.2 TLSv1.3;
|
||||
|
||||
root /var/www/rustingface;
|
||||
index index.html;
|
||||
|
||||
absolute_redirect off;
|
||||
|
||||
access_log /var/log/nginx/rustingface.com.access.log;
|
||||
error_log /var/log/nginx/rustingface.com.error.log;
|
||||
|
||||
# A public origin gets the headers a public origin needs. The app loads
|
||||
# nothing cross-origin, so the policy can be tight.
|
||||
add_header X-Content-Type-Options "nosniff" always;
|
||||
add_header X-Frame-Options "DENY" always;
|
||||
add_header Referrer-Policy "no-referrer" always;
|
||||
add_header Strict-Transport-Security "max-age=31536000" always;
|
||||
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header Connection "";
|
||||
proxy_set_header X-Real-IP $proxy_protocol_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_protocol_addr;
|
||||
proxy_set_header X-Forwarded-Proto https;
|
||||
proxy_set_header Accept-Encoding $http_accept_encoding;
|
||||
|
||||
proxy_buffering off;
|
||||
proxy_request_buffering off;
|
||||
client_max_body_size 0;
|
||||
proxy_connect_timeout 15s;
|
||||
proxy_read_timeout 300s;
|
||||
proxy_send_timeout 300s;
|
||||
|
||||
# --- never public ------------------------------------------------------
|
||||
|
||||
# /metrics is exempt from the service's own auth so a mesh Prometheus can
|
||||
# scrape it without a credential. That exemption is fine on rf.internal
|
||||
# and unacceptable here, so the public vhost closes it explicitly rather
|
||||
# than relying on the service to.
|
||||
location = /metrics { return 404; }
|
||||
|
||||
# --- the service -------------------------------------------------------
|
||||
|
||||
# Reachable, but the service answers 401 without a token. Published so a
|
||||
# token holder can pull from anywhere, and so an anonymous client gets a
|
||||
# 401 it can act on rather than a 404 that reads as "no such model".
|
||||
location ^~ /api/ { proxy_pass http://rustingface; }
|
||||
location ^~ /v1/ { proxy_pass http://rustingface; }
|
||||
location ~ /resolve/ { proxy_pass http://rustingface; }
|
||||
|
||||
location = /healthz { proxy_pass http://rustingface; }
|
||||
|
||||
# --- the app -----------------------------------------------------------
|
||||
|
||||
location /assets/ {
|
||||
gzip_static on;
|
||||
add_header Cache-Control "public, max-age=31536000, immutable";
|
||||
}
|
||||
|
||||
location = /index.html {
|
||||
add_header Cache-Control "no-cache";
|
||||
}
|
||||
|
||||
location / {
|
||||
try_files $uri $uri/ /index.html;
|
||||
}
|
||||
|
||||
gzip on;
|
||||
gzip_types text/css application/javascript application/json image/svg+xml;
|
||||
gzip_min_length 1024;
|
||||
}
|
||||
@@ -6,7 +6,7 @@
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use rustingface_entities::config::AuthMode;
|
||||
use rustingface_entities::config::{Anonymous, AuthMode};
|
||||
use rustingface_entities::error::{Error, Result};
|
||||
use subtle::ConstantTimeEq;
|
||||
|
||||
@@ -15,6 +15,7 @@ use subtle::ConstantTimeEq;
|
||||
pub struct Tokens {
|
||||
mode: AuthMode,
|
||||
accepted: Vec<String>,
|
||||
anonymous: Anonymous,
|
||||
}
|
||||
|
||||
impl Tokens {
|
||||
@@ -23,6 +24,15 @@ impl Tokens {
|
||||
/// One token per line; blank lines and `#` comments are ignored so an
|
||||
/// operator can annotate who each token belongs to.
|
||||
pub fn new(mode: AuthMode, token_file_contents: Option<&str>) -> Result<Self> {
|
||||
Self::with_anonymous(mode, token_file_contents, Anonymous::Nothing)
|
||||
}
|
||||
|
||||
/// As [`Tokens::new`], stating what an unauthenticated request may reach.
|
||||
pub fn with_anonymous(
|
||||
mode: AuthMode,
|
||||
token_file_contents: Option<&str>,
|
||||
anonymous: Anonymous,
|
||||
) -> Result<Self> {
|
||||
let accepted: Vec<String> = token_file_contents
|
||||
.unwrap_or_default()
|
||||
.lines()
|
||||
@@ -38,7 +48,16 @@ impl Tokens {
|
||||
.into(),
|
||||
));
|
||||
}
|
||||
Ok(Self { mode, accepted })
|
||||
Ok(Self {
|
||||
mode,
|
||||
accepted,
|
||||
anonymous,
|
||||
})
|
||||
}
|
||||
|
||||
/// Whether a request with no valid token may read the catalogue.
|
||||
pub fn anonymous_may_browse(&self) -> bool {
|
||||
self.anonymous == Anonymous::Catalog
|
||||
}
|
||||
|
||||
/// Whether any authentication is required.
|
||||
@@ -81,6 +100,7 @@ impl Default for Tokens {
|
||||
Self {
|
||||
mode: AuthMode::None,
|
||||
accepted: Vec::new(),
|
||||
anonymous: Anonymous::Nothing,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -163,6 +163,56 @@ pub async fn detail(State(state): State<AppState>, rest: String) -> Result<Respo
|
||||
.into_response())
|
||||
}
|
||||
|
||||
/// `GET /v1/repos/{type}/{repo_id}/card`
|
||||
///
|
||||
/// The model card, as markdown, read straight out of the bucket.
|
||||
///
|
||||
/// The card is catalogue material, so it lives on the catalogue surface
|
||||
/// rather than being read through `/resolve/` like any other file. That is
|
||||
/// what lets an anonymous visitor read a card without also being able to
|
||||
/// download the weights beside it — and it means browsing never causes a
|
||||
/// fetch, because this only ever reads what is already stored.
|
||||
pub async fn card_markdown(
|
||||
state: &AppState,
|
||||
repo_type: RepoType,
|
||||
repo_id: &RepoId,
|
||||
) -> Result<Response, ApiError> {
|
||||
let commit = match state.registry.load_ref(repo_type, repo_id, "main").await? {
|
||||
Some(record) => Some(record.commit),
|
||||
None => state
|
||||
.registry
|
||||
.list_refs(repo_type, repo_id)
|
||||
.await?
|
||||
.first()
|
||||
.map(|r| r.commit.clone()),
|
||||
};
|
||||
let Some(commit) = commit else {
|
||||
return Err(ApiError(Error::RevisionNotFound(format!(
|
||||
"{repo_id} has no pinned revision"
|
||||
))));
|
||||
};
|
||||
|
||||
let manifest = state
|
||||
.registry
|
||||
.load_manifest(repo_type, repo_id, &commit)
|
||||
.await?
|
||||
.ok_or_else(|| Error::RevisionNotFound(format!("no manifest for {repo_id}@{commit}")))?;
|
||||
|
||||
// A card that was never fetched is a 404, not a fetch. The registry
|
||||
// records what it has actually seen, and browsing must not change that.
|
||||
let entry = manifest.entry("README.md").ok_or_else(|| {
|
||||
Error::EntryNotFound(format!("no README.md is stored for {repo_id}@{commit}"))
|
||||
})?;
|
||||
|
||||
let markdown = state.registry.read_stored_text(entry).await?;
|
||||
telemetry::request("v1_card", 200);
|
||||
Ok((
|
||||
[(http::header::CONTENT_TYPE, "text/markdown; charset=utf-8")],
|
||||
markdown,
|
||||
)
|
||||
.into_response())
|
||||
}
|
||||
|
||||
/// `GET /v1/status`
|
||||
///
|
||||
/// What this instance is, for a viewer deciding whether it can be trusted to
|
||||
|
||||
@@ -65,12 +65,23 @@ async fn authenticate(State(state): State<AppState>, request: Request, next: Nex
|
||||
if path == "/healthz" || path == "/metrics" {
|
||||
return next.run(request).await;
|
||||
}
|
||||
|
||||
let presented = request
|
||||
.headers()
|
||||
.get(http::header::AUTHORIZATION)
|
||||
.and_then(|v| v.to_str().ok());
|
||||
|
||||
if let Err(err) = state.tokens.check(presented) {
|
||||
return ApiError(err).into_response();
|
||||
// `anonymous = "catalog"` opens the `/v1/` surface to a request with
|
||||
// no valid token. It is safe to open precisely because every handler
|
||||
// beneath it reads the bucket and nothing else: an anonymous visitor
|
||||
// can see what is stored but cannot cause anything to be stored, and
|
||||
// cannot download a weight — which is what would make serving this
|
||||
// publicly an act of redistribution.
|
||||
let catalogue = path == "/v1" || path.starts_with("/v1/");
|
||||
if !(catalogue && state.tokens.anonymous_may_browse()) {
|
||||
return ApiError(err).into_response();
|
||||
}
|
||||
}
|
||||
next.run(request).await
|
||||
}
|
||||
@@ -103,6 +114,16 @@ async fn v1_repo_detail(
|
||||
)
|
||||
.await;
|
||||
}
|
||||
// `…/card` is the only suffix this surface takes; anything else is the
|
||||
// repository detail.
|
||||
if let Some(head) = rest.trim_end_matches('/').strip_suffix("/card") {
|
||||
let segments: Vec<&str> = head.trim_matches('/').split('/').collect();
|
||||
if segments.len() >= 2 {
|
||||
let repo_type: RepoType = segments[0].parse()?;
|
||||
let repo_id: rustingface_entities::repo::RepoId = segments[1..].join("/").parse()?;
|
||||
return handlers::inventory::card_markdown(&state, repo_type, &repo_id).await;
|
||||
}
|
||||
}
|
||||
handlers::inventory::detail(state, rest).await
|
||||
}
|
||||
|
||||
|
||||
@@ -1088,6 +1088,216 @@ async fn the_v1_surface_does_not_shadow_a_repo_named_like_it() {
|
||||
assert_eq!(body, b"{}");
|
||||
}
|
||||
|
||||
/// Build an instance with bearer auth and an anonymous tier.
|
||||
fn gated_instance(
|
||||
bucket: &std::path::Path,
|
||||
hub: Option<&FakeHub>,
|
||||
anonymous: rustingface_entities::config::Anonymous,
|
||||
) -> axum::Router {
|
||||
use rustingface_entities::config::{AuthMode, Config};
|
||||
let mut config = Config::default();
|
||||
config.storage.local_path = Some(bucket.to_owned());
|
||||
config.auth.mode = AuthMode::Bearer;
|
||||
config.auth.anonymous = anonymous;
|
||||
match hub {
|
||||
Some(hub) => {
|
||||
config.upstream.enabled = true;
|
||||
config.upstream.endpoint = hub.base_url.clone();
|
||||
}
|
||||
None => config.upstream.enabled = false,
|
||||
}
|
||||
|
||||
let store: std::sync::Arc<dyn rustingface_core::ports::Store> =
|
||||
std::sync::Arc::new(rustingface_data::ObjectStoreAdapter::local(bucket).unwrap());
|
||||
let upstream: std::sync::Arc<dyn rustingface_core::ports::Upstream> = if config.upstream.enabled
|
||||
{
|
||||
std::sync::Arc::new(rustingface_data::HubClient::new(&config.upstream, None).unwrap())
|
||||
} else {
|
||||
std::sync::Arc::new(rustingface_data::SealedUpstream::new(
|
||||
config.upstream.endpoint.clone(),
|
||||
))
|
||||
};
|
||||
let registry = rustingface_core::Registry::new(store, upstream, config.clone()).unwrap();
|
||||
let tokens = rustingface_api::auth::Tokens::with_anonymous(
|
||||
config.auth.mode,
|
||||
Some("hf_operator\n"),
|
||||
anonymous,
|
||||
)
|
||||
.unwrap();
|
||||
rustingface_api::router(rustingface_api::AppState::new(
|
||||
registry,
|
||||
std::sync::Arc::new(tokens),
|
||||
None,
|
||||
))
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn an_anonymous_visitor_may_browse_the_catalogue_but_not_download() {
|
||||
use rustingface_entities::config::Anonymous;
|
||||
|
||||
let hub = FakeHub::start().await;
|
||||
let weights = publish_model(&hub).await;
|
||||
hub.publish(
|
||||
"Qwen/Qwen3-32B",
|
||||
"main",
|
||||
COMMIT_A,
|
||||
&[
|
||||
("config.json", HubFile::plain(r#"{"model_type":"qwen3"}"#)),
|
||||
("model.safetensors", HubFile::lfs(&weights)),
|
||||
(
|
||||
"README.md",
|
||||
HubFile::plain(
|
||||
"# Qwen3
|
||||
|
||||
A **model card**.
|
||||
",
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
.await;
|
||||
|
||||
// Warm the bucket as the operator would, with a token.
|
||||
let bucket = bucket_dir("anon-catalog");
|
||||
let warm = instance(&bucket, Some(&hub), |_| {});
|
||||
for path in ["config.json", "model.safetensors", "README.md"] {
|
||||
let uri = format!("/Qwen/Qwen3-32B/resolve/main/{path}");
|
||||
assert_eq!(
|
||||
call(&warm.app, "GET", &uri, &[]).await.0,
|
||||
StatusCode::OK,
|
||||
"{uri}"
|
||||
);
|
||||
}
|
||||
|
||||
let app = gated_instance(&bucket, Some(&hub), Anonymous::Catalog);
|
||||
let token: &[(&str, &str)] = &[("authorization", "Bearer hf_operator")];
|
||||
|
||||
// The catalogue, the app's data, and the model card are all readable
|
||||
// without a token: that is the shop window.
|
||||
for uri in [
|
||||
"/v1/status",
|
||||
"/v1/repos",
|
||||
"/v1/repos/models/Qwen/Qwen3-32B",
|
||||
"/v1/repos/models/Qwen/Qwen3-32B/card",
|
||||
] {
|
||||
assert_eq!(
|
||||
call(&app, "GET", uri, &[]).await.0,
|
||||
StatusCode::OK,
|
||||
"{uri} must be readable anonymously"
|
||||
);
|
||||
}
|
||||
let (_, _, card) = call(&app, "GET", "/v1/repos/models/Qwen/Qwen3-32B/card", &[]).await;
|
||||
assert!(String::from_utf8_lossy(&card).contains("A **model card**"));
|
||||
|
||||
// Weights are not. Neither is the Hub-compatible surface a client would
|
||||
// use to pull them.
|
||||
for uri in [
|
||||
"/Qwen/Qwen3-32B/resolve/main/model.safetensors",
|
||||
"/Qwen/Qwen3-32B/resolve/main/config.json",
|
||||
"/api/models/Qwen/Qwen3-32B",
|
||||
"/api/whoami-v2",
|
||||
] {
|
||||
assert_eq!(
|
||||
call(&app, "GET", uri, &[]).await.0,
|
||||
StatusCode::UNAUTHORIZED,
|
||||
"{uri} must require a token"
|
||||
);
|
||||
assert_eq!(
|
||||
call(&app, "GET", uri, token).await.0,
|
||||
StatusCode::OK,
|
||||
"{uri} must work with one"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn browsing_the_catalogue_anonymously_cannot_cause_an_upstream_fetch() {
|
||||
use rustingface_entities::config::Anonymous;
|
||||
|
||||
let hub = FakeHub::start().await;
|
||||
publish_model(&hub).await;
|
||||
let bucket = bucket_dir("anon-no-fetch");
|
||||
let app = gated_instance(&bucket, Some(&hub), Anonymous::Catalog);
|
||||
hub.reset_counts();
|
||||
|
||||
// Every catalogue route, against a bucket that holds nothing. None of
|
||||
// them may reach upstream — that is what keeps an anonymous visitor from
|
||||
// filling the bucket.
|
||||
for uri in [
|
||||
"/v1/status",
|
||||
"/v1/repos",
|
||||
"/v1/repos?q=qwen",
|
||||
"/v1/repos/models/Qwen/Qwen3-32B",
|
||||
"/v1/repos/models/Qwen/Qwen3-32B/card",
|
||||
] {
|
||||
call(&app, "GET", uri, &[]).await;
|
||||
}
|
||||
assert_eq!(
|
||||
hub.requests(),
|
||||
0,
|
||||
"the catalogue surface must never consult upstream"
|
||||
);
|
||||
assert!(
|
||||
!bucket.join("repos").exists(),
|
||||
"and must never create a repository record"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn the_default_anonymous_tier_closes_the_catalogue_too() {
|
||||
use rustingface_entities::config::Anonymous;
|
||||
|
||||
let bucket = bucket_dir("anon-nothing");
|
||||
let app = gated_instance(&bucket, None, Anonymous::Nothing);
|
||||
|
||||
// Opening the catalogue must be a deliberate choice, not what happens by
|
||||
// default when somebody turns bearer auth on.
|
||||
for uri in ["/v1/status", "/v1/repos"] {
|
||||
assert_eq!(
|
||||
call(&app, "GET", uri, &[]).await.0,
|
||||
StatusCode::UNAUTHORIZED,
|
||||
"{uri}"
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
call(&app, "GET", "/healthz", &[]).await.0,
|
||||
StatusCode::OK,
|
||||
"the liveness probe stays open regardless"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_card_that_was_never_fetched_is_a_404_rather_than_a_fetch() {
|
||||
let hub = FakeHub::start().await;
|
||||
publish_model(&hub).await;
|
||||
let bucket = bucket_dir("card-miss");
|
||||
let rf = instance(&bucket, Some(&hub), |_| {});
|
||||
|
||||
// Pull one file so the repo exists, but not its README.
|
||||
assert_eq!(
|
||||
call(
|
||||
&rf.app,
|
||||
"GET",
|
||||
"/Qwen/Qwen3-32B/resolve/main/config.json",
|
||||
&[]
|
||||
)
|
||||
.await
|
||||
.0,
|
||||
StatusCode::OK
|
||||
);
|
||||
hub.reset_counts();
|
||||
|
||||
let (status, headers, _) =
|
||||
call(&rf.app, "GET", "/v1/repos/models/Qwen/Qwen3-32B/card", &[]).await;
|
||||
assert_eq!(status, StatusCode::NOT_FOUND);
|
||||
assert_eq!(headers["x-error-code"], "EntryNotFound");
|
||||
assert_eq!(
|
||||
hub.requests(),
|
||||
0,
|
||||
"a missing card is not a reason to fetch one"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn whoami_answers_so_hf_auth_login_can_complete() {
|
||||
let bucket = bucket_dir("whoami");
|
||||
|
||||
@@ -349,7 +349,7 @@ impl Registry {
|
||||
repo_type: RepoType,
|
||||
repo_id: &RepoId,
|
||||
commit: &str,
|
||||
mut mutate: F,
|
||||
mutate: F,
|
||||
) -> Result<Manifest>
|
||||
where
|
||||
F: FnMut(&mut Manifest),
|
||||
@@ -362,49 +362,83 @@ impl Registry {
|
||||
.entry(manifest_key.clone())
|
||||
.or_default(),
|
||||
);
|
||||
let _guard = lock.lock().await;
|
||||
|
||||
let result = {
|
||||
let _guard = lock.lock().await;
|
||||
self.update_manifest_locked(repo_type, repo_id, commit, &manifest_key, mutate)
|
||||
.await
|
||||
};
|
||||
|
||||
// Retire the map entry only once nobody else holds or is waiting on
|
||||
// this mutex. Removing it inside the critical section -- which is what
|
||||
// this used to do -- hands a *fresh* mutex to the next arrival while a
|
||||
// queued waiter still holds the old one, and the two then run
|
||||
// concurrently. That defeats the lock exactly when there is contention,
|
||||
// which is the only time it matters.
|
||||
//
|
||||
// Dropping our own reference first leaves the map holding the last one,
|
||||
// and `remove_if` runs under the shard lock that `entry` also takes, so
|
||||
// a caller cannot slip in between the count and the removal.
|
||||
drop(lock);
|
||||
self.inner
|
||||
.manifest_locks
|
||||
.remove_if(&manifest_key, |_, held| Arc::strong_count(held) == 1);
|
||||
|
||||
result
|
||||
}
|
||||
|
||||
/// The body of [`Registry::update_manifest`], with the per-key lock held.
|
||||
async fn update_manifest_locked<F>(
|
||||
&self,
|
||||
repo_type: RepoType,
|
||||
repo_id: &RepoId,
|
||||
commit: &str,
|
||||
manifest_key: &str,
|
||||
mut mutate: F,
|
||||
) -> Result<Manifest>
|
||||
where
|
||||
F: FnMut(&mut Manifest),
|
||||
{
|
||||
let mut attempt = 0;
|
||||
loop {
|
||||
let (mut manifest, version) =
|
||||
match self.inner.store.get_versioned(&manifest_key).await? {
|
||||
Some((bytes, version)) => {
|
||||
let manifest: Manifest = serde_json::from_slice(&bytes).map_err(|e| {
|
||||
Error::Storage(format!("{manifest_key} is not a valid manifest: {e}"))
|
||||
})?;
|
||||
(manifest, Some(version))
|
||||
}
|
||||
None => (
|
||||
Manifest::new(
|
||||
repo_type,
|
||||
repo_id.clone(),
|
||||
commit.to_owned(),
|
||||
Some(self.inner.upstream.endpoint().to_owned()),
|
||||
),
|
||||
None,
|
||||
let (mut manifest, version) = match self.inner.store.get_versioned(manifest_key).await?
|
||||
{
|
||||
Some((bytes, version)) => {
|
||||
let manifest: Manifest = serde_json::from_slice(&bytes).map_err(|e| {
|
||||
Error::Storage(format!("{manifest_key} is not a valid manifest: {e}"))
|
||||
})?;
|
||||
(manifest, Some(version))
|
||||
}
|
||||
None => (
|
||||
Manifest::new(
|
||||
repo_type,
|
||||
repo_id.clone(),
|
||||
commit.to_owned(),
|
||||
Some(self.inner.upstream.endpoint().to_owned()),
|
||||
),
|
||||
};
|
||||
None,
|
||||
),
|
||||
};
|
||||
mutate(&mut manifest);
|
||||
|
||||
let body = Bytes::from(encode_json(&manifest)?);
|
||||
let won = self
|
||||
.inner
|
||||
.store
|
||||
.put_if_version(&manifest_key, body, version.as_ref())
|
||||
.put_if_version(manifest_key, body, version.as_ref())
|
||||
.await?;
|
||||
|
||||
if won && self.inner.store.supports_compare_and_set() {
|
||||
// The write was conditional on what we read, so nothing can
|
||||
// have been merged away.
|
||||
self.inner.manifest_locks.remove(&manifest_key);
|
||||
return Ok(manifest);
|
||||
}
|
||||
|
||||
if won {
|
||||
// No conditional write available: confirm our contribution
|
||||
// actually survived rather than assuming it did. This is the
|
||||
// difference between a served file being recorded and a
|
||||
// sealed instance later claiming never to have seen it.
|
||||
// difference between a served file being recorded and a sealed
|
||||
// instance later claiming never to have seen it.
|
||||
let landed = self.load_manifest(repo_type, repo_id, commit).await?;
|
||||
let intact = landed.as_ref().is_some_and(|current| {
|
||||
manifest
|
||||
@@ -413,14 +447,12 @@ impl Registry {
|
||||
.all(|e| current.entry(&e.path).is_some())
|
||||
});
|
||||
if intact {
|
||||
self.inner.manifest_locks.remove(&manifest_key);
|
||||
return Ok(landed.unwrap_or(manifest));
|
||||
}
|
||||
}
|
||||
|
||||
attempt += 1;
|
||||
if attempt > self.inner.config.storage.manifest_write_retries {
|
||||
self.inner.manifest_locks.remove(&manifest_key);
|
||||
return Err(Error::Storage(format!(
|
||||
"{manifest_key} could not be updated after {attempt} attempts: another writer \
|
||||
kept winning the race. Nothing was lost -- the blob is stored -- but this \
|
||||
@@ -760,6 +792,36 @@ impl Registry {
|
||||
}
|
||||
}
|
||||
|
||||
/// Read a stored entry's bytes as UTF-8.
|
||||
///
|
||||
/// For small documents the catalogue serves directly — model cards. The
|
||||
/// ceiling is a guard, not a policy: a card is prose, and a repository
|
||||
/// that put a gigabyte of something else at `README.md` should not be
|
||||
/// able to make this allocate it.
|
||||
pub async fn read_stored_text(&self, entry: &Entry) -> Result<String> {
|
||||
const CEILING: u64 = 4 * 1024 * 1024;
|
||||
if entry.size > CEILING {
|
||||
return Err(Error::BadRequest(format!(
|
||||
"{} is {} bytes, larger than this endpoint serves; fetch it as a file instead",
|
||||
entry.path, entry.size
|
||||
)));
|
||||
}
|
||||
let key = blob_key(entry)?;
|
||||
if self.inner.store.head(&key).await?.is_none() {
|
||||
return Err(Error::EntryNotFound(format!(
|
||||
"{} is recorded but its blob is not in the bucket",
|
||||
entry.path
|
||||
)));
|
||||
}
|
||||
let mut blob = self.inner.store.read_blob(&key, None).await?;
|
||||
let mut bytes = Vec::with_capacity(entry.size as usize);
|
||||
while let Some(chunk) = futures::StreamExt::next(&mut blob.stream).await {
|
||||
bytes.extend_from_slice(&chunk?);
|
||||
}
|
||||
String::from_utf8(bytes)
|
||||
.map_err(|_| Error::BadRequest(format!("{} is not valid UTF-8", entry.path)))
|
||||
}
|
||||
|
||||
/// The recorded entry for a path, if the manifest has one.
|
||||
async fn stored_entry(
|
||||
&self,
|
||||
|
||||
@@ -182,6 +182,22 @@ pub enum AuthMode {
|
||||
Bearer,
|
||||
}
|
||||
|
||||
/// What a request carrying no valid token may reach when `mode = "bearer"`.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum Anonymous {
|
||||
/// Nothing. Every path needs a token.
|
||||
Nothing,
|
||||
/// The `/v1/` catalogue and the web app only.
|
||||
///
|
||||
/// That surface reads the bucket and nothing else: it never consults
|
||||
/// upstream, so it cannot cause a byte to be stored. Blob downloads and
|
||||
/// the Hub-compatible `/api/` surface still need a token, which is what
|
||||
/// keeps an anonymous visitor from filling the bucket — or from
|
||||
/// redistributing weights that are licensed to the operator.
|
||||
Catalog,
|
||||
}
|
||||
|
||||
/// Client-facing auth settings. Distinct from `upstream.token_file`, which is
|
||||
/// never echoed downstream.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
@@ -190,6 +206,9 @@ pub struct Auth {
|
||||
pub mode: AuthMode,
|
||||
/// One accepted token per line; `#` comments and blanks ignored.
|
||||
pub token_file: Option<PathBuf>,
|
||||
/// What an unauthenticated request may reach. Ignored when
|
||||
/// `mode = "none"`, where everything is open anyway.
|
||||
pub anonymous: Anonymous,
|
||||
}
|
||||
|
||||
impl Default for Auth {
|
||||
@@ -197,6 +216,7 @@ impl Default for Auth {
|
||||
Self {
|
||||
mode: AuthMode::None,
|
||||
token_file: None,
|
||||
anonymous: Anonymous::Nothing,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -12,7 +12,7 @@ pub mod manifest;
|
||||
pub mod refs;
|
||||
pub mod repo;
|
||||
|
||||
pub use config::Config;
|
||||
pub use config::{Anonymous, Config};
|
||||
pub use error::{Error, ErrorCode, Result};
|
||||
pub use manifest::{Entry, MANIFEST_SCHEMA, Manifest};
|
||||
pub use refs::{PinnedBy, REF_SCHEMA, RefRecord};
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
//! cannot write to.
|
||||
|
||||
use anyhow::{Result, bail};
|
||||
use rustingface_entities::config::{AuthMode, BlobDelivery, RefResolution};
|
||||
use rustingface_entities::config::{Anonymous, AuthMode, BlobDelivery, RefResolution};
|
||||
use rustingface_entities::repo::RepoType;
|
||||
|
||||
use crate::cmd::human_bytes;
|
||||
@@ -64,7 +64,15 @@ pub async fn run(config: rustingface_entities::config::Config) -> Result<()> {
|
||||
}
|
||||
);
|
||||
println!(" ref resolution {:?}", config.policy.ref_resolution);
|
||||
println!(" client auth {:?}", config.auth.mode);
|
||||
println!(
|
||||
" client auth {:?}{}",
|
||||
config.auth.mode,
|
||||
match (config.auth.mode, config.auth.anonymous) {
|
||||
(AuthMode::Bearer, Anonymous::Catalog) => ", anonymous may browse the catalogue",
|
||||
(AuthMode::Bearer, Anonymous::Nothing) => ", nothing is anonymous",
|
||||
(AuthMode::None, _) => ", everything is open",
|
||||
}
|
||||
);
|
||||
println!(
|
||||
" allowlist {}",
|
||||
if config.policy.allowlist.is_empty() {
|
||||
@@ -112,6 +120,22 @@ pub async fn run(config: rustingface_entities::config::Config) -> Result<()> {
|
||||
);
|
||||
}
|
||||
|
||||
// A catalogue open to anonymous visitors is a public face, and a public
|
||||
// face with allow_new_repos still on means any token holder can grow the
|
||||
// bucket without bound.
|
||||
if config.auth.anonymous == Anonymous::Catalog
|
||||
&& config.auth.mode == AuthMode::Bearer
|
||||
&& config.policy.allow_new_repos
|
||||
{
|
||||
advisories.push(
|
||||
"the catalogue is open to anonymous visitors and policy.allow_new_repos is true, \
|
||||
so any token holder can cause an arbitrary repository to be fetched and stored. \
|
||||
Set allow_new_repos = false and warm the bucket with `rustingface fetch`, or \
|
||||
bound it with policy.allowlist."
|
||||
.to_owned(),
|
||||
);
|
||||
}
|
||||
|
||||
if config.auth.mode == AuthMode::None && config.server.listen.starts_with("0.0.0.0") {
|
||||
problems.push(
|
||||
"auth.mode = \"none\" with a wildcard bind publishes the registry on every address \
|
||||
|
||||
@@ -24,7 +24,11 @@ pub async fn run(config: Config) -> Result<()> {
|
||||
read_optional_secret(config.auth.token_file.as_ref()).context("auth.token_file")?
|
||||
}
|
||||
};
|
||||
let tokens = Tokens::new(config.auth.mode, token_file.as_deref())?;
|
||||
let tokens = Tokens::with_anonymous(
|
||||
config.auth.mode,
|
||||
token_file.as_deref(),
|
||||
config.auth.anonymous,
|
||||
)?;
|
||||
|
||||
// Say this at startup, not only when somebody runs `doctor`: the operator
|
||||
// who needs to hear it is the one who set an upstream token and left the
|
||||
@@ -59,6 +63,7 @@ pub async fn run(config: Config) -> Result<()> {
|
||||
upstream = if config.upstream.enabled { config.upstream.endpoint.as_str() } else { "sealed" },
|
||||
delivery = ?config.storage.blob_delivery,
|
||||
auth = ?config.auth.mode,
|
||||
anonymous = ?config.auth.anonymous,
|
||||
"rustingface listening"
|
||||
);
|
||||
|
||||
|
||||
@@ -153,6 +153,17 @@ The two settings worth understanding:
|
||||
which breaks guarantee 4 and warns at startup.
|
||||
- **`upstream.enabled`** — `false` seals the registry. No upstream client is
|
||||
constructed at all, so there is no code path that could reach the network.
|
||||
- **`auth.mode` / `auth.anonymous`** — `bearer` requires a token from
|
||||
`auth.token_file`; `anonymous = "catalog"` then carves out the `/v1/`
|
||||
surface so an unauthenticated visitor can browse the inventory and read
|
||||
model cards, but cannot download a weight. That carve-out is safe because
|
||||
every handler under `/v1/` reads the bucket and never consults upstream, so
|
||||
an anonymous visitor cannot cause anything to be stored.
|
||||
- **`policy.allow_new_repos`** — `false` restricts serving to repositories
|
||||
already in the bucket. This, **not** auth, is what bounds how large the
|
||||
bucket gets: a token says *who*, not *what*, and any token holder can
|
||||
otherwise cause an arbitrary model to be fetched. Growing the registry
|
||||
becomes an operator action (`rustingface fetch`).
|
||||
|
||||
---
|
||||
|
||||
@@ -185,7 +196,8 @@ operator typing `gc`.
|
||||
| --- | --- |
|
||||
| Service | `bob.hanzalova.internal:20482` |
|
||||
| Object storage | MinIO on `caveman.kosherinata.internal:9000`, bucket `rustingface` |
|
||||
| Ingress | `rf.internal`, via the hanzalova edge proxy (mesh-only, no public record) |
|
||||
| Ingress (mesh) | `rf.internal`, internal CA, via the hanzalova edge proxy |
|
||||
| Ingress (public) | `rustingface.com`, Let's Encrypt, same proxy and webroot |
|
||||
| Frontend | `/var/www/rustingface` on `hanzalova`, served by the same vhost |
|
||||
| Port | `20482`, derived from the service name per `architecture/port-allocations.md` §3 |
|
||||
|
||||
@@ -205,7 +217,20 @@ One-time host provisioning is `script/infra-setup.sh`, run by an operator from
|
||||
a workstation with full sudo. It is idempotent and skips unreachable hosts.
|
||||
|
||||
Required Gitea repo secrets: `RSYNC_SSH_KEY`, `S3_ACCESS_KEY_ID`,
|
||||
`S3_SECRET_ACCESS_KEY`, `HF_TOKEN`.
|
||||
`S3_SECRET_ACCESS_KEY`, `CLIENT_TOKENS` (one accepted client token per line),
|
||||
and `HF_TOKEN` (optional; only for gated repositories).
|
||||
|
||||
### The two audiences
|
||||
|
||||
`rustingface.com` is a shop window: an operator evaluating whether to run their
|
||||
own instance can browse the catalogue and read model cards without a
|
||||
credential. Downloads need a token, on both names — auth is enforced by the
|
||||
service, not per-vhost, so a token is required on `rf.internal` too.
|
||||
|
||||
The public vhost additionally closes `/metrics`, which the service leaves
|
||||
auth-exempt so a mesh Prometheus can scrape it without a credential. That
|
||||
exemption is fine on the mesh and unacceptable publicly, so nginx closes it
|
||||
rather than the service relaxing it.
|
||||
|
||||
The service holds no durable state, which is what makes the systemd hardening
|
||||
easy: `ProtectSystem=strict` with no `ReadWritePaths` and no `StateDirectory`
|
||||
|
||||
+92
-12
@@ -13,8 +13,14 @@
|
||||
# What it does:
|
||||
# 1. bob — gitea_ci user, runner key, scoped sudoers, SELinux port label
|
||||
# 2. caveman — the rustingface bucket and a scoped MinIO service account
|
||||
# 3. hanzalova— rf.internal cert, nginx vhost, webroot, step@ renewal timer,
|
||||
# and the gitea_ci account CI uses to ship the built frontend
|
||||
# 3. hanzalova— rf.internal cert (internal CA) and rustingface.com cert
|
||||
# (Let's Encrypt), both vhosts, the webroot, the renewal
|
||||
# timers, and the gitea_ci account CI uses to ship the frontend
|
||||
#
|
||||
# Public DNS is NOT touched here. The Cloudflare token can rewrite DNS for
|
||||
# every zone on the account, including MX records for domains whose mail we
|
||||
# host (public-dns.md §1), so publishing the name is a separate, deliberate
|
||||
# operator step. This script prints the exact call and verifies the result.
|
||||
#
|
||||
# Application config is NOT shipped here. The deploy workflow renders it from
|
||||
# Gitea secrets on every deploy.
|
||||
@@ -30,6 +36,9 @@ readonly MINIO_URL=http://caveman.kosherinata.internal:9000
|
||||
readonly BUCKET=rustingface
|
||||
readonly VHOST=rf
|
||||
readonly WEBROOT=/var/www/rustingface
|
||||
readonly PUBLIC_NAME=rustingface.com
|
||||
readonly SITE_INDIRECTION=nh.thgttg.com # hanzalova's WAN address
|
||||
readonly CERTBOT_CREDENTIALS=/root/.certbot-internal
|
||||
readonly RUNNER_PUBKEY="${RUNNER_PUBKEY:-$HOME/.ssh/id_gitea_ci.pub}"
|
||||
readonly PROVISIONER_PASSWORD="${PROVISIONER_PASSWORD:-$HOME/.step/secrets/provisioner}"
|
||||
readonly ROOT_CA=/etc/pki/ca-trust/source/anchors/root-internal.pem
|
||||
@@ -107,13 +116,14 @@ gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/${APP}/config.toml
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/${APP}/s3-access-key
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/${APP}/s3-secret-key
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/${APP}/hf-token
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/${APP}/client-tokens
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/systemd/system/${APP}.service
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/sysusers.d/${APP}.conf
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/firewalld/services/${APP}.xml
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemd-sysusers
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/install -d -o root -g ${APP} -m 0750 /etc/${APP}
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/chown -R root\:${APP} /etc/${APP}
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/chmod 0640 /etc/${APP}/config.toml /etc/${APP}/s3-access-key /etc/${APP}/s3-secret-key /etc/${APP}/hf-token
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/chmod 0640 /etc/${APP}/config.toml /etc/${APP}/s3-access-key /etc/${APP}/s3-secret-key /etc/${APP}/hf-token /etc/${APP}/client-tokens
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/chmod 0755 /usr/local/bin/${APP}
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/sbin/restorecon -R /usr/local/bin/${APP} /etc/${APP}
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl daemon-reload
|
||||
@@ -242,19 +252,61 @@ provision_proxy() {
|
||||
ssh "$PROXY_HOST" "sudo systemctl enable --now step@${VHOST}.timer"
|
||||
ok "renewal timer enabled"
|
||||
|
||||
# The vhost references the cert paths, so it is only installed after the
|
||||
# cert exists: nginx -t fails on a missing ssl_certificate and that blocks
|
||||
# every unrelated reload on the host.
|
||||
# The public cert. Idempotent: --keep-until-expiring makes a repeat run a
|
||||
# no-op while the lineage is still valid. DNS-01 means this works before
|
||||
# the name resolves anywhere, so the cert can exist before the record does.
|
||||
local have_public
|
||||
have_public=$(ssh "$PROXY_HOST" \
|
||||
"sudo test -d /etc/letsencrypt/live/${PUBLIC_NAME} && echo yes || echo no" | tail -1)
|
||||
if [ "$have_public" = no ]; then
|
||||
if ! ssh "$PROXY_HOST" "sudo test -f ${CERTBOT_CREDENTIALS}"; then
|
||||
die "${CERTBOT_CREDENTIALS} is missing on ${PROXY_HOST}; cannot issue ${PUBLIC_NAME}"
|
||||
fi
|
||||
ssh "$PROXY_HOST" "sudo certbot certonly \
|
||||
-m ops@${PUBLIC_NAME} --agree-tos --no-eff-email --noninteractive \
|
||||
--cert-name ${PUBLIC_NAME} \
|
||||
--key-type ecdsa \
|
||||
--dns-cloudflare \
|
||||
--dns-cloudflare-credentials ${CERTBOT_CREDENTIALS} \
|
||||
--dns-cloudflare-propagation-seconds 60 \
|
||||
--keep-until-expiring \
|
||||
-d ${PUBLIC_NAME} -d www.${PUBLIC_NAME}"
|
||||
ok "issued ${PUBLIC_NAME}"
|
||||
else
|
||||
ok "${PUBLIC_NAME} cert is present"
|
||||
fi
|
||||
|
||||
# certbot renews on its own timer, but a reload is only a request and a
|
||||
# failed one leaves the old cert on the wire for up to 90 days. The hook
|
||||
# must therefore leave a trace rather than swallow the failure.
|
||||
ssh "$PROXY_HOST" 'sudo bash -s' <<'REMOTE'
|
||||
set -euo pipefail
|
||||
install -d -m 0755 /etc/letsencrypt/renewal-hooks/deploy
|
||||
cat > /etc/letsencrypt/renewal-hooks/deploy/reload-nginx.sh <<'HOOK'
|
||||
#!/bin/sh
|
||||
systemctl reload nginx || logger -t reload-nginx -p daemon.err \
|
||||
"nginx reload failed after certbot renewal; cert on disk is newer than the one served"
|
||||
HOOK
|
||||
chmod +x /etc/letsencrypt/renewal-hooks/deploy/reload-nginx.sh
|
||||
REMOTE
|
||||
ok "certbot deploy hook installed"
|
||||
|
||||
# The vhosts reference cert paths, so they are only installed once both
|
||||
# certs exist: nginx -t fails on a missing ssl_certificate, and that
|
||||
# blocks every unrelated reload on this shared proxy.
|
||||
rsync -az --rsync-path='sudo rsync' --mkpath \
|
||||
asset/nginx/${VHOST}.internal.conf \
|
||||
"$PROXY_HOST:/etc/nginx/sites-available/${VHOST}.internal.conf"
|
||||
asset/nginx/rustingface-upstream.conf \
|
||||
"$PROXY_HOST:/etc/nginx/conf.d/rustingface-upstream.conf"
|
||||
for conf in ${VHOST}.internal.conf ${PUBLIC_NAME}.conf; do
|
||||
rsync -az --rsync-path='sudo rsync' --mkpath \
|
||||
"asset/nginx/${conf}" "$PROXY_HOST:/etc/nginx/sites-available/${conf}"
|
||||
ssh "$PROXY_HOST" "sudo ln -sfn ../sites-available/${conf} /etc/nginx/sites-enabled/${conf}"
|
||||
done
|
||||
ssh "$PROXY_HOST" "
|
||||
set -euo pipefail
|
||||
sudo ln -sfn ../sites-available/${VHOST}.internal.conf \
|
||||
/etc/nginx/sites-enabled/${VHOST}.internal.conf
|
||||
sudo nginx -t
|
||||
sudo systemctl reload nginx"
|
||||
ok "vhost installed and nginx reloaded"
|
||||
ok "both vhosts installed and nginx reloaded"
|
||||
|
||||
# A reload is a request, not a guarantee: nginx keeps its old cycle if it
|
||||
# cannot rebind, and then serves the certificate it loaded then. Compare
|
||||
@@ -320,6 +372,10 @@ gitea_ci ALL=(root) NOPASSWD: /usr/bin/install -d -o root -g root -m 0755 ${WEBR
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/sbin/restorecon -R ${WEBROOT}
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/sbin/nginx -t
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl reload nginx
|
||||
# The deploy's health check asks whether the public certificate exists
|
||||
# before probing the public vhost; /etc/letsencrypt/live is root-only, so
|
||||
# an unprivileged test silently returns false and would skip the check.
|
||||
gitea_ci ALL=(root) NOPASSWD: /usr/bin/test -d /etc/letsencrypt/live/rustingface.com
|
||||
SUDOERS
|
||||
chmod 0440 /etc/sudoers.d/.${APP}_gitea_ci.new
|
||||
visudo -cf /etc/sudoers.d/.${APP}_gitea_ci.new
|
||||
@@ -327,7 +383,31 @@ mv /etc/sudoers.d/.${APP}_gitea_ci.new /etc/sudoers.d/${APP}_gitea_ci
|
||||
REMOTE
|
||||
ok "webroot ${WEBROOT} and scoped sudoers on the proxy"
|
||||
|
||||
info "DNS"
|
||||
info "public DNS (${PUBLIC_NAME})"
|
||||
if getent hosts "${PUBLIC_NAME}" >/dev/null; then
|
||||
ok "${PUBLIC_NAME} resolves to $(getent hosts "${PUBLIC_NAME}" | awk '{print $1}' | head -1)"
|
||||
else
|
||||
echo " ${PUBLIC_NAME} does not resolve yet. It is a zone apex, so it takes an"
|
||||
echo " A record rather than a CNAME (public-dns.md §3). From ${PROXY_HOST}:"
|
||||
echo
|
||||
echo " TOKEN=\$(sudo grep -oP '(?<=dns_cloudflare_api_token\s=\s).*' \\"
|
||||
echo " ${CERTBOT_CREDENTIALS} | tr -d \"'\" \")"
|
||||
echo " zid=\$(curl -sS -H \"Authorization: Bearer \$TOKEN\" \\"
|
||||
echo " 'https://api.cloudflare.com/client/v4/zones?name=${PUBLIC_NAME}' | jq -r '.result[0].id')"
|
||||
echo " # apex: A record to this site's WAN address"
|
||||
echo " curl -sS -X POST \"https://api.cloudflare.com/client/v4/zones/\$zid/dns_records\" \\"
|
||||
echo " -H \"Authorization: Bearer \$TOKEN\" -H 'Content-Type: application/json' \\"
|
||||
echo " --data '{\"type\":\"A\",\"name\":\"${PUBLIC_NAME}\",\"content\":\"$(dig +short ${SITE_INDIRECTION} A | head -1)\",\"ttl\":1,\"proxied\":false,\"comment\":\"rustingface\"}'"
|
||||
echo " # www: CNAME to the site indirection, so a re-homed site is one record"
|
||||
echo " curl -sS -X POST \"https://api.cloudflare.com/client/v4/zones/\$zid/dns_records\" \\"
|
||||
echo " -H \"Authorization: Bearer \$TOKEN\" -H 'Content-Type: application/json' \\"
|
||||
echo " --data '{\"type\":\"CNAME\",\"name\":\"www.${PUBLIC_NAME}\",\"content\":\"${SITE_INDIRECTION}\",\"ttl\":1,\"proxied\":false,\"comment\":\"rustingface\"}'"
|
||||
echo
|
||||
echo " Run these from ${PROXY_HOST}: the token is already there, and copying it"
|
||||
echo " off the host spreads a credential that can edit every zone we own."
|
||||
fi
|
||||
|
||||
info "mesh DNS"
|
||||
if getent hosts "${VHOST}.internal" >/dev/null; then
|
||||
ok "${VHOST}.internal resolves"
|
||||
return 0
|
||||
|
||||
+11
-9
@@ -114,22 +114,24 @@ export function fetchRepo(
|
||||
}
|
||||
|
||||
/**
|
||||
* The model card, read through the ordinary resolve path.
|
||||
* The model card, from the catalogue surface.
|
||||
*
|
||||
* That is the same URL a client uses, so a card rendering here is direct
|
||||
* evidence the blob path works. Returns `null` when the card was never
|
||||
* fetched: a registry records what it has actually seen, and a repo pulled
|
||||
* without its README simply has no card to show.
|
||||
* Read through `/v1/` rather than `/resolve/` deliberately. The card is
|
||||
* catalogue material, and putting it here means an anonymous visitor
|
||||
* evaluating rustingface can read a card without also being able to download
|
||||
* the weights beside it — and that browsing never causes an upstream fetch,
|
||||
* because the catalogue only ever reads what is already stored.
|
||||
*
|
||||
* Returns `null` when no card was fetched: a registry records what it has
|
||||
* actually seen, and a repo pulled without its README simply has none.
|
||||
*/
|
||||
export async function fetchCard(
|
||||
repoType: string,
|
||||
repoId: string,
|
||||
revision: string,
|
||||
signal?: AbortSignal,
|
||||
): Promise<string | null> {
|
||||
const prefix = repoType === 'datasets' ? 'datasets/' : ''
|
||||
const response = await fetch(`/${prefix}${repoId}/resolve/${revision}/README.md`, { signal })
|
||||
const response = await fetch(`/v1/repos/${repoType}/${repoId}/card`, { signal })
|
||||
if (response.status === 404) return null
|
||||
if (!response.ok) throw new ApiError(response.status, `could not read the model card`)
|
||||
if (!response.ok) throw new ApiError(response.status, 'could not read the model card')
|
||||
return await response.text()
|
||||
}
|
||||
|
||||
@@ -40,14 +40,7 @@ export default function Repo() {
|
||||
|
||||
// A missing card is not a failure: a repo pulled without its README simply
|
||||
// has none to show.
|
||||
const card = detail.commit
|
||||
? await fetchCard(
|
||||
detail.repo_type,
|
||||
detail.repo_id,
|
||||
detail.default_ref ?? detail.commit,
|
||||
signal,
|
||||
).catch(() => null)
|
||||
: null
|
||||
const card = await fetchCard(detail.repo_type, detail.repo_id, signal).catch(() => null)
|
||||
|
||||
return { detail, card }
|
||||
})
|
||||
@@ -169,7 +162,8 @@ export default function Repo() {
|
||||
</>
|
||||
)}
|
||||
Only files this registry has actually fetched are listed — those are
|
||||
the ones it can serve with no upstream.
|
||||
the ones it can serve with no upstream. Downloading one may require a
|
||||
token; browsing does not.
|
||||
</p>
|
||||
{detail.files.length === 0 ? (
|
||||
<p className="muted">No file has been fetched from this repository yet.</p>
|
||||
|
||||
Reference in New Issue
Block a user