From cb1ca8b6afae12c79e7950b441b9b7b4b20b00d4 Mon Sep 17 00:00:00 2001 From: rob thijssen Date: Mon, 31 Aug 2026 10:46:25 +0300 Subject: [PATCH] feat(rustingface): implement the registry, admin CLI and deployment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Scaffolds the workspace per architecture/generic.md §1 and implements phases 0-3 of doc/spec.md §12. Crates: entities manifest/ref/repo schemas, bucket key layout, config, the X-Error-Code taxonomy. No I/O. core resolver, freeze pinning, single-flight, the streaming tee, policy, gc/verify/refresh. Defines the Store and Upstream ports. data object_store (S3 + local) and reqwest Hub adapters. api the axum surface: resolve, model/dataset info, tree, refs, whoami, metrics, bearer auth, range handling. bin one binary: serve plus fetch/pin/refresh/list/show/rm/gc/ verify/doctor. Deployment targets bob.hanzalova.internal:20482 (port derived per architecture/port-allocations.md §3), storing to the MinIO on caveman.kosherinata.internal, fronted by hanzalova at rf.internal. Ships the sysusers drop-in, hardened unit, firewalld service, nginx vhost, config template, infra-setup.sh and the Gitea Actions ci/deploy/conformance workflows. Testing: 112 unit and integration tests, including the sovereignty suite (cold fetch, sealed replay, single-flight, client disconnect, range resume, freeze stability, gc-after-rm, digest mismatch), plus a conformance suite driving a pinned huggingface_hub against a real Hub. Deviations from the spec, all deliberate: - one binary with subcommands (spec §9) rather than generic.md's separate -api and -cli binaries; the library split is unchanged. - a dedicated sysusers account and hardened unit (generic.md §8) rather than the spec's illustrative DynamicUser unit. - manifests carry an optional repo_tree recorded verbatim, resolving spec §13's "record whole, filter on read" question for the tree endpoint as well as model-info. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01XZG2i4AmfSqE97EJGBVb64 --- .gitea/workflows/ci.yml | 29 + .gitea/workflows/conformance.yml | 30 + .gitea/workflows/deploy.yml | 185 ++ .gitignore | 1 + Cargo.lock | 3238 +++++++++++++++++++ Cargo.toml | 54 + asset/config/config.toml.example | 19 + asset/config/config.toml.tmpl | 59 + asset/firewalld/rustingface.xml | 6 + asset/nginx/rf.internal.conf | 57 + asset/systemd/rustingface.service | 59 + asset/systemd/rustingface.sysusers.conf | 2 + crates/rustingface-api/Cargo.toml | 36 + crates/rustingface-api/src/auth.rs | 130 + crates/rustingface-api/src/error.rs | 120 + crates/rustingface-api/src/handlers/blob.rs | 175 + crates/rustingface-api/src/handlers/meta.rs | 132 + crates/rustingface-api/src/handlers/mod.rs | 5 + crates/rustingface-api/src/handlers/ops.rs | 37 + crates/rustingface-api/src/lib.rs | 123 + crates/rustingface-api/src/path.rs | 264 ++ crates/rustingface-api/src/range.rs | 114 + crates/rustingface-api/src/state.rs | 59 + crates/rustingface-api/src/telemetry.rs | 46 + crates/rustingface-api/tests/sovereignty.rs | 953 ++++++ crates/rustingface-api/tests/support/mod.rs | 470 +++ crates/rustingface-core/Cargo.toml | 28 + crates/rustingface-core/src/admin.rs | 389 +++ crates/rustingface-core/src/digest.rs | 88 + crates/rustingface-core/src/fetch.rs | 347 ++ crates/rustingface-core/src/flight.rs | 301 ++ crates/rustingface-core/src/lib.rs | 18 + crates/rustingface-core/src/policy.rs | 116 + crates/rustingface-core/src/ports.rs | 234 ++ crates/rustingface-core/src/registry.rs | 1078 ++++++ crates/rustingface-data/Cargo.toml | 26 + crates/rustingface-data/src/lib.rs | 11 + crates/rustingface-data/src/store.rs | 300 ++ crates/rustingface-data/src/upstream.rs | 431 +++ crates/rustingface-entities/Cargo.toml | 20 + crates/rustingface-entities/src/config.rs | 408 +++ crates/rustingface-entities/src/error.rs | 201 ++ crates/rustingface-entities/src/key.rs | 184 ++ crates/rustingface-entities/src/lib.rs | 19 + crates/rustingface-entities/src/manifest.rs | 192 ++ crates/rustingface-entities/src/refs.rs | 96 + crates/rustingface-entities/src/repo.rs | 232 ++ crates/rustingface/Cargo.toml | 38 + crates/rustingface/src/cmd/doctor.rs | 145 + crates/rustingface/src/cmd/fetch.rs | 123 + crates/rustingface/src/cmd/gc.rs | 65 + crates/rustingface/src/cmd/list.rs | 96 + crates/rustingface/src/cmd/mod.rs | 69 + crates/rustingface/src/cmd/pin.rs | 88 + crates/rustingface/src/cmd/rm.rs | 51 + crates/rustingface/src/cmd/serve.rs | 70 + crates/rustingface/src/config.rs | 139 + crates/rustingface/src/logging.rs | 45 + crates/rustingface/src/main.rs | 185 ++ crates/rustingface/src/wire.rs | 48 + rust-toolchain.toml | 4 + script/infra-setup.sh | 295 ++ test/conformance/conformance.py | 555 ++++ test/conformance/requirements.txt | 9 + 64 files changed, 13147 insertions(+) create mode 100644 .gitea/workflows/ci.yml create mode 100644 .gitea/workflows/conformance.yml create mode 100644 .gitea/workflows/deploy.yml create mode 100644 .gitignore create mode 100644 Cargo.lock create mode 100644 Cargo.toml create mode 100644 asset/config/config.toml.example create mode 100644 asset/config/config.toml.tmpl create mode 100644 asset/firewalld/rustingface.xml create mode 100644 asset/nginx/rf.internal.conf create mode 100644 asset/systemd/rustingface.service create mode 100644 asset/systemd/rustingface.sysusers.conf create mode 100644 crates/rustingface-api/Cargo.toml create mode 100644 crates/rustingface-api/src/auth.rs create mode 100644 crates/rustingface-api/src/error.rs create mode 100644 crates/rustingface-api/src/handlers/blob.rs create mode 100644 crates/rustingface-api/src/handlers/meta.rs create mode 100644 crates/rustingface-api/src/handlers/mod.rs create mode 100644 crates/rustingface-api/src/handlers/ops.rs create mode 100644 crates/rustingface-api/src/lib.rs create mode 100644 crates/rustingface-api/src/path.rs create mode 100644 crates/rustingface-api/src/range.rs create mode 100644 crates/rustingface-api/src/state.rs create mode 100644 crates/rustingface-api/src/telemetry.rs create mode 100644 crates/rustingface-api/tests/sovereignty.rs create mode 100644 crates/rustingface-api/tests/support/mod.rs create mode 100644 crates/rustingface-core/Cargo.toml create mode 100644 crates/rustingface-core/src/admin.rs create mode 100644 crates/rustingface-core/src/digest.rs create mode 100644 crates/rustingface-core/src/fetch.rs create mode 100644 crates/rustingface-core/src/flight.rs create mode 100644 crates/rustingface-core/src/lib.rs create mode 100644 crates/rustingface-core/src/policy.rs create mode 100644 crates/rustingface-core/src/ports.rs create mode 100644 crates/rustingface-core/src/registry.rs create mode 100644 crates/rustingface-data/Cargo.toml create mode 100644 crates/rustingface-data/src/lib.rs create mode 100644 crates/rustingface-data/src/store.rs create mode 100644 crates/rustingface-data/src/upstream.rs create mode 100644 crates/rustingface-entities/Cargo.toml create mode 100644 crates/rustingface-entities/src/config.rs create mode 100644 crates/rustingface-entities/src/error.rs create mode 100644 crates/rustingface-entities/src/key.rs create mode 100644 crates/rustingface-entities/src/lib.rs create mode 100644 crates/rustingface-entities/src/manifest.rs create mode 100644 crates/rustingface-entities/src/refs.rs create mode 100644 crates/rustingface-entities/src/repo.rs create mode 100644 crates/rustingface/Cargo.toml create mode 100644 crates/rustingface/src/cmd/doctor.rs create mode 100644 crates/rustingface/src/cmd/fetch.rs create mode 100644 crates/rustingface/src/cmd/gc.rs create mode 100644 crates/rustingface/src/cmd/list.rs create mode 100644 crates/rustingface/src/cmd/mod.rs create mode 100644 crates/rustingface/src/cmd/pin.rs create mode 100644 crates/rustingface/src/cmd/rm.rs create mode 100644 crates/rustingface/src/cmd/serve.rs create mode 100644 crates/rustingface/src/config.rs create mode 100644 crates/rustingface/src/logging.rs create mode 100644 crates/rustingface/src/main.rs create mode 100644 crates/rustingface/src/wire.rs create mode 100644 rust-toolchain.toml create mode 100755 script/infra-setup.sh create mode 100755 test/conformance/conformance.py create mode 100644 test/conformance/requirements.txt diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml new file mode 100644 index 0000000..7367c77 --- /dev/null +++ b/.gitea/workflows/ci.yml @@ -0,0 +1,29 @@ +name: ci + +on: + push: + branches-ignore: [main] + pull_request: + workflow_dispatch: + +jobs: + check: + runs-on: rust + steps: + - uses: actions/checkout@v4 + + - name: format + run: cargo fmt --all -- --check + + - name: clippy + run: cargo clippy --workspace --all-targets --all-features -- -D warnings + + - name: test + run: cargo test --workspace --all-features + + # A musl build is what actually ships, so a target-only breakage should + # fail here rather than in the deploy workflow. + - name: build (musl) + run: | + rustup target add x86_64-unknown-linux-musl + cargo build --release --target x86_64-unknown-linux-musl --bin rustingface diff --git a/.gitea/workflows/conformance.yml b/.gitea/workflows/conformance.yml new file mode 100644 index 0000000..d83e74b --- /dev/null +++ b/.gitea/workflows/conformance.yml @@ -0,0 +1,30 @@ +name: conformance + +on: + workflow_dispatch: + schedule: + # Weekly. The clients are pinned, so this catches upstream Hub changes + # rather than client upgrades; a client upgrade is a deliberate commit and + # a spec-review trigger. + - cron: "0 4 * * 1" + +jobs: + conformance: + runs-on: rust + steps: + - uses: actions/checkout@v4 + + - name: build + run: cargo build --release --bin rustingface + + - name: install pinned clients + run: | + python3 -m venv .venv + .venv/bin/pip install --upgrade pip + .venv/bin/pip install -r test/conformance/requirements.txt + + # Runs against a directory rather than a bucket. The S3 path is covered + # by the `doctor` step of the deploy workflow, which exercises the real + # MinIO on caveman with the real credentials. + - name: conformance + run: .venv/bin/python test/conformance/conformance.py --binary target/release/rustingface diff --git a/.gitea/workflows/deploy.yml b/.gitea/workflows/deploy.yml new file mode 100644 index 0000000..e9254ed --- /dev/null +++ b/.gitea/workflows/deploy.yml @@ -0,0 +1,185 @@ +name: deploy + +on: + push: { branches: [main] } + workflow_dispatch: + +concurrency: + group: deploy + cancel-in-progress: false + +env: + # --- infra truth: hosts, ports and paths live here, not in a manifest --- + SERVICE_HOST: bob.hanzalova.internal + # Bind the mesh address rather than a wildcard: the reverse proxy is on a + # different host, so loopback will not do, and a wildcard bind on a host that + # may later gain another interface would publish the registry there too. + LISTEN_ADDR: 10.6.0.193:20482 + APP_PORT: "20482" + S3_ENDPOINT: http://caveman.kosherinata.internal:9000 + S3_BUCKET: rustingface + UPSTREAM_ENABLED: "true" + +jobs: + build: + runs-on: rust + steps: + - uses: actions/checkout@v4 + + # The gate runs before anything is built for deployment, so a broken + # commit on main never reaches a host. + - name: format + run: cargo fmt --all -- --check + + - name: clippy + run: cargo clippy --workspace --all-targets --all-features -- -D warnings + + - name: test + run: cargo test --workspace --all-features + + # Static musl: the runner is Fedora 44 and the target is Fedora 43, so a + # dynamically linked binary could reference a newer glibc than bob has. + - name: build + run: | + rustup target add x86_64-unknown-linux-musl + cargo build --release --target x86_64-unknown-linux-musl --bin rustingface + + - uses: actions/upload-artifact@v3 + with: + name: rustingface + path: target/x86_64-unknown-linux-musl/release/rustingface + + deploy: + needs: build + runs-on: fedora-43 + steps: + - uses: actions/checkout@v4 + - uses: actions/download-artifact@v3 + with: + name: rustingface + path: dist + + - name: authenticate to the target + run: | + install -d -m 0700 ~/.ssh + printf '%s\n' "${{ secrets.RSYNC_SSH_KEY }}" > ~/.ssh/id_gitea_ci + chmod 600 ~/.ssh/id_gitea_ci + cat > ~/.ssh/config <&2 + exit 1 + fi + if [ ! -s dist/s3-access-key ] || [ ! -s dist/s3-secret-key ]; then + echo "S3 credentials are empty; set the S3_ACCESS_KEY_ID and S3_SECRET_ACCESS_KEY secrets" >&2 + exit 1 + fi + + - name: create the service account and its config directory + run: | + rsync -az --rsync-path='sudo rsync' --mkpath \ + asset/systemd/rustingface.sysusers.conf \ + "$SERVICE_HOST:/etc/sysusers.d/rustingface.conf" + ssh "$SERVICE_HOST" ' + set -euo pipefail + sudo systemd-sysusers + sudo install -d -o root -g rustingface -m 0750 /etc/rustingface' + + - name: ship the binary, config and credentials + run: | + set -euo pipefail + rsync -az --rsync-path='sudo rsync' \ + dist/rustingface "$SERVICE_HOST:/usr/local/bin/rustingface" + for f in config.toml s3-access-key s3-secret-key hf-token; do + rsync -az --rsync-path='sudo rsync' --mkpath \ + "dist/$f" "$SERVICE_HOST:/etc/rustingface/$f" + done + rsync -az --rsync-path='sudo rsync' --mkpath \ + asset/systemd/rustingface.service \ + "$SERVICE_HOST:/etc/systemd/system/rustingface.service" + ssh "$SERVICE_HOST" ' + set -euo pipefail + sudo chmod 0755 /usr/local/bin/rustingface + sudo chown -R root:rustingface /etc/rustingface + sudo chmod 0640 /etc/rustingface/config.toml /etc/rustingface/s3-access-key /etc/rustingface/s3-secret-key /etc/rustingface/hf-token + sudo restorecon -R /usr/local/bin/rustingface /etc/rustingface' + + - name: firewalld + run: | + set -euo pipefail + # rsync the XML first: firewalld only learns a freshly-shipped custom + # service after --reload, and querying it before that fails. + rsync -az --rsync-path='sudo rsync' --mkpath \ + asset/firewalld/rustingface.xml \ + "$SERVICE_HOST:/etc/firewalld/services/rustingface.xml" + ssh "$SERVICE_HOST" ' + set -euo pipefail + sudo firewall-cmd --reload + zone=$(sudo firewall-cmd --get-default-zone) + if sudo firewall-cmd --zone="$zone" --query-service=rustingface; then + echo "rustingface already enabled in $zone" + else + sudo firewall-cmd --permanent --zone="$zone" --add-service=rustingface + sudo firewall-cmd --zone="$zone" --add-service=rustingface + fi' + + - name: restart + run: | + ssh "$SERVICE_HOST" ' + set -euo pipefail + sudo systemctl daemon-reload + sudo systemctl enable rustingface.service + sudo systemctl restart rustingface.service' + + - name: health check + run: | + set -euo pipefail + ssh "$SERVICE_HOST" 'sudo systemctl is-active rustingface.service' + for attempt in $(seq 1 20); do + if ssh "$SERVICE_HOST" "curl -fsS http://$LISTEN_ADDR/healthz"; then + echo "healthy after $attempt attempt(s)" + exit 0 + fi + sleep 2 + done + echo "rustingface did not answer /healthz" >&2 + exit 1 + + - name: doctor + run: | + # Proves the deployment can actually reach and write to the bucket, + # which a liveness probe deliberately does not. + ssh "$SERVICE_HOST" \ + 'sudo -u rustingface /usr/local/bin/rustingface --config /etc/rustingface/config.toml doctor' + + - name: journal + if: always() + run: ssh "$SERVICE_HOST" 'journalctl -u rustingface.service -n 200 --no-pager' diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ea8c4bf --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +/target diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..2119a67 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,3238 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "android_system_properties" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc" +dependencies = [ + "libc", +] + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "async-trait" +version = "0.1.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "atomic" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89cbf775b137e9b968e67227ef7f775587cde3fd31b0d8599dbd0f598a48340" +dependencies = [ + "bytemuck", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "aws-lc-rs" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce2b2dcc879c3bae0d371e77c99f2238400ef24ec001394befa67b6e543add9e" +dependencies = [ + "aws-lc-sys", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.44.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f09fae7be8bb3174e05c6afdb34199e6dc0c7c04ba9fa237b1967adfbde27483" +dependencies = [ + "cc", + "cmake", + "dunce", + "fs_extra", + "pkg-config", +] + +[[package]] +name = "axum" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" +dependencies = [ + "axum-core", + "bytes", + "form_urlencoded", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "serde_core", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tower", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-core" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "sync_wrapper", + "tower-layer", + "tower-service", +] + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bitflags" +version = "2.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "bstr" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bb31b46c14244e20ee9984b11bf5c992b91fb6939fea616e3512c8baecdbe5f" +dependencies = [ + "memchr", + "serde_core", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "bytemuck" +version = "1.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "cc" +version = "1.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ad534f4357a5264cce5019c989cf66a4f0dc4e0d1b1d15f8aacec0ff7360273" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cesu8" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cfg_aliases" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" + +[[package]] +name = "chacha20" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.1", + "rand_core 0.10.1", +] + +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "num-traits", + "serde", + "windows-link", +] + +[[package]] +name = "clap" +version = "4.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "473c7e07f409a8d772161724aa8db6a765a2532a70f9667eeb7b49d3d02fbdca" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b48fea5a88e9ae728a2dcbedbfc0e730f7d60da42e1cb049a83c9fb8b789889" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d012d2b9d65aca7f18f4d9878a045bc17899bba951561ba5ec3c2ba1eed9a061" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + +[[package]] +name = "cmake" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" +dependencies = [ + "cc", +] + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "combine" +version = "4.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfc320937d09e6de266b31b9afb480f197d7a861be86be7cb2ea7e5d1bfffc5e" +dependencies = [ + "bytes", + "memchr", +] + +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "cpufeatures" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" +dependencies = [ + "libc", +] + +[[package]] +name = "crc" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9710d3b3739c2e349eb44fe848ad0b7c8cb1e42bd87ee49371df2f7acaf3e675" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" + +[[package]] +name = "crc-fast" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fd92aca2c6001b1bf5ba0ff84ee74ec8501b52bbef0cac80bf25a6c1d87a83d" +dependencies = [ + "crc", + "digest 0.10.7", + "rustversion", + "spin", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "dashmap" +version = "6.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6361d5c062261c78a176addb82d4c821ae42bed6089de0e12603cd25de2059c" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core", +] + +[[package]] +name = "deranged" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" +dependencies = [ + "powerfmt", + "serde_core", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer 0.10.4", + "crypto-common 0.1.7", +] + +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.1", + "crypto-common 0.2.2", +] + +[[package]] +name = "displaydoc" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + +[[package]] +name = "either" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34" + +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "evmap" +version = "11.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b8874945f036109c72242964c1174cf99434e30cfa45bf45fedc983f50046f8" +dependencies = [ + "hashbag", + "left-right", + "smallvec", +] + +[[package]] +name = "figment" +version = "0.10.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cb01cd46b0cf372153850f4c6c272d9cbea2da513e07538405148f95bd789f3" +dependencies = [ + "atomic", + "pear", + "serde", + "toml 0.8.23", + "uncased", + "version_check", +] + +[[package]] +name = "find-msvc-tools" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d45db016d36b838f563236e9193d0ee6ce38f3f68b6c94e914b4929c96bbb890" + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + +[[package]] +name = "futures" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a31d2a3fbaaeb2af2368bbdd904aa8e812d3c04a1ee10d3171f52d556e5d0a3" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-executor" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "031b47cf1a3c6cc8bc2fc76cd437f521619387907d469316e7c0bc278f1f5432" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" + +[[package]] +name = "futures-macro" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "futures-sink" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "generator" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3b854b0e584ead1a33f18b2fcad7cf7be18b3875c78816b753639aa501513ae" +dependencies = [ + "cc", + "cfg-if", + "libc", + "log", + "rustversion", + "windows-link", + "windows-result", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi 5.3.0", + "wasip2", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi 6.0.0", + "rand_core 0.10.1", + "wasm-bindgen", +] + +[[package]] +name = "globset" +version = "0.4.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e47d37d2ae4464254884b60ab7071be2b876a9c35b696bd018ddcc76847309cd" +dependencies = [ + "aho-corasick", + "bstr", + "log", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "h2" +version = "0.4.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef8e5e5a340588f4452631496976cf8636d4a7ecf600239fdc27615d2530bc16" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbag" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7040a10f52cba493ddb09926e15d10a9d8a28043708a405931fe4c6f19fac064" + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "foldhash", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "http" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "humantime" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15" + +[[package]] +name = "humantime-serde" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57a3db5ea5923d99402c94e9feb261dc5ee9b4efa158b0315f788cf549cc200c" +dependencies = [ + "humantime", + "serde", +] + +[[package]] +name = "hybrid-array" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "707114b52a152fa7bdb290cd7cd5912d9467273b6d74e21b8d81aca1f8533f6b" +dependencies = [ + "typenum", +] + +[[package]] +name = "hyper" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27b501faa50e7a26c3d3560ca625132f4078a17771f4810baf70475ae48cbe43" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "tokio", + "tokio-rustls", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "icu_collections" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6b649701667bbe825c3b7e6388cb521c23d88644678e83c0c4d0a621a34b43" +dependencies = [ + "displaydoc", + "potential_utf", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edba7861004dd3714265b4db54a3c390e880ab658fec5f7db895fae2046b5bb6" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f6c8828b67bf8908d82127b2054ea1b4427ff0230ee9141c54251934ab1b599" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a" + +[[package]] +name = "icu_properties" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "020bfc02fe870ec3a66d93e677ccca0562506e5872c650f893269e08615d74ec" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "616c294cf8d725c6afcd8f55abc17c56464ef6211f9ed59cccffe534129c77af" + +[[package]] +name = "icu_provider" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85962cf0ce02e1e0a629cc34e7ca3e373ce20dda4c4d7294bbd0bf1fdb59e614" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "2.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07aa2048142242915a31d35844fb311e0e53fcca590c3a0a40dcf1b841fa09eb" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", +] + +[[package]] +name = "inlinable_string" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8fae54786f62fb2918dcfae3d568594e50eb9b5c25bf04371af6fe7516452fb" + +[[package]] +name = "ipnet" +version = "2.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a756c3fac73139e83f14c2d742155dd2b78d3ee56597b419a0579b7bdd6dd78" + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itertools" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jni" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a87aa2bb7d2af34197c04845522473242e1aa17c12f4935d5856491a7fb8c97" +dependencies = [ + "cesu8", + "cfg-if", + "combine", + "jni-sys 0.3.1", + "log", + "thiserror 1.0.69", + "walkdir", + "windows-sys 0.45.0", +] + +[[package]] +name = "jni-sys" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41a652e1f9b6e0275df1f15b32661cf0d4b78d4d87ddec5e0c3c20f097433258" +dependencies = [ + "jni-sys 0.4.1", +] + +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn 2.0.119", +] + +[[package]] +name = "jobserver" +version = "0.1.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" +dependencies = [ + "getrandom 0.4.3", + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e0c1080212aad755ea003d18543e8768dd432c48819efd73a7bf1e39b7a5a3a" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "left-right" +version = "0.11.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bc015ded5d9b3054dbbdb63332cdd6ee42352ccef19e911e25117490e2f48ee" +dependencies = [ + "crossbeam-utils", + "loom", + "slab", +] + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "litemap" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + +[[package]] +name = "loom" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "419e0dc8046cb947daa77eb95ae174acfbddb7673b4151f56d1eed8e93fbfaca" +dependencies = [ + "cfg-if", + "generator", + "scoped-tls", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + +[[package]] +name = "matchit" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" + +[[package]] +name = "md-5" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69b6441f590336821bb897fb28fc622898ccceb1d6cea3fde5ea86b090c4de98" +dependencies = [ + "cfg-if", + "digest 0.11.3", +] + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "metrics" +version = "0.24.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89550ee9f79e88fef3119de263694973a8adb26c21d75322164fb8c493039fe2" +dependencies = [ + "portable-atomic", + "rapidhash", +] + +[[package]] +name = "metrics-exporter-prometheus" +version = "0.18.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1db0d8f1fc9e62caebd0319e11eaec5822b0186c171568f0480b46a0137f9108" +dependencies = [ + "base64", + "evmap", + "indexmap", + "metrics", + "metrics-util", + "quanta", + "thiserror 2.0.20", +] + +[[package]] +name = "metrics-util" +version = "0.20.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96f8722f8562635f92f8ed992f26df0532266eb03d5202607c20c0d7e9745e13" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", + "hashbrown 0.16.1", + "metrics", + "quanta", + "rand 0.9.5", + "rand_xoshiro", + "rapidhash", + "sketches-ddsketch", +] + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "mio" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "nix" +version = "0.31.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" +dependencies = [ + "bitflags", + "cfg-if", + "cfg_aliases", + "libc", +] + +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "object_store" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d354792e39fa5f0009e47623cf8b15b099bf9a652fa55c6f817fe28ac84fea50" +dependencies = [ + "async-trait", + "aws-lc-rs", + "base64", + "bytes", + "chrono", + "crc-fast", + "form_urlencoded", + "futures-channel", + "futures-core", + "futures-util", + "http", + "http-body-util", + "humantime", + "hyper", + "itertools", + "md-5", + "nix", + "parking_lot", + "percent-encoding", + "quick-xml", + "rand 0.10.2", + "reqwest", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "thiserror 2.0.20", + "tokio", + "tracing", + "url", + "walkdir", + "wasm-bindgen-futures", + "web-time", + "windows-sys 0.61.2", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "openssl-probe" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "pear" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdeeaa00ce488657faba8ebf44ab9361f9365a97bd39ffb8a60663f57ff4b467" +dependencies = [ + "inlinable_string", + "pear_codegen", + "yansi", +] + +[[package]] +name = "pear_codegen" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bab5b985dc082b345f812b7df84e1bef27e7207b39e448439ba8bd69c93f147" +dependencies = [ + "proc-macro2", + "proc-macro2-diagnostics", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pkg-config" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" + +[[package]] +name = "portable-atomic" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" + +[[package]] +name = "potential_utf" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" +dependencies = [ + "zerovec", +] + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "proc-macro2-diagnostics" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "version_check", + "yansi", +] + +[[package]] +name = "quanta" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3ab5a9d756f0d97bdc89019bd2e4ea098cf9cde50ee7564dde6b81ccc8f06c7" +dependencies = [ + "crossbeam-utils", + "libc", + "once_cell", + "raw-cpuid", + "wasi", + "web-sys", + "winapi", +] + +[[package]] +name = "quick-xml" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1" +dependencies = [ + "memchr", + "serde", +] + +[[package]] +name = "quinn" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c1a41e437b6bbd489372cd4971de128e85c855f56c57f283d20ff016cf7c0a8" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror 2.0.20", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04759210543be93709136e28212294a659ef5001836ff4eab4d663e4529bba83" +dependencies = [ + "aws-lc-rs", + "bytes", + "getrandom 0.4.3", + "lru-slab", + "rand 0.10.2", + "rand_pcg", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror 2.0.20", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35a133f956daabe89a61a685c2649f13d82d5aa4bd5d12d1277e1072a21c0694" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.61.2", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" +dependencies = [ + "rand_chacha", + "rand_core 0.9.5", +] + +[[package]] +name = "rand" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +dependencies = [ + "chacha20", + "getrandom 0.4.3", + "rand_core 0.10.1", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "rand_pcg" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" +dependencies = [ + "rand_core 0.10.1", +] + +[[package]] +name = "rand_xoshiro" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f703f4665700daf5512dcca5f43afa6af89f09db47fb56be587f80636bda2d41" +dependencies = [ + "rand_core 0.9.5", +] + +[[package]] +name = "rapidhash" +version = "4.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5da7e78a036ce858e8d55b7e7dc8ba3a88b78350fd2155d3591bbd966b58589e" +dependencies = [ + "rustversion", +] + +[[package]] +name = "raw-cpuid" +version = "11.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186" +dependencies = [ + "bitflags", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "reqwest" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04e9018c9d814e5f30cc16a0f03271aeab3571e609612d9fe78c1aa8d11c2f62" +dependencies = [ + "base64", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "mime", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-native-certs", + "rustls-pki-types", + "rustls-platform-verifier", + "serde", + "serde_json", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-util", + "tower", + "tower-http 0.6.11", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + +[[package]] +name = "rustingface" +version = "0.1.0" +dependencies = [ + "anyhow", + "axum", + "bytes", + "clap", + "figment", + "futures", + "http-body-util", + "reqwest", + "rustingface-api", + "rustingface-core", + "rustingface-data", + "rustingface-entities", + "serde_json", + "time", + "tokio", + "toml 0.9.12+spec-1.1.0", + "tower", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "rustingface-api" +version = "0.1.0" +dependencies = [ + "axum", + "bytes", + "futures", + "hex", + "http", + "http-body-util", + "metrics", + "metrics-exporter-prometheus", + "rustingface-core", + "rustingface-data", + "rustingface-entities", + "serde", + "serde_json", + "sha2", + "subtle", + "time", + "tokio", + "tower", + "tower-http 0.7.0", + "tracing", +] + +[[package]] +name = "rustingface-core" +version = "0.1.0" +dependencies = [ + "async-trait", + "bytes", + "dashmap", + "futures", + "globset", + "hex", + "rustingface-entities", + "serde", + "serde_json", + "sha2", + "time", + "tokio", + "tracing", +] + +[[package]] +name = "rustingface-data" +version = "0.1.0" +dependencies = [ + "async-trait", + "bytes", + "futures", + "http", + "object_store", + "reqwest", + "rustingface-core", + "rustingface-entities", + "serde_json", + "tokio", + "tracing", +] + +[[package]] +name = "rustingface-entities" +version = "0.1.0" +dependencies = [ + "humantime-serde", + "percent-encoding", + "serde", + "serde_json", + "thiserror 2.0.20", + "time", + "toml 0.9.12+spec-1.1.0", +] + +[[package]] +name = "rustls" +version = "0.23.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0283386ce02abc0151e1761d08802dfe86c173b0b494af5cbc086574e453da06" +dependencies = [ + "aws-lc-rs", + "once_cell", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pki-types" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" +dependencies = [ + "web-time", + "zeroize", +] + +[[package]] +name = "rustls-platform-verifier" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d99feebc72bae7ab76ba994bb5e121b8d83d910ca40b36e0921f53becc41784" +dependencies = [ + "core-foundation", + "core-foundation-sys", + "jni", + "log", + "once_cell", + "rustls", + "rustls-native-certs", + "rustls-platform-verifier-android", + "rustls-webpki", + "security-framework", + "security-framework-sys", + "webpki-root-certs", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls-platform-verifier-android" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" + +[[package]] +name = "rustls-webpki" +version = "0.103.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" +dependencies = [ + "aws-lc-rs", + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "schannel" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "scoped-tls" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "security-framework" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "indexmap", + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_path_to_error" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" +dependencies = [ + "itoa", + "serde", + "serde_core", +] + +[[package]] +name = "serde_spanned" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" +dependencies = [ + "serde", +] + +[[package]] +name = "serde_spanned" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6662b5879511e06e8999a8a235d848113e942c9124f211511b16466ee2995f26" +dependencies = [ + "serde_core", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "sketches-ddsketch" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c6f73aeb92d671e0cc4dca167e59b2deb6387c375391bc99ee743f326994a2b" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" + +[[package]] +name = "socket2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "spin" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "023a211cb3138dbc438680b32560ad89f699977624c9f8dbb95a47d5b4c07dd3" + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6275cddf4610d1775e6d1fe9469b2e77d0f39fd98fb7450901b821e0c53649f" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" +dependencies = [ + "thiserror-impl 2.0.20", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "thread_local" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad99c4c6d32803332c548b1af0540b357b3f5fc0be8f6c6bfe8b2e6ae784070" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "time" +version = "0.3.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9e442fc33d7fdb45aa9bfeb312c095964abdf596f7567261062b2a7107aaabd" +dependencies = [ + "deranged", + "itoa", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b36ee98fd31ec7426d599183e8fe26932a8dc1fb76ddb6214d05493377d34ca" + +[[package]] +name = "time-macros" +version = "0.2.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71e552d1249bf61ac2a52db88179fd0673def1e1ad8243a00d9ec9ed71fee3dd" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "tinystr" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinyvec" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" +dependencies = [ + "bytes", + "libc", + "mio", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "libc", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "toml" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" +dependencies = [ + "serde", + "serde_spanned 0.6.9", + "toml_datetime 0.6.11", + "toml_edit", +] + +[[package]] +name = "toml" +version = "0.9.12+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf92845e79fc2e2def6a5d828f0801e29a2f8acc037becc5ab08595c7d5e9863" +dependencies = [ + "indexmap", + "serde_core", + "serde_spanned 1.1.1", + "toml_datetime 0.7.5+spec-1.1.0", + "toml_parser", + "toml_writer", + "winnow 0.7.15", +] + +[[package]] +name = "toml_datetime" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_datetime" +version = "0.7.5+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92e1cfed4a3038bc5a127e35a2d360f145e1f4b971b551a2ba5fd7aedf7e1347" +dependencies = [ + "serde_core", +] + +[[package]] +name = "toml_edit" +version = "0.22.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" +dependencies = [ + "indexmap", + "serde", + "serde_spanned 0.6.9", + "toml_datetime 0.6.11", + "toml_write", + "winnow 0.7.15", +] + +[[package]] +name = "toml_parser" +version = "1.1.3+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d38ac1cf9b95face32296c0a3ede1fdc270627c9d9c02a7274dd6d960dc4d56" +dependencies = [ + "winnow 1.0.4", +] + +[[package]] +name = "toml_write" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" + +[[package]] +name = "toml_writer" +version = "1.1.2+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d56353a2a665ad0f41a421187180aab746c8c325620617ad883a99a1cbe66d2" + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-http" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +dependencies = [ + "bitflags", + "bytes", + "futures-util", + "http", + "http-body", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", + "url", +] + +[[package]] +name = "tower-http" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b11f75e912b0c2be01b63d8cf8057b8c3f97cf34abb3d431a3a4c8675498e233" +dependencies = [ + "bitflags", + "bytes", + "http", + "http-body", + "percent-encoding", + "pin-project-lite", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-serde" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "704b1aeb7be0d0a84fc9828cae51dab5970fee5088f83d1dd7ee6f6246fc6ff1" +dependencies = [ + "serde", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "serde", + "serde_json", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", + "tracing-serde", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "uncased" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1b88fcfe09e89d3866a5c11019378088af2d24c3fbd4f0543f96b479ec90697" +dependencies = [ + "version_check", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.1+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0562428422c63773dad2c345a1882263bbf4d65cf3f42e90921f787ef5ad58e7" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b70935747edd64d89de3efa29d73789b806c15798f8e7dca4d8ac356b50ce70" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b7777d5cc23d0e91404e53ce2d5e8ec7acae3026b16233dba62cd3246457950" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77775f8f3f7217702089053b94958f8f54061a3f663417df76e19cbdcca29bc1" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e11d33f857dc2fb11b8bc75aee111aa9cbeb12cd9f25efd3d4c2a3dd4e235284" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 2.0.119", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef64dbcc55df09c7e5a46182d181c2cfa3e925f3da937ea764728b4bbb9dcbf" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasm-streams" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "web-sys" +version = "0.3.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c435338968042f4f59a557f690a253676d47ce13ceb55d70100e7facf6620a30" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-root-certs" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b96554aa2acc8ccdb7e1c9a58a7a68dd5d13bccc69cd124cb09406db612a1c9b" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets 0.42.2", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +dependencies = [ + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "winnow" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" +dependencies = [ + "memchr", +] + +[[package]] +name = "winnow" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" + +[[package]] +name = "wit-bindgen" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59" + +[[package]] +name = "writeable" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" + +[[package]] +name = "yansi" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" + +[[package]] +name = "yoke" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.8.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "556764e583adb45a9f8d413c2a147fa7e8d821e48e12b14fd560b607998b75eb" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2ab42fc20575779bd240faa45f94a74256f755c0fa9e89f0ede20d91d0cdfc1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" + +[[package]] +name = "zerotrie" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0464e17806c1d976d5cba29399c7f08e516e279e2ba493f63123b5fca67dd8" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..b6bbb82 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,54 @@ +[workspace] +resolver = "3" +members = ["crates/*"] + +[workspace.package] +version = "0.1.0" +edition = "2024" +rust-version = "1.85" +license = "GPL-3.0-or-later" +authors = ["Rob Thijssen "] +repository = "https://git.lair.cafe/grenade/rustingface" + +[workspace.dependencies] +rustingface-entities = { path = "crates/rustingface-entities", version = "=0.1.0" } +rustingface-core = { path = "crates/rustingface-core", version = "=0.1.0" } +rustingface-data = { path = "crates/rustingface-data", version = "=0.1.0" } +rustingface-api = { path = "crates/rustingface-api", version = "=0.1.0" } + +anyhow = "1" +async-trait = "0.1" +axum = { version = "0.8", default-features = false, features = ["http1", "http2", "json", "tokio", "query", "matched-path"] } +bytes = "1" +clap = { version = "4", features = ["derive", "env"] } +dashmap = "6" +figment = { version = "0.10", features = ["toml", "env"] } +futures = "0.3" +globset = "0.4" +hex = "0.4" +http = "1" +http-body-util = "0.1" +humantime-serde = "1" +metrics = "0.24" +metrics-exporter-prometheus = { version = "0.18", default-features = false } +object_store = { version = "0.14", features = ["aws"] } +percent-encoding = "2" +reqwest = { version = "0.13", default-features = false, features = ["rustls", "rustls-native-certs", "stream", "http2", "charset", "json"] } +serde = { version = "1", features = ["derive"] } +serde_json = { version = "1", features = ["preserve_order"] } +sha2 = "0.10" +subtle = "2" +thiserror = "2" +time = { version = "0.3", features = ["formatting", "parsing", "serde", "macros"] } +tokio = { version = "1", features = ["full"] } +tokio-util = { version = "0.7", features = ["io"] } +toml = "0.9" +tower = "0.5" +tower-http = { version = "0.7", features = ["trace", "normalize-path"] } +tracing = "0.1" +tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] } + +[profile.release] +lto = "thin" +codegen-units = 1 +strip = "symbols" diff --git a/asset/config/config.toml.example b/asset/config/config.toml.example new file mode 100644 index 0000000..5d2ec83 --- /dev/null +++ b/asset/config/config.toml.example @@ -0,0 +1,19 @@ +# A local development configuration. +# +# Uses a directory instead of a bucket, which forfeits state portability and is +# only appropriate for trying things out. Run with: +# +# cargo run -- --config asset/config/config.toml.example serve + +[server] +listen = "127.0.0.1:20482" + +[storage] +local_path = "/tmp/rustingface-dev" + +[upstream] +enabled = true +endpoint = "https://huggingface.co" + +[observability] +log_format = "text" diff --git a/asset/config/config.toml.tmpl b/asset/config/config.toml.tmpl new file mode 100644 index 0000000..f87d067 --- /dev/null +++ b/asset/config/config.toml.tmpl @@ -0,0 +1,59 @@ +# rustingface configuration. +# +# Rendered from this template by the deploy workflow, substituting {{...}} +# placeholders from Gitea repo secrets and workflow env. The rendered file is +# never committed; this template is. +# +# Every key here can be overridden per-key by environment, e.g. +# RUSTINGFACE__SERVER__LISTEN. Secrets are named by path, never inline. + +[server] +# Bind the mesh address, not loopback: the reverse proxy (hanzalova) is on a +# different host from the service (bob). firewalld bounds who may reach it. +listen = "{{LISTEN_ADDR}}" +external_url = "https://rf.internal" +request_timeout = "300s" +client_stall_timeout = "60s" + +[storage] +endpoint = "{{S3_ENDPOINT}}" +bucket = "{{S3_BUCKET}}" +region = "us-east-1" +path_style = true +access_key_id_file = "/etc/rustingface/s3-access-key" +secret_access_key_file = "/etc/rustingface/s3-secret-key" +# proxy: stream bytes through rustingface. redirect: 302 to a presigned URL, +# which takes rustingface out of the data path but exposes the object store's +# address to every client. +blob_delivery = "proxy" +presign_ttl = "15m" +multipart_part_size = "16MiB" + +[upstream] +# false seals the registry: it serves only what is already in the bucket, and +# constructs no upstream client at all. +enabled = {{UPSTREAM_ENABLED}} +endpoint = "https://huggingface.co" +token_file = "/etc/rustingface/hf-token" +connect_timeout = "10s" +read_timeout = "60s" +# Do not change. rustingface implements no Xet client; a Xet-backed fetch would +# leave the bucket unreadable without one, which forfeits state portability. +disable_xet = true + +[policy] +# freeze: the first resolution of a mutable ref is recorded and reused +# indefinitely. Moving to a newer revision is `rustingface refresh`. +ref_resolution = "freeze" +allow_new_repos = true +allowlist = [] + +[auth] +# The vhost is mesh-only and there is no per-client identity to enforce yet. +# Switch to "bearer" and populate /etc/rustingface/client-tokens to change that. +mode = "none" +token_file = "/etc/rustingface/client-tokens" + +[observability] +metrics = true +log_format = "json" diff --git a/asset/firewalld/rustingface.xml b/asset/firewalld/rustingface.xml new file mode 100644 index 0000000..218928c --- /dev/null +++ b/asset/firewalld/rustingface.xml @@ -0,0 +1,6 @@ + + + rustingface + Sovereign HuggingFace-compatible model registry. Serves model and dataset blobs to inference hosts on the mesh; fronted by the hanzalova edge proxy at rf.internal. + + diff --git a/asset/nginx/rf.internal.conf b/asset/nginx/rf.internal.conf new file mode 100644 index 0000000..21f92a8 --- /dev/null +++ b/asset/nginx/rf.internal.conf @@ -0,0 +1,57 @@ +# rf.internal — the mesh vhost for rustingface. +# +# Installed on hanzalova (the office edge proxy); the service itself runs on +# bob. TCP :443 belongs to the stream SNI router, which hands connections to +# the https tier on 127.0.0.1:14443, so this vhost listens there and reads the +# real client address from PROXY protocol. + +upstream rustingface { + server bob.hanzalova.internal:20482; + keepalive 8; +} + +server { + listen 127.0.0.1:14443 ssl proxy_protocol; + server_name rf.internal; + + ssl_certificate /etc/nginx/tls/cert/rf.internal.pem; + ssl_certificate_key /etc/nginx/tls/key/rf.internal.pem; + ssl_trusted_certificate /etc/pki/ca-trust/source/anchors/root-internal.pem; + ssl_protocols TLSv1.3; + + # The vhost believes it is serving 127.0.0.1:14443, so any redirect it + # generates itself would carry that unreachable port. + absolute_redirect off; + + access_log /var/log/nginx/rf.internal.access.log; + error_log /var/log/nginx/rf.internal.error.log; + + location / { + proxy_pass http://rustingface; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $proxy_protocol_addr; + proxy_set_header X-Forwarded-For $proxy_protocol_addr; + proxy_set_header X-Forwarded-Proto https; + + # Model weights are measured in tens of gigabytes. Buffering them to + # disk on the proxy would double the I/O and stall the client until the + # whole file had landed, so stream straight through. + proxy_buffering off; + proxy_request_buffering off; + client_max_body_size 0; + + # A cold 40GB pull through this vhost is one long-lived response. The + # read timeout bounds silence between chunks, not total duration, so it + # stays modest; the send timeout must tolerate a slow client. + proxy_connect_timeout 15s; + proxy_read_timeout 300s; + proxy_send_timeout 300s; + + # Never re-compress. The client sends `Accept-Encoding: identity` + # precisely so the sizes it learns are real. + gzip off; + proxy_set_header Accept-Encoding $http_accept_encoding; + } +} diff --git a/asset/systemd/rustingface.service b/asset/systemd/rustingface.service new file mode 100644 index 0000000..ce74c94 --- /dev/null +++ b/asset/systemd/rustingface.service @@ -0,0 +1,59 @@ +[Unit] +Description=rustingface sovereign model registry +Documentation=https://git.lair.cafe/grenade/rustingface +After=network-online.target +Wants=network-online.target + +[Service] +# Type=exec, not notify: the binary sends no sd_notify, and a notify unit that +# never sends READY=1 blocks `systemctl restart` until TimeoutStartSec. +Type=exec +User=rustingface +Group=rustingface +ExecStart=/usr/local/bin/rustingface serve --config /etc/rustingface/config.toml +Restart=on-failure +RestartSec=5s + +# Credentials are passed by systemd rather than read from disk by the service, +# so the secret files stay root-owned and never appear in the unit's namespace +# except under $CREDENTIALS_DIRECTORY. The config names these basenames. +LoadCredential=s3-access-key:/etc/rustingface/s3-access-key +LoadCredential=s3-secret-key:/etc/rustingface/s3-secret-key +LoadCredential=hf-token:/etc/rustingface/hf-token + +# Long transfers must not be killed mid-flight on stop; a detached upload that +# is cut short leaves an orphan blob, which is safe but wasteful. +TimeoutStopSec=120 + +# Hardening. The process holds no durable state — the bucket is the whole of it +# — so there is no StateDirectory to protect and ProtectSystem can stay strict +# with no ReadWritePaths at all. +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +PrivateDevices=true +ProtectKernelTunables=true +ProtectKernelModules=true +ProtectControlGroups=true +ProtectClock=true +ProtectHostname=true +ProtectProc=invisible +RestrictRealtime=true +RestrictSUIDSGID=true +RestrictNamespaces=true +LockPersonality=true +MemoryDenyWriteExecute=true +SystemCallFilter=@system-service +SystemCallArchitectures=native +CapabilityBoundingSet= +AmbientCapabilities= +RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 +UMask=0077 + +# Concurrent multi-gigabyte transfers each hold a handful of sockets and part +# buffers; the default 1024 is comfortable but the headroom costs nothing. +LimitNOFILE=16384 + +[Install] +WantedBy=multi-user.target diff --git a/asset/systemd/rustingface.sysusers.conf b/asset/systemd/rustingface.sysusers.conf new file mode 100644 index 0000000..f86a5d2 --- /dev/null +++ b/asset/systemd/rustingface.sysusers.conf @@ -0,0 +1,2 @@ +#Type Name ID GECOS Home directory Shell +u rustingface - "rustingface model registry service" /var/lib/rustingface /usr/sbin/nologin diff --git a/crates/rustingface-api/Cargo.toml b/crates/rustingface-api/Cargo.toml new file mode 100644 index 0000000..ef22833 --- /dev/null +++ b/crates/rustingface-api/Cargo.toml @@ -0,0 +1,36 @@ +[package] +name = "rustingface-api" +description = "The axum HTTP surface: Hub-compatible resolve, metadata and header replay." +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +authors.workspace = true +repository.workspace = true + +[dependencies] +rustingface-core.workspace = true +rustingface-entities.workspace = true + +axum.workspace = true +bytes.workspace = true +futures.workspace = true +http.workspace = true +metrics.workspace = true +metrics-exporter-prometheus.workspace = true +serde.workspace = true +serde_json.workspace = true +subtle.workspace = true +time.workspace = true +tokio.workspace = true +tower.workspace = true +tower-http.workspace = true +tracing.workspace = true + +[dev-dependencies] +hex.workspace = true +http-body-util.workspace = true +rustingface-data.workspace = true +sha2.workspace = true +tokio = { workspace = true, features = ["macros", "rt-multi-thread", "test-util"] } +tower = { workspace = true, features = ["util"] } diff --git a/crates/rustingface-api/src/auth.rs b/crates/rustingface-api/src/auth.rs new file mode 100644 index 0000000..dbf51ee --- /dev/null +++ b/crates/rustingface-api/src/auth.rs @@ -0,0 +1,130 @@ +//! Client-facing bearer auth. +//! +//! Distinct from `upstream.token_file`, which authenticates rustingface *to* +//! the Hub and is never echoed downstream. This is who may pull from +//! rustingface. + +use std::sync::Arc; + +use rustingface_entities::config::AuthMode; +use rustingface_entities::error::{Error, Result}; +use subtle::ConstantTimeEq; + +/// The accepted client tokens. +#[derive(Debug)] +pub struct Tokens { + mode: AuthMode, + accepted: Vec, +} + +impl Tokens { + /// Build from `[auth]` and the contents of the token file. + /// + /// One token per line; blank lines and `#` comments are ignored so an + /// operator can annotate who each token belongs to. + pub fn new(mode: AuthMode, token_file_contents: Option<&str>) -> Result { + let accepted: Vec = token_file_contents + .unwrap_or_default() + .lines() + .map(str::trim) + .filter(|line| !line.is_empty() && !line.starts_with('#')) + .map(str::to_owned) + .collect(); + + if mode == AuthMode::Bearer && accepted.is_empty() { + return Err(Error::Config( + "auth.mode = \"bearer\" but auth.token_file holds no tokens; \ + every client would be refused" + .into(), + )); + } + Ok(Self { mode, accepted }) + } + + /// Whether any authentication is required. + pub fn required(&self) -> bool { + self.mode == AuthMode::Bearer + } + + /// Check an `Authorization` header value. + /// + /// Comparison is constant-time and always runs against every configured + /// token, so a rejection tells an attacker nothing about how close a guess + /// was or how many tokens exist. + pub fn check(&self, authorization: Option<&str>) -> Result<()> { + if !self.required() { + return Ok(()); + } + let presented = authorization + .and_then(|v| v.strip_prefix("Bearer ")) + .or_else(|| authorization.and_then(|v| v.strip_prefix("bearer "))) + .map(str::trim) + .unwrap_or_default(); + + let mut matched = 0u8; + for token in &self.accepted { + matched |= token.as_bytes().ct_eq(presented.as_bytes()).unwrap_u8(); + } + if matched == 1 { + Ok(()) + } else { + Err(Error::Unauthorized) + } + } +} + +/// Shared handle to the accepted tokens. +pub type SharedTokens = Arc; + +impl Default for Tokens { + fn default() -> Self { + Self { + mode: AuthMode::None, + accepted: Vec::new(), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn no_auth_admits_every_request() { + let tokens = Tokens::new(AuthMode::None, None).unwrap(); + assert!(!tokens.required()); + assert!(tokens.check(None).is_ok()); + assert!(tokens.check(Some("Bearer nonsense")).is_ok()); + } + + #[test] + fn bearer_admits_a_configured_token_and_refuses_others() { + let tokens = Tokens::new( + AuthMode::Bearer, + Some("# the beast box\nhf_alpha\n\nhf_beta\n"), + ) + .unwrap(); + assert!(tokens.check(Some("Bearer hf_alpha")).is_ok()); + assert!(tokens.check(Some("Bearer hf_beta")).is_ok()); + assert!(tokens.check(Some("Bearer hf_gamma")).is_err()); + assert!(tokens.check(None).is_err()); + assert!( + tokens.check(Some("hf_alpha")).is_err(), + "the scheme is required" + ); + } + + #[test] + fn comments_and_blanks_are_not_themselves_tokens() { + let tokens = Tokens::new(AuthMode::Bearer, Some("# comment\nreal\n")).unwrap(); + assert!(tokens.check(Some("Bearer # comment")).is_err()); + assert!(tokens.check(Some("Bearer ")).is_err()); + assert!(tokens.check(Some("Bearer real")).is_ok()); + } + + #[test] + fn bearer_with_an_empty_token_file_fails_at_startup() { + let err = Tokens::new(AuthMode::Bearer, Some("# nothing but comments\n")).unwrap_err(); + assert!(matches!(err, Error::Config(_)), "{err}"); + } +} diff --git a/crates/rustingface-api/src/error.rs b/crates/rustingface-api/src/error.rs new file mode 100644 index 0000000..53fdada --- /dev/null +++ b/crates/rustingface-api/src/error.rs @@ -0,0 +1,120 @@ +//! Turning an [`Error`] into the response a Hub client expects. +//! +//! `huggingface_hub` branches on the `X-Error-Code` header to raise typed +//! exceptions, so emitting it is not cosmetic. The status alone would collapse +//! "no such repo", "no such revision" and "no such file" into one 404 that the +//! client cannot tell apart. + +use axum::response::{IntoResponse, Response}; +use http::{StatusCode, header}; +use rustingface_entities::error::Error; + +/// Header name the client reads to classify a failure. +pub const X_ERROR_CODE: &str = "x-error-code"; +/// Header the Hub uses to carry a human-readable reason. +pub const X_ERROR_MESSAGE: &str = "x-error-message"; + +/// The exact `X-Error-Message` the client matches to raise `DisabledRepoError`. +/// +/// Verified against the pinned `huggingface_hub` by the conformance suite: +/// unlike the others, a disabled repo is dispatched on the *message*, not on +/// `X-Error-Code`, so emitting only the code would have the client raise a +/// generic `HfHubHTTPError` instead. +pub const DISABLED_REPO_MESSAGE: &str = "Access to this resource is disabled."; + +/// Wrapper giving [`Error`] an axum response. +pub struct ApiError(pub Error); + +impl From for ApiError { + fn from(err: Error) -> Self { + Self(err) + } +} + +impl IntoResponse for ApiError { + fn into_response(self) -> Response { + let ApiError(err) = self; + let status = + StatusCode::from_u16(err.status()).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR); + let message = err.to_string(); + // A disabled repo is dispatched by the client on this exact message + // rather than on X-Error-Code; everything else carries its own reason. + let wire_message = match &err { + Error::DisabledRepo(_) => DISABLED_REPO_MESSAGE.to_owned(), + _ => sanitise(&message), + }; + + let mut response = + (status, axum::Json(serde_json::json!({ "error": message }))).into_response(); + + if let Some(code) = err.code() + && let Ok(value) = header::HeaderValue::from_str(code.as_str()) + { + response.headers_mut().insert(X_ERROR_CODE, value); + } + // Header values must be visible ASCII; a path or upstream detail could + // carry anything, so a message that will not encode is simply dropped + // rather than replacing the response with a 500. + if let Ok(value) = header::HeaderValue::from_str(&wire_message) { + response.headers_mut().insert(X_ERROR_MESSAGE, value); + } + + if err.status() >= 500 { + tracing::error!(%err, "request failed"); + } else { + tracing::debug!(%err, "request refused"); + } + response + } +} + +/// Reduce a message to characters a header value can carry. +fn sanitise(message: &str) -> String { + message + .chars() + .map(|c| if (' '..='~').contains(&c) { c } else { ' ' }) + .take(512) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_missing_entry_carries_its_code_and_a_404() { + let response = ApiError(Error::EntryNotFound("config.json".into())).into_response(); + assert_eq!(response.status(), StatusCode::NOT_FOUND); + assert_eq!(response.headers()[X_ERROR_CODE], "EntryNotFound"); + } + + #[test] + fn a_gated_repo_is_a_403_with_its_own_code() { + let response = ApiError(Error::GatedRepo("terms".into())).into_response(); + assert_eq!(response.status(), StatusCode::FORBIDDEN); + assert_eq!(response.headers()[X_ERROR_CODE], "GatedRepo"); + } + + #[test] + fn an_upstream_auth_failure_is_a_401_with_no_code() { + let response = ApiError(Error::UpstreamUnauthorized("bad token".into())).into_response(); + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + assert!(!response.headers().contains_key(X_ERROR_CODE)); + } + + #[test] + fn a_disabled_repo_carries_the_exact_message_the_client_matches_on() { + // The client dispatches DisabledRepoError on this string rather than + // on X-Error-Code, so it must go out verbatim. + let response = ApiError(Error::DisabledRepo("upstream said so".into())).into_response(); + assert_eq!(response.status(), StatusCode::FORBIDDEN); + assert_eq!(response.headers()[X_ERROR_MESSAGE], DISABLED_REPO_MESSAGE); + } + + #[test] + fn a_message_with_a_newline_still_produces_a_valid_header() { + let response = ApiError(Error::EntryNotFound("a\nb\u{1f600}".into())).into_response(); + let value = response.headers()[X_ERROR_MESSAGE].to_str().unwrap(); + assert!(!value.contains('\n'), "{value:?}"); + } +} diff --git a/crates/rustingface-api/src/handlers/blob.rs b/crates/rustingface-api/src/handlers/blob.rs new file mode 100644 index 0000000..32e5b15 --- /dev/null +++ b/crates/rustingface-api/src/handlers/blob.rs @@ -0,0 +1,175 @@ +//! The resolve path: `HEAD` and `GET` for file bytes. +//! +//! This is the part clients actually depend on. Every header below is either +//! upstream's value replayed verbatim or a true fact about the bytes being +//! served; nothing here is normalised, recomputed or guessed. + +use std::time::Instant; + +use axum::body::Body as AxumBody; +use axum::extract::State; +use axum::response::{IntoResponse, Response}; +use futures::StreamExt; +use http::{HeaderMap, HeaderValue, StatusCode, header}; +use rustingface_core::{Body, ResolvedFile, Source}; +use rustingface_entities::error::{Error, Result}; +use rustingface_entities::repo::RepoType; + +use crate::error::ApiError; +use crate::path::{ResolvePath, parse_resolve}; +use crate::state::AppState; +use crate::{range, telemetry}; + +/// `X-Repo-Commit`: the commit the revision resolved to. +pub const X_REPO_COMMIT: &str = "x-repo-commit"; +/// `X-Linked-Etag`: upstream's LFS etag, which clients prefer over `ETag`. +pub const X_LINKED_ETAG: &str = "x-linked-etag"; +/// `X-Linked-Size`: upstream's LFS size. +pub const X_LINKED_SIZE: &str = "x-linked-size"; + +/// `HEAD /{repo_id}/resolve/{revision}/{filename}`. +pub async fn head( + State(state): State, + repo_type: RepoType, + rest: String, +) -> std::result::Result { + let ResolvePath { + repo_id, + revision, + filename, + } = parse_resolve(&rest)?; + + let file = state + .registry + .head_file(repo_type, &repo_id, &revision, &filename) + .await?; + + let mut headers = metadata_headers(&file)?; + headers.insert(header::CONTENT_LENGTH, HeaderValue::from(file.size)); + + // In redirect mode a HEAD answers 302 with the metadata still attached: + // the client reads its metadata off this response and then downloads from + // `Location`, so moving the headers onto the presigned request loses them. + if let Body::Redirect(location) = &file.body { + headers.insert(header::LOCATION, header_value(location)?); + telemetry::request("resolve_head", 302); + return Ok((StatusCode::FOUND, headers).into_response()); + } + + telemetry::request("resolve_head", 200); + Ok((StatusCode::OK, headers).into_response()) +} + +/// `GET /{repo_id}/resolve/{revision}/{filename}`. +pub async fn get( + State(state): State, + repo_type: RepoType, + request_headers: HeaderMap, + rest: String, +) -> std::result::Result { + let started = Instant::now(); + let ResolvePath { + repo_id, + revision, + filename, + } = parse_resolve(&rest)?; + + // The range is parsed against the stored length, which is not known until + // the file resolves. Ask for the object first, then narrow: a cold file + // must be fetched and stored whole before any range of it is served. + let requested_range = request_headers + .get(header::RANGE) + .and_then(|v| v.to_str().ok()) + .map(str::to_owned); + + let probe = match &requested_range { + None => None, + Some(spec) => { + let head = state + .registry + .head_file(repo_type, &repo_id, &revision, &filename) + .await?; + range::parse(spec, head.size)? + } + }; + + let (file, source) = state + .registry + .get_file(repo_type, &repo_id, &revision, &filename, probe) + .await?; + + telemetry::in_flight(state.registry.in_flight()); + if source == Source::Upstream { + telemetry::upstream_fetch("ok", started.elapsed().as_secs_f64()); + } + + let mut headers = metadata_headers(&file)?; + + match file.body { + Body::Redirect(location) => { + headers.insert(header::CONTENT_LENGTH, HeaderValue::from(file.size)); + headers.insert(header::LOCATION, header_value(&location)?); + telemetry::request("resolve_get", 302); + Ok((StatusCode::FOUND, headers).into_response()) + } + Body::Stream { stream, len, range } => { + headers.insert(header::CONTENT_LENGTH, HeaderValue::from(len)); + let status = match &range { + Some(range) => { + headers.insert( + header::CONTENT_RANGE, + header_value(&range::content_range(range, file.size))?, + ); + StatusCode::PARTIAL_CONTENT + } + None => StatusCode::OK, + }; + telemetry::served(source, len); + telemetry::request("resolve_get", status.as_u16()); + let body = AxumBody::from_stream( + stream.map(|chunk| chunk.map_err(|err| std::io::Error::other(err.to_string()))), + ); + Ok((status, headers, body).into_response()) + } + Body::None => Err(ApiError(Error::Internal( + "a GET resolved to a metadata-only response".into(), + ))), + } +} + +/// Build the header set both `HEAD` and `GET` must carry. +/// +/// `Content-Encoding` is deliberately absent and must stay that way: the +/// client sends `Accept-Encoding: identity` precisely so the sizes it learns +/// are real, and compressing this path corrupts its size accounting. +fn metadata_headers(file: &ResolvedFile) -> Result { + let mut headers = HeaderMap::new(); + headers.insert(X_REPO_COMMIT, header_value(&file.commit)?); + headers.insert(header::ACCEPT_RANGES, HeaderValue::from_static("bytes")); + headers.insert( + header::CONTENT_TYPE, + HeaderValue::from_static("application/octet-stream"), + ); + + // Emitted exactly as upstream sent them, quotes included, and omitted + // entirely where upstream omitted them. The client prefers the linked + // etag and falls back to the plain one, so normalising either would move + // a cache entry that a directly-populated cache would not have moved. + if let Some(etag) = &file.etag { + headers.insert(header::ETAG, header_value(etag)?); + } + if let Some(linked) = &file.linked_etag { + headers.insert(X_LINKED_ETAG, header_value(linked)?); + } + if let Some(size) = file.linked_size { + headers.insert(X_LINKED_SIZE, HeaderValue::from(size)); + } + Ok(headers) +} + +/// A header value, refusing anything that cannot be encoded rather than +/// silently dropping a value a client depends on. +fn header_value(value: &str) -> Result { + HeaderValue::from_str(value) + .map_err(|_| Error::Internal(format!("{value:?} cannot be sent as a header value"))) +} diff --git a/crates/rustingface-api/src/handlers/meta.rs b/crates/rustingface-api/src/handlers/meta.rs new file mode 100644 index 0000000..974d0e8 --- /dev/null +++ b/crates/rustingface-api/src/handlers/meta.rs @@ -0,0 +1,132 @@ +//! The metadata surface: model/dataset info, refs and tree. +//! +//! Answered from the manifest of the resolved commit, which is upstream's own +//! document recorded verbatim. In sealed mode the file lists are narrowed to +//! what is actually stored, so a client is never told about a file it cannot +//! then fetch. + +use axum::extract::{Query, State}; +use axum::response::{IntoResponse, Response}; +use serde::Deserialize; +use serde_json::json; + +use rustingface_entities::repo::RepoType; + +use crate::error::ApiError; +use crate::path::{ApiPath, parse_api}; +use crate::state::AppState; +use crate::telemetry; + +/// The default revision when a request names none. +const DEFAULT_REVISION: &str = "main"; + +/// Query parameters the tree endpoint accepts. +#[derive(Debug, Default, Deserialize)] +pub struct TreeQuery { + /// List every descendant rather than one level. + #[serde(default)] + pub recursive: Option, + /// Include blob details. rustingface always records the expanded form, so + /// this is accepted and ignored rather than changing the answer. + #[serde(default)] + pub expand: Option, +} + +/// Whether a Hub-style boolean query parameter is set. +fn truthy(value: &Option) -> bool { + match value.as_deref() { + None => false, + Some("") => true, + Some(v) => !matches!(v.to_ascii_lowercase().as_str(), "0" | "false" | "no"), + } +} + +/// `GET /api/{models,datasets}/…` +pub async fn api( + State(state): State, + repo_type: RepoType, + Query(query): Query, + rest: String, +) -> std::result::Result { + match parse_api(&rest)? { + ApiPath::Info { repo_id } => { + let info = state + .registry + .repo_info(repo_type, &repo_id, DEFAULT_REVISION) + .await?; + telemetry::request("api_info", 200); + Ok(axum::Json(info).into_response()) + } + ApiPath::Revision { repo_id, revision } => { + let info = state + .registry + .repo_info(repo_type, &repo_id, &revision) + .await?; + telemetry::request("api_info", 200); + Ok(axum::Json(info).into_response()) + } + ApiPath::Refs { repo_id } => { + let refs = state.registry.list_refs(repo_type, &repo_id).await?; + // The Hub reports branches and tags separately. rustingface stores + // one flat namespace of pins and cannot tell them apart after the + // fact, so everything it pinned is reported as a branch and the + // tag list stays empty rather than being guessed at. + let branches: Vec<_> = refs + .iter() + .map(|r| { + json!({ + "name": r.ref_name, + "ref": format!("refs/heads/{}", r.ref_name), + "targetCommit": r.commit, + }) + }) + .collect(); + telemetry::request("api_refs", 200); + Ok(axum::Json(json!({ "branches": branches, "tags": [] })).into_response()) + } + ApiPath::Tree { + repo_id, + revision, + path, + } => { + let tree = state + .registry + .repo_tree( + repo_type, + &repo_id, + &revision, + &path, + truthy(&query.recursive), + ) + .await?; + telemetry::request("api_tree", 200); + Ok(axum::Json(tree).into_response()) + } + } +} + +/// `GET /api/whoami-v2` +pub async fn whoami(State(state): State) -> Response { + telemetry::request("whoami", 200); + axum::Json((*state.identity).clone()).into_response() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_bare_query_flag_counts_as_set() { + assert!(truthy(&Some(String::new()))); + assert!(truthy(&Some("1".into()))); + assert!(truthy(&Some("true".into()))); + } + + #[test] + fn an_absent_or_falsey_flag_does_not() { + assert!(!truthy(&None)); + assert!(!truthy(&Some("0".into()))); + assert!(!truthy(&Some("false".into()))); + assert!(!truthy(&Some("FALSE".into()))); + } +} diff --git a/crates/rustingface-api/src/handlers/mod.rs b/crates/rustingface-api/src/handlers/mod.rs new file mode 100644 index 0000000..5e1932a --- /dev/null +++ b/crates/rustingface-api/src/handlers/mod.rs @@ -0,0 +1,5 @@ +//! HTTP handlers. + +pub mod blob; +pub mod meta; +pub mod ops; diff --git a/crates/rustingface-api/src/handlers/ops.rs b/crates/rustingface-api/src/handlers/ops.rs new file mode 100644 index 0000000..debb3f5 --- /dev/null +++ b/crates/rustingface-api/src/handlers/ops.rs @@ -0,0 +1,37 @@ +//! Operational endpoints: liveness and metrics. + +use axum::extract::State; +use axum::response::{IntoResponse, Response}; +use http::StatusCode; + +use crate::state::AppState; + +/// `GET /healthz` +/// +/// Answers from process state only. It deliberately does not probe the bucket: +/// a health check that fails when object storage hiccups would have systemd +/// restart a process that is fine, which fixes nothing and drops in-flight +/// transfers. `rustingface doctor` is the check that touches storage. +pub async fn healthz() -> Response { + (StatusCode::OK, "ok\n").into_response() +} + +/// `GET /metrics` +pub async fn metrics(State(state): State) -> Response { + match &state.metrics { + Some(handle) => ( + StatusCode::OK, + [( + http::header::CONTENT_TYPE, + "text/plain; version=0.0.4; charset=utf-8", + )], + handle.render(), + ) + .into_response(), + None => ( + StatusCode::NOT_FOUND, + "metrics are disabled (observability.metrics = false)\n", + ) + .into_response(), + } +} diff --git a/crates/rustingface-api/src/lib.rs b/crates/rustingface-api/src/lib.rs new file mode 100644 index 0000000..1c65b4f --- /dev/null +++ b/crates/rustingface-api/src/lib.rs @@ -0,0 +1,123 @@ +//! The HTTP surface. +//! +//! Route shapes come from the Hub, not from what is convenient to express in a +//! router. Repo ids, revisions and file paths all contain slashes, so the +//! routes here are wildcards and the disambiguation lives in [`path`], where +//! the rules can be stated and tested. + +pub mod auth; +pub mod error; +pub mod handlers; +pub mod path; +pub mod range; +pub mod state; +pub mod telemetry; + +use axum::extract::{Query, Request, State}; +use axum::middleware::{self, Next}; +use axum::response::{IntoResponse, Response}; +use axum::routing::get; +use axum::{Router, http::HeaderMap}; +use http::Method; +use rustingface_entities::error::Error; +use rustingface_entities::repo::RepoType; +use tower_http::trace::TraceLayer; + +pub use error::ApiError; +pub use state::AppState; + +/// Build the router. +/// +/// Ordering matters: the catch-all resolve route must be registered last, or +/// it would shadow `/api/…` and `/metrics`. +pub fn router(state: AppState) -> Router { + Router::new() + .route("/healthz", get(handlers::ops::healthz)) + .route("/metrics", get(handlers::ops::metrics)) + .route("/api/whoami-v2", get(handlers::meta::whoami)) + .route("/api/models/{*rest}", get(api_models)) + .route("/api/datasets/{*rest}", get(api_datasets)) + .route( + "/datasets/{*rest}", + get(resolve_datasets).head(resolve_datasets), + ) + .route("/{*rest}", get(resolve_models).head(resolve_models)) + .layer(middleware::from_fn_with_state(state.clone(), authenticate)) + .layer(TraceLayer::new_for_http()) + .with_state(state) +} + +/// Reject unauthenticated requests when `auth.mode = "bearer"`. +/// +/// `/healthz` and `/metrics` stay open: they carry no repository data, and a +/// liveness probe that needs a credential is a liveness probe that silently +/// stops working when the credential rotates. +async fn authenticate(State(state): State, request: Request, next: Next) -> Response { + let path = request.uri().path(); + if path == "/healthz" || path == "/metrics" { + return next.run(request).await; + } + let presented = request + .headers() + .get(http::header::AUTHORIZATION) + .and_then(|v| v.to_str().ok()); + if let Err(err) = state.tokens.check(presented) { + return ApiError(err).into_response(); + } + next.run(request).await +} + +/// `GET /api/models/…` +async fn api_models( + state: State, + query: Query, + axum::extract::Path(rest): axum::extract::Path, +) -> Result { + handlers::meta::api(state, RepoType::Models, query, rest).await +} + +/// `GET /api/datasets/…` +async fn api_datasets( + state: State, + query: Query, + axum::extract::Path(rest): axum::extract::Path, +) -> Result { + handlers::meta::api(state, RepoType::Datasets, query, rest).await +} + +/// `GET`/`HEAD /datasets/{repo_id}/resolve/…` +async fn resolve_datasets( + state: State, + method: Method, + headers: HeaderMap, + axum::extract::Path(rest): axum::extract::Path, +) -> Result { + dispatch(state, RepoType::Datasets, method, headers, rest).await +} + +/// `GET`/`HEAD /{repo_id}/resolve/…` +async fn resolve_models( + state: State, + method: Method, + headers: HeaderMap, + axum::extract::Path(rest): axum::extract::Path, +) -> Result { + dispatch(state, RepoType::Models, method, headers, rest).await +} + +/// Route a resolve request by method. +async fn dispatch( + state: State, + repo_type: RepoType, + method: Method, + headers: HeaderMap, + rest: String, +) -> Result { + match method { + Method::HEAD => handlers::blob::head(state, repo_type, rest).await, + Method::GET => handlers::blob::get(state, repo_type, headers, rest).await, + other => Err(ApiError(Error::BadRequest(format!( + "{other} is not supported on the resolve path; rustingface is read-only" + )))), + } +} diff --git a/crates/rustingface-api/src/path.rs b/crates/rustingface-api/src/path.rs new file mode 100644 index 0000000..6decf61 --- /dev/null +++ b/crates/rustingface-api/src/path.rs @@ -0,0 +1,264 @@ +//! Hub URL parsing. +//! +//! Repo ids, revisions and file paths all contain slashes, and the Hub's URL +//! shapes are only unambiguous to a server that knows its own repos. These +//! parsers encode the disambiguation rules explicitly rather than leaning on +//! router pattern matching, which cannot express them. + +use rustingface_entities::error::{Error, Result}; +use rustingface_entities::refs::is_commit_sha; +use rustingface_entities::repo::RepoId; + +/// A parsed resolve path: `{repo_id}/resolve/{revision}/{filename}`. +#[derive(Debug, PartialEq, Eq)] +pub struct ResolvePath { + pub repo_id: RepoId, + pub revision: String, + pub filename: String, +} + +/// Split `{repo_id}/resolve/{revision}/{filename}`. +/// +/// The revision is the ambiguous part. A 40-hex segment is a commit; a +/// `refs/`-prefixed revision takes three segments (`refs/pr/1`, +/// `refs/convert/parquet`); anything else takes one. A branch name containing +/// a slash and not starting with `refs/` is therefore read as a one-segment +/// revision followed by a path, which is the same guess the Hub's own clients +/// make when constructing these URLs. +pub fn parse_resolve(rest: &str) -> Result { + let rest = rest.trim_start_matches('/'); + let (repo_part, tail) = rest + .split_once("/resolve/") + .ok_or_else(|| Error::BadRequest(format!("{rest:?} is not a resolve path")))?; + + let repo_id: RepoId = repo_part.parse()?; + let segments: Vec<&str> = tail.split('/').collect(); + + let revision_len = if segments.first() == Some(&"refs") { + 3 + } else { + 1 + }; + if segments.len() <= revision_len { + return Err(Error::BadRequest(format!( + "{rest:?} names a revision but no file" + ))); + } + let revision = segments[..revision_len].join("/"); + let filename = segments[revision_len..].join("/"); + if filename.is_empty() { + return Err(Error::BadRequest(format!( + "{rest:?} names an empty file path" + ))); + } + let _ = is_commit_sha(&revision); + Ok(ResolvePath { + repo_id, + revision, + filename, + }) +} + +/// The API request shapes rustingface serves. +#[derive(Debug, PartialEq, Eq)] +pub enum ApiPath { + /// `/api/{type}/{repo_id}` — info for the default ref. + Info { repo_id: RepoId }, + /// `/api/{type}/{repo_id}/revision/{revision}`. + Revision { repo_id: RepoId, revision: String }, + /// `/api/{type}/{repo_id}/refs`. + Refs { repo_id: RepoId }, + /// `/api/{type}/{repo_id}/tree/{revision}/{path}`. + Tree { + repo_id: RepoId, + revision: String, + path: String, + }, +} + +/// Segments that end a repo id in an API path. +const MARKERS: [&str; 3] = ["revision", "refs", "tree"]; + +/// Split the part of an `/api/{type}/…` path after the type. +/// +/// A repo id is one or two segments, so the marker is looked for at offset 1 +/// or 2 only. That is what keeps a repo genuinely named `tree` from being +/// mistaken for the tree endpoint of a bare-named repo. +pub fn parse_api(rest: &str) -> Result { + let rest = rest.trim_matches('/'); + if rest.is_empty() { + return Err(Error::BadRequest("no repository named".into())); + } + let segments: Vec<&str> = rest.split('/').collect(); + + let marker = (1..=2) + .filter(|i| *i < segments.len()) + .find(|i| MARKERS.contains(&segments[*i])); + + let Some(i) = marker else { + return Ok(ApiPath::Info { + repo_id: segments[..segments.len().min(2)].join("/").parse()?, + }); + }; + + let repo_id: RepoId = segments[..i].join("/").parse()?; + let tail = &segments[i + 1..]; + match segments[i] { + "refs" => Ok(ApiPath::Refs { repo_id }), + "revision" => { + if tail.is_empty() { + return Err(Error::BadRequest("no revision named".into())); + } + Ok(ApiPath::Revision { + repo_id, + revision: tail.join("/"), + }) + } + "tree" => { + if tail.is_empty() { + return Err(Error::BadRequest("no revision named".into())); + } + let revision_len = if tail.first() == Some(&"refs") { 3 } else { 1 }; + if tail.len() < revision_len { + return Err(Error::BadRequest("incomplete revision".into())); + } + Ok(ApiPath::Tree { + repo_id, + revision: tail[..revision_len].join("/"), + path: tail[revision_len..].join("/"), + }) + } + other => Err(Error::BadRequest(format!( + "unknown API path segment {other:?}" + ))), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn resolve(s: &str) -> ResolvePath { + parse_resolve(s).unwrap() + } + + #[test] + fn a_namespaced_resolve_path_splits_into_three_parts() { + let p = resolve("Qwen/Qwen3-32B/resolve/main/config.json"); + assert_eq!(p.repo_id.to_string(), "Qwen/Qwen3-32B"); + assert_eq!(p.revision, "main"); + assert_eq!(p.filename, "config.json"); + } + + #[test] + fn a_bare_named_repo_resolves_too() { + let p = resolve("gpt2/resolve/main/config.json"); + assert_eq!(p.repo_id.to_string(), "gpt2"); + assert_eq!(p.revision, "main"); + } + + #[test] + fn a_nested_filename_keeps_all_its_segments() { + let p = resolve("Qwen/Qwen3-32B/resolve/main/onnx/model/weights.onnx"); + assert_eq!(p.filename, "onnx/model/weights.onnx"); + } + + #[test] + fn a_commit_sha_revision_is_a_single_segment() { + let sha = "8f2a1c9e0b7d4f6a3e5c2b8d9f0a1c3e5b7d9f01"; + let p = resolve(&format!("Qwen/Qwen3-32B/resolve/{sha}/a/b.json")); + assert_eq!(p.revision, sha); + assert_eq!(p.filename, "a/b.json"); + } + + #[test] + fn a_refs_revision_takes_three_segments() { + let p = resolve("Qwen/Qwen3-32B/resolve/refs/pr/1/config.json"); + assert_eq!(p.revision, "refs/pr/1"); + assert_eq!(p.filename, "config.json"); + + let p = resolve("Qwen/Qwen3-32B/resolve/refs/convert/parquet/data/train.parquet"); + assert_eq!(p.revision, "refs/convert/parquet"); + assert_eq!(p.filename, "data/train.parquet"); + } + + #[test] + fn a_resolve_path_with_no_filename_is_a_bad_request() { + assert!(parse_resolve("Qwen/Qwen3-32B/resolve/main").is_err()); + assert!(parse_resolve("Qwen/Qwen3-32B/config.json").is_err()); + } + + #[test] + fn api_info_paths_parse_with_and_without_a_namespace() { + assert_eq!( + parse_api("Qwen/Qwen3-32B").unwrap(), + ApiPath::Info { + repo_id: "Qwen/Qwen3-32B".parse().unwrap() + } + ); + assert_eq!( + parse_api("gpt2").unwrap(), + ApiPath::Info { + repo_id: "gpt2".parse().unwrap() + } + ); + } + + #[test] + fn api_revision_refs_and_tree_paths_parse() { + assert_eq!( + parse_api("Qwen/Qwen3-32B/revision/main").unwrap(), + ApiPath::Revision { + repo_id: "Qwen/Qwen3-32B".parse().unwrap(), + revision: "main".into() + } + ); + assert_eq!( + parse_api("Qwen/Qwen3-32B/refs").unwrap(), + ApiPath::Refs { + repo_id: "Qwen/Qwen3-32B".parse().unwrap() + } + ); + assert_eq!( + parse_api("gpt2/tree/main/onnx").unwrap(), + ApiPath::Tree { + repo_id: "gpt2".parse().unwrap(), + revision: "main".into(), + path: "onnx".into() + } + ); + } + + #[test] + fn a_tree_path_with_a_refs_revision_splits_correctly() { + assert_eq!( + parse_api("a/b/tree/refs/pr/1/sub/dir").unwrap(), + ApiPath::Tree { + repo_id: "a/b".parse().unwrap(), + revision: "refs/pr/1".into(), + path: "sub/dir".into() + } + ); + } + + #[test] + fn a_marker_word_inside_a_repo_name_is_not_treated_as_a_marker() { + // `refs` here is the *name* of a repo in the `datasets-org` namespace, + // reached at offset 1, so it is a marker. At offset 2 it would be past + // any possible repo id and is not looked for at all. + assert_eq!( + parse_api("owner/tree-sitter").unwrap(), + ApiPath::Info { + repo_id: "owner/tree-sitter".parse().unwrap() + } + ); + } + + #[test] + fn incomplete_api_paths_are_bad_requests_not_panics() { + assert!(parse_api("").is_err()); + assert!(parse_api("a/b/revision").is_err()); + assert!(parse_api("a/b/tree").is_err()); + assert!(parse_api("a/b/tree/refs/pr").is_err()); + } +} diff --git a/crates/rustingface-api/src/range.rs b/crates/rustingface-api/src/range.rs new file mode 100644 index 0000000..9ef53e3 --- /dev/null +++ b/crates/rustingface-api/src/range.rs @@ -0,0 +1,114 @@ +//! `Range` header handling. +//! +//! Range support is what makes an interrupted 40GB download resumable rather +//! than restartable, so the proxy path honours it (spec §6). Only a single +//! byte range is supported; a multi-range request is answered whole, which is +//! always a legal response. + +use std::ops::Range; + +use rustingface_entities::error::{Error, Result}; + +/// Parse a single-range `Range: bytes=…` header against a known length. +/// +/// Returns `Ok(None)` for anything that is not a single byte range, so the +/// caller serves the whole object. +pub fn parse(header: &str, total: u64) -> Result>> { + let Some(spec) = header.trim().strip_prefix("bytes=") else { + return Ok(None); + }; + if spec.contains(',') { + return Ok(None); + } + let (start, end) = spec + .split_once('-') + .ok_or_else(|| Error::BadRequest(format!("malformed Range header {header:?}")))?; + + let unsatisfiable = + || Error::RangeNotSatisfiable(format!("{header:?} against a {total}-byte object")); + + let range = match (start.trim(), end.trim()) { + // `bytes=-N`: the final N bytes. + ("", suffix) => { + let n: u64 = suffix.parse().map_err(|_| unsatisfiable())?; + if n == 0 { + return Err(unsatisfiable()); + } + total.saturating_sub(n)..total + } + // `bytes=N-`: from N to the end. + (first, "") => { + let start: u64 = first.parse().map_err(|_| unsatisfiable())?; + start..total + } + // `bytes=N-M`, inclusive of M. + (first, last) => { + let start: u64 = first.parse().map_err(|_| unsatisfiable())?; + let end: u64 = last.parse().map_err(|_| unsatisfiable())?; + if end < start { + return Err(unsatisfiable()); + } + start..end.saturating_add(1).min(total) + } + }; + + if range.start >= total || range.start >= range.end { + return Err(unsatisfiable()); + } + Ok(Some(range)) +} + +/// The `Content-Range` value for a satisfied range. +pub fn content_range(range: &Range, total: u64) -> String { + format!("bytes {}-{}/{total}", range.start, range.end - 1) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_bounded_range_is_inclusive_of_its_last_byte() { + assert_eq!(parse("bytes=0-99", 1000).unwrap(), Some(0..100)); + assert_eq!(parse("bytes=100-199", 1000).unwrap(), Some(100..200)); + } + + #[test] + fn an_open_ended_range_runs_to_the_end() { + assert_eq!(parse("bytes=500-", 1000).unwrap(), Some(500..1000)); + } + + #[test] + fn a_suffix_range_takes_the_final_bytes() { + assert_eq!(parse("bytes=-100", 1000).unwrap(), Some(900..1000)); + assert_eq!( + parse("bytes=-5000", 1000).unwrap(), + Some(0..1000), + "a suffix longer than the object is the whole object" + ); + } + + #[test] + fn a_range_past_the_end_is_refused_rather_than_silently_truncated() { + let err = parse("bytes=1000-1100", 1000).unwrap_err(); + assert_eq!(err.status(), 416); + assert_eq!(parse("bytes=200-100", 1000).unwrap_err().status(), 416); + } + + #[test] + fn a_range_ending_past_the_end_is_clamped() { + assert_eq!(parse("bytes=900-2000", 1000).unwrap(), Some(900..1000)); + } + + #[test] + fn unsupported_range_forms_fall_back_to_the_whole_object() { + assert_eq!(parse("bytes=0-10,20-30", 1000).unwrap(), None); + assert_eq!(parse("items=0-10", 1000).unwrap(), None); + } + + #[test] + fn content_range_reports_the_inclusive_last_byte() { + assert_eq!(content_range(&(0..100), 1000), "bytes 0-99/1000"); + assert_eq!(content_range(&(900..1000), 1000), "bytes 900-999/1000"); + } +} diff --git a/crates/rustingface-api/src/state.rs b/crates/rustingface-api/src/state.rs new file mode 100644 index 0000000..674128e --- /dev/null +++ b/crates/rustingface-api/src/state.rs @@ -0,0 +1,59 @@ +//! Shared handler state. + +use std::sync::Arc; + +use metrics_exporter_prometheus::PrometheusHandle; +use rustingface_core::Registry; + +use crate::auth::Tokens; + +/// Everything a handler needs, cheap to clone. +#[derive(Clone)] +pub struct AppState { + pub registry: Registry, + pub tokens: Arc, + /// `None` when `observability.metrics = false`. + pub metrics: Option, + /// Static identity served from `/api/whoami-v2`. + pub identity: Arc, +} + +impl AppState { + /// Assemble state around a registry. + pub fn new(registry: Registry, tokens: Arc, metrics: Option) -> Self { + Self { + registry, + tokens, + metrics, + identity: Arc::new(whoami_identity()), + } + } +} + +/// The `/api/whoami-v2` document. +/// +/// `hf auth login` calls this to confirm a token works and refuses to store +/// one when it does not. rustingface has no user model, so it answers with a +/// fixed identity whose only job is to let that flow complete. +fn whoami_identity() -> serde_json::Value { + serde_json::json!({ + "type": "user", + "id": "rustingface", + "name": "rustingface", + "fullname": "rustingface registry", + "email": serde_json::Value::Null, + "emailVerified": true, + "canPay": false, + "isPro": false, + "periodEnd": serde_json::Value::Null, + "avatarUrl": "", + "orgs": [], + "auth": { + "type": "access_token", + "accessToken": { + "displayName": "rustingface", + "role": "read", + }, + }, + }) +} diff --git a/crates/rustingface-api/src/telemetry.rs b/crates/rustingface-api/src/telemetry.rs new file mode 100644 index 0000000..9cb3892 --- /dev/null +++ b/crates/rustingface-api/src/telemetry.rs @@ -0,0 +1,46 @@ +//! Prometheus metrics. +//! +//! Label sets are deliberately small. A metric keyed on repo id would give a +//! registry serving a few hundred models a few hundred time series per +//! counter, which is a cost paid forever for a question the manifests already +//! answer. + +use metrics::{counter, gauge, histogram}; +use metrics_exporter_prometheus::{PrometheusBuilder, PrometheusHandle}; +use rustingface_core::Source; +use rustingface_entities::error::{Error, Result}; + +/// Install the recorder and return the handle `/metrics` renders from. +pub fn install() -> Result { + PrometheusBuilder::new() + .set_buckets(&[0.005, 0.025, 0.1, 0.5, 1.0, 5.0, 30.0, 120.0, 600.0, 3600.0]) + .map_err(|e| Error::Config(format!("metrics buckets: {e}")))? + .install_recorder() + .map_err(|e| Error::Config(format!("metrics recorder: {e}"))) +} + +/// Record a completed request. +pub fn request(route: &'static str, status: u16) { + counter!("rustingface_requests_total", "route" => route, "status" => status.to_string()) + .increment(1); +} + +/// Record where a served body came from. +pub fn served(source: Source, bytes: u64) { + let label = match source { + Source::Bucket => "bucket", + Source::Upstream => "upstream", + }; + counter!("rustingface_bytes_served_total", "source" => label).increment(bytes); +} + +/// Record the outcome of an upstream fetch. +pub fn upstream_fetch(result: &'static str, seconds: f64) { + counter!("rustingface_upstream_fetches_total", "result" => result).increment(1); + histogram!("rustingface_upstream_fetch_seconds").record(seconds); +} + +/// Publish how many fetches are in flight. +pub fn in_flight(count: usize) { + gauge!("rustingface_fetches_in_flight").set(count as f64); +} diff --git a/crates/rustingface-api/tests/sovereignty.rs b/crates/rustingface-api/tests/sovereignty.rs new file mode 100644 index 0000000..7c0d517 --- /dev/null +++ b/crates/rustingface-api/tests/sovereignty.rs @@ -0,0 +1,953 @@ +//! The tests that matter (spec §11). +//! +//! Each one asserts a contractual guarantee rather than an implementation +//! detail: bytes land in the bucket, a sealed instance serves them with no +//! upstream in existence, a fleet-wide rollout produces one upstream fetch, +//! and a client hitting Ctrl-C does not throw away the transfer. + +mod support; + +use std::time::Duration; + +use axum::body::Body; +use http::{Request, StatusCode}; +use http_body_util::BodyExt; +use rustingface_entities::key; +use rustingface_entities::repo::{RepoId, RepoType}; +use support::{FakeHub, HubFile, bucket_dir, instance}; +use tower::ServiceExt; + +const COMMIT_A: &str = "8f2a1c9e0b7d4f6a3e5c2b8d9f0a1c3e5b7d9f01"; +const COMMIT_B: &str = "1122334455667788990011223344556677889900"; + +fn qwen() -> RepoId { + "Qwen/Qwen3-32B".parse().unwrap() +} + +/// A model with one small config and one LFS-backed weights file. +async fn publish_model(hub: &FakeHub) -> Vec { + let weights: Vec = (0..300_000u32).map(|i| (i % 251) as u8).collect(); + hub.publish( + "Qwen/Qwen3-32B", + "main", + COMMIT_A, + &[ + ("config.json", HubFile::plain(r#"{"model_type":"qwen3"}"#)), + ("model.safetensors", HubFile::lfs(&weights)), + ], + ) + .await; + weights +} + +/// Issue a request against an instance and read the whole response. +async fn call( + app: &axum::Router, + method: &str, + uri: &str, + headers: &[(&str, &str)], +) -> (StatusCode, http::HeaderMap, Vec) { + let mut request = Request::builder().method(method).uri(uri); + for (name, value) in headers { + request = request.header(*name, *value); + } + let response = app + .clone() + .oneshot(request.body(Body::empty()).unwrap()) + .await + .expect("the router must always answer"); + let status = response.status(); + let headers = response.headers().clone(); + let body = response + .into_body() + .collect() + .await + .expect("reading the response body") + .to_bytes() + .to_vec(); + (status, headers, body) +} + +/// As [`call`], but tolerating a body that aborts mid-stream. +/// +/// Once headers are on the wire a failure can only be signalled by ending the +/// body short of `Content-Length`, which is what a client detects. Returning +/// the error rather than panicking lets a test assert on that. +async fn call_streaming(app: &axum::Router, uri: &str) -> (StatusCode, Result, String>) { + use futures::StreamExt; + let response = app + .clone() + .oneshot(Request::builder().uri(uri).body(Body::empty()).unwrap()) + .await + .expect("the router must always answer"); + let status = response.status(); + let mut stream = response.into_body().into_data_stream(); + let mut body = Vec::new(); + while let Some(chunk) = stream.next().await { + match chunk { + Ok(chunk) => body.extend_from_slice(&chunk), + Err(err) => return (status, Err(err.to_string())), + } + } + (status, Ok(body)) +} + +#[tokio::test] +async fn a_cold_fetch_serves_the_bytes_and_leaves_them_in_the_bucket() { + let hub = FakeHub::start().await; + let weights = publish_model(&hub).await; + let bucket = bucket_dir("cold"); + let rf = instance(&bucket, Some(&hub), |_| {}); + + let (status, headers, body) = call( + &rf.app, + "GET", + "/Qwen/Qwen3-32B/resolve/main/model.safetensors", + &[], + ) + .await; + + assert_eq!(status, StatusCode::OK); + assert_eq!(body, weights, "the client receives the file it asked for"); + + // The header contract: upstream's values, replayed exactly. + assert_eq!(headers["x-repo-commit"], COMMIT_A); + assert_eq!(headers["accept-ranges"], "bytes"); + assert_eq!(headers["content-length"], weights.len().to_string()); + assert!( + !headers.contains_key("content-encoding"), + "compressing this path corrupts the client's size accounting" + ); + let oid = { + use sha2::{Digest, Sha256}; + hex::encode(Sha256::digest(&weights)) + }; + assert_eq!(headers["etag"], format!("\"{oid}\""), "quotes are kept"); + assert_eq!(headers["x-linked-etag"], format!("\"{oid}\"")); + assert_eq!(headers["x-linked-size"], weights.len().to_string()); + + // Guarantee 1: durably stored before the response completed. + let manifest = rf + .registry + .load_manifest(RepoType::Models, &qwen(), COMMIT_A) + .await + .unwrap() + .expect("a manifest must exist after a served fetch"); + let entry = manifest.entry("model.safetensors").expect("the entry"); + assert_eq!(entry.size, weights.len() as u64); + assert_eq!(entry.digest, format!("sha256:{oid}")); + assert!(entry.lfs); + + let blob = bucket.join(key::blob(&oid)); + assert!(blob.exists(), "the blob is at its content-addressed key"); + assert_eq!(std::fs::read(&blob).unwrap(), weights); +} + +#[tokio::test] +async fn a_sealed_instance_serves_the_same_bytes_with_no_upstream_at_all() { + let hub = FakeHub::start().await; + let weights = publish_model(&hub).await; + let bucket = bucket_dir("sealed"); + + { + let rf = instance(&bucket, Some(&hub), |_| {}); + for path in ["config.json", "model.safetensors"] { + let (status, _, _) = call( + &rf.app, + "GET", + &format!("/Qwen/Qwen3-32B/resolve/main/{path}"), + &[], + ) + .await; + assert_eq!(status, StatusCode::OK); + } + } + + // Step 6 of the spec's test: the host is gone. A brand-new process, a + // brand-new instance, no upstream client in existence, same bucket. + let sealed = instance(&bucket, None, |_| {}); + hub.reset_counts(); + + let (status, headers, body) = call( + &sealed.app, + "GET", + "/Qwen/Qwen3-32B/resolve/main/model.safetensors", + &[], + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!(body, weights, "identical bytes, from the bucket alone"); + assert_eq!(headers["x-repo-commit"], COMMIT_A, "and identical metadata"); + assert_eq!( + hub.requests(), + 0, + "a sealed instance must not touch upstream even once" + ); + + // And the ref still resolves, because the pin is in the bucket. + let (status, _, _) = call( + &sealed.app, + "HEAD", + "/Qwen/Qwen3-32B/resolve/main/config.json", + &[], + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!(hub.requests(), 0); +} + +#[tokio::test] +async fn a_sealed_instance_refuses_loudly_for_anything_it_never_stored() { + let bucket = bucket_dir("sealed-miss"); + let sealed = instance(&bucket, None, |_| {}); + + let (status, headers, _) = call( + &sealed.app, + "GET", + "/Qwen/Qwen3-32B/resolve/main/model.safetensors", + &[], + ) + .await; + assert_eq!(status, StatusCode::NOT_FOUND); + assert_eq!( + headers["x-error-code"], "RevisionNotFound", + "a typed code, not an empty file or a partial stream" + ); +} + +#[tokio::test] +async fn eight_concurrent_clients_produce_exactly_one_upstream_fetch() { + let hub = FakeHub::start().await; + let weights = publish_model(&hub).await; + // Slow enough that all eight are in flight together. + hub.throttle(16 * 1024, Duration::from_millis(10)).await; + + let bucket = bucket_dir("singleflight"); + let rf = instance(&bucket, Some(&hub), |_| {}); + hub.reset_counts(); + + let mut clients = Vec::new(); + for _ in 0..8 { + let app = rf.app.clone(); + clients.push(tokio::spawn(async move { + call( + &app, + "GET", + "/Qwen/Qwen3-32B/resolve/main/model.safetensors", + &[], + ) + .await + })); + } + + for client in clients { + let (status, _, body) = client.await.unwrap(); + assert_eq!(status, StatusCode::OK); + assert_eq!(body, weights, "every client gets the whole, correct file"); + } + + assert_eq!( + hub.file_gets(), + 1, + "a fleet-wide rollout must not become eight concurrent pulls" + ); +} + +#[tokio::test] +async fn a_client_that_disconnects_mid_transfer_still_leaves_the_file_stored() { + let hub = FakeHub::start().await; + let weights = publish_model(&hub).await; + hub.throttle(8 * 1024, Duration::from_millis(5)).await; + + let bucket = bucket_dir("disconnect"); + let rf = instance(&bucket, Some(&hub), |_| {}); + + // Read a little, then drop the response: the client hit Ctrl-C. + let response = rf + .app + .clone() + .oneshot( + Request::builder() + .uri("/Qwen/Qwen3-32B/resolve/main/model.safetensors") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let mut body = response.into_body().into_data_stream(); + use futures::StreamExt; + let first = body.next().await.expect("at least one chunk").unwrap(); + assert!(first.len() < weights.len(), "the transfer is still running"); + drop(body); + + // Discarding 30GB because a client hit Ctrl-C is the single most annoying + // possible behaviour; the transfer must finish regardless. + let oid = { + use sha2::{Digest, Sha256}; + hex::encode(Sha256::digest(&weights)) + }; + let blob = bucket.join(key::blob(&oid)); + for _ in 0..400 { + if blob.exists() && std::fs::metadata(&blob).unwrap().len() == weights.len() as u64 { + break; + } + tokio::time::sleep(Duration::from_millis(25)).await; + } + assert_eq!( + std::fs::read(&blob).unwrap(), + weights, + "the detached transfer ran to completion" + ); + + // And the manifest caught up too, so it is servable, not just present. + for _ in 0..200 { + if rf + .registry + .load_manifest(RepoType::Models, &qwen(), COMMIT_A) + .await + .unwrap() + .and_then(|m| m.entry("model.safetensors").cloned()) + .is_some() + { + break; + } + tokio::time::sleep(Duration::from_millis(25)).await; + } + let sealed = instance(&bucket, None, |_| {}); + let (status, _, served) = call( + &sealed.app, + "GET", + "/Qwen/Qwen3-32B/resolve/main/model.safetensors", + &[], + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!(served, weights); +} + +#[tokio::test] +async fn a_ranged_request_resumes_from_the_offset_it_asks_for() { + let hub = FakeHub::start().await; + let weights = publish_model(&hub).await; + let bucket = bucket_dir("range"); + let rf = instance(&bucket, Some(&hub), |_| {}); + + // Cold, with a range: the whole file must be stored before any part of it + // is served, or the bucket would hold a fragment. + let (status, headers, body) = call( + &rf.app, + "GET", + "/Qwen/Qwen3-32B/resolve/main/model.safetensors", + &[("range", "bytes=100000-100999")], + ) + .await; + assert_eq!(status, StatusCode::PARTIAL_CONTENT); + assert_eq!(body, weights[100_000..101_000]); + assert_eq!( + headers["content-range"], + format!("bytes 100000-100999/{}", weights.len()) + ); + assert_eq!(headers["content-length"], "1000"); + + let oid = { + use sha2::{Digest, Sha256}; + hex::encode(Sha256::digest(&weights)) + }; + assert_eq!( + std::fs::metadata(bucket.join(key::blob(&oid))) + .unwrap() + .len(), + weights.len() as u64, + "a ranged cold request stores the whole file, never a fragment" + ); + + // Warm, resuming to the end. + let (status, _, tail) = call( + &rf.app, + "GET", + "/Qwen/Qwen3-32B/resolve/main/model.safetensors", + &[("range", "bytes=299000-")], + ) + .await; + assert_eq!(status, StatusCode::PARTIAL_CONTENT); + assert_eq!(tail, weights[299_000..]); + + // And a range past the end is refused, not silently truncated. + let (status, _, _) = call( + &rf.app, + "GET", + "/Qwen/Qwen3-32B/resolve/main/model.safetensors", + &[("range", "bytes=999999999-")], + ) + .await; + assert_eq!(status, StatusCode::RANGE_NOT_SATISFIABLE); +} + +#[tokio::test] +async fn a_gated_repo_answers_403_with_the_code_the_client_branches_on() { + let hub = FakeHub::start().await; + publish_model(&hub).await; + hub.gate("Qwen/Qwen3-32B").await; + + let bucket = bucket_dir("gated"); + let rf = instance(&bucket, Some(&hub), |_| {}); + + let (status, headers, _) = call( + &rf.app, + "GET", + "/Qwen/Qwen3-32B/resolve/main/config.json", + &[], + ) + .await; + assert_eq!(status, StatusCode::FORBIDDEN); + assert_eq!(headers["x-error-code"], "GatedRepo"); +} + +#[tokio::test] +async fn a_frozen_ref_keeps_its_commit_after_upstream_moves() { + let hub = FakeHub::start().await; + publish_model(&hub).await; + let bucket = bucket_dir("freeze"); + let rf = instance(&bucket, Some(&hub), |_| {}); + + let (status, headers, first) = call( + &rf.app, + "GET", + "/Qwen/Qwen3-32B/resolve/main/config.json", + &[], + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!(headers["x-repo-commit"], COMMIT_A); + + // Upstream publishes a new commit and moves `main` onto it. + hub.publish( + "Qwen/Qwen3-32B", + "main", + COMMIT_B, + &[("config.json", HubFile::plain(r#"{"model_type":"qwen4"}"#))], + ) + .await; + + let (status, headers, again) = call( + &rf.app, + "GET", + "/Qwen/Qwen3-32B/resolve/main/config.json", + &[], + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!( + headers["x-repo-commit"], COMMIT_A, + "guarantee 4: upstream cannot change what the fleet resolves to" + ); + assert_eq!(again, first); + + // Moving is an operator action, and it is the only thing that moves it. + let (before, after) = rf + .registry + .refresh(RepoType::Models, &qwen(), "main") + .await + .unwrap(); + assert_eq!(before.as_deref(), Some(COMMIT_A)); + assert_eq!(after, COMMIT_B); + + let (_, headers, updated) = call( + &rf.app, + "GET", + "/Qwen/Qwen3-32B/resolve/main/config.json", + &[], + ) + .await; + assert_eq!(headers["x-repo-commit"], COMMIT_B); + assert_ne!(updated, first); + + // The old commit stays servable by SHA: nothing that pinned it broke. + let (status, _, pinned) = call( + &rf.app, + "GET", + &format!("/Qwen/Qwen3-32B/resolve/{COMMIT_A}/config.json"), + &[], + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!(pinned, first); +} + +#[tokio::test] +async fn gc_after_rm_reclaims_exactly_the_right_blobs_and_no_others() { + let hub = FakeHub::start().await; + publish_model(&hub).await; + hub.publish( + "meta-llama/Llama-3-8B", + "main", + COMMIT_B, + &[("config.json", HubFile::plain(r#"{"model_type":"llama"}"#))], + ) + .await; + + let bucket = bucket_dir("gc"); + let rf = instance(&bucket, Some(&hub), |_| {}); + + for uri in [ + "/Qwen/Qwen3-32B/resolve/main/config.json", + "/Qwen/Qwen3-32B/resolve/main/model.safetensors", + "/meta-llama/Llama-3-8B/resolve/main/config.json", + ] { + let (status, _, _) = call(&rf.app, "GET", uri, &[]).await; + assert_eq!(status, StatusCode::OK, "{uri}"); + } + + // Nothing is reclaimable while every blob is referenced. + let dry = rf.registry.gc(true).await.unwrap(); + assert_eq!(dry.scanned, 3); + assert!( + dry.deleted.is_empty(), + "a referenced blob is never reclaimable" + ); + + let llama: RepoId = "meta-llama/Llama-3-8B".parse().unwrap(); + let llama_digest = rf + .registry + .load_manifest(RepoType::Models, &llama, COMMIT_B) + .await + .unwrap() + .unwrap() + .entry("config.json") + .unwrap() + .digest + .clone(); + + rf.registry + .remove(RepoType::Models, &llama, None) + .await + .unwrap(); + + // rm detaches; it does not delete bytes. + assert_eq!( + rf.store.list(key::BLOB_PREFIX).await.unwrap().len(), + 3, + "rm must not touch blobs" + ); + + let dry = rf.registry.gc(true).await.unwrap(); + assert_eq!( + dry.deleted.len(), + 1, + "exactly the detached blob is reclaimable" + ); + assert!( + !std::path::Path::new(&bucket.join(&dry.deleted[0])) + .to_string_lossy() + .is_empty() + ); + assert!( + dry.deleted[0].ends_with(llama_digest.trim_start_matches("sha256:")), + "and it is the right one" + ); + assert!( + bucket.join(&dry.deleted[0]).exists(), + "a dry run deletes nothing" + ); + + let done = rf.registry.gc(false).await.unwrap(); + assert_eq!(done.deleted.len(), 1); + assert!(!bucket.join(&done.deleted[0]).exists()); + assert_eq!(rf.store.list(key::BLOB_PREFIX).await.unwrap().len(), 2); + + // The repo that was kept is untouched and still serves, sealed. + let sealed = instance(&bucket, None, |_| {}); + for uri in [ + "/Qwen/Qwen3-32B/resolve/main/config.json", + "/Qwen/Qwen3-32B/resolve/main/model.safetensors", + ] { + let (status, _, _) = call(&sealed.app, "GET", uri, &[]).await; + assert_eq!(status, StatusCode::OK, "{uri} must survive an unrelated gc"); + } +} + +#[tokio::test] +async fn a_digest_that_does_not_match_the_advertised_oid_stores_nothing() { + let hub = FakeHub::start().await; + let body = b"these bytes do not hash to the advertised oid".to_vec(); + hub.publish( + "Qwen/Qwen3-32B", + "main", + COMMIT_A, + &[("model.safetensors", HubFile::corrupt(&body))], + ) + .await; + + let bucket = bucket_dir("mismatch"); + let rf = instance(&bucket, Some(&hub), |_| {}); + + // The response cannot be a clean 200 with the wrong bytes. Headers are + // already on the wire by the time the digest is known, so the failure + // surfaces as a body that ends short of its Content-Length — which is + // exactly what the client detects and retries on. + let (_, body) = call_streaming(&rf.app, "/Qwen/Qwen3-32B/resolve/main/model.safetensors").await; + let err = body.expect_err("a mismatched transfer must not complete cleanly"); + assert!(err.contains("digest mismatch"), "{err}"); + + tokio::time::sleep(Duration::from_millis(200)).await; + assert!( + !bucket.join(key::blob(&"b".repeat(64))).exists(), + "a mismatched transfer must not leave bytes at the advertised key" + ); + let manifest = rf + .registry + .load_manifest(RepoType::Models, &qwen(), COMMIT_A) + .await + .unwrap(); + assert!( + manifest.is_none_or(|m| m.entry("model.safetensors").is_none()), + "and must never be recorded" + ); +} + +#[tokio::test] +async fn verify_reports_a_blob_whose_bytes_stopped_matching_its_key() { + let hub = FakeHub::start().await; + let weights = publish_model(&hub).await; + let bucket = bucket_dir("verify"); + let rf = instance(&bucket, Some(&hub), |_| {}); + + let (status, _, _) = call( + &rf.app, + "GET", + "/Qwen/Qwen3-32B/resolve/main/model.safetensors", + &[], + ) + .await; + assert_eq!(status, StatusCode::OK); + + let clean = rf.registry.verify(None).await.unwrap(); + assert_eq!(clean.blobs, 1); + assert!(clean.corrupt.is_empty() && clean.missing.is_empty()); + + // Corrupt the stored bytes behind rustingface's back, as bit rot would. + let oid = { + use sha2::{Digest, Sha256}; + hex::encode(Sha256::digest(&weights)) + }; + let mut damaged = weights.clone(); + damaged[0] ^= 0xff; + std::fs::write(bucket.join(key::blob(&oid)), &damaged).unwrap(); + + let dirty = rf.registry.verify(None).await.unwrap(); + assert_eq!( + dirty.corrupt.len(), + 1, + "verify re-reads bytes, it does not trust metadata" + ); + assert_eq!(dirty.corrupt[0].expected, format!("sha256:{oid}")); +} + +#[tokio::test] +async fn metadata_endpoints_answer_from_the_bucket_once_recorded() { + let hub = FakeHub::start().await; + publish_model(&hub).await; + let bucket = bucket_dir("meta"); + let rf = instance(&bucket, Some(&hub), |_| {}); + + let (status, _, body) = call(&rf.app, "GET", "/api/models/Qwen/Qwen3-32B", &[]).await; + assert_eq!(status, StatusCode::OK); + let info: serde_json::Value = serde_json::from_slice(&body).unwrap(); + assert_eq!(info["sha"], COMMIT_A); + assert_eq!(info["siblings"].as_array().unwrap().len(), 2); + + let (status, _, body) = call( + &rf.app, + "GET", + "/api/models/Qwen/Qwen3-32B/tree/main?recursive=1", + &[], + ) + .await; + assert_eq!(status, StatusCode::OK); + let tree: serde_json::Value = serde_json::from_slice(&body).unwrap(); + assert_eq!(tree.as_array().unwrap().len(), 2); + + let (status, _, body) = call(&rf.app, "GET", "/api/models/Qwen/Qwen3-32B/refs", &[]).await; + assert_eq!(status, StatusCode::OK); + let refs: serde_json::Value = serde_json::from_slice(&body).unwrap(); + assert_eq!(refs["branches"][0]["name"], "main"); + assert_eq!(refs["branches"][0]["targetCommit"], COMMIT_A); + + // Fetch one of the two files, then seal. The info document must stop + // advertising the file that can no longer be served. + let (status, _, _) = call( + &rf.app, + "GET", + "/Qwen/Qwen3-32B/resolve/main/config.json", + &[], + ) + .await; + assert_eq!(status, StatusCode::OK); + + let sealed = instance(&bucket, None, |_| {}); + let (status, _, body) = call(&sealed.app, "GET", "/api/models/Qwen/Qwen3-32B", &[]).await; + assert_eq!(status, StatusCode::OK); + let info: serde_json::Value = serde_json::from_slice(&body).unwrap(); + let siblings = info["siblings"].as_array().unwrap(); + assert_eq!( + siblings.len(), + 1, + "sealed mode advertises only what it can serve" + ); + assert_eq!(siblings[0]["rfilename"], "config.json"); +} + +#[tokio::test] +async fn datasets_take_the_same_path_under_their_own_prefix() { + let hub = FakeHub::start().await; + hub.publish( + "HuggingFaceFW/fineweb", + "main", + COMMIT_A, + &[("data/train.parquet", HubFile::lfs(b"parquet bytes"))], + ) + .await; + + let bucket = bucket_dir("datasets"); + let rf = instance(&bucket, Some(&hub), |_| {}); + + let (status, headers, body) = call( + &rf.app, + "GET", + "/datasets/HuggingFaceFW/fineweb/resolve/main/data/train.parquet", + &[], + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!(body, b"parquet bytes"); + assert_eq!(headers["x-repo-commit"], COMMIT_A); + + let manifests = rf.store.list("manifests/datasets/").await.unwrap(); + assert_eq!( + manifests.len(), + 1, + "datasets are stored under their own prefix" + ); + assert!(rf.store.list("manifests/models/").await.unwrap().is_empty()); +} + +#[tokio::test] +async fn deduplication_records_a_second_repo_without_transferring_again() { + let hub = FakeHub::start().await; + let weights: Vec = (0..200_000u32).map(|i| (i % 97) as u8).collect(); + for repo in ["org-a/model", "org-b/mirror"] { + hub.publish( + repo, + "main", + COMMIT_A, + &[("model.safetensors", HubFile::lfs(&weights))], + ) + .await; + } + + let bucket = bucket_dir("dedup"); + let rf = instance(&bucket, Some(&hub), |_| {}); + + let (status, _, first) = call( + &rf.app, + "GET", + "/org-a/model/resolve/main/model.safetensors", + &[], + ) + .await; + assert_eq!(status, StatusCode::OK); + hub.reset_counts(); + + let (status, _, second) = call( + &rf.app, + "GET", + "/org-b/mirror/resolve/main/model.safetensors", + &[], + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!(second, first); + assert_eq!( + rf.store.list(key::BLOB_PREFIX).await.unwrap().len(), + 1, + "identical bytes occupy the bucket once" + ); + + // gc must keep the shared blob while either repo still references it. + rf.registry + .remove(RepoType::Models, &"org-a/model".parse().unwrap(), None) + .await + .unwrap(); + let reclaimed = rf.registry.gc(false).await.unwrap(); + assert!( + reclaimed.deleted.is_empty(), + "a blob shared with a surviving repo is not garbage" + ); + let (status, _, _) = call( + &rf.app, + "GET", + "/org-b/mirror/resolve/main/model.safetensors", + &[], + ) + .await; + assert_eq!(status, StatusCode::OK); +} + +#[tokio::test] +async fn a_repo_outside_the_allowlist_is_refused_before_any_upstream_contact() { + let hub = FakeHub::start().await; + publish_model(&hub).await; + let bucket = bucket_dir("policy"); + let rf = instance(&bucket, Some(&hub), |cfg| { + cfg.policy.allowlist = vec!["meta-llama/*".into()]; + }); + hub.reset_counts(); + + let (status, _, _) = call( + &rf.app, + "GET", + "/Qwen/Qwen3-32B/resolve/main/config.json", + &[], + ) + .await; + assert_eq!(status, StatusCode::FORBIDDEN); + assert_eq!(hub.requests(), 0, "policy is checked before the network is"); +} + +#[tokio::test] +async fn bearer_auth_refuses_an_unknown_token_but_never_the_health_probe() { + let hub = FakeHub::start().await; + publish_model(&hub).await; + let bucket = bucket_dir("auth"); + + let mut config = rustingface_entities::config::Config::default(); + config.storage.local_path = Some(bucket.clone()); + config.upstream.enabled = true; + config.upstream.endpoint = hub.base_url.clone(); + config.auth.mode = rustingface_entities::config::AuthMode::Bearer; + + let store: std::sync::Arc = + std::sync::Arc::new(rustingface_data::ObjectStoreAdapter::local(&bucket).unwrap()); + let upstream: std::sync::Arc = + std::sync::Arc::new(rustingface_data::HubClient::new(&config.upstream, None).unwrap()); + let registry = rustingface_core::Registry::new(store, upstream, config.clone()).unwrap(); + let tokens = + rustingface_api::auth::Tokens::new(config.auth.mode, Some("# beast\nhf_good\n")).unwrap(); + let app = rustingface_api::router(rustingface_api::AppState::new( + registry, + std::sync::Arc::new(tokens), + None, + )); + + let uri = "/Qwen/Qwen3-32B/resolve/main/config.json"; + assert_eq!( + call(&app, "GET", uri, &[]).await.0, + StatusCode::UNAUTHORIZED + ); + assert_eq!( + call(&app, "GET", uri, &[("authorization", "Bearer hf_wrong")]) + .await + .0, + StatusCode::UNAUTHORIZED + ); + assert_eq!( + call(&app, "GET", uri, &[("authorization", "Bearer hf_good")]) + .await + .0, + StatusCode::OK + ); + assert_eq!( + call(&app, "GET", "/healthz", &[]).await.0, + StatusCode::OK, + "a liveness probe that needs a credential silently stops working when it rotates" + ); +} + +#[tokio::test] +async fn whoami_answers_so_hf_auth_login_can_complete() { + let bucket = bucket_dir("whoami"); + let rf = instance(&bucket, None, |_| {}); + let (status, _, body) = call(&rf.app, "GET", "/api/whoami-v2", &[]).await; + assert_eq!(status, StatusCode::OK); + let identity: serde_json::Value = serde_json::from_slice(&body).unwrap(); + assert_eq!(identity["type"], "user"); + assert_eq!(identity["name"], "rustingface"); +} + +#[tokio::test] +async fn a_head_replays_metadata_without_transferring_the_body() { + let hub = FakeHub::start().await; + let weights = publish_model(&hub).await; + let bucket = bucket_dir("head"); + let rf = instance(&bucket, Some(&hub), |_| {}); + + let (status, headers, body) = call( + &rf.app, + "HEAD", + "/Qwen/Qwen3-32B/resolve/main/model.safetensors", + &[], + ) + .await; + assert_eq!(status, StatusCode::OK); + assert!(body.is_empty()); + assert_eq!(headers["content-length"], weights.len().to_string()); + assert_eq!(headers["x-repo-commit"], COMMIT_A); + assert_eq!( + hub.file_gets(), + 0, + "a HEAD must not pull the body it is asking about" + ); +} + +#[tokio::test] +async fn state_is_portable_because_the_bucket_is_the_whole_of_it() { + let hub = FakeHub::start().await; + let weights = publish_model(&hub).await; + let source = bucket_dir("portable-src"); + + { + let rf = instance(&source, Some(&hub), |_| {}); + for path in ["config.json", "model.safetensors"] { + let (status, _, _) = call( + &rf.app, + "GET", + &format!("/Qwen/Qwen3-32B/resolve/main/{path}"), + &[], + ) + .await; + assert_eq!(status, StatusCode::OK); + } + } + + // Copy the bucket somewhere else entirely, as re-pointing a reinstalled + // host at the same bucket would. There is no second datastore to carry. + let destination = bucket_dir("portable-dst"); + copy_tree(&source, &destination); + + let moved = instance(&destination, None, |_| {}); + let (status, headers, body) = call( + &moved.app, + "GET", + "/Qwen/Qwen3-32B/resolve/main/model.safetensors", + &[], + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!(body, weights); + assert_eq!(headers["x-repo-commit"], COMMIT_A); +} + +/// Recursively copy a directory tree. +fn copy_tree(from: &std::path::Path, to: &std::path::Path) { + std::fs::create_dir_all(to).unwrap(); + for entry in std::fs::read_dir(from).unwrap() { + let entry = entry.unwrap(); + let target = to.join(entry.file_name()); + if entry.file_type().unwrap().is_dir() { + copy_tree(&entry.path(), &target); + } else { + std::fs::copy(entry.path(), &target).unwrap(); + } + } +} diff --git a/crates/rustingface-api/tests/support/mod.rs b/crates/rustingface-api/tests/support/mod.rs new file mode 100644 index 0000000..0632fbf --- /dev/null +++ b/crates/rustingface-api/tests/support/mod.rs @@ -0,0 +1,470 @@ +//! A fake Hub, and a rustingface wired to it. +//! +//! The tests that matter are about behaviour against a real client protocol, +//! so the fake upstream speaks the same headers the Hub does — `X-Repo-Commit`, +//! `X-Linked-Etag`, `X-Linked-Size`, `X-Error-Code` — and counts what it was +//! asked for, which is how single-flight is proved rather than assumed. + +#![allow(dead_code)] + +use std::collections::HashMap; +use std::net::SocketAddr; +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::time::Duration; + +use axum::Router; +use axum::body::Body; +use axum::extract::{Path, State}; +use axum::response::{IntoResponse, Response}; +use axum::routing::get; +use http::{HeaderMap, Method, StatusCode}; +use rustingface_api::auth::Tokens; +use rustingface_api::{AppState, router}; +use rustingface_core::Registry; +use rustingface_core::ports::{Store, Upstream}; +use rustingface_data::{HubClient, ObjectStoreAdapter, SealedUpstream}; +use rustingface_entities::config::{AuthMode, Config}; +use tokio::sync::RwLock; + +/// One file the fake Hub serves. +#[derive(Clone)] +pub struct HubFile { + pub body: Vec, + /// `ETag`, verbatim, quotes included. + pub etag: String, + /// `X-Linked-Etag`, present for LFS-backed files. + pub linked_etag: Option, + pub linked_size: Option, +} + +impl HubFile { + /// A small text file, the shape `config.json` takes upstream: a git blob + /// etag and no LFS headers. + pub fn plain(body: &str) -> Self { + Self { + body: body.as_bytes().to_vec(), + etag: format!("\"{:040x}\"", body.len()), + linked_etag: None, + linked_size: None, + } + } + + /// An LFS-backed file, whose linked etag is the sha256 of its bytes. + pub fn lfs(body: &[u8]) -> Self { + use sha2::{Digest, Sha256}; + let oid = hex::encode(Sha256::digest(body)); + Self { + body: body.to_vec(), + etag: format!("\"{oid}\""), + linked_etag: Some(format!("\"{oid}\"")), + linked_size: Some(body.len() as u64), + } + } + + /// An LFS-backed file whose advertised oid does not match its bytes. + pub fn corrupt(body: &[u8]) -> Self { + let mut file = Self::lfs(body); + file.linked_etag = Some(format!("\"{}\"", "b".repeat(64))); + file + } +} + +/// One revision of one repo. +#[derive(Clone, Default)] +pub struct HubRevision { + pub files: HashMap, +} + +/// The fake Hub's mutable state. +#[derive(Default)] +pub struct HubState { + /// `repo_id` -> ref name -> commit. + pub refs: HashMap>, + /// `repo_id` -> commit -> revision. + pub revisions: HashMap>, + /// Repos that answer 403 with `X-Error-Code: GatedRepo`. + pub gated: Vec, + /// Delay inserted between body chunks, to keep a transfer observable. + pub chunk_delay: Duration, + /// Bytes per body chunk. + pub chunk_size: usize, +} + +/// A running fake Hub. +pub struct FakeHub { + pub base_url: String, + pub state: Arc>, + /// Every `GET` on the resolve path. This is the number single-flight is + /// asserted against. + pub file_gets: Arc, + /// Every request of any kind, so a sealed test can prove zero contact. + pub requests: Arc, + shutdown: Option>, +} + +impl FakeHub { + /// Start a fake Hub on an ephemeral port. + pub async fn start() -> Self { + let state = Arc::new(RwLock::new(HubState { + chunk_size: 64 * 1024, + ..Default::default() + })); + let file_gets = Arc::new(AtomicUsize::new(0)); + let requests = Arc::new(AtomicUsize::new(0)); + + let hub = HubHandle { + state: Arc::clone(&state), + file_gets: Arc::clone(&file_gets), + requests: Arc::clone(&requests), + }; + + let app = Router::new() + .route("/api/models/{*rest}", get(api)) + .route("/api/datasets/{*rest}", get(api)) + .route("/{*rest}", get(resolve).head(resolve)) + .with_state(hub); + + let listener = tokio::net::TcpListener::bind::("127.0.0.1:0".parse().unwrap()) + .await + .expect("binding the fake hub"); + let addr = listener.local_addr().unwrap(); + let (tx, rx) = tokio::sync::oneshot::channel(); + tokio::spawn(async move { + let _ = axum::serve(listener, app) + .with_graceful_shutdown(async { + let _ = rx.await; + }) + .await; + }); + + Self { + base_url: format!("http://{addr}"), + state, + file_gets, + requests, + shutdown: Some(tx), + } + } + + /// Publish a revision and point a ref at it. + pub async fn publish( + &self, + repo: &str, + ref_name: &str, + commit: &str, + files: &[(&str, HubFile)], + ) { + let mut state = self.state.write().await; + state + .refs + .entry(repo.to_owned()) + .or_default() + .insert(ref_name.to_owned(), commit.to_owned()); + let revision = HubRevision { + files: files + .iter() + .map(|(path, file)| ((*path).to_owned(), file.clone())) + .collect(), + }; + state + .revisions + .entry(repo.to_owned()) + .or_default() + .insert(commit.to_owned(), revision); + } + + /// Move a ref to a different commit, as an upstream force-push would. + pub async fn move_ref(&self, repo: &str, ref_name: &str, commit: &str) { + self.state + .write() + .await + .refs + .entry(repo.to_owned()) + .or_default() + .insert(ref_name.to_owned(), commit.to_owned()); + } + + /// Make a repo answer 403 `GatedRepo`. + pub async fn gate(&self, repo: &str) { + self.state.write().await.gated.push(repo.to_owned()); + } + + /// Slow the body down so a transfer stays observable mid-flight. + pub async fn throttle(&self, chunk_size: usize, delay: Duration) { + let mut state = self.state.write().await; + state.chunk_size = chunk_size; + state.chunk_delay = delay; + } + + /// `GET`s on the resolve path so far. + pub fn file_gets(&self) -> usize { + self.file_gets.load(Ordering::SeqCst) + } + + /// Requests of any kind so far. + pub fn requests(&self) -> usize { + self.requests.load(Ordering::SeqCst) + } + + /// Forget everything counted so far. + pub fn reset_counts(&self) { + self.file_gets.store(0, Ordering::SeqCst); + self.requests.store(0, Ordering::SeqCst); + } + + /// Stop answering, as null-routing the Hub at the site router would. + pub fn stop(&mut self) { + if let Some(tx) = self.shutdown.take() { + let _ = tx.send(()); + } + } +} + +#[derive(Clone)] +struct HubHandle { + state: Arc>, + file_gets: Arc, + requests: Arc, +} + +/// Reject a gated repo the way the Hub does. +fn gated() -> Response { + ( + StatusCode::FORBIDDEN, + [("x-error-code", "GatedRepo")], + "access to this repo is gated", + ) + .into_response() +} + +/// Report a missing entity with the code the client branches on. +fn not_found(code: &'static str) -> Response { + (StatusCode::NOT_FOUND, [("x-error-code", code)], code).into_response() +} + +/// The fake Hub's `/api/{type}/…` surface. +async fn api(State(hub): State, Path(rest): Path) -> Response { + hub.requests.fetch_add(1, Ordering::SeqCst); + let state = hub.state.read().await; + let segments: Vec<&str> = rest.split('/').collect(); + + let marker = (1..=2) + .filter(|i| *i < segments.len()) + .find(|i| matches!(segments[*i], "revision" | "tree")); + let Some(i) = marker else { + return not_found("RepoNotFound"); + }; + let repo = segments[..i].join("/"); + let revision = segments.get(i + 1).copied().unwrap_or("main"); + + if state.gated.contains(&repo) { + return gated(); + } + let Some(commit) = resolve_commit(&state, &repo, revision) else { + return not_found("RevisionNotFound"); + }; + let Some(rev) = state.revisions.get(&repo).and_then(|r| r.get(&commit)) else { + return not_found("RevisionNotFound"); + }; + + let mut paths: Vec<&String> = rev.files.keys().collect(); + paths.sort(); + + if segments[i] == "tree" { + let tree: Vec<_> = paths + .iter() + .map(|path| { + let file = &rev.files[*path]; + serde_json::json!({ + "type": "file", + "path": path, + "size": file.body.len(), + "oid": file.etag.trim_matches('"'), + }) + }) + .collect(); + return axum::Json(tree).into_response(); + } + + axum::Json(serde_json::json!({ + "id": repo, + "sha": commit, + "lastModified": "2026-06-11T00:00:00.000Z", + "private": false, + "gated": false, + "tags": ["text-generation"], + "siblings": paths.iter().map(|p| serde_json::json!({"rfilename": p})).collect::>(), + })) + .into_response() +} + +/// The fake Hub's resolve path. +async fn resolve( + State(hub): State, + method: Method, + Path(rest): Path, +) -> Response { + hub.requests.fetch_add(1, Ordering::SeqCst); + let state = hub.state.read().await; + + let rest = rest.strip_prefix("datasets/").unwrap_or(&rest); + let Some((repo, tail)) = rest.split_once("/resolve/") else { + return not_found("RepoNotFound"); + }; + if state.gated.contains(&repo.to_owned()) { + return gated(); + } + let Some((revision, path)) = tail.split_once('/') else { + return not_found("EntryNotFound"); + }; + let revision = percent_decode(revision); + + let Some(commit) = resolve_commit(&state, repo, &revision) else { + return not_found("RevisionNotFound"); + }; + let Some(file) = state + .revisions + .get(repo) + .and_then(|r| r.get(&commit)) + .and_then(|r| r.files.get(path)) + .cloned() + else { + return not_found("EntryNotFound"); + }; + + let mut headers = HeaderMap::new(); + headers.insert("x-repo-commit", commit.parse().unwrap()); + headers.insert("etag", file.etag.parse().unwrap()); + if let Some(linked) = &file.linked_etag { + headers.insert("x-linked-etag", linked.parse().unwrap()); + } + if let Some(size) = file.linked_size { + headers.insert("x-linked-size", size.to_string().parse().unwrap()); + } + headers.insert( + "content-length", + file.body.len().to_string().parse().unwrap(), + ); + + if method == Method::HEAD { + return (StatusCode::OK, headers).into_response(); + } + hub.file_gets.fetch_add(1, Ordering::SeqCst); + + let chunk_size = state.chunk_size.max(1); + let delay = state.chunk_delay; + let body = file.body.clone(); + drop(state); + + let stream = futures::stream::unfold(0usize, move |offset| { + let body = body.clone(); + async move { + if offset >= body.len() { + return None; + } + if !delay.is_zero() { + tokio::time::sleep(delay).await; + } + let end = (offset + chunk_size).min(body.len()); + let chunk = bytes::Bytes::copy_from_slice(&body[offset..end]); + Some((Ok::<_, std::io::Error>(chunk), end)) + } + }); + + (StatusCode::OK, headers, Body::from_stream(stream)).into_response() +} + +/// Resolve a ref or pass a commit through. +fn resolve_commit(state: &HubState, repo: &str, revision: &str) -> Option { + if let Some(commit) = state.refs.get(repo).and_then(|r| r.get(revision)) { + return Some(commit.clone()); + } + state + .revisions + .get(repo) + .and_then(|r| r.contains_key(revision).then(|| revision.to_owned())) +} + +/// Undo the percent-encoding the resolve URL applies to a revision. +fn percent_decode(value: &str) -> String { + let mut out = String::with_capacity(value.len()); + let bytes = value.as_bytes(); + let mut i = 0; + while i < bytes.len() { + if bytes[i] == b'%' && i + 2 < bytes.len() { + if let Ok(byte) = u8::from_str_radix(&value[i + 1..i + 3], 16) { + out.push(byte as char); + i += 3; + continue; + } + } + out.push(bytes[i] as char); + i += 1; + } + out +} + +/// A rustingface instance under test, over a temporary bucket directory. +pub struct Instance { + pub app: axum::Router, + pub registry: Registry, + pub store: Arc, + pub config: Config, + pub bucket: std::path::PathBuf, +} + +/// Build an instance against `hub`, applying `tweak` to the config first. +pub fn instance( + bucket: &std::path::Path, + hub: Option<&FakeHub>, + tweak: impl FnOnce(&mut Config), +) -> Instance { + let mut config = Config::default(); + config.storage.local_path = Some(bucket.to_owned()); + config.storage.multipart_part_size = 5 * 1024 * 1024; + config.server.client_stall_timeout = Duration::from_secs(2); + match hub { + Some(hub) => { + config.upstream.enabled = true; + config.upstream.endpoint = hub.base_url.clone(); + } + None => config.upstream.enabled = false, + } + tweak(&mut config); + config.validate().expect("test config must be valid"); + + let store: Arc = Arc::new(ObjectStoreAdapter::local(bucket).expect("local store")); + let upstream: Arc = if config.upstream.enabled { + Arc::new(HubClient::new(&config.upstream, None).expect("hub client")) + } else { + Arc::new(SealedUpstream::new(config.upstream.endpoint.clone())) + }; + + let registry = Registry::new(Arc::clone(&store), upstream, config.clone()).expect("registry"); + let tokens = Tokens::new(config.auth.mode, None) + .unwrap_or_else(|_| Tokens::new(AuthMode::None, None).expect("permissive tokens")); + let app = router(AppState::new(registry.clone(), Arc::new(tokens), None)); + + Instance { + app, + registry, + store, + config, + bucket: bucket.to_owned(), + } +} + +/// A fresh directory to use as a bucket. +pub fn bucket_dir(name: &str) -> std::path::PathBuf { + let dir = std::env::temp_dir().join(format!( + "rustingface-test-{name}-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + std::fs::create_dir_all(&dir).expect("creating the test bucket"); + dir +} diff --git a/crates/rustingface-core/Cargo.toml b/crates/rustingface-core/Cargo.toml new file mode 100644 index 0000000..6036bba --- /dev/null +++ b/crates/rustingface-core/Cargo.toml @@ -0,0 +1,28 @@ +[package] +name = "rustingface-core" +description = "Resolver, single-flight fetch, retention policy and admin operations for rustingface." +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +authors.workspace = true +repository.workspace = true + +[dependencies] +rustingface-entities.workspace = true + +async-trait.workspace = true +bytes.workspace = true +dashmap.workspace = true +futures.workspace = true +globset.workspace = true +hex.workspace = true +serde.workspace = true +serde_json.workspace = true +sha2.workspace = true +time.workspace = true +tokio.workspace = true +tracing.workspace = true + +[dev-dependencies] +tokio = { workspace = true, features = ["macros", "rt-multi-thread", "test-util"] } diff --git a/crates/rustingface-core/src/admin.rs b/crates/rustingface-core/src/admin.rs new file mode 100644 index 0000000..cb2641a --- /dev/null +++ b/crates/rustingface-core/src/admin.rs @@ -0,0 +1,389 @@ +//! Operator actions: the ones that remove things, and the ones that check. +//! +//! `rm` and `gc` are deliberately separate. Removal detaches metadata; +//! reclamation is a second, explicit, dry-runnable step. Nothing in this file +//! deletes blob bytes except [`Registry::gc`], and only when it is asked to. + +use std::collections::BTreeSet; + +use bytes::Bytes; +use futures::StreamExt; +use rustingface_entities::error::{Error, Result}; +use rustingface_entities::key; +use rustingface_entities::manifest::Manifest; +use rustingface_entities::repo::{RepoId, RepoType}; +use tracing::{info, warn}; + +use crate::digest::{self, Digest}; +use crate::registry::Registry; + +/// A stored repo, summarised for `rustingface list`. +#[derive(Debug, Clone)] +pub struct RepoSummary { + pub repo_type: RepoType, + pub repo_id: RepoId, + /// Ref name to pinned commit, sorted by ref name. + pub refs: Vec<(String, String)>, + /// Every recorded commit. + pub commits: Vec, + /// Distinct files recorded across all commits. + pub files: usize, + /// Bytes referenced, counting each distinct blob once. + pub bytes: u64, +} + +/// What `rustingface rm` detached. +#[derive(Debug, Clone, Default)] +pub struct Removed { + pub manifests: Vec, + pub refs: Vec, + pub repo_record: bool, +} + +/// What `rustingface gc` found, and did or would do. +#[derive(Debug, Clone, Default)] +pub struct Reclaimed { + pub scanned: usize, + pub referenced: usize, + pub deleted: Vec, + pub bytes: u64, + pub dry_run: bool, +} + +/// One blob that failed `rustingface verify`. +#[derive(Debug, Clone)] +pub struct Corruption { + pub key: String, + pub expected: String, + pub found: String, +} + +/// What `rustingface verify` checked. +#[derive(Debug, Clone, Default)] +pub struct Verified { + pub blobs: usize, + pub bytes: u64, + pub missing: Vec, + pub corrupt: Vec, +} + +impl Registry { + /// Summarise a stored repo. + pub async fn summarise(&self, repo_type: RepoType, repo_id: &RepoId) -> Result { + let manifests = self.list_manifests(repo_type, repo_id).await?; + let refs = self + .list_refs(repo_type, repo_id) + .await? + .into_iter() + .map(|r| (r.ref_name, r.commit)) + .collect(); + + // Count each distinct blob once: the same weights shared by two + // revisions occupy the bucket once, and a summary that double-counts + // them would misreport what `gc` could ever reclaim. + let mut seen = BTreeSet::new(); + let mut bytes = 0; + let mut files = BTreeSet::new(); + for manifest in &manifests { + for entry in &manifest.entries { + files.insert(entry.path.clone()); + if seen.insert(entry.digest.clone()) { + bytes += entry.size; + } + } + } + + Ok(RepoSummary { + repo_type, + repo_id: repo_id.clone(), + refs, + commits: manifests.iter().map(|m| m.commit.clone()).collect(), + files: files.len(), + bytes, + }) + } + + /// Detach a repo, or one revision of it, from the bucket. + /// + /// Blobs are untouched. Reclaiming them is [`Registry::gc`], which the + /// operator runs separately and can dry-run first. + pub async fn remove( + &self, + repo_type: RepoType, + repo_id: &RepoId, + revision: Option<&str>, + ) -> Result { + let mut removed = Removed::default(); + + let Some(revision) = revision else { + for object in self + .store() + .list(&key::manifest_prefix(repo_type, repo_id)) + .await? + { + self.store().delete(&object.key).await?; + removed.manifests.push(object.key); + } + for object in self + .store() + .list(&key::ref_prefix(repo_type, repo_id)) + .await? + { + self.store().delete(&object.key).await?; + removed.refs.push(object.key); + } + let repo_key = key::repo(repo_type, repo_id); + if self.store().head(&repo_key).await?.is_some() { + self.store().delete(&repo_key).await?; + removed.repo_record = true; + } + info!(%repo_id, "removed repository metadata; blobs remain until `gc`"); + return Ok(removed); + }; + + // A single revision: drop its manifest, and any ref still pointing at + // it, so nothing resolves to a commit that is no longer described. + let commit = self + .resolve_stored_revision(repo_type, repo_id, revision) + .await?; + let manifest_key = key::manifest(repo_type, repo_id, &commit); + if self.store().head(&manifest_key).await?.is_some() { + self.store().delete(&manifest_key).await?; + removed.manifests.push(manifest_key); + } + for record in self.list_refs(repo_type, repo_id).await? { + if record.commit == commit { + let key = key::ref_key(repo_type, repo_id, &record.ref_name); + self.store().delete(&key).await?; + removed.refs.push(key); + } + } + info!(%repo_id, commit, "removed revision metadata; blobs remain until `gc`"); + Ok(removed) + } + + /// Resolve a revision using only what is stored, never upstream. + /// + /// Admin actions that remove things must not have the side effect of + /// pulling a new pin from upstream first. + pub async fn resolve_stored_revision( + &self, + repo_type: RepoType, + repo_id: &RepoId, + revision: &str, + ) -> Result { + if rustingface_entities::refs::is_commit_sha(revision) { + return Ok(revision.to_owned()); + } + self.load_ref(repo_type, repo_id, revision) + .await? + .map(|r| r.commit) + .ok_or_else(|| { + Error::RevisionNotFound(format!( + "{repo_id}@{revision} is not pinned in this bucket" + )) + }) + } + + /// Every digest referenced by any manifest in the bucket. + pub async fn referenced_digests(&self) -> Result> { + let mut digests = BTreeSet::new(); + for object in self.store().list("manifests/").await? { + let Some(bytes) = self.store().get(&object.key).await? else { + continue; + }; + match serde_json::from_slice::(&bytes) { + Ok(manifest) => { + for entry in manifest.entries { + if let Some(hex) = digest::unprefixed(&entry.digest) { + digests.insert(hex.to_owned()); + } + } + } + // A manifest we cannot parse is not proof that its blobs are + // unreferenced, and `gc` must never delete on a guess. + Err(err) => { + return Err(Error::Storage(format!( + "{} could not be parsed ({err}); refusing to compute a reachable set \ + from an incomplete scan", + object.key + ))); + } + } + } + Ok(digests) + } + + /// Delete blobs no manifest references. + /// + /// The reachable set is computed from a complete scan of every manifest, + /// and a manifest that fails to parse aborts the whole run rather than + /// silently shrinking that set. + pub async fn gc(&self, dry_run: bool) -> Result { + let referenced = self.referenced_digests().await?; + let blobs = self.store().list(key::BLOB_PREFIX).await?; + + let mut result = Reclaimed { + scanned: blobs.len(), + referenced: referenced.len(), + dry_run, + ..Default::default() + }; + + for blob in blobs { + let Some(hex) = key::blob_digest(&blob.key) else { + warn!(key = %blob.key, "unrecognised key under the blob prefix; leaving it alone"); + continue; + }; + if referenced.contains(hex) { + continue; + } + result.bytes += blob.size; + if !dry_run { + self.store().delete(&blob.key).await?; + } + result.deleted.push(blob.key); + } + + info!( + scanned = result.scanned, + referenced = result.referenced, + reclaimed = result.deleted.len(), + bytes = result.bytes, + dry_run, + "gc complete" + ); + Ok(result) + } + + /// Re-read stored blobs and check them against their content-addressed + /// keys. + /// + /// `repo` narrows the check to one repository; without it every blob in + /// the bucket is read. + pub async fn verify(&self, repo: Option<(RepoType, &RepoId)>) -> Result { + let mut result = Verified::default(); + + let targets: Vec<(String, u64)> = match repo { + None => self + .store() + .list(key::BLOB_PREFIX) + .await? + .into_iter() + .map(|o| (o.key, o.size)) + .collect(), + Some((repo_type, repo_id)) => { + let mut seen = BTreeSet::new(); + let mut targets = Vec::new(); + for manifest in self.list_manifests(repo_type, repo_id).await? { + for entry in manifest.entries { + let Some(hex) = digest::unprefixed(&entry.digest) else { + continue; + }; + if !seen.insert(hex.to_owned()) { + continue; + } + let key = key::blob(hex); + match self.store().head(&key).await? { + Some(size) => targets.push((key, size)), + None => result.missing.push(key), + } + } + } + targets + } + }; + + for (key, size) in targets { + let Some(expected) = key::blob_digest(&key).map(str::to_owned) else { + continue; + }; + let mut read = self.store().read_blob(&key, None).await?; + let mut hasher = Digest::new(); + while let Some(chunk) = read.stream.next().await { + hasher.update(&chunk?); + } + let found = hasher.hex(); + result.blobs += 1; + result.bytes += size; + if found != expected { + result.corrupt.push(Corruption { + key, + expected: digest::prefixed(&expected), + found: digest::prefixed(&found), + }); + } + } + + if result.corrupt.is_empty() && result.missing.is_empty() { + info!( + blobs = result.blobs, + bytes = result.bytes, + "verify complete; no problems found" + ); + } else { + warn!( + corrupt = result.corrupt.len(), + missing = result.missing.len(), + "verify found problems" + ); + } + Ok(result) + } + + /// Re-resolve a ref upstream and repin it. + /// + /// This is the whole point of `freeze` mode: moving to a newer revision is + /// an operator action, so upstream cannot change what a fleet resolves to. + pub async fn refresh( + &self, + repo_type: RepoType, + repo_id: &RepoId, + ref_name: &str, + ) -> Result<(Option, String)> { + if !self.upstream().enabled() { + return Err(Error::Config( + "refresh needs upstream; this instance is sealed (upstream.enabled = false)".into(), + )); + } + let before = self + .load_ref(repo_type, repo_id, ref_name) + .await? + .map(|r| r.commit); + let after = self + .upstream() + .resolve_ref(repo_type, repo_id, ref_name) + .await?; + self.write_ref( + repo_type, + repo_id, + ref_name, + &after, + rustingface_entities::refs::PinnedBy::Operator, + ) + .await?; + self.ensure_repo_record(repo_type, repo_id).await?; + Ok((before, after)) + } + + /// A round-trip probe of the bucket: write, read back, delete. + /// + /// Used by `doctor` to prove credentials and permissions rather than + /// merely that a TCP connection opened. + pub async fn probe_store(&self) -> Result<()> { + let key = format!("doctor/{}", crate::registry::probe_id()); + let payload = Bytes::from_static(b"rustingface doctor probe"); + self.store().put(&key, payload.clone()).await?; + let read = self.store().get(&key).await?; + self.store().delete(&key).await?; + match read { + Some(bytes) if bytes == payload => Ok(()), + Some(_) => Err(Error::Storage( + "the doctor probe read back different bytes than it wrote".into(), + )), + None => Err(Error::Storage( + "the doctor probe wrote an object that could not then be read".into(), + )), + } + } +} diff --git a/crates/rustingface-core/src/digest.rs b/crates/rustingface-core/src/digest.rs new file mode 100644 index 0000000..de3542a --- /dev/null +++ b/crates/rustingface-core/src/digest.rs @@ -0,0 +1,88 @@ +//! Incremental SHA-256 over a streaming body. +//! +//! The digest is computed over the same bytes that go to both sinks, so a +//! mismatch against upstream's LFS oid is detected before the manifest that +//! would reference the blob is written. + +use sha2::{Digest as _, Sha256}; + +/// A running SHA-256 that yields the `sha256:` form used as a blob key. +#[derive(Default)] +pub struct Digest { + hasher: Sha256, + len: u64, +} + +impl Digest { + /// A fresh, empty digest. + pub fn new() -> Self { + Self::default() + } + + /// Absorb the next chunk. + pub fn update(&mut self, chunk: &[u8]) { + self.hasher.update(chunk); + self.len += chunk.len() as u64; + } + + /// Bytes absorbed so far. + pub fn len(&self) -> u64 { + self.len + } + + /// Whether nothing has been absorbed. + pub fn is_empty(&self) -> bool { + self.len == 0 + } + + /// The lowercase hex digest. + pub fn hex(self) -> String { + hex::encode(self.hasher.finalize()) + } +} + +/// Render a hex digest in the `sha256:` form stored in manifests. +pub fn prefixed(hex: &str) -> String { + format!("sha256:{hex}") +} + +/// Strip the `sha256:` prefix from a manifest digest. +pub fn unprefixed(digest: &str) -> Option<&str> { + digest.strip_prefix("sha256:") +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The well-known SHA-256 of the empty input, as a canary that this is + /// really SHA-256 and not something else with the same output width. + const EMPTY: &str = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + + #[test] + fn an_empty_stream_hashes_to_the_known_empty_digest() { + let d = Digest::new(); + assert!(d.is_empty()); + assert_eq!(d.hex(), EMPTY); + } + + #[test] + fn chunking_does_not_change_the_digest_or_the_length() { + let mut whole = Digest::new(); + whole.update(b"hello world"); + + let mut split = Digest::new(); + for chunk in [b"hel".as_slice(), b"lo ", b"world"] { + split.update(chunk); + } + assert_eq!(split.len(), 11); + assert_eq!(whole.hex(), split.hex()); + } + + #[test] + fn the_prefixed_form_round_trips() { + let hex = "a".repeat(64); + assert_eq!(unprefixed(&prefixed(&hex)), Some(hex.as_str())); + assert_eq!(unprefixed("md5:abc"), None); + } +} diff --git a/crates/rustingface-core/src/fetch.rs b/crates/rustingface-core/src/fetch.rs new file mode 100644 index 0000000..8182249 --- /dev/null +++ b/crates/rustingface-core/src/fetch.rs @@ -0,0 +1,347 @@ +//! The fetch path: one upstream read, two sinks. +//! +//! The upstream body is consumed exactly once and written concurrently to the +//! client response and to a multipart upload, with the digest computed over +//! the same bytes. The transfer runs in a detached task, so a client that hits +//! Ctrl-C at 30GB does not discard 30GB: the retention guarantee is better +//! served by finishing (spec §7). + +use std::sync::Arc; + +use bytes::Bytes; +use futures::StreamExt; +use rustingface_entities::error::{Error, Result}; +use rustingface_entities::key; +use rustingface_entities::manifest::Entry; +use rustingface_entities::repo::{RepoId, RepoType}; +use tokio::sync::{mpsc, oneshot}; +use tracing::{debug, info, warn}; + +use crate::digest::{self, Digest}; +use crate::flight::Outcome; +use crate::ports::{Store, UpstreamHead}; +use crate::registry::{FetchTask, LeadResult, Registry, TEE_DEPTH}; + +/// Where a transfer's bytes are being written while the digest is unknown. +enum Sink { + /// Straight to the final content-addressed key, because upstream gave an + /// LFS oid and the key is therefore known up front. + Direct { key: String }, + /// Buffered in memory, for a small file with no oid. + Buffered, + /// To a staging key, to be moved once the digest is known. + Staged { key: String }, +} + +impl Registry { + /// Lead an upstream fetch for one file. + /// + /// `blocking` forces the transfer to run to completion before returning, + /// which is what a ranged request on a cold file needs: a partial blob + /// must never reach the bucket. + pub(crate) async fn lead_fetch( + &self, + repo_type: RepoType, + repo_id: &RepoId, + commit: &str, + path: &str, + blocking: bool, + ) -> Result { + let body = self + .upstream() + .get_file(repo_type, repo_id, commit, path) + .await?; + let head = body.head.clone(); + + // Deduplication: when upstream named the oid and we already hold those + // bytes, the transfer is unnecessary. Dropping the response body here + // abandons the download rather than paying for it twice. + if let Some(oid) = head.expected_sha256() + && self.store().head(&key::blob(&oid)).await?.is_some() + { + drop(body); + let size = self.store().head(&key::blob(&oid)).await?.unwrap_or(0); + let entry = entry_from(path, size, &oid, &head); + info!(%repo_id, commit, path, oid, "blob already stored under this digest; recording without transfer"); + self.record_entry(repo_type, repo_id, commit, entry.clone()) + .await?; + return Ok(LeadResult::Deduplicated(entry)); + } + + let sink = self.choose_sink(&head); + let (client_tx, client_rx) = mpsc::channel::>(TEE_DEPTH); + let (outcome_tx, outcome_rx) = oneshot::channel::(); + + let transfer = Transfer { + store: Arc::clone(self.store()), + registry_repo_type: repo_type, + repo_id: repo_id.clone(), + commit: commit.to_owned(), + path: path.to_owned(), + head: head.clone(), + sink, + part_size: self.config().storage.multipart_part_size, + stall_timeout: self.config().server.client_stall_timeout, + }; + + if blocking { + // Nobody is reading the tee, so drop the client half up front and + // let the transfer run at full speed. + drop(client_rx); + let entry = transfer.run(self, body.stream, client_tx).await?; + return Ok(LeadResult::Completed(entry)); + } + + let registry = self.clone(); + tokio::spawn(async move { + let result = transfer.run(®istry, body.stream, client_tx).await; + let outcome = match &result { + Ok(entry) => Outcome::Stored(entry.clone()), + Err(err) => Outcome::from(err), + }; + let _ = outcome_tx.send(outcome); + }); + + Ok(LeadResult::Streaming { + head, + rx: client_rx, + task: FetchTask::new(outcome_rx), + }) + } + + /// Pick where the bytes go while the digest is still unknown. + fn choose_sink(&self, head: &UpstreamHead) -> Sink { + if let Some(oid) = head.expected_sha256() { + return Sink::Direct { + key: key::blob(&oid), + }; + } + let size = head.linked_size.or(head.content_length); + match size { + Some(size) if size <= crate::registry::INLINE_BUFFER_LIMIT => Sink::Buffered, + _ => Sink::Staged { + key: format!("staging/{}", staging_id()), + }, + } + } +} + +/// One transfer in progress. +struct Transfer { + store: Arc, + registry_repo_type: RepoType, + repo_id: RepoId, + commit: String, + path: String, + head: UpstreamHead, + sink: Sink, + part_size: u64, + stall_timeout: std::time::Duration, +} + +impl Transfer { + /// Consume the upstream body, writing to storage and the client at once. + async fn run( + self, + registry: &Registry, + mut stream: futures::stream::BoxStream<'static, Result>, + client: mpsc::Sender>, + ) -> Result { + let mut hasher = Digest::new(); + let mut client = Some(client); + let mut buffered: Vec = Vec::new(); + + let mut writer = match &self.sink { + Sink::Buffered => None, + Sink::Direct { key } | Sink::Staged { key } => { + Some(self.store.write_blob(key, self.part_size).await?) + } + }; + + let transfer = async { + while let Some(chunk) = stream.next().await { + let chunk = chunk?; + hasher.update(&chunk); + + match &mut writer { + Some(writer) => writer.write(chunk.clone()).await?, + None => buffered.push(chunk.clone()), + } + + // Tee to the client, dropping it — not the transfer — if it has + // gone away or stopped reading. + if let Some(tx) = &client { + match tokio::time::timeout(self.stall_timeout, tx.send(Ok(chunk))).await { + Ok(Ok(())) => {} + Ok(Err(_)) => { + debug!(path = %self.path, "client disconnected; finishing the transfer without it"); + client = None; + } + Err(_) => { + warn!( + path = %self.path, + "client stopped reading for longer than server.client_stall_timeout; \ + detaching it and finishing the transfer" + ); + client = None; + } + } + } + } + Ok::<(), Error>(()) + }; + + if let Err(err) = transfer.await { + if let Some(writer) = writer { + let _ = writer.abort().await; + } + if let Some(tx) = &client { + let _ = tx.send(Err(Error::Upstream(err.to_string()))).await; + } + return Err(err); + } + + let size = hasher.len(); + let hex = hasher.hex(); + + // Verify before anything is completed or referenced: a mismatch must + // abort the upload, not land bytes that a later manifest could point at. + if let Some(expected) = self.head.expected_sha256() + && expected != hex + { + if let Some(writer) = writer { + let _ = writer.abort().await; + } + let err = Error::DigestMismatch { + path: self.path.clone(), + expected: digest::prefixed(&expected), + actual: digest::prefixed(&hex), + }; + if let Some(tx) = &client { + let _ = tx.send(Err(Error::Upstream(err.to_string()))).await; + } + return Err(err); + } + + let final_key = key::blob(&hex); + + // Phase 1: the blob is durable. A crash after this point leaks an + // orphan, which `gc` reclaims; nothing references it yet. + match (writer, &self.sink) { + (Some(writer), Sink::Direct { .. }) => writer.finish().await?, + (Some(writer), Sink::Staged { key: staged }) => { + writer.finish().await?; + if self.store.head(&final_key).await?.is_none() { + self.store.copy(staged, &final_key).await.map_err(|e| { + Error::Storage(format!( + "staged {size}-byte blob for {} could not be moved to its \ + content-addressed key ({e}). Files this large normally arrive with an \ + LFS oid and skip staging entirely; the staged object is left at {staged} \ + for recovery.", + self.path + )) + })?; + } + self.store.delete(staged).await?; + } + (None, Sink::Buffered) => { + if self.store.head(&final_key).await?.is_none() { + let mut whole = Vec::with_capacity(size as usize); + for chunk in &buffered { + whole.extend_from_slice(chunk); + } + self.store.put(&final_key, Bytes::from(whole)).await?; + } + } + (writer, sink) => { + if let Some(writer) = writer { + let _ = writer.abort().await; + } + let _ = sink; + return Err(Error::Internal("transfer sink and writer disagree".into())); + } + } + + // Phase 2: only now may a manifest reference it. + let entry = entry_from(&self.path, size, &hex, &self.head); + registry + .record_entry( + self.registry_repo_type, + &self.repo_id, + &self.commit, + entry.clone(), + ) + .await?; + + info!( + repo_id = %self.repo_id, commit = %self.commit, path = %self.path, size, + detached = client.is_none(), + "stored blob and recorded manifest entry" + ); + Ok(entry) + } +} + +/// Build the manifest entry for a completed transfer, carrying upstream's +/// values through unchanged. +fn entry_from(path: &str, size: u64, hex: &str, head: &UpstreamHead) -> Entry { + Entry { + path: path.to_owned(), + size, + digest: digest::prefixed(hex), + etag: head.etag.clone(), + linked_etag: head.linked_etag.clone(), + linked_size: head.linked_size, + lfs: head.lfs, + } +} + +/// A collision-resistant staging suffix, without pulling in a uuid crate for +/// a name nothing else ever reads. +fn staging_id() -> String { + use std::collections::hash_map::RandomState; + use std::hash::{BuildHasher, Hasher}; + let mut hasher = RandomState::new().build_hasher(); + hasher.write_u128( + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_nanos(), + ); + format!( + "{:016x}{:016x}", + hasher.finish(), + RandomState::new().build_hasher().finish() + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn an_entry_carries_upstream_values_through_unchanged() { + let head = UpstreamHead { + commit: Some("abc".into()), + etag: Some("\"71ab09\"".into()), + linked_etag: Some("\"71ab0900\"".into()), + linked_size: Some(4_831_838_208), + content_length: Some(4_831_838_208), + lfs: true, + }; + let entry = entry_from("model.safetensors", 4_831_838_208, &"7".repeat(64), &head); + assert_eq!(entry.etag.as_deref(), Some("\"71ab09\""), "quotes are kept"); + assert_eq!(entry.linked_etag.as_deref(), Some("\"71ab0900\"")); + assert_eq!(entry.linked_size, Some(4_831_838_208)); + assert!(entry.lfs); + assert_eq!(entry.digest, format!("sha256:{}", "7".repeat(64))); + } + + #[test] + fn staging_ids_do_not_repeat() { + let a = staging_id(); + let b = staging_id(); + assert_ne!(a, b); + assert_eq!(a.len(), 32); + } +} diff --git a/crates/rustingface-core/src/flight.rs b/crates/rustingface-core/src/flight.rs new file mode 100644 index 0000000..1cd8de0 --- /dev/null +++ b/crates/rustingface-core/src/flight.rs @@ -0,0 +1,301 @@ +//! Single-flight: exactly one upstream fetch per uncached blob. +//! +//! Without this, a fleet-wide rollout of a new model produces N concurrent +//! 40GB pulls (spec §7). The map is keyed on +//! `(repo_type, repo_id, commit, path)` — the commit, not the ref, so two +//! clients naming `main` and its resolved SHA share a flight. +//! +//! The leader tees bytes to its own response while uploading; followers wait +//! for the flight to finish and then read the completed blob from the bucket. +//! Streaming live to a follower that arrived mid-body would mean handing it a +//! stream already missing its first chunks, so followers deliberately take the +//! slower, correct path. + +use std::sync::Arc; + +use dashmap::DashMap; +use rustingface_entities::error::{Error, Result}; +use rustingface_entities::manifest::Entry; +use rustingface_entities::repo::{RepoId, RepoType}; +use tokio::sync::watch; + +/// Identifies one uncached blob. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct FlightKey { + pub repo_type: RepoType, + pub repo_id: RepoId, + pub commit: String, + pub path: String, +} + +impl FlightKey { + /// Key a fetch by its resolved commit, so ref and SHA requests coalesce. + pub fn new( + repo_type: RepoType, + repo_id: RepoId, + commit: impl Into, + path: impl Into, + ) -> Self { + Self { + repo_type, + repo_id, + commit: commit.into(), + path: path.into(), + } + } +} + +/// How a completed flight ended. +#[derive(Debug, Clone)] +pub enum Outcome { + /// The blob is durably stored and the manifest references it. + Stored(Entry), + /// The fetch failed. Carries the rendered error, since [`Error`] is not + /// `Clone` and every waiter needs a copy. + Failed { + label: &'static str, + message: String, + }, +} + +impl Outcome { + /// Reconstitute a failure as an error for a waiter to return. + pub fn into_result(self) -> Result { + match self { + Self::Stored(entry) => Ok(entry), + Self::Failed { label, message } => Err(match label { + "repo_not_found" => Error::RepoNotFound(message), + "revision_not_found" => Error::RevisionNotFound(message), + "entry_not_found" => Error::EntryNotFound(message), + "gated" => Error::GatedRepo(message), + "disabled" => Error::DisabledRepo(message), + "upstream_unauthorized" => Error::UpstreamUnauthorized(message), + "storage" => Error::Storage(message), + _ => Error::Upstream(message), + }), + } + } +} + +impl From<&Error> for Outcome { + fn from(err: &Error) -> Self { + Self::Failed { + label: err.label(), + message: err.to_string(), + } + } +} + +/// A flight in progress. Waiters hold one of these and await the outcome. +#[derive(Debug)] +pub struct Flight { + rx: watch::Receiver>, +} + +impl Flight { + /// Block until the leader's fetch completes, then yield its outcome. + /// + /// A leader that panics drops its sender; that closes the channel and + /// every waiter gets a definite error rather than hanging forever. + pub async fn wait(mut self) -> Result { + loop { + if let Some(outcome) = self.rx.borrow_and_update().clone() { + return outcome.into_result(); + } + if self.rx.changed().await.is_err() { + return Err(Error::Internal( + "the upstream fetch this request was waiting on ended without reporting an outcome".into(), + )); + } + } + } +} + +/// The leader's half: publishing an outcome and clearing the map entry. +/// +/// Dropping this without calling [`Lead::finish`] still removes the map entry +/// and closes the channel, so a panicking leader cannot wedge later requests +/// on a key that will never complete. +pub struct Lead { + map: Arc>>>, + key: FlightKey, + tx: watch::Sender>, +} + +impl Lead { + /// Publish the outcome to every waiter and retire the key. + pub fn finish(self, outcome: Outcome) { + self.map.remove(&self.key); + let _ = self.tx.send(Some(outcome)); + } +} + +impl Drop for Lead { + fn drop(&mut self) { + self.map.remove(&self.key); + } +} + +/// Either this caller leads the fetch, or it waits on one already running. +pub enum Role { + Leader(Lead), + Follower(Flight), +} + +/// The in-flight registry. +#[derive(Debug, Default)] +pub struct Flights { + map: Arc>>>, +} + +impl Flights { + /// An empty registry. + pub fn new() -> Self { + Self::default() + } + + /// Claim leadership of `key`, or join the flight already leading it. + /// + /// The claim is made under DashMap's per-shard lock, so exactly one caller + /// can become leader for a key no matter how many arrive at once. + pub fn join(&self, key: FlightKey) -> Role { + use dashmap::mapref::entry::Entry as MapEntry; + match self.map.entry(key.clone()) { + MapEntry::Occupied(occupied) => Role::Follower(Flight { + rx: occupied.get().clone(), + }), + MapEntry::Vacant(vacant) => { + let (tx, rx) = watch::channel(None); + vacant.insert(rx); + Role::Leader(Lead { + map: Arc::clone(&self.map), + key, + tx, + }) + } + } + } + + /// How many fetches are in flight, for the metrics gauge. + pub fn len(&self) -> usize { + self.map.len() + } + + /// Whether nothing is in flight. + pub fn is_empty(&self) -> bool { + self.map.is_empty() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn key(path: &str) -> FlightKey { + FlightKey::new( + RepoType::Models, + "Qwen/Qwen3-32B".parse().unwrap(), + "8f2ac31", + path, + ) + } + + fn entry(path: &str) -> Entry { + Entry { + path: path.into(), + size: 4, + digest: format!("sha256:{}", "a".repeat(64)), + etag: Some("\"abc\"".into()), + linked_etag: None, + linked_size: None, + lfs: false, + } + } + + #[tokio::test] + async fn eight_concurrent_callers_produce_exactly_one_leader() { + let flights = Arc::new(Flights::new()); + let mut leaders = 0; + let mut followers = Vec::new(); + for _ in 0..8 { + match flights.join(key("config.json")) { + Role::Leader(lead) => { + leaders += 1; + // Hand the outcome over only after every caller has joined. + followers.push(tokio::spawn(async move { + tokio::task::yield_now().await; + lead.finish(Outcome::Stored(entry("config.json"))); + Ok(entry("config.json")) + })); + } + Role::Follower(flight) => { + followers.push(tokio::spawn(flight.wait())); + } + } + } + assert_eq!(leaders, 1, "exactly one upstream fetch"); + for handle in followers { + let stored = handle.await.unwrap().unwrap(); + assert_eq!(stored.path, "config.json"); + } + assert!( + flights.is_empty(), + "the key is retired once the flight ends" + ); + } + + #[tokio::test] + async fn different_paths_do_not_share_a_flight() { + let flights = Flights::new(); + let a = flights.join(key("a.json")); + let b = flights.join(key("b.json")); + assert!(matches!(a, Role::Leader(_))); + assert!(matches!(b, Role::Leader(_))); + assert_eq!(flights.len(), 2, "one flight per path, not one per repo"); + } + + #[tokio::test] + async fn a_failure_reaches_every_follower_as_the_same_error() { + let flights = Flights::new(); + let Role::Leader(lead) = flights.join(key("x")) else { + panic!("first caller must lead"); + }; + let Role::Follower(flight) = flights.join(key("x")) else { + panic!("second caller must follow"); + }; + lead.finish(Outcome::from(&Error::GatedRepo( + "terms not accepted".into(), + ))); + let err = flight.wait().await.unwrap_err(); + assert!(matches!(err, Error::GatedRepo(_)), "{err}"); + assert_eq!(err.status(), 403); + } + + #[tokio::test] + async fn a_leader_that_vanishes_does_not_hang_its_followers() { + let flights = Flights::new(); + let Role::Leader(lead) = flights.join(key("x")) else { + panic!("first caller must lead"); + }; + let Role::Follower(flight) = flights.join(key("x")) else { + panic!("second caller must follow"); + }; + drop(lead); + let err = flight.wait().await.unwrap_err(); + assert!(matches!(err, Error::Internal(_)), "{err}"); + assert!(flights.is_empty(), "a dropped lead still retires the key"); + } + + #[tokio::test] + async fn a_key_can_be_led_again_after_its_flight_ends() { + let flights = Flights::new(); + let Role::Leader(lead) = flights.join(key("x")) else { + panic!("first caller must lead"); + }; + lead.finish(Outcome::Stored(entry("x"))); + assert!( + matches!(flights.join(key("x")), Role::Leader(_)), + "a later miss on the same key starts a new flight" + ); + } +} diff --git a/crates/rustingface-core/src/lib.rs b/crates/rustingface-core/src/lib.rs new file mode 100644 index 0000000..340a649 --- /dev/null +++ b/crates/rustingface-core/src/lib.rs @@ -0,0 +1,18 @@ +//! Resolution, retention and delivery for rustingface. +//! +//! This crate holds the logic: how a revision becomes a commit, how a miss +//! becomes exactly one upstream fetch, how bytes reach the bucket before +//! anything references them, and what an operator may remove. It defines the +//! two ports it needs — [`ports::Store`] and [`ports::Upstream`] — and leaves +//! their implementation to `rustingface-data`. + +pub mod admin; +pub mod digest; +mod fetch; +pub mod flight; +pub mod policy; +pub mod ports; +pub mod registry; + +pub use policy::Policy; +pub use registry::{Body, Registry, ResolvedFile, Source}; diff --git a/crates/rustingface-core/src/policy.rs b/crates/rustingface-core/src/policy.rs new file mode 100644 index 0000000..f8f6311 --- /dev/null +++ b/crates/rustingface-core/src/policy.rs @@ -0,0 +1,116 @@ +//! Retention policy: what rustingface is willing to fetch that it does not +//! already hold. +//! +//! Policy never restricts what is *served* from the bucket. Anything already +//! stored stays servable regardless of a later allowlist change, because +//! guarantee 3 does not have a policy exception. + +use globset::{Glob, GlobSet, GlobSetBuilder}; +use rustingface_entities::config::Policy as PolicyConfig; +use rustingface_entities::error::{Error, Result}; +use rustingface_entities::repo::RepoId; + +/// Compiled form of `[policy]`. +#[derive(Debug)] +pub struct Policy { + allowlist: Option, + allow_new_repos: bool, +} + +impl Policy { + /// Compile the globs once, at startup, so a bad pattern fails loudly + /// before the listener binds rather than on the first miss. + pub fn new(cfg: &PolicyConfig) -> Result { + let allowlist = if cfg.allowlist.is_empty() { + None + } else { + let mut builder = GlobSetBuilder::new(); + for pattern in &cfg.allowlist { + let glob = Glob::new(pattern).map_err(|e| { + Error::Config(format!("policy.allowlist pattern {pattern:?}: {e}")) + })?; + builder.add(glob); + } + Some( + builder + .build() + .map_err(|e| Error::Config(format!("policy.allowlist: {e}")))?, + ) + }; + Ok(Self { + allowlist, + allow_new_repos: cfg.allow_new_repos, + }) + } + + /// Whether a repo may be fetched from upstream. + /// + /// `known` says whether the bucket already holds a repo record, which is + /// what `allow_new_repos = false` gates on. + pub fn admits_fetch(&self, repo_id: &RepoId, known: bool) -> Result<()> { + if let Some(set) = &self.allowlist + && !set.is_match(repo_id.to_string()) + { + return Err(Error::PolicyDenied(format!( + "{repo_id} is not in policy.allowlist" + ))); + } + if !self.allow_new_repos && !known { + return Err(Error::PolicyDenied(format!( + "{repo_id} is not already stored and policy.allow_new_repos is false" + ))); + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn policy(allowlist: &[&str], allow_new_repos: bool) -> Policy { + Policy::new(&PolicyConfig { + ref_resolution: rustingface_entities::config::RefResolution::Freeze, + allow_new_repos, + allowlist: allowlist.iter().map(|s| s.to_string()).collect(), + }) + .unwrap() + } + + fn id(s: &str) -> RepoId { + s.parse().unwrap() + } + + #[test] + fn an_empty_allowlist_admits_everything() { + let p = policy(&[], true); + assert!(p.admits_fetch(&id("Qwen/Qwen3-32B"), false).is_ok()); + assert!(p.admits_fetch(&id("gpt2"), false).is_ok()); + } + + #[test] + fn a_namespace_glob_admits_only_that_namespace() { + let p = policy(&["Qwen/*", "meta-llama/*"], true); + assert!(p.admits_fetch(&id("Qwen/Qwen3-32B"), false).is_ok()); + assert!(p.admits_fetch(&id("meta-llama/Llama-3-8B"), false).is_ok()); + assert!(p.admits_fetch(&id("someone/else"), false).is_err()); + } + + #[test] + fn allow_new_repos_false_still_admits_a_repo_already_stored() { + let p = policy(&[], false); + assert!(p.admits_fetch(&id("Qwen/Qwen3-32B"), true).is_ok()); + assert!(p.admits_fetch(&id("Qwen/Qwen3-32B"), false).is_err()); + } + + #[test] + fn a_malformed_glob_fails_at_construction_not_at_request_time() { + let err = Policy::new(&PolicyConfig { + ref_resolution: rustingface_entities::config::RefResolution::Freeze, + allow_new_repos: true, + allowlist: vec!["Qwen/[".into()], + }) + .unwrap_err(); + assert!(matches!(err, Error::Config(_)), "{err}"); + } +} diff --git a/crates/rustingface-core/src/ports.rs b/crates/rustingface-core/src/ports.rs new file mode 100644 index 0000000..e32c5a0 --- /dev/null +++ b/crates/rustingface-core/src/ports.rs @@ -0,0 +1,234 @@ +//! Ports: the two things rustingface needs from the outside world. +//! +//! [`Store`] is the bucket — the entire durable state of the system. +//! [`Upstream`] is the Hub, consulted on miss only and not at all when sealed. +//! Both are implemented in `rustingface-data`; nothing in this crate performs +//! I/O of its own. + +use std::ops::Range; +use std::time::Duration; + +use async_trait::async_trait; +use bytes::Bytes; +use futures::stream::BoxStream; +use rustingface_entities::error::Result; +use serde_json::Value; + +use rustingface_entities::repo::{RepoId, RepoType}; + +/// Metadata about one stored object. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct StoredObject { + pub key: String, + pub size: u64, +} + +/// A readable blob: the byte stream plus what the store said about it. +pub struct BlobRead { + /// Bytes of the requested range, in order. + pub stream: BoxStream<'static, Result>, + /// Length of this read, which is the range length when one was requested. + pub len: u64, + /// Full length of the object, regardless of the range read. + pub total: u64, +} + +/// A streaming multipart upload. +/// +/// Dropping without [`BlobWriter::finish`] leaves an incomplete multipart +/// upload; call [`BlobWriter::abort`] on the failure path so the backend can +/// reclaim the parts. +#[async_trait] +pub trait BlobWriter: Send { + /// Buffer bytes, flushing whole parts as they accumulate. + async fn write(&mut self, chunk: Bytes) -> Result<()>; + /// Flush the tail and complete the upload. + async fn finish(self: Box) -> Result<()>; + /// Discard the upload and any parts already sent. + async fn abort(self: Box) -> Result<()>; +} + +/// The bucket. +#[async_trait] +pub trait Store: Send + Sync + 'static { + /// Fetch a small object whole. `Ok(None)` when it does not exist. + async fn get(&self, key: &str) -> Result>; + + /// Overwrite an object. + async fn put(&self, key: &str, body: Bytes) -> Result<()>; + + /// Write only if the key is absent. Returns `false` if it already existed. + /// + /// Backed by a conditional put where the backend supports it, which is + /// what makes ref freezing atomically first-write-wins. + async fn put_if_absent(&self, key: &str, body: Bytes) -> Result; + + /// Whether the backend's [`Store::put_if_absent`] is genuinely atomic. A + /// `false` here is a correctness caveat worth logging, not a failure. + fn conditional_put_is_atomic(&self) -> bool { + true + } + + /// Delete an object. Deleting a key that is already absent succeeds. + async fn delete(&self, key: &str) -> Result<()>; + + /// Server-side copy, used to move a staged blob to its content-addressed + /// key once the digest is known. + async fn copy(&self, from: &str, to: &str) -> Result<()>; + + /// Size of an object, or `None` when absent. + async fn head(&self, key: &str) -> Result>; + + /// Every object under `prefix`. + async fn list(&self, prefix: &str) -> Result>; + + /// Open a blob for reading, optionally a byte range. + async fn read_blob(&self, key: &str, range: Option>) -> Result; + + /// Begin a multipart upload, flushing whole parts of `part_size` bytes. + async fn write_blob(&self, key: &str, part_size: u64) -> Result>; + + /// A presigned GET URL, when the backend can mint one. + async fn presign_get(&self, key: &str, ttl: Duration) -> Result; +} + +/// What upstream said about a file, captured verbatim for replay. +/// +/// Every string field here is stored and re-emitted unchanged. rustingface +/// never computes a git blob SHA-1 and never derives an etag from content. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct UpstreamHead { + /// `X-Repo-Commit`. + pub commit: Option, + /// `ETag`, quotes included. + pub etag: Option, + /// `X-Linked-Etag`, quotes included. + pub linked_etag: Option, + /// `X-Linked-Size`. + pub linked_size: Option, + /// `Content-Length` of the resolved resource. + pub content_length: Option, + /// Whether upstream served this through the Git LFS bridge. + pub lfs: bool, +} + +impl UpstreamHead { + /// The etag a client would key its cache on: `X-Linked-Etag` when present, + /// otherwise `ETag` (spec §6). + pub fn effective_etag(&self) -> Option<&str> { + self.linked_etag.as_deref().or(self.etag.as_deref()) + } + + /// The sha256 upstream promised for this file, when it gave one. + /// + /// For LFS-backed files the linked etag *is* the LFS oid, so a completed + /// transfer can be checked against it before anything references the blob. + pub fn expected_sha256(&self) -> Option { + let raw = self.linked_etag.as_deref()?.trim_matches('"'); + (raw.len() == 64 && raw.bytes().all(|b| b.is_ascii_hexdigit())) + .then(|| raw.to_ascii_lowercase()) + } +} + +/// An in-progress upstream download. +pub struct UpstreamBody { + pub head: UpstreamHead, + pub stream: BoxStream<'static, Result>, +} + +/// The Hub. +#[async_trait] +pub trait Upstream: Send + Sync + 'static { + /// `false` when the service is sealed. Nothing else on this trait is + /// called when it is. + fn enabled(&self) -> bool; + + /// The configured endpoint, recorded into manifests for provenance. + fn endpoint(&self) -> &str; + + /// Resolve a ref name to a commit SHA. + async fn resolve_ref( + &self, + repo_type: RepoType, + repo_id: &RepoId, + revision: &str, + ) -> Result; + + /// `HEAD` on the resolve path, for metadata without transferring bytes. + async fn head_file( + &self, + repo_type: RepoType, + repo_id: &RepoId, + revision: &str, + path: &str, + ) -> Result; + + /// `GET` on the resolve path, returning headers and the body stream. + async fn get_file( + &self, + repo_type: RepoType, + repo_id: &RepoId, + revision: &str, + path: &str, + ) -> Result; + + /// The `/api/{type}/{id}/revision/{rev}` document, verbatim. + async fn repo_info( + &self, + repo_type: RepoType, + repo_id: &RepoId, + revision: &str, + ) -> Result; + + /// The recursive, expanded `/api/{type}/{id}/tree/{rev}` document, verbatim. + async fn repo_tree( + &self, + repo_type: RepoType, + repo_id: &RepoId, + revision: &str, + ) -> Result; +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn linked_etag_wins_over_etag() { + let head = UpstreamHead { + etag: Some("\"aaa\"".into()), + linked_etag: Some("\"bbb\"".into()), + ..Default::default() + }; + assert_eq!(head.effective_etag(), Some("\"bbb\"")); + } + + #[test] + fn a_plain_etag_is_used_when_upstream_omits_the_linked_one() { + let head = UpstreamHead { + etag: Some("\"aaa\"".into()), + ..Default::default() + }; + assert_eq!(head.effective_etag(), Some("\"aaa\"")); + } + + #[test] + fn an_lfs_oid_is_recovered_from_the_linked_etag() { + let oid = "71ab09".repeat(10) + "abcd"; + let head = UpstreamHead { + linked_etag: Some(format!("\"{oid}\"")), + ..Default::default() + }; + assert_eq!(head.expected_sha256().as_deref(), Some(oid.as_str())); + } + + #[test] + fn a_git_blob_etag_is_not_mistaken_for_a_sha256() { + // Git blob SHA-1s are 40 hex characters; only a 64-hex value is an oid. + let head = UpstreamHead { + linked_etag: Some("\"8f2a1c9e0b7d4f6a3e5c2b8d9f0a1c3e5b7d9f01\"".into()), + ..Default::default() + }; + assert_eq!(head.expected_sha256(), None); + } +} diff --git a/crates/rustingface-core/src/registry.rs b/crates/rustingface-core/src/registry.rs new file mode 100644 index 0000000..5b93048 --- /dev/null +++ b/crates/rustingface-core/src/registry.rs @@ -0,0 +1,1078 @@ +//! The registry: resolution, retention and delivery. +//! +//! Everything here obeys one ordering rule (spec §4): the blob upload +//! completes before any manifest references it. A crash between the two leaks +//! an orphan blob, which `gc` reclaims. The inverse — a manifest entry +//! pointing at bytes that were never fully written — is a silent correctness +//! failure, and is unreachable by construction. + +use std::ops::Range; +use std::sync::Arc; + +use bytes::Bytes; +use futures::stream::BoxStream; +use rustingface_entities::config::{BlobDelivery, Config, RefResolution}; +use rustingface_entities::error::{Error, Result}; +use rustingface_entities::key; +use rustingface_entities::manifest::{Entry, Manifest}; +use rustingface_entities::refs::{PinnedBy, RefRecord, is_commit_sha}; +use rustingface_entities::repo::{RepoId, RepoRecord, RepoType}; +use serde_json::Value; +use tokio::sync::mpsc; +use tracing::{debug, info, warn}; + +use crate::digest; +use crate::flight::{FlightKey, Flights, Outcome, Role}; +use crate::policy::Policy; +use crate::ports::{BlobRead, Store, Upstream, UpstreamHead}; + +/// Chunks buffered between the fetch task and a proxying client. +/// +/// Bounded, so a slow client applies backpressure to the transfer rather than +/// growing the heap. A client that stops reading altogether is detached after +/// `server.client_stall_timeout` and the transfer finishes without it. +pub(crate) const TEE_DEPTH: usize = 32; + +/// Files below this size are buffered whole when upstream gave no LFS oid, so +/// their content-addressed key is known before a single byte is uploaded. +pub(crate) const INLINE_BUFFER_LIMIT: u64 = 32 * 1024 * 1024; + +/// How the body of a resolved file reaches the client. +pub enum Body { + /// `HEAD`: metadata only. + None, + /// A byte stream, either read back from the bucket or teed from upstream. + Stream { + stream: BoxStream<'static, Result>, + /// Length of this stream, which is the range length when ranged. + len: u64, + /// The satisfied range, when the request carried one. + range: Option>, + }, + /// `302` to a presigned URL on object storage. + Redirect(String), +} + +/// A resolved file: everything the HTTP layer needs to answer. +/// +/// The etag fields are upstream's, verbatim. They are replayed rather than +/// recomputed because client cache layouts are keyed on them (spec §2). +pub struct ResolvedFile { + pub commit: String, + pub etag: Option, + pub linked_etag: Option, + pub linked_size: Option, + /// True byte length of the whole file. + pub size: u64, + pub body: Body, +} + +/// Where a served body came from, for metrics. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Source { + Bucket, + Upstream, +} + +/// The registry, wired to a bucket and (optionally) a Hub. +/// +/// A cheap handle: cloning shares one set of connections, one flight registry +/// and one compiled policy, so a detached transfer can hold its own copy. +#[derive(Clone)] +pub struct Registry { + inner: Arc, +} + +struct Inner { + store: Arc, + upstream: Arc, + policy: Policy, + config: Config, + flights: Flights, +} + +impl Registry { + /// Assemble a registry. Fails if `[policy]` does not compile. + pub fn new(store: Arc, upstream: Arc, config: Config) -> Result { + let policy = Policy::new(&config.policy)?; + if config.policy.ref_resolution == RefResolution::Follow { + warn!( + "policy.ref_resolution = \"follow\": mutable refs will be re-resolved upstream, \ + so a revision your fleet pinned today can move tomorrow. This breaks the \ + reference-stability guarantee and is intended for development only." + ); + } + if !store.conditional_put_is_atomic() { + warn!( + "the object store does not support conditional put; ref freezing falls back to \ + read-then-write and a concurrent first resolution could race. Both writers \ + record the same commit in the common case; correct any divergence with \ + `rustingface pin`." + ); + } + Ok(Self { + inner: Arc::new(Inner { + store, + upstream, + policy, + config, + flights: Flights::new(), + }), + }) + } + + /// The effective configuration. + pub fn config(&self) -> &Config { + &self.inner.config + } + + /// The bucket, for admin operations that work on raw keys. + pub fn store(&self) -> &Arc { + &self.inner.store + } + + /// The Hub adapter. + pub fn upstream(&self) -> &Arc { + &self.inner.upstream + } + + /// Fetches currently in flight, for the metrics gauge. + pub fn in_flight(&self) -> usize { + self.inner.flights.len() + } + + // ---------------------------------------------------------------- refs + + /// Resolve a revision to a commit SHA. + /// + /// A 40-hex revision is already immutable and is used as-is; it is never + /// written to `refs/`. A ref name resolves through the stored pin, and + /// only on a miss (or in `follow` mode) is upstream consulted. + pub async fn resolve_revision( + &self, + repo_type: RepoType, + repo_id: &RepoId, + revision: &str, + ) -> Result { + if is_commit_sha(revision) { + return Ok(revision.to_owned()); + } + + let stored = self.load_ref(repo_type, repo_id, revision).await?; + + if self.inner.config.policy.ref_resolution == RefResolution::Follow + && self.inner.upstream.enabled() + { + match self + .inner + .upstream + .resolve_ref(repo_type, repo_id, revision) + .await + { + Ok(commit) => { + if stored.as_ref().is_none_or(|r| r.commit != commit) { + self.write_ref(repo_type, repo_id, revision, &commit, PinnedBy::AutoFreeze) + .await?; + } + return Ok(commit); + } + Err(err) if stored.is_some() => { + warn!(%repo_id, revision, %err, "upstream unreachable in follow mode; using the stored pin"); + } + Err(err) => return Err(err), + } + } + + if let Some(record) = stored { + return Ok(record.commit); + } + + if !self.inner.upstream.enabled() { + return Err(Error::RevisionNotFound(format!( + "{repo_id}@{revision} has never been resolved and upstream is disabled" + ))); + } + + let known = self.repo_is_known(repo_type, repo_id).await?; + self.inner.policy.admits_fetch(repo_id, known)?; + + let commit = self + .inner + .upstream + .resolve_ref(repo_type, repo_id, revision) + .await?; + self.freeze_ref(repo_type, repo_id, revision, &commit) + .await?; + self.ensure_repo_record(repo_type, repo_id).await?; + Ok(commit) + } + + /// The stored pin for a ref, if there is one. + pub async fn load_ref( + &self, + repo_type: RepoType, + repo_id: &RepoId, + ref_name: &str, + ) -> Result> { + let key = key::ref_key(repo_type, repo_id, ref_name); + let Some(bytes) = self.inner.store.get(&key).await? else { + return Ok(None); + }; + serde_json::from_slice(&bytes) + .map(Some) + .map_err(|e| Error::Storage(format!("{key} is not a valid ref record: {e}"))) + } + + /// Every stored ref for a repo, sorted by name. + pub async fn list_refs(&self, repo_type: RepoType, repo_id: &RepoId) -> Result> { + let prefix = key::ref_prefix(repo_type, repo_id); + let mut records = Vec::new(); + for object in self.inner.store.list(&prefix).await? { + if let Some(bytes) = self.inner.store.get(&object.key).await? + && let Ok(record) = serde_json::from_slice::(&bytes) + { + records.push(record); + } + } + records.sort_by(|a, b| a.ref_name.cmp(&b.ref_name)); + Ok(records) + } + + /// Record the first resolution of a ref, first-write-wins. + /// + /// A conditional put makes the freeze atomic where the backend supports + /// it. On a lost race the winner's value is returned, so two clients + /// racing the same cold ref still agree on one commit. + async fn freeze_ref( + &self, + repo_type: RepoType, + repo_id: &RepoId, + ref_name: &str, + commit: &str, + ) -> Result { + let record = RefRecord::new(ref_name, commit, PinnedBy::AutoFreeze); + let key = key::ref_key(repo_type, repo_id, ref_name); + let body = Bytes::from(encode_json(&record)?); + if self.inner.store.put_if_absent(&key, body).await? { + info!(%repo_id, ref_name, commit, "froze ref"); + return Ok(commit.to_owned()); + } + match self.load_ref(repo_type, repo_id, ref_name).await? { + Some(existing) => { + if existing.commit != commit { + warn!( + %repo_id, ref_name, ours = commit, theirs = %existing.commit, + "lost the freeze race to a concurrent resolution; using the stored pin" + ); + } + Ok(existing.commit) + } + None => Ok(commit.to_owned()), + } + } + + /// Overwrite a ref. Only ever reached from an explicit operator action. + pub async fn write_ref( + &self, + repo_type: RepoType, + repo_id: &RepoId, + ref_name: &str, + commit: &str, + pinned_by: PinnedBy, + ) -> Result<()> { + let record = RefRecord::new(ref_name, commit, pinned_by); + let key = key::ref_key(repo_type, repo_id, ref_name); + self.inner + .store + .put(&key, Bytes::from(encode_json(&record)?)) + .await?; + info!(%repo_id, ref_name, commit, pinned_by = pinned_by.as_str(), "pinned ref"); + Ok(()) + } + + // ------------------------------------------------------------ manifests + + /// The manifest for one commit, if it has been recorded. + pub async fn load_manifest( + &self, + repo_type: RepoType, + repo_id: &RepoId, + commit: &str, + ) -> Result> { + let key = key::manifest(repo_type, repo_id, commit); + let Some(bytes) = self.inner.store.get(&key).await? else { + return Ok(None); + }; + serde_json::from_slice(&bytes) + .map(Some) + .map_err(|e| Error::Storage(format!("{key} is not a valid manifest: {e}"))) + } + + /// Every recorded commit for a repo. + pub async fn list_manifests( + &self, + repo_type: RepoType, + repo_id: &RepoId, + ) -> Result> { + let prefix = key::manifest_prefix(repo_type, repo_id); + let mut manifests = Vec::new(); + for object in self.inner.store.list(&prefix).await? { + if let Some(bytes) = self.inner.store.get(&object.key).await? + && let Ok(manifest) = serde_json::from_slice::(&bytes) + { + manifests.push(manifest); + } + } + manifests.sort_by(|a, b| a.commit.cmp(&b.commit)); + Ok(manifests) + } + + /// Apply `mutate` to the manifest for a commit and persist the result. + /// + /// Concurrent writers are resolved by read-merge-write with a bounded + /// retry: after writing, the manifest is re-read, and if a concurrent + /// writer's copy is missing anything we contributed, we merge and write + /// again. Entries are deterministic for a given commit, so the merged set + /// is the same whichever writer lands last. + async fn update_manifest( + &self, + repo_type: RepoType, + repo_id: &RepoId, + commit: &str, + mut mutate: F, + ) -> Result + where + F: FnMut(&mut Manifest), + { + let manifest_key = key::manifest(repo_type, repo_id, commit); + let mut attempt = 0; + loop { + let mut manifest = self + .load_manifest(repo_type, repo_id, commit) + .await? + .unwrap_or_else(|| { + Manifest::new( + repo_type, + repo_id.clone(), + commit.to_owned(), + Some(self.inner.upstream.endpoint().to_owned()), + ) + }); + mutate(&mut manifest); + self.inner + .store + .put(&manifest_key, Bytes::from(encode_json(&manifest)?)) + .await?; + + // Confirm our contribution survived a concurrent write. + let landed = self.load_manifest(repo_type, repo_id, commit).await?; + let lost = match &landed { + Some(current) => manifest + .entries + .iter() + .any(|e| current.entry(&e.path).is_none()), + None => true, + }; + if !lost { + return Ok(landed.unwrap_or(manifest)); + } + attempt += 1; + if attempt > self.inner.config.storage.manifest_write_retries { + return Err(Error::Storage(format!( + "{manifest_key} kept losing entries to concurrent writers after {attempt} attempts" + ))); + } + debug!(key = %manifest_key, attempt, "manifest write raced; merging and retrying"); + } + } + + /// Record one file into a commit's manifest. + pub async fn record_entry( + &self, + repo_type: RepoType, + repo_id: &RepoId, + commit: &str, + entry: Entry, + ) -> Result<()> { + self.update_manifest(repo_type, repo_id, commit, |m| m.upsert(entry.clone())) + .await?; + Ok(()) + } + + // ------------------------------------------------------------ repo info + + /// Whether the bucket already holds a record for this repo. + pub async fn repo_is_known(&self, repo_type: RepoType, repo_id: &RepoId) -> Result { + Ok(self + .inner + .store + .head(&key::repo(repo_type, repo_id)) + .await? + .is_some()) + } + + /// Write a repo record if there isn't one, so `list` is a prefix scan. + pub async fn ensure_repo_record(&self, repo_type: RepoType, repo_id: &RepoId) -> Result<()> { + let key = key::repo(repo_type, repo_id); + if self.inner.store.head(&key).await?.is_some() { + return Ok(()); + } + let record = RepoRecord::new( + repo_type, + repo_id.clone(), + Some(self.inner.upstream.endpoint().to_owned()), + ); + self.inner + .store + .put_if_absent(&key, Bytes::from(encode_json(&record)?)) + .await?; + Ok(()) + } + + /// Every repo of a type the bucket knows about. + pub async fn list_repos(&self, repo_type: RepoType) -> Result> { + let mut repos: Vec = self + .inner + .store + .list(&key::repo_type_prefix(repo_type)) + .await? + .iter() + .filter_map(|o| key::repo_from_key(&o.key).map(|(_, id)| id)) + .collect(); + repos.sort(); + Ok(repos) + } + + /// The model- or dataset-info document for a revision. + /// + /// Recorded whole and filtered on read, so a field a client asks for later + /// was never dropped at record time (spec §13). In sealed mode `siblings` + /// is narrowed to what is actually stored: advertising a file that cannot + /// be served would be a lie the client acts on. + pub async fn repo_info( + &self, + repo_type: RepoType, + repo_id: &RepoId, + revision: &str, + ) -> Result { + let commit = self.resolve_revision(repo_type, repo_id, revision).await?; + let manifest = self.ensure_repo_meta(repo_type, repo_id, &commit).await?; + let Some(meta) = manifest.repo_meta.clone() else { + return Err(Error::RevisionNotFound(format!( + "no repository metadata recorded for {repo_id}@{commit}" + ))); + }; + Ok(if self.inner.config.sealed() { + narrow_siblings(meta, &manifest) + } else { + meta + }) + } + + /// The tree listing for a revision. + pub async fn repo_tree( + &self, + repo_type: RepoType, + repo_id: &RepoId, + revision: &str, + path_prefix: &str, + recursive: bool, + ) -> Result { + let commit = self.resolve_revision(repo_type, repo_id, revision).await?; + let manifest = self.ensure_repo_tree(repo_type, repo_id, &commit).await?; + let entries = tree_entries(&manifest, self.inner.config.sealed()); + Ok(Value::Array(filter_tree(entries, path_prefix, recursive))) + } + + /// Load the manifest for a commit, fetching and recording its metadata if + /// this is the first time anyone has asked. + async fn ensure_repo_meta( + &self, + repo_type: RepoType, + repo_id: &RepoId, + commit: &str, + ) -> Result { + let existing = self.load_manifest(repo_type, repo_id, commit).await?; + if let Some(manifest) = &existing + && manifest.repo_meta.is_some() + { + return Ok(existing.unwrap()); + } + if !self.inner.upstream.enabled() { + return existing.ok_or_else(|| { + Error::RevisionNotFound(format!( + "{repo_id}@{commit} is not stored and upstream is disabled" + )) + }); + } + let known = self.repo_is_known(repo_type, repo_id).await?; + self.inner.policy.admits_fetch(repo_id, known)?; + let meta = self + .inner + .upstream + .repo_info(repo_type, repo_id, commit) + .await?; + self.ensure_repo_record(repo_type, repo_id).await?; + self.update_manifest(repo_type, repo_id, commit, |m| { + m.repo_meta = Some(meta.clone()); + }) + .await + } + + /// As [`Registry::ensure_repo_meta`], for the tree document. + async fn ensure_repo_tree( + &self, + repo_type: RepoType, + repo_id: &RepoId, + commit: &str, + ) -> Result { + let manifest = self.ensure_repo_meta(repo_type, repo_id, commit).await?; + if manifest.repo_tree.is_some() || !self.inner.upstream.enabled() { + return Ok(manifest); + } + let tree = self + .inner + .upstream + .repo_tree(repo_type, repo_id, commit) + .await?; + self.update_manifest(repo_type, repo_id, commit, |m| { + m.repo_tree = Some(tree.clone()); + }) + .await + } + + // ----------------------------------------------------------------- files + + /// Answer a `HEAD` on the resolve path. + /// + /// A miss consults upstream for metadata but transfers no bytes and + /// records no manifest entry: there is no digest to record, and an entry + /// without a blob would make a later sealed-mode `HEAD` promise a file the + /// `GET` could not deliver. + pub async fn head_file( + &self, + repo_type: RepoType, + repo_id: &RepoId, + revision: &str, + path: &str, + ) -> Result { + let commit = self.resolve_revision(repo_type, repo_id, revision).await?; + + if let Some(entry) = self.stored_entry(repo_type, repo_id, &commit, path).await? { + let key = blob_key(&entry)?; + if let Some(size) = self.inner.store.head(&key).await? { + // In redirect mode the metadata headers ride on the 302: the + // client reads them off this response and then downloads from + // `Location`, so they cannot move to the presigned request. + let body = if self.inner.config.storage.blob_delivery == BlobDelivery::Redirect { + Body::Redirect( + self.inner + .store + .presign_get(&key, self.inner.config.storage.presign_ttl) + .await?, + ) + } else { + Body::None + }; + return Ok(ResolvedFile { + commit, + etag: entry.etag, + linked_etag: entry.linked_etag, + linked_size: entry.linked_size, + size, + body, + }); + } + warn!( + %repo_id, commit, path, digest = %entry.digest, + "manifest references a blob that is not in the bucket; treating as a miss" + ); + } + + if !self.inner.upstream.enabled() { + return Err(Error::EntryNotFound(format!( + "{repo_id}@{commit}/{path} is not stored and upstream is disabled" + ))); + } + let known = self.repo_is_known(repo_type, repo_id).await?; + self.inner.policy.admits_fetch(repo_id, known)?; + let head = self + .inner + .upstream + .head_file(repo_type, repo_id, &commit, path) + .await?; + Ok(ResolvedFile { + commit, + size: head.linked_size.or(head.content_length).unwrap_or(0), + etag: head.etag, + linked_etag: head.linked_etag, + linked_size: head.linked_size, + body: Body::None, + }) + } + + /// Answer a `GET` on the resolve path. + /// + /// On a miss with a `Range` header the whole file is fetched and stored + /// first, then the range is served from the bucket: storing a fragment + /// would break the retention guarantee for every later request. + pub async fn get_file( + &self, + repo_type: RepoType, + repo_id: &RepoId, + revision: &str, + path: &str, + range: Option>, + ) -> Result<(ResolvedFile, Source)> { + let commit = self.resolve_revision(repo_type, repo_id, revision).await?; + + if let Some(entry) = self.stored_entry(repo_type, repo_id, &commit, path).await? + && let Some(file) = self.serve_stored(&entry, &commit, range.clone()).await? + { + return Ok((file, Source::Bucket)); + } + + if !self.inner.upstream.enabled() { + return Err(Error::EntryNotFound(format!( + "{repo_id}@{commit}/{path} is not stored and upstream is disabled" + ))); + } + let known = self.repo_is_known(repo_type, repo_id).await?; + self.inner.policy.admits_fetch(repo_id, known)?; + + let key = FlightKey::new(repo_type, repo_id.clone(), &commit, path); + match self.inner.flights.join(key) { + Role::Follower(flight) => { + debug!(%repo_id, commit, path, "joining an in-flight upstream fetch"); + let entry = flight.wait().await?; + let file = self + .serve_stored(&entry, &commit, range) + .await? + .ok_or_else(|| { + Error::Storage(format!( + "the fetch of {repo_id}@{commit}/{path} reported success but its blob is absent" + )) + })?; + Ok((file, Source::Bucket)) + } + Role::Leader(lead) => { + let outcome = self + .lead_fetch(repo_type, repo_id, &commit, path, range.is_some()) + .await; + match outcome { + Err(err) => { + lead.finish(Outcome::from(&err)); + Err(err) + } + Ok(LeadResult::Deduplicated(entry)) => { + lead.finish(Outcome::Stored(entry.clone())); + let file = self + .serve_stored(&entry, &commit, range) + .await? + .ok_or_else(|| { + Error::Storage(format!( + "the blob for {repo_id}@{commit}/{path} vanished between check and read" + )) + })?; + Ok((file, Source::Bucket)) + } + Ok(LeadResult::Streaming { head, rx, task }) => { + // The upload task owns the flight from here: it + // publishes the outcome whether or not this client + // stays connected. + task.attach(lead); + let size = head.linked_size.or(head.content_length).unwrap_or(0); + Ok(( + ResolvedFile { + commit, + size, + etag: head.etag, + linked_etag: head.linked_etag, + linked_size: head.linked_size, + body: Body::Stream { + stream: Box::pin(tokio_stream_of(rx)), + len: size, + range: None, + }, + }, + Source::Upstream, + )) + } + Ok(LeadResult::Completed(entry)) => { + lead.finish(Outcome::Stored(entry.clone())); + let file = self + .serve_stored(&entry, &commit, range) + .await? + .ok_or_else(|| { + Error::Storage(format!( + "the blob for {repo_id}@{commit}/{path} vanished after being stored" + )) + })?; + Ok((file, Source::Upstream)) + } + } + } + } + } + + /// The recorded entry for a path, if the manifest has one. + async fn stored_entry( + &self, + repo_type: RepoType, + repo_id: &RepoId, + commit: &str, + path: &str, + ) -> Result> { + Ok(self + .load_manifest(repo_type, repo_id, commit) + .await? + .and_then(|m| m.entry(path).cloned())) + } + + /// Build a response from a stored blob, or `None` if the blob is missing. + async fn serve_stored( + &self, + entry: &Entry, + commit: &str, + range: Option>, + ) -> Result> { + let key = blob_key(entry)?; + let Some(total) = self.inner.store.head(&key).await? else { + warn!(path = %entry.path, digest = %entry.digest, "manifest entry has no blob; treating as a miss"); + return Ok(None); + }; + + let range = match range { + None => None, + Some(r) => Some(clamp_range(r, total).ok_or_else(|| { + Error::RangeNotSatisfiable(format!( + "{} is {total} bytes; the requested range starts past the end", + entry.path + )) + })?), + }; + + let body = if self.inner.config.storage.blob_delivery == BlobDelivery::Redirect + && range.is_none() + { + Body::Redirect( + self.inner + .store + .presign_get(&key, self.inner.config.storage.presign_ttl) + .await?, + ) + } else { + let BlobRead { stream, len, .. } = + self.inner.store.read_blob(&key, range.clone()).await?; + Body::Stream { + stream, + len, + range: range.clone(), + } + }; + + Ok(Some(ResolvedFile { + commit: commit.to_owned(), + etag: entry.etag.clone(), + linked_etag: entry.linked_etag.clone(), + linked_size: entry.linked_size, + size: total, + body, + })) + } +} + +/// What leading a fetch produced. +pub(crate) enum LeadResult { + /// The bytes were already in the bucket under this digest; only the + /// manifest entry was missing. + Deduplicated(Entry), + /// The transfer is running and the client is being teed into. + Streaming { + head: UpstreamHead, + rx: mpsc::Receiver>, + task: FetchTask, + }, + /// The transfer ran to completion before the response was built, because + /// the client asked for a range and a partial blob may not be stored. + Completed(Entry), +} + +/// Handle to a detached transfer, which owns the flight once attached. +pub(crate) struct FetchTask { + outcome: tokio::sync::oneshot::Receiver, +} + +impl FetchTask { + /// Wrap the channel the transfer reports its outcome on. + pub(crate) fn new(outcome: tokio::sync::oneshot::Receiver) -> Self { + Self { outcome } + } + + /// Hand the flight to the transfer, so waiters are released when the + /// upload finishes rather than when the leader's client disconnects. + fn attach(self, lead: crate::flight::Lead) { + tokio::spawn(async move { + match self.outcome.await { + Ok(outcome) => lead.finish(outcome), + Err(_) => lead.finish(Outcome::Failed { + label: "internal", + message: "the transfer task ended without reporting an outcome".into(), + }), + } + }); + } +} + +/// Wrap an mpsc receiver as a stream of results. +fn tokio_stream_of( + mut rx: mpsc::Receiver>, +) -> impl futures::Stream> { + futures::stream::poll_fn(move |cx| rx.poll_recv(cx)) +} + +/// The blob key for an entry, rejecting a digest we did not write. +fn blob_key(entry: &Entry) -> Result { + let hex = digest::unprefixed(&entry.digest).ok_or_else(|| { + Error::Storage(format!( + "manifest entry {} carries digest {:?}, which is not a sha256 blob key", + entry.path, entry.digest + )) + })?; + Ok(key::blob(hex)) +} + +/// Clamp a requested range to the object, or `None` if it starts past the end. +fn clamp_range(range: Range, total: u64) -> Option> { + if range.start >= total || range.start >= range.end { + return None; + } + Some(range.start..range.end.min(total)) +} + +/// A short random suffix for a `doctor` probe key. +pub(crate) fn probe_id() -> String { + use std::collections::hash_map::RandomState; + use std::hash::{BuildHasher, Hasher}; + format!("{:016x}", RandomState::new().build_hasher().finish()) +} + +/// Serialise a document as the pretty JSON stored in the bucket. +/// +/// Pretty-printed on purpose: the bucket is meant to be inspectable with +/// nothing but `mc cat`. +fn encode_json(value: &T) -> Result> { + serde_json::to_vec_pretty(value).map_err(|e| Error::Internal(format!("serialising: {e}"))) +} + +/// Narrow `siblings` in a repo-info document to what is actually stored. +fn narrow_siblings(mut meta: Value, manifest: &Manifest) -> Value { + if let Some(siblings) = meta.get_mut("siblings") + && let Some(array) = siblings.as_array_mut() + { + array.retain(|s| { + s.get("rfilename") + .and_then(Value::as_str) + .is_some_and(|p| manifest.entry(p).is_some()) + }); + } + meta +} + +/// The tree listing for a manifest. +/// +/// While upstream is reachable the recorded tree is replayed whole, matching +/// what a client would have seen from the Hub. Sealed, it is narrowed to +/// stored entries, and where no tree was ever recorded it is synthesised from +/// them. +fn tree_entries(manifest: &Manifest, sealed: bool) -> Vec { + let synthesised = || { + manifest + .entries + .iter() + .map(|e| { + serde_json::json!({ + "type": "file", + "path": e.path, + "size": e.size, + "oid": digest::unprefixed(&e.digest).unwrap_or(&e.digest), + "lfs": if e.lfs { + serde_json::json!({ "oid": digest::unprefixed(&e.digest), "size": e.size }) + } else { + Value::Null + }, + }) + }) + .collect::>() + }; + + let Some(recorded) = manifest.repo_tree.as_ref().and_then(Value::as_array) else { + return synthesised(); + }; + if !sealed { + return recorded.clone(); + } + recorded + .iter() + .filter(|item| { + item.get("path") + .and_then(Value::as_str) + .is_some_and(|p| manifest.entry(p).is_some()) + }) + .cloned() + .collect() +} + +/// Apply the `path` prefix and `recursive` query to a tree listing. +fn filter_tree(entries: Vec, prefix: &str, recursive: bool) -> Vec { + let prefix = prefix.trim_matches('/'); + entries + .into_iter() + .filter(|item| { + let Some(path) = item.get("path").and_then(Value::as_str) else { + return false; + }; + let rest = if prefix.is_empty() { + Some(path) + } else { + path.strip_prefix(prefix).and_then(|r| r.strip_prefix('/')) + }; + match rest { + None => false, + Some(rest) => recursive || !rest.contains('/'), + } + }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use rustingface_entities::manifest::Entry; + + fn entry(path: &str) -> Entry { + Entry { + path: path.into(), + size: 10, + digest: format!("sha256:{}", "a".repeat(64)), + etag: None, + linked_etag: None, + linked_size: None, + lfs: false, + } + } + + fn manifest_with(paths: &[&str]) -> Manifest { + let mut m = Manifest::new( + RepoType::Models, + "Qwen/Qwen3-32B".parse().unwrap(), + "abc".into(), + None, + ); + for p in paths { + m.upsert(entry(p)); + } + m + } + + #[test] + fn a_range_is_clamped_to_the_object_length() { + assert_eq!(clamp_range(0..100, 50), Some(0..50)); + assert_eq!(clamp_range(10..20, 50), Some(10..20)); + assert_eq!( + clamp_range(50..60, 50), + None, + "a range starting at EOF is unsatisfiable" + ); + assert_eq!(clamp_range(60..70, 50), None); + assert_eq!( + clamp_range(10..10, 50), + None, + "an empty range is unsatisfiable" + ); + } + + #[test] + fn sealed_mode_narrows_siblings_to_what_is_stored() { + let manifest = manifest_with(&["config.json"]); + let meta = serde_json::json!({ + "id": "Qwen/Qwen3-32B", + "siblings": [{"rfilename": "config.json"}, {"rfilename": "model.safetensors"}], + }); + let narrowed = narrow_siblings(meta, &manifest); + let siblings = narrowed["siblings"].as_array().unwrap(); + assert_eq!( + siblings.len(), + 1, + "an unfetchable file is not advertised when sealed" + ); + assert_eq!(siblings[0]["rfilename"], "config.json"); + } + + #[test] + fn an_unrecorded_tree_is_synthesised_from_entries() { + let manifest = manifest_with(&["config.json", "a/b.json"]); + let tree = tree_entries(&manifest, false); + assert_eq!(tree.len(), 2); + assert_eq!(tree[0]["path"], "a/b.json"); + assert_eq!(tree[0]["size"], 10); + assert_eq!(tree[0]["type"], "file"); + } + + #[test] + fn a_recorded_tree_is_replayed_whole_and_narrowed_only_when_sealed() { + let mut manifest = manifest_with(&["config.json"]); + manifest.repo_tree = Some(serde_json::json!([ + {"type": "file", "path": "config.json", "size": 10}, + {"type": "file", "path": "model.safetensors", "size": 4_000_000}, + ])); + assert_eq!( + tree_entries(&manifest, false).len(), + 2, + "open: replayed verbatim" + ); + let sealed = tree_entries(&manifest, true); + assert_eq!(sealed.len(), 1, "sealed: only what can actually be served"); + assert_eq!(sealed[0]["path"], "config.json"); + } + + #[test] + fn a_non_recursive_tree_lists_one_level_only() { + let entries = vec![ + serde_json::json!({"path": "config.json"}), + serde_json::json!({"path": "sub/a.json"}), + serde_json::json!({"path": "sub/deep/b.json"}), + ]; + let top = filter_tree(entries.clone(), "", false); + assert_eq!(top.len(), 1); + assert_eq!(top[0]["path"], "config.json"); + + let all = filter_tree(entries.clone(), "", true); + assert_eq!(all.len(), 3); + + let sub = filter_tree(entries.clone(), "sub", false); + assert_eq!(sub.len(), 1); + assert_eq!(sub[0]["path"], "sub/a.json"); + + let sub_deep = filter_tree(entries, "sub", true); + assert_eq!(sub_deep.len(), 2); + } + + #[test] + fn a_prefix_matches_whole_segments_only() { + let entries = vec![ + serde_json::json!({"path": "subdir/a.json"}), + serde_json::json!({"path": "sub/a.json"}), + ]; + let sub = filter_tree(entries, "sub", true); + assert_eq!(sub.len(), 1, "`sub` must not match `subdir`"); + assert_eq!(sub[0]["path"], "sub/a.json"); + } + + #[test] + fn a_non_sha256_digest_is_refused_rather_than_turned_into_a_key() { + let mut e = entry("config.json"); + e.digest = "md5:deadbeef".into(); + assert!(blob_key(&e).is_err()); + } +} diff --git a/crates/rustingface-data/Cargo.toml b/crates/rustingface-data/Cargo.toml new file mode 100644 index 0000000..4bc8021 --- /dev/null +++ b/crates/rustingface-data/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "rustingface-data" +description = "Object-storage and upstream-Hub adapters implementing the rustingface-core ports." +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +authors.workspace = true +repository.workspace = true + +[dependencies] +rustingface-core.workspace = true +rustingface-entities.workspace = true + +async-trait.workspace = true +bytes.workspace = true +futures.workspace = true +object_store.workspace = true +reqwest.workspace = true +serde_json.workspace = true +tokio.workspace = true +tracing.workspace = true +http.workspace = true + +[dev-dependencies] +tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } diff --git a/crates/rustingface-data/src/lib.rs b/crates/rustingface-data/src/lib.rs new file mode 100644 index 0000000..e23bbed --- /dev/null +++ b/crates/rustingface-data/src/lib.rs @@ -0,0 +1,11 @@ +//! Adapters for the ports `rustingface-core` defines. +//! +//! Two implementations live here and nowhere else: object storage, and the +//! upstream Hub. Everything that talks to a network or a filesystem on +//! rustingface's behalf is in this crate. + +pub mod store; +pub mod upstream; + +pub use store::ObjectStoreAdapter; +pub use upstream::{HubClient, SealedUpstream}; diff --git a/crates/rustingface-data/src/store.rs b/crates/rustingface-data/src/store.rs new file mode 100644 index 0000000..40c5763 --- /dev/null +++ b/crates/rustingface-data/src/store.rs @@ -0,0 +1,300 @@ +//! Object-storage adapter. +//! +//! The bucket is the entire durable state of the system (guarantee 5), so this +//! adapter deliberately stays thin: keys in, bytes out, no schema knowledge and +//! no caching layer that could disagree with what is actually stored. + +use std::ops::Range; +use std::path::Path as FsPath; +use std::sync::Arc; +use std::time::Duration; + +use async_trait::async_trait; +use bytes::Bytes; +use futures::StreamExt; +use object_store::aws::{AmazonS3, AmazonS3Builder}; +use object_store::local::LocalFileSystem; +use object_store::path::Path; +use object_store::signer::Signer; +use object_store::{ + GetOptions, GetRange, ObjectStore, ObjectStoreExt, PutMode, PutOptions, PutPayload, + WriteMultipart, +}; +use rustingface_core::ports::{BlobRead, BlobWriter, Store, StoredObject}; +use rustingface_entities::config::Storage as StorageConfig; +use rustingface_entities::error::{Error, Result}; +use tracing::debug; + +/// A [`Store`] backed by `object_store`. +pub struct ObjectStoreAdapter { + inner: Arc, + signer: Option>, +} + +impl ObjectStoreAdapter { + /// Build from `[storage]`, resolving credentials from their files. + /// + /// `local_path` selects a filesystem store, which is what the test suite + /// and a single-host trial use. It forfeits guarantee 5 — a local + /// directory is not a bucket you can re-point a new host at — so the + /// caller warns when it is in play. + pub fn new( + cfg: &StorageConfig, + access_key_id: Option, + secret_access_key: Option, + ) -> Result { + if let Some(path) = &cfg.local_path { + std::fs::create_dir_all(path).map_err(|e| { + Error::Config(format!("storage.local_path {}: {e}", path.display())) + })?; + let store = LocalFileSystem::new_with_prefix(path).map_err(|e| { + Error::Config(format!("storage.local_path {}: {e}", path.display())) + })?; + return Ok(Self { + inner: Arc::new(store), + signer: None, + }); + } + + let mut builder = AmazonS3Builder::new() + .with_bucket_name(&cfg.bucket) + .with_region(&cfg.region) + .with_virtual_hosted_style_request(!cfg.path_style); + + if let Some(endpoint) = &cfg.endpoint { + // MinIO on the mesh is reached over plain HTTP; the mesh itself is + // the confidentiality boundary. Allowing it must be explicit + // rather than a silent downgrade, so it keys off the scheme the + // operator actually configured. + if endpoint.starts_with("http://") { + builder = builder.with_allow_http(true); + } + builder = builder.with_endpoint(endpoint); + } + if let Some(key) = access_key_id { + builder = builder.with_access_key_id(key); + } + if let Some(secret) = secret_access_key { + builder = builder.with_secret_access_key(secret); + } + + let s3 = builder + .build() + .map_err(|e| Error::Config(format!("object storage: {e}")))?; + let s3 = Arc::new(s3); + Ok(Self { + inner: Arc::clone(&s3) as Arc, + signer: Some(s3), + }) + } + + /// A filesystem-backed adapter, for tests. + pub fn local(path: &FsPath) -> Result { + std::fs::create_dir_all(path) + .map_err(|e| Error::Config(format!("{}: {e}", path.display())))?; + let store = LocalFileSystem::new_with_prefix(path) + .map_err(|e| Error::Config(format!("{}: {e}", path.display())))?; + Ok(Self { + inner: Arc::new(store), + signer: None, + }) + } +} + +/// Map an `object_store` error, preserving "absent" as a distinct outcome. +fn absent(err: &object_store::Error) -> bool { + matches!(err, object_store::Error::NotFound { .. }) +} + +fn storage(context: &str, err: object_store::Error) -> Error { + Error::Storage(format!("{context}: {err}")) +} + +#[async_trait] +impl Store for ObjectStoreAdapter { + async fn get(&self, key: &str) -> Result> { + match self.inner.get(&Path::from(key)).await { + Ok(result) => result + .bytes() + .await + .map(Some) + .map_err(|e| storage(&format!("reading {key}"), e)), + Err(err) if absent(&err) => Ok(None), + Err(err) => Err(storage(&format!("reading {key}"), err)), + } + } + + async fn put(&self, key: &str, body: Bytes) -> Result<()> { + self.inner + .put(&Path::from(key), PutPayload::from_bytes(body)) + .await + .map(|_| ()) + .map_err(|e| storage(&format!("writing {key}"), e)) + } + + async fn put_if_absent(&self, key: &str, body: Bytes) -> Result { + let opts = PutOptions { + mode: PutMode::Create, + ..Default::default() + }; + match self + .inner + .put_opts(&Path::from(key), PutPayload::from_bytes(body), opts) + .await + { + Ok(_) => Ok(true), + Err(object_store::Error::AlreadyExists { .. }) => Ok(false), + Err(object_store::Error::Precondition { .. }) => Ok(false), + Err(err) => Err(storage(&format!("conditionally writing {key}"), err)), + } + } + + fn conditional_put_is_atomic(&self) -> bool { + // S3 with `If-None-Match: *`, which MinIO supports, and the local + // filesystem store, which uses an atomic link. Both are genuinely + // first-write-wins. + true + } + + async fn delete(&self, key: &str) -> Result<()> { + match self.inner.delete(&Path::from(key)).await { + Ok(()) => Ok(()), + Err(err) if absent(&err) => Ok(()), + Err(err) => Err(storage(&format!("deleting {key}"), err)), + } + } + + async fn copy(&self, from: &str, to: &str) -> Result<()> { + self.inner + .copy(&Path::from(from), &Path::from(to)) + .await + .map_err(|e| storage(&format!("copying {from} to {to}"), e)) + } + + async fn head(&self, key: &str) -> Result> { + match self.inner.head(&Path::from(key)).await { + Ok(meta) => Ok(Some(meta.size)), + Err(err) if absent(&err) => Ok(None), + Err(err) => Err(storage(&format!("stat of {key}"), err)), + } + } + + async fn list(&self, prefix: &str) -> Result> { + let mut stream = self.inner.list(Some(&Path::from(prefix))); + let mut objects = Vec::new(); + while let Some(meta) = stream.next().await { + let meta = meta.map_err(|e| storage(&format!("listing {prefix}"), e))?; + objects.push(StoredObject { + key: meta.location.as_ref().to_owned(), + size: meta.size, + }); + } + objects.sort_by(|a, b| a.key.cmp(&b.key)); + Ok(objects) + } + + async fn read_blob(&self, key: &str, range: Option>) -> Result { + let opts = GetOptions { + range: range.clone().map(GetRange::Bounded), + ..Default::default() + }; + let result = self + .inner + .get_opts(&Path::from(key), opts) + .await + .map_err(|e| storage(&format!("reading blob {key}"), e))?; + + let total = result.meta.size; + let len = result.range.end - result.range.start; + let key_for_errors = key.to_owned(); + let stream = result.into_stream().map(move |chunk| { + chunk.map_err(|e| storage(&format!("streaming blob {key_for_errors}"), e)) + }); + + Ok(BlobRead { + stream: Box::pin(stream), + len, + total, + }) + } + + async fn write_blob(&self, key: &str, part_size: u64) -> Result> { + let upload = self + .inner + .put_multipart(&Path::from(key)) + .await + .map_err(|e| storage(&format!("starting multipart upload of {key}"), e))?; + Ok(Box::new(MultipartBlobWriter { + key: key.to_owned(), + inner: Some(WriteMultipart::new_with_chunk_size( + upload, + part_size as usize, + )), + })) + } + + async fn presign_get(&self, key: &str, ttl: Duration) -> Result { + let signer = self.signer.as_ref().ok_or_else(|| { + Error::Config( + "blob_delivery = \"redirect\" needs an S3 backend; the configured store cannot presign" + .into(), + ) + })?; + signer + .signed_url(http::Method::GET, &Path::from(key), ttl) + .await + .map(|url| url.to_string()) + .map_err(|e| storage(&format!("presigning {key}"), e)) + } +} + +/// A streaming multipart upload. +struct MultipartBlobWriter { + key: String, + inner: Option, +} + +impl MultipartBlobWriter { + fn take(&mut self, what: &str) -> Result { + self.inner.take().ok_or_else(|| { + Error::Internal(format!("{what} called twice on the upload of {}", self.key)) + }) + } +} + +#[async_trait] +impl BlobWriter for MultipartBlobWriter { + async fn write(&mut self, chunk: Bytes) -> Result<()> { + let writer = self + .inner + .as_mut() + .ok_or_else(|| Error::Internal(format!("write after finish on {}", self.key)))?; + // Bound the number of parts in flight so a fast upstream cannot grow + // the heap without limit when the backend is the slower side. + writer + .wait_for_capacity(8) + .await + .map_err(|e| storage(&format!("uploading {}", self.key), e))?; + writer.put(chunk); + Ok(()) + } + + async fn finish(mut self: Box) -> Result<()> { + let writer = self.take("finish")?; + writer + .finish() + .await + .map(|_| ()) + .map_err(|e| storage(&format!("completing upload of {}", self.key), e)) + } + + async fn abort(mut self: Box) -> Result<()> { + let key = self.key.clone(); + let writer = self.take("abort")?; + debug!(%key, "aborting multipart upload"); + writer + .abort() + .await + .map_err(|e| storage(&format!("aborting upload of {key}"), e)) + } +} diff --git a/crates/rustingface-data/src/upstream.rs b/crates/rustingface-data/src/upstream.rs new file mode 100644 index 0000000..5f3e443 --- /dev/null +++ b/crates/rustingface-data/src/upstream.rs @@ -0,0 +1,431 @@ +//! The upstream Hub adapter. +//! +//! Consulted on miss only. Everything it returns that a client can observe is +//! carried through verbatim: this adapter parses headers into fields, it never +//! normalises or recomputes them. + +use async_trait::async_trait; +use futures::StreamExt; +use reqwest::header::{ACCEPT_ENCODING, AUTHORIZATION, ETAG, HeaderMap, HeaderValue, USER_AGENT}; +use reqwest::{Client, Response, StatusCode}; +use rustingface_core::ports::{Upstream, UpstreamBody, UpstreamHead}; +use rustingface_entities::error::{Error, Result}; +use rustingface_entities::repo::{RepoId, RepoType}; +use serde_json::Value; +use tracing::debug; + +/// `X-Repo-Commit`: the commit a revision resolved to. +const X_REPO_COMMIT: &str = "x-repo-commit"; +/// `X-Linked-Etag`: the LFS object's etag, which clients prefer over `ETag`. +const X_LINKED_ETAG: &str = "x-linked-etag"; +/// `X-Linked-Size`: the LFS object's true size. +const X_LINKED_SIZE: &str = "x-linked-size"; +/// `X-Error-Code`: upstream's own typed error label. +const X_ERROR_CODE: &str = "x-error-code"; + +/// A [`Upstream`] backed by `reqwest`. +pub struct HubClient { + client: Client, + endpoint: String, + enabled: bool, + token: Option, +} + +impl HubClient { + /// Build from `[upstream]`. + /// + /// The client always advertises `Accept-Encoding: identity` so upstream + /// reports real byte lengths rather than compressed ones, matching what + /// `huggingface_hub` itself asks for and keeping size accounting honest. + pub fn new( + cfg: &rustingface_entities::config::Upstream, + token: Option, + ) -> Result { + let mut headers = HeaderMap::new(); + headers.insert(ACCEPT_ENCODING, HeaderValue::from_static("identity")); + headers.insert( + USER_AGENT, + HeaderValue::from_str(&format!("rustingface/{}", env!("CARGO_PKG_VERSION"))) + .map_err(|e| Error::Config(format!("user agent: {e}")))?, + ); + if cfg.disable_xet { + // Never advertise Xet capability upstream, so the Hub answers over + // the Git LFS bridge with a whole-file URL. Storing whole blobs is + // what keeps the bucket readable without this binary (spec §8). + headers.insert("x-xet-capable", HeaderValue::from_static("false")); + } + + let client = Client::builder() + .default_headers(headers) + .connect_timeout(cfg.connect_timeout) + .read_timeout(cfg.read_timeout) + .build() + .map_err(|e| Error::Config(format!("upstream client: {e}")))?; + + Ok(Self { + client, + endpoint: cfg.endpoint.trim_end_matches('/').to_owned(), + enabled: cfg.enabled, + token, + }) + } + + /// The resolve URL for a file. + fn resolve_url( + &self, + repo_type: RepoType, + repo_id: &RepoId, + revision: &str, + path: &str, + ) -> String { + let prefix = match repo_type { + RepoType::Models => String::new(), + RepoType::Datasets => "datasets/".into(), + }; + format!( + "{}/{prefix}{repo_id}/resolve/{}/{}", + self.endpoint, + urlencode_segment(revision), + path.split('/') + .map(urlencode_segment) + .collect::>() + .join("/"), + ) + } + + /// The API URL for a repo revision. + fn api_url(&self, repo_type: RepoType, repo_id: &RepoId, suffix: &str) -> String { + format!( + "{}/api/{}/{repo_id}{suffix}", + self.endpoint, + repo_type.as_str() + ) + } + + /// Attach the operator's upstream token. + /// + /// Used on outbound requests only. It is never echoed downstream and never + /// logged: a gated repo works because *this operator's* token accepted the + /// terms, and the stored copy is for that operator's own use. + fn authorize(&self, request: reqwest::RequestBuilder) -> reqwest::RequestBuilder { + match &self.token { + Some(token) => request.header(AUTHORIZATION, format!("Bearer {token}")), + None => request, + } + } + + /// Turn a non-success upstream status into the error it maps to. + async fn check(&self, response: Response, what: &str) -> Result { + let status = response.status(); + if status.is_success() { + return Ok(response); + } + let code = response + .headers() + .get(X_ERROR_CODE) + .and_then(|v| v.to_str().ok()) + .unwrap_or_default() + .to_owned(); + let detail = format!("{what}: upstream returned {status}"); + Err(match (status, code.as_str()) { + (StatusCode::UNAUTHORIZED, _) => Error::UpstreamUnauthorized(detail), + (StatusCode::FORBIDDEN, "DisabledRepo") => Error::DisabledRepo(detail), + (StatusCode::FORBIDDEN, _) => Error::GatedRepo(detail), + (StatusCode::NOT_FOUND, "RevisionNotFound") => Error::RevisionNotFound(detail), + (StatusCode::NOT_FOUND, "EntryNotFound") => Error::EntryNotFound(detail), + (StatusCode::NOT_FOUND, _) => Error::RepoNotFound(detail), + _ => Error::Upstream(detail), + }) + } + + /// Fetch and parse a JSON document from the Hub API. + async fn get_json(&self, url: &str, what: &str) -> Result { + let response = self + .authorize(self.client.get(url)) + .send() + .await + .map_err(|e| Error::Upstream(format!("{what}: {e}")))?; + let response = self.check(response, what).await?; + response + .json() + .await + .map_err(|e| Error::Upstream(format!("{what}: response was not JSON: {e}"))) + } +} + +/// Read the replayable metadata out of an upstream response. +fn head_of(response: &Response) -> UpstreamHead { + let headers = response.headers(); + let string = |name: &str| { + headers + .get(name) + .and_then(|v| v.to_str().ok()) + .map(str::to_owned) + }; + let linked_etag = string(X_LINKED_ETAG); + UpstreamHead { + commit: string(X_REPO_COMMIT), + etag: headers + .get(ETAG) + .and_then(|v| v.to_str().ok()) + .map(str::to_owned), + lfs: linked_etag.is_some(), + linked_etag, + linked_size: string(X_LINKED_SIZE).and_then(|v| v.parse().ok()), + content_length: response.content_length(), + } +} + +/// Percent-encode one URL path segment. +fn urlencode_segment(segment: &str) -> String { + segment + .bytes() + .map(|b| match b { + b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => { + (b as char).to_string() + } + other => format!("%{other:02X}"), + }) + .collect() +} + +#[async_trait] +impl Upstream for HubClient { + fn enabled(&self) -> bool { + self.enabled + } + + fn endpoint(&self) -> &str { + &self.endpoint + } + + async fn resolve_ref( + &self, + repo_type: RepoType, + repo_id: &RepoId, + revision: &str, + ) -> Result { + // The revision endpoint answers with the commit in `sha`, and does so + // without transferring any file bytes. + let url = self.api_url( + repo_type, + repo_id, + &format!("/revision/{}", urlencode_segment(revision)), + ); + let what = format!("resolving {repo_id}@{revision}"); + let doc = self.get_json(&url, &what).await?; + doc.get("sha") + .and_then(Value::as_str) + .map(str::to_owned) + .ok_or_else(|| { + Error::RevisionNotFound(format!( + "{repo_id}@{revision}: upstream's revision document carried no `sha`" + )) + }) + } + + async fn head_file( + &self, + repo_type: RepoType, + repo_id: &RepoId, + revision: &str, + path: &str, + ) -> Result { + let url = self.resolve_url(repo_type, repo_id, revision, path); + let what = format!("HEAD {repo_id}@{revision}/{path}"); + let response = self + .authorize(self.client.head(&url)) + .send() + .await + .map_err(|e| Error::Upstream(format!("{what}: {e}")))?; + let response = self.check(response, &what).await?; + Ok(head_of(&response)) + } + + async fn get_file( + &self, + repo_type: RepoType, + repo_id: &RepoId, + revision: &str, + path: &str, + ) -> Result { + let url = self.resolve_url(repo_type, repo_id, revision, path); + let what = format!("GET {repo_id}@{revision}/{path}"); + let response = self + .authorize(self.client.get(&url)) + .send() + .await + .map_err(|e| Error::Upstream(format!("{what}: {e}")))?; + let response = self.check(response, &what).await?; + let head = head_of(&response); + debug!(%url, size = ?head.content_length, lfs = head.lfs, "upstream transfer starting"); + + let stream = response + .bytes_stream() + .map(move |chunk| chunk.map_err(|e| Error::Upstream(format!("{what}: {e}")))); + + Ok(UpstreamBody { + head, + stream: Box::pin(stream), + }) + } + + async fn repo_info( + &self, + repo_type: RepoType, + repo_id: &RepoId, + revision: &str, + ) -> Result { + let url = self.api_url( + repo_type, + repo_id, + &format!("/revision/{}", urlencode_segment(revision)), + ); + self.get_json(&url, &format!("info for {repo_id}@{revision}")) + .await + } + + async fn repo_tree( + &self, + repo_type: RepoType, + repo_id: &RepoId, + revision: &str, + ) -> Result { + let url = format!( + "{}?recursive=1&expand=1", + self.api_url( + repo_type, + repo_id, + &format!("/tree/{}", urlencode_segment(revision)) + ) + ); + self.get_json(&url, &format!("tree for {repo_id}@{revision}")) + .await + } +} + +/// An [`Upstream`] that refuses every call. +/// +/// Installed when `upstream.enabled = false`. Sealed mode is not "upstream +/// that happens to be unreachable": there is no client, no token loaded and no +/// code path that could reach the network. +pub struct SealedUpstream { + endpoint: String, +} + +impl SealedUpstream { + /// A sealed adapter that still records where the data originally came + /// from, for manifest provenance. + pub fn new(endpoint: impl Into) -> Self { + Self { + endpoint: endpoint.into(), + } + } + + fn sealed(&self, what: &str) -> Result { + Err(Error::RepoNotFound(format!( + "{what} is not stored, and this instance is sealed (upstream.enabled = false)" + ))) + } +} + +#[async_trait] +impl Upstream for SealedUpstream { + fn enabled(&self) -> bool { + false + } + + fn endpoint(&self) -> &str { + &self.endpoint + } + + async fn resolve_ref(&self, _: RepoType, repo_id: &RepoId, revision: &str) -> Result { + self.sealed(&format!("{repo_id}@{revision}")) + } + + async fn head_file( + &self, + _: RepoType, + repo_id: &RepoId, + revision: &str, + path: &str, + ) -> Result { + self.sealed(&format!("{repo_id}@{revision}/{path}")) + } + + async fn get_file( + &self, + _: RepoType, + repo_id: &RepoId, + revision: &str, + path: &str, + ) -> Result { + self.sealed(&format!("{repo_id}@{revision}/{path}")) + } + + async fn repo_info(&self, _: RepoType, repo_id: &RepoId, revision: &str) -> Result { + self.sealed(&format!("{repo_id}@{revision}")) + } + + async fn repo_tree(&self, _: RepoType, repo_id: &RepoId, revision: &str) -> Result { + self.sealed(&format!("{repo_id}@{revision}")) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn client() -> HubClient { + HubClient::new( + &rustingface_entities::config::Upstream::default(), + Some("secret".into()), + ) + .unwrap() + } + + #[test] + fn model_resolve_urls_have_no_type_prefix_and_dataset_urls_do() { + let c = client(); + let id: RepoId = "Qwen/Qwen3-32B".parse().unwrap(); + assert_eq!( + c.resolve_url(RepoType::Models, &id, "main", "config.json"), + "https://huggingface.co/Qwen/Qwen3-32B/resolve/main/config.json" + ); + assert_eq!( + c.resolve_url(RepoType::Datasets, &id, "main", "data/train.parquet"), + "https://huggingface.co/datasets/Qwen/Qwen3-32B/resolve/main/data/train.parquet" + ); + } + + #[test] + fn path_separators_survive_encoding_but_other_specials_do_not() { + let c = client(); + let id: RepoId = "gpt2".parse().unwrap(); + let url = c.resolve_url(RepoType::Models, &id, "refs/pr/1", "a b/c.json"); + assert!(url.contains("/resolve/refs%2Fpr%2F1/"), "{url}"); + assert!(url.ends_with("/a%20b/c.json"), "{url}"); + } + + #[test] + fn a_sealed_upstream_refuses_without_reaching_the_network() { + let sealed = SealedUpstream::new("https://huggingface.co"); + assert!(!sealed.enabled()); + assert_eq!(sealed.endpoint(), "https://huggingface.co"); + } + + #[tokio::test] + async fn every_sealed_call_is_a_not_found_naming_the_seal() { + let sealed = SealedUpstream::new("https://huggingface.co"); + let id: RepoId = "gpt2".parse().unwrap(); + let err = match sealed + .get_file(RepoType::Models, &id, "main", "config.json") + .await + { + Err(err) => err, + Ok(_) => panic!("a sealed upstream must never produce a body"), + }; + assert_eq!(err.status(), 404); + assert!(err.to_string().contains("sealed"), "{err}"); + } +} diff --git a/crates/rustingface-entities/Cargo.toml b/crates/rustingface-entities/Cargo.toml new file mode 100644 index 0000000..19976b8 --- /dev/null +++ b/crates/rustingface-entities/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "rustingface-entities" +description = "Domain types, manifest/ref schemas, configuration and error taxonomy for rustingface." +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +authors.workspace = true +repository.workspace = true + +[dependencies] +humantime-serde.workspace = true +percent-encoding.workspace = true +serde.workspace = true +serde_json.workspace = true +thiserror.workspace = true +time.workspace = true + +[dev-dependencies] +toml.workspace = true diff --git a/crates/rustingface-entities/src/config.rs b/crates/rustingface-entities/src/config.rs new file mode 100644 index 0000000..fcbb756 --- /dev/null +++ b/crates/rustingface-entities/src/config.rs @@ -0,0 +1,408 @@ +//! Configuration: one TOML file, per-key environment overrides, secrets from +//! files (spec §5). +//! +//! Secrets are never inline. Every credential is named by a path so it can be +//! managed by `pass` or systemd credentials and stay out of both the config +//! file and source control. + +use std::path::PathBuf; +use std::time::Duration; + +use serde::{Deserialize, Deserializer, Serialize}; + +use crate::error::Error; + +/// Default config path. Overridable with `--config`. +pub const DEFAULT_CONFIG_PATH: &str = "/etc/rustingface/config.toml"; + +/// Environment prefix for per-key overrides, e.g. +/// `RUSTINGFACE__SERVER__LISTEN`. +pub const ENV_PREFIX: &str = "RUSTINGFACE__"; + +/// The whole of rustingface's configuration. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct Config { + pub server: Server, + pub storage: Storage, + pub upstream: Upstream, + pub policy: Policy, + pub auth: Auth, + pub observability: Observability, +} + +/// HTTP listener settings. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct Server { + /// Address to bind. Bind the mesh address when the reverse proxy lives on + /// another host; loopback only when it is co-resident. + pub listen: String, + /// How this service is reached from outside, used to build absolute URLs. + pub external_url: Option, + /// Ceiling on a single request, including a cold multi-gigabyte fetch. + #[serde(with = "humantime_serde")] + pub request_timeout: Duration, + /// How long a stalled client may hold up the tee before the transfer is + /// detached and finished without it. + #[serde(with = "humantime_serde")] + pub client_stall_timeout: Duration, +} + +impl Default for Server { + fn default() -> Self { + Self { + listen: "127.0.0.1:20482".into(), + external_url: None, + request_timeout: Duration::from_secs(300), + client_stall_timeout: Duration::from_secs(60), + } + } +} + +/// How a blob body reaches the client. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum BlobDelivery { + /// Stream the bytes through rustingface. + Proxy, + /// Answer `302` with a presigned URL to object storage. + Redirect, +} + +/// Object storage settings. The bucket is the entire system state. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct Storage { + pub endpoint: Option, + pub bucket: String, + pub region: String, + pub path_style: bool, + /// Use a local directory instead of S3. Test and single-host use only; + /// it forfeits guarantee 5. + pub local_path: Option, + pub access_key_id_file: Option, + pub secret_access_key_file: Option, + pub blob_delivery: BlobDelivery, + #[serde(with = "humantime_serde")] + pub presign_ttl: Duration, + #[serde(deserialize_with = "de_byte_size")] + pub multipart_part_size: u64, + /// Bounded retries for the read-merge-write that resolves concurrent + /// manifest writes for the same commit. + pub manifest_write_retries: u32, +} + +impl Default for Storage { + fn default() -> Self { + Self { + endpoint: None, + bucket: "rustingface".into(), + region: "us-east-1".into(), + path_style: true, + local_path: None, + access_key_id_file: None, + secret_access_key_file: None, + blob_delivery: BlobDelivery::Proxy, + presign_ttl: Duration::from_secs(15 * 60), + multipart_part_size: 16 * 1024 * 1024, + manifest_write_retries: 5, + } + } +} + +/// The upstream Hub. Consulted on miss only, and not at all when sealed. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct Upstream { + /// `false` seals the service: it serves only what is already stored. + pub enabled: bool, + pub endpoint: String, + pub token_file: Option, + #[serde(with = "humantime_serde")] + pub connect_timeout: Duration, + #[serde(with = "humantime_serde")] + pub read_timeout: Duration, + /// Keep Xet off outbound fetches so whole files arrive via the LFS bridge + /// and the bucket stays independently readable (spec §8). Do not change. + pub disable_xet: bool, +} + +impl Default for Upstream { + fn default() -> Self { + Self { + enabled: true, + endpoint: "https://huggingface.co".into(), + token_file: None, + connect_timeout: Duration::from_secs(10), + read_timeout: Duration::from_secs(60), + disable_xet: true, + } + } +} + +/// How a mutable ref resolves on second and later requests. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum RefResolution { + /// First resolution wins, forever, until an operator repins. + Freeze, + /// Re-resolve upstream when reachable. Breaks guarantee 4; development + /// only, and warned about at startup. + Follow, +} + +/// What rustingface is willing to fetch. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct Policy { + pub ref_resolution: RefResolution, + /// `false` restricts fetching to repos already present in the bucket. + pub allow_new_repos: bool, + /// Optional glob allowlist over `repo_id`, e.g. `["Qwen/*"]`. Empty + /// allows everything. + pub allowlist: Vec, +} + +impl Default for Policy { + fn default() -> Self { + Self { + ref_resolution: RefResolution::Freeze, + allow_new_repos: true, + allowlist: Vec::new(), + } + } +} + +/// Downstream client authentication. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum AuthMode { + None, + Bearer, +} + +/// Client-facing auth settings. Distinct from `upstream.token_file`, which is +/// never echoed downstream. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct Auth { + pub mode: AuthMode, + /// One accepted token per line; `#` comments and blanks ignored. + pub token_file: Option, +} + +impl Default for Auth { + fn default() -> Self { + Self { + mode: AuthMode::None, + token_file: None, + } + } +} + +/// Logging and metrics. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct Observability { + /// Serve Prometheus text format at `/metrics`. + pub metrics: bool, + pub log_format: LogFormat, +} + +impl Default for Observability { + fn default() -> Self { + Self { + metrics: true, + log_format: LogFormat::Json, + } + } +} + +/// Log encoding. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum LogFormat { + Json, + Text, +} + +impl Config { + /// Reject configurations that are internally inconsistent, before the + /// listener binds and the first client depends on them. + pub fn validate(&self) -> Result<(), Error> { + if self.storage.local_path.is_none() && self.storage.bucket.is_empty() { + return Err(Error::Config( + "storage.bucket must be set unless storage.local_path is used".into(), + )); + } + if self.storage.multipart_part_size < 5 * 1024 * 1024 { + return Err(Error::Config( + "storage.multipart_part_size must be at least 5MiB, the S3 minimum part size" + .into(), + )); + } + if self.storage.blob_delivery == BlobDelivery::Redirect && self.storage.local_path.is_some() + { + return Err(Error::Config( + "storage.blob_delivery = \"redirect\" requires S3; a local store cannot presign" + .into(), + )); + } + if self.auth.mode == AuthMode::Bearer && self.auth.token_file.is_none() { + return Err(Error::Config( + "auth.mode = \"bearer\" requires auth.token_file".into(), + )); + } + if !self.upstream.disable_xet { + return Err(Error::Config( + "upstream.disable_xet = false is unsupported: rustingface implements no Xet client, \ + and a Xet-backed fetch would leave the bucket unreadable without one (spec §8)" + .into(), + )); + } + Ok(()) + } + + /// Whether the service is sealed: no upstream, serve only what is stored. + pub fn sealed(&self) -> bool { + !self.upstream.enabled + } +} + +/// Parse `16MiB`, `1GB`, `524288` and friends into bytes. +pub fn parse_byte_size(s: &str) -> Result { + let s = s.trim(); + let split = s + .find(|c: char| !c.is_ascii_digit() && c != '_' && c != '.') + .unwrap_or(s.len()); + let (digits, unit) = s.split_at(split); + let digits = digits.replace('_', ""); + let value: f64 = digits + .parse() + .map_err(|_| Error::Config(format!("{s:?} is not a byte size")))?; + let multiplier: f64 = match unit.trim().to_ascii_lowercase().as_str() { + "" | "b" => 1.0, + "k" | "kb" => 1_000.0, + "ki" | "kib" => 1024.0, + "m" | "mb" => 1_000_000.0, + "mi" | "mib" => 1024.0 * 1024.0, + "g" | "gb" => 1_000_000_000.0, + "gi" | "gib" => 1024.0 * 1024.0 * 1024.0, + other => { + return Err(Error::Config(format!( + "unknown byte-size unit {other:?} in {s:?}" + ))); + } + }; + Ok((value * multiplier) as u64) +} + +/// Accept either a bare integer or a suffixed string for a byte size, so +/// `16MiB` in TOML and `16777216` from an environment override both work. +fn de_byte_size<'de, D: Deserializer<'de>>(d: D) -> Result { + #[derive(Deserialize)] + #[serde(untagged)] + enum Repr { + Int(u64), + Str(String), + } + match Repr::deserialize(d)? { + Repr::Int(n) => Ok(n), + Repr::Str(s) => parse_byte_size(&s).map_err(serde::de::Error::custom), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn byte_sizes_parse_binary_and_decimal_units() { + assert_eq!(parse_byte_size("16MiB").unwrap(), 16 * 1024 * 1024); + assert_eq!(parse_byte_size("16MB").unwrap(), 16_000_000); + assert_eq!(parse_byte_size("1GiB").unwrap(), 1024 * 1024 * 1024); + assert_eq!(parse_byte_size("5242880").unwrap(), 5_242_880); + assert_eq!(parse_byte_size(" 8 KiB ").unwrap(), 8192); + assert!(parse_byte_size("16 parsecs").is_err()); + } + + #[test] + fn the_documented_example_config_parses() { + let toml = r#" + [server] + listen = "127.0.0.1:20482" + external_url = "https://rf.internal" + request_timeout = "300s" + + [storage] + endpoint = "http://caveman.kosherinata.internal:9000" + bucket = "rustingface" + region = "us-east-1" + path_style = true + blob_delivery = "proxy" + presign_ttl = "15m" + multipart_part_size = "16MiB" + + [upstream] + enabled = true + endpoint = "https://huggingface.co" + connect_timeout = "10s" + read_timeout = "60s" + disable_xet = true + + [policy] + ref_resolution = "freeze" + allow_new_repos = true + allowlist = ["Qwen/*"] + + [auth] + mode = "none" + + [observability] + metrics = true + log_format = "json" + "#; + let cfg: Config = toml::from_str(toml).unwrap(); + cfg.validate().unwrap(); + assert_eq!(cfg.storage.multipart_part_size, 16 * 1024 * 1024); + assert_eq!(cfg.policy.ref_resolution, RefResolution::Freeze); + assert_eq!(cfg.server.request_timeout, Duration::from_secs(300)); + assert!(!cfg.sealed()); + } + + #[test] + fn defaults_are_valid_and_freeze_by_default() { + let cfg = Config::default(); + cfg.validate().unwrap(); + assert_eq!(cfg.policy.ref_resolution, RefResolution::Freeze); + assert!(cfg.upstream.disable_xet); + } + + #[test] + fn a_part_size_below_the_s3_minimum_is_refused() { + let mut cfg = Config::default(); + cfg.storage.multipart_part_size = 1024; + assert!(cfg.validate().is_err()); + } + + #[test] + fn bearer_without_a_token_file_is_refused() { + let mut cfg = Config::default(); + cfg.auth.mode = AuthMode::Bearer; + assert!(cfg.validate().is_err()); + } + + #[test] + fn enabling_xet_is_refused() { + let mut cfg = Config::default(); + cfg.upstream.disable_xet = false; + assert!(cfg.validate().is_err()); + } + + #[test] + fn unknown_keys_are_rejected_rather_than_silently_ignored() { + let err = toml::from_str::("[server]\nlissen = \"x\"\n").unwrap_err(); + assert!(err.to_string().contains("lissen"), "{err}"); + } +} diff --git a/crates/rustingface-entities/src/error.rs b/crates/rustingface-entities/src/error.rs new file mode 100644 index 0000000..a32d810 --- /dev/null +++ b/crates/rustingface-entities/src/error.rs @@ -0,0 +1,201 @@ +//! Error taxonomy. +//! +//! `huggingface_hub` maps the `X-Error-Code` response header onto typed +//! exceptions, so the header is part of the wire contract and not a nicety. +//! The strings below are client-internal constants upstream and should be +//! re-checked whenever the pinned client version moves (spec §6). + +use std::fmt; + +/// Convenience alias used throughout the workspace. +pub type Result = std::result::Result; + +/// The `X-Error-Code` values rustingface emits. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ErrorCode { + RepoNotFound, + RevisionNotFound, + EntryNotFound, + GatedRepo, + DisabledRepo, +} + +impl ErrorCode { + /// The header value, exactly as the client compares it. + pub const fn as_str(self) -> &'static str { + match self { + Self::RepoNotFound => "RepoNotFound", + Self::RevisionNotFound => "RevisionNotFound", + Self::EntryNotFound => "EntryNotFound", + Self::GatedRepo => "GatedRepo", + Self::DisabledRepo => "DisabledRepo", + } + } +} + +impl fmt::Display for ErrorCode { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.as_str()) + } +} + +/// Everything that can go wrong on the way to a response. +/// +/// Failures are loud by construction: there is no variant that yields an empty +/// body or a partial stream, because a silently short file is the one outcome +/// this service exists to prevent. +#[derive(Debug, thiserror::Error)] +pub enum Error { + /// The repo is not stored and upstream is disabled or unreachable. + #[error("repo not found: {0}")] + RepoNotFound(String), + + /// The revision is unknown and cannot be resolved. + #[error("revision not found: {0}")] + RevisionNotFound(String), + + /// The path is not in the manifest and cannot be fetched. + #[error("entry not found: {0}")] + EntryNotFound(String), + + /// Upstream returned 403 for a gated repo. + #[error("gated repo: {0}")] + GatedRepo(String), + + /// Upstream returned 403 for a disabled repo. + #[error("disabled repo: {0}")] + DisabledRepo(String), + + /// Upstream authentication failed (bad or missing `upstream.token_file`). + #[error("upstream authentication failed: {0}")] + UpstreamUnauthorized(String), + + /// The client presented no or an unrecognised bearer token. + #[error("unauthorized")] + Unauthorized, + + /// Policy (allowlist or `allow_new_repos`) refused the request. + #[error("forbidden by policy: {0}")] + PolicyDenied(String), + + /// A malformed request path, range header or repo id. + #[error("bad request: {0}")] + BadRequest(String), + + /// A `Range` header that cannot be satisfied against the stored length. + #[error("range not satisfiable: {0}")] + RangeNotSatisfiable(String), + + /// The stored bytes did not hash to the digest upstream promised. + #[error("digest mismatch for {path}: upstream said {expected}, got {actual}")] + DigestMismatch { + path: String, + expected: String, + actual: String, + }, + + /// Object storage failed. + #[error("storage: {0}")] + Storage(String), + + /// The upstream request failed at the transport layer, or returned a + /// status rustingface has no specific mapping for. + #[error("upstream: {0}")] + Upstream(String), + + /// Configuration could not be loaded or is internally inconsistent. + #[error("configuration: {0}")] + Config(String), + + /// An invariant broke. These are bugs. + #[error("internal: {0}")] + Internal(String), +} + +impl Error { + /// The `X-Error-Code` header value for this error, if it has one. + pub fn code(&self) -> Option { + match self { + Self::RepoNotFound(_) => Some(ErrorCode::RepoNotFound), + Self::RevisionNotFound(_) => Some(ErrorCode::RevisionNotFound), + Self::EntryNotFound(_) => Some(ErrorCode::EntryNotFound), + Self::GatedRepo(_) => Some(ErrorCode::GatedRepo), + Self::DisabledRepo(_) => Some(ErrorCode::DisabledRepo), + _ => None, + } + } + + /// The HTTP status this error is served as. + pub fn status(&self) -> u16 { + match self { + Self::RepoNotFound(_) | Self::RevisionNotFound(_) | Self::EntryNotFound(_) => 404, + Self::GatedRepo(_) | Self::DisabledRepo(_) | Self::PolicyDenied(_) => 403, + Self::UpstreamUnauthorized(_) | Self::Unauthorized => 401, + Self::BadRequest(_) => 400, + Self::RangeNotSatisfiable(_) => 416, + Self::DigestMismatch { .. } | Self::Storage(_) | Self::Upstream(_) => 502, + Self::Config(_) | Self::Internal(_) => 500, + } + } + + /// A short, stable label for metrics, so cardinality stays bounded. + pub fn label(&self) -> &'static str { + match self { + Self::RepoNotFound(_) => "repo_not_found", + Self::RevisionNotFound(_) => "revision_not_found", + Self::EntryNotFound(_) => "entry_not_found", + Self::GatedRepo(_) => "gated", + Self::DisabledRepo(_) => "disabled", + Self::UpstreamUnauthorized(_) => "upstream_unauthorized", + Self::Unauthorized => "unauthorized", + Self::PolicyDenied(_) => "policy_denied", + Self::BadRequest(_) => "bad_request", + Self::RangeNotSatisfiable(_) => "range_not_satisfiable", + Self::DigestMismatch { .. } => "digest_mismatch", + Self::Storage(_) => "storage", + Self::Upstream(_) => "upstream", + Self::Config(_) => "config", + Self::Internal(_) => "internal", + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_documented_status_and_code_pairs_hold() { + let cases: Vec<(Error, u16, Option<&str>)> = vec![ + (Error::RepoNotFound("a".into()), 404, Some("RepoNotFound")), + ( + Error::RevisionNotFound("a".into()), + 404, + Some("RevisionNotFound"), + ), + (Error::EntryNotFound("a".into()), 404, Some("EntryNotFound")), + (Error::GatedRepo("a".into()), 403, Some("GatedRepo")), + (Error::UpstreamUnauthorized("a".into()), 401, None), + ]; + for (err, status, code) in cases { + assert_eq!(err.status(), status, "{err}"); + assert_eq!(err.code().map(ErrorCode::as_str), code, "{err}"); + } + } + + #[test] + fn transport_and_integrity_failures_are_bad_gateway_not_not_found() { + assert_eq!(Error::Upstream("connect".into()).status(), 502); + assert_eq!(Error::Storage("timeout".into()).status(), 502); + assert_eq!( + Error::DigestMismatch { + path: "model.safetensors".into(), + expected: "sha256:aa".into(), + actual: "sha256:bb".into(), + } + .status(), + 502, + "a corrupt fetch must never look like a missing file" + ); + } +} diff --git a/crates/rustingface-entities/src/key.rs b/crates/rustingface-entities/src/key.rs new file mode 100644 index 0000000..69f07ca --- /dev/null +++ b/crates/rustingface-entities/src/key.rs @@ -0,0 +1,184 @@ +//! Bucket key layout. +//! +//! Prefixes are stable and human-inspectable, so an operator with `mc ls` can +//! answer any question about system state without this binary (spec §4). + +use percent_encoding::{AsciiSet, CONTROLS, utf8_percent_encode}; + +use crate::repo::{RepoId, RepoType}; + +/// Characters escaped in a ref name before it becomes a key segment. +/// +/// Real Hub refs contain slashes (`refs/pr/1`, `refs/convert/parquet`), which +/// would otherwise silently become directory levels and make the ref listing +/// for a repo non-flat. +const REF_ESCAPE: &AsciiSet = &CONTROLS.add(b'/').add(b'%').add(b'?').add(b'#').add(b' '); + +/// Key of a content-addressed blob. +/// +/// Two levels of hex fanout keep large repos off a single key prefix on +/// backends that shard by key range. +pub fn blob(digest_hex: &str) -> String { + debug_assert_eq!(digest_hex.len(), 64, "blob keys are sha256 hex"); + format!( + "blobs/sha256/{}/{}/{}", + &digest_hex[0..2], + &digest_hex[2..4], + digest_hex + ) +} + +/// Extract the hex digest from a blob key, if it is one. +pub fn blob_digest(key: &str) -> Option<&str> { + let rest = key.strip_prefix("blobs/sha256/")?; + let hex = rest.rsplit('/').next()?; + (hex.len() == 64 && hex.bytes().all(|b| b.is_ascii_hexdigit())).then_some(hex) +} + +/// Prefix under which every blob lives. +pub const BLOB_PREFIX: &str = "blobs/sha256/"; + +/// Prefix holding every manifest for a repo. +pub fn manifest_prefix(repo_type: RepoType, repo_id: &RepoId) -> String { + format!( + "manifests/{}/{}/{}/", + repo_type.as_str(), + repo_id.namespace_segment(), + repo_id.name() + ) +} + +/// Key of the manifest for one commit. +pub fn manifest(repo_type: RepoType, repo_id: &RepoId, commit: &str) -> String { + format!("{}{commit}.json", manifest_prefix(repo_type, repo_id)) +} + +/// Prefix holding every ref for a repo. +pub fn ref_prefix(repo_type: RepoType, repo_id: &RepoId) -> String { + format!( + "refs/{}/{}/{}/", + repo_type.as_str(), + repo_id.namespace_segment(), + repo_id.name() + ) +} + +/// Key of one ref record. +pub fn ref_key(repo_type: RepoType, repo_id: &RepoId, ref_name: &str) -> String { + format!( + "{}{}.json", + ref_prefix(repo_type, repo_id), + utf8_percent_encode(ref_name, REF_ESCAPE) + ) +} + +/// Recover a ref name from its key, reversing the escaping. +pub fn ref_name_from_key(key: &str) -> Option { + let file = key.rsplit('/').next()?.strip_suffix(".json")?; + percent_encoding::percent_decode_str(file) + .decode_utf8() + .ok() + .map(|s| s.into_owned()) +} + +/// Key of a repo record. +pub fn repo(repo_type: RepoType, repo_id: &RepoId) -> String { + format!( + "repos/{}/{}/{}/repo.json", + repo_type.as_str(), + repo_id.namespace_segment(), + repo_id.name() + ) +} + +/// Prefix holding every repo record of one type. +pub fn repo_type_prefix(repo_type: RepoType) -> String { + format!("repos/{}/", repo_type.as_str()) +} + +/// Recover `(repo_type, repo_id)` from a repo record key. +pub fn repo_from_key(key: &str) -> Option<(RepoType, RepoId)> { + let rest = key.strip_prefix("repos/")?.strip_suffix("/repo.json")?; + let mut parts = rest.split('/'); + let repo_type = parts.next()?.parse().ok()?; + let namespace = parts.next()?; + let name = parts.next()?; + if parts.next().is_some() { + return None; + } + Some((repo_type, RepoId::from_segments(namespace, name).ok()?)) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn qwen() -> RepoId { + "Qwen/Qwen3-32B".parse().unwrap() + } + + #[test] + fn blob_key_fans_out_two_levels_and_round_trips() { + let hex = "9c1eff00112233445566778899aabbccddeeff00112233445566778899aabbcc"; + let key = blob(hex); + assert_eq!( + key, + format!("blobs/sha256/9c/1e/{hex}"), + "two-level fanout on the first four hex characters" + ); + assert_eq!(blob_digest(&key), Some(hex)); + } + + #[test] + fn blob_digest_rejects_non_blob_keys() { + assert_eq!( + blob_digest("manifests/models/Qwen/Qwen3-32B/abc.json"), + None + ); + assert_eq!(blob_digest("blobs/sha256/aa/bb/short"), None); + } + + #[test] + fn manifest_and_ref_keys_use_the_documented_layout() { + assert_eq!( + manifest(RepoType::Models, &qwen(), "8f2ac31"), + "manifests/models/Qwen/Qwen3-32B/8f2ac31.json" + ); + assert_eq!( + ref_key(RepoType::Models, &qwen(), "main"), + "refs/models/Qwen/Qwen3-32B/main.json" + ); + } + + #[test] + fn slashed_ref_names_stay_one_key_segment() { + let key = ref_key(RepoType::Datasets, &qwen(), "refs/pr/1"); + assert_eq!(key, "refs/datasets/Qwen/Qwen3-32B/refs%2Fpr%2F1.json"); + assert_eq!( + key.strip_prefix(&ref_prefix(RepoType::Datasets, &qwen())) + .unwrap() + .matches('/') + .count(), + 0, + "the ref listing for a repo stays flat" + ); + assert_eq!(ref_name_from_key(&key).as_deref(), Some("refs/pr/1")); + } + + #[test] + fn bare_named_repos_round_trip_through_the_repo_key() { + let id: RepoId = "gpt2".parse().unwrap(); + let key = repo(RepoType::Models, &id); + assert_eq!(key, "repos/models/__/gpt2/repo.json"); + let (rt, back) = repo_from_key(&key).unwrap(); + assert_eq!(rt, RepoType::Models); + assert_eq!(back.to_string(), "gpt2"); + } + + #[test] + fn repo_from_key_rejects_malformed_keys() { + assert!(repo_from_key("repos/models/Qwen/repo.json").is_none()); + assert!(repo_from_key("repos/spaces/a/b/repo.json").is_none()); + assert!(repo_from_key("manifests/models/a/b/repo.json").is_none()); + } +} diff --git a/crates/rustingface-entities/src/lib.rs b/crates/rustingface-entities/src/lib.rs new file mode 100644 index 0000000..add77f5 --- /dev/null +++ b/crates/rustingface-entities/src/lib.rs @@ -0,0 +1,19 @@ +//! Domain types for rustingface. +//! +//! This crate is pure: it defines the on-disk schemas (manifest, ref, repo +//! record), the bucket key layout, the configuration shape and the error +//! taxonomy that the HTTP surface replays to clients. It performs no I/O and +//! pulls in no async runtime. + +pub mod config; +pub mod error; +pub mod key; +pub mod manifest; +pub mod refs; +pub mod repo; + +pub use config::Config; +pub use error::{Error, ErrorCode, Result}; +pub use manifest::{Entry, MANIFEST_SCHEMA, Manifest}; +pub use refs::{PinnedBy, REF_SCHEMA, RefRecord}; +pub use repo::{REPO_SCHEMA, RepoId, RepoRecord, RepoType}; diff --git a/crates/rustingface-entities/src/manifest.rs b/crates/rustingface-entities/src/manifest.rs new file mode 100644 index 0000000..ae1636c --- /dev/null +++ b/crates/rustingface-entities/src/manifest.rs @@ -0,0 +1,192 @@ +//! The manifest: the immutable record of what upstream said about one commit. +//! +//! rustingface never derives these values. Every field sourced from upstream is +//! stored exactly as received and replayed byte-identically (spec §2), because +//! client cache layouts are keyed on them. + +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use time::OffsetDateTime; + +use crate::repo::{RepoId, RepoType}; + +/// Schema version of `manifests/…/.json`. +pub const MANIFEST_SCHEMA: u32 = 1; + +/// One recorded file within a commit. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Entry { + /// Repo-relative path, exactly as the client requested it. + pub path: String, + /// True byte length of the stored blob. + pub size: u64, + /// rustingface blob key digest, `sha256:<64 hex>`. + pub digest: String, + /// Upstream `ETag`, verbatim, quotes included. + pub etag: Option, + /// Upstream `X-Linked-Etag`, verbatim. `None` if upstream omitted it. + pub linked_etag: Option, + /// Upstream `X-Linked-Size`, verbatim. `None` if upstream omitted it. + pub linked_size: Option, + /// Whether upstream served this via the LFS bridge. + pub lfs: bool, +} + +/// The manifest for one `(repo, commit)` pair. +/// +/// `entries` is append-only and expected to be partial: clients fetch subsets +/// of a repo, and rustingface records what it has actually seen. A path absent +/// from `entries` is a miss, not a 404, while upstream is reachable. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Manifest { + pub schema: u32, + pub repo_type: RepoType, + pub repo_id: RepoId, + pub commit: String, + #[serde(with = "time::serde::rfc3339")] + pub recorded_at: OffsetDateTime, + pub upstream: Option, + /// The upstream `/api/{type}/{id}/revision/{commit}` response, verbatim. + /// + /// Stored whole and filtered on read (spec §13): recording the full + /// document means a field a client asks for later was never dropped. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub repo_meta: Option, + /// The upstream `/api/{type}/{id}/tree/{commit}?recursive=1&expand=1` + /// response, verbatim, when one has been recorded. + /// + /// Additive to schema 1: absent on manifests written before a tree was + /// requested, and the tree handler falls back to `repo_meta.siblings`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub repo_tree: Option, + #[serde(default)] + pub entries: Vec, +} + +impl Manifest { + /// An empty manifest for a commit whose metadata has just been recorded. + pub fn new( + repo_type: RepoType, + repo_id: RepoId, + commit: String, + upstream: Option, + ) -> Self { + Self { + schema: MANIFEST_SCHEMA, + repo_type, + repo_id, + commit, + recorded_at: OffsetDateTime::now_utc(), + upstream, + repo_meta: None, + repo_tree: None, + entries: Vec::new(), + } + } + + /// The recorded entry for `path`, if any. + pub fn entry(&self, path: &str) -> Option<&Entry> { + self.entries.iter().find(|e| e.path == path) + } + + /// Insert or replace an entry, keeping `entries` sorted by path. + /// + /// Entries are deterministic for a given commit, so a replace is a no-op in + /// content and last-writer-wins is correct (spec §4). + pub fn upsert(&mut self, entry: Entry) { + match self.entries.binary_search_by(|e| e.path.cmp(&entry.path)) { + Ok(i) => self.entries[i] = entry, + Err(i) => self.entries.insert(i, entry), + } + } + + /// Merge `other`'s entries and metadata into `self`. + /// + /// Used by the read-merge-write retry that resolves concurrent manifest + /// writes for the same commit. + pub fn merge_from(&mut self, other: &Manifest) { + for entry in &other.entries { + if self.entry(&entry.path).is_none() { + self.upsert(entry.clone()); + } + } + if self.repo_meta.is_none() { + self.repo_meta.clone_from(&other.repo_meta); + } + if self.repo_tree.is_none() { + self.repo_tree.clone_from(&other.repo_tree); + } + } + + /// Total stored bytes across recorded entries. + pub fn stored_bytes(&self) -> u64 { + self.entries.iter().map(|e| e.size).sum() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn entry(path: &str, size: u64) -> Entry { + Entry { + path: path.into(), + size, + digest: format!("sha256:{:064x}", size), + etag: Some("\"abc\"".into()), + linked_etag: None, + linked_size: None, + lfs: false, + } + } + + fn manifest() -> Manifest { + Manifest::new( + RepoType::Models, + "Qwen/Qwen3-32B".parse().unwrap(), + "8f2ac31".into(), + Some("https://huggingface.co".into()), + ) + } + + #[test] + fn upsert_keeps_entries_sorted_and_replaces_in_place() { + let mut m = manifest(); + m.upsert(entry("model.safetensors", 2)); + m.upsert(entry("config.json", 1)); + m.upsert(entry("config.json", 3)); + assert_eq!(m.entries.len(), 2); + assert_eq!(m.entries[0].path, "config.json"); + assert_eq!(m.entries[0].size, 3); + assert_eq!(m.entries[1].path, "model.safetensors"); + } + + #[test] + fn merge_takes_missing_entries_but_keeps_local_ones() { + let mut a = manifest(); + a.upsert(entry("config.json", 1)); + let mut b = manifest(); + b.upsert(entry("config.json", 99)); + b.upsert(entry("tokenizer.json", 2)); + b.repo_meta = Some(serde_json::json!({"id": "Qwen/Qwen3-32B"})); + + a.merge_from(&b); + assert_eq!(a.entries.len(), 2); + assert_eq!(a.entry("config.json").unwrap().size, 1, "local entry wins"); + assert!(a.repo_meta.is_some(), "absent metadata is adopted"); + } + + #[test] + fn round_trips_through_json_with_optional_fields_absent() { + let mut m = manifest(); + m.upsert(entry("config.json", 1204)); + let json = serde_json::to_string(&m).unwrap(); + assert!( + !json.contains("repo_meta"), + "absent optionals stay out of the document" + ); + let back: Manifest = serde_json::from_str(&json).unwrap(); + assert_eq!(back.commit, m.commit); + assert_eq!(back.entries, m.entries); + } +} diff --git a/crates/rustingface-entities/src/refs.rs b/crates/rustingface-entities/src/refs.rs new file mode 100644 index 0000000..a4a9a03 --- /dev/null +++ b/crates/rustingface-entities/src/refs.rs @@ -0,0 +1,96 @@ +//! Refs: the only mutable objects in the bucket. +//! +//! A ref record pins a mutable name (`main`, a tag, `refs/pr/1`) to the commit +//! it resolved to the first time it was seen. Guarantee 4 — reference +//! stability — is entirely this file plus a first-write-wins put. + +use serde::{Deserialize, Serialize}; +use time::OffsetDateTime; + +/// Schema version of `refs/…/.json`. +pub const REF_SCHEMA: u32 = 1; + +/// What caused a ref to hold its current value. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum PinnedBy { + /// First resolution through the resolve path, frozen automatically. + AutoFreeze, + /// An explicit `rustingface pin` or `rustingface refresh`. + Operator, +} + +impl PinnedBy { + /// The string as it appears in the stored document. + pub const fn as_str(self) -> &'static str { + match self { + Self::AutoFreeze => "auto-freeze", + Self::Operator => "operator", + } + } +} + +/// The `refs////.json` record. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RefRecord { + pub schema: u32, + #[serde(rename = "ref")] + pub ref_name: String, + pub commit: String, + #[serde(with = "time::serde::rfc3339")] + pub pinned_at: OffsetDateTime, + pub pinned_by: PinnedBy, +} + +impl RefRecord { + /// A pin recorded now. + pub fn new( + ref_name: impl Into, + commit: impl Into, + pinned_by: PinnedBy, + ) -> Self { + Self { + schema: REF_SCHEMA, + ref_name: ref_name.into(), + commit: commit.into(), + pinned_at: OffsetDateTime::now_utc(), + pinned_by, + } + } +} + +/// Whether `revision` is a commit SHA rather than a ref name. +/// +/// The Hub uses 40-hex git SHA-1 commit ids. A revision that looks like one is +/// used directly and never written to `refs/`, which is what keeps immutable +/// revisions off the mutable-object path entirely. +pub fn is_commit_sha(revision: &str) -> bool { + revision.len() == 40 && revision.bytes().all(|b| b.is_ascii_hexdigit()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn recognises_a_commit_sha() { + assert!(is_commit_sha("8f2a1c9e0b7d4f6a3e5c2b8d9f0a1c3e5b7d9f01")); + } + + #[test] + fn rejects_ref_names_and_short_or_long_hex() { + assert!(!is_commit_sha("main")); + assert!(!is_commit_sha("v1.0")); + assert!(!is_commit_sha("8f2a1c9e")); + assert!(!is_commit_sha("8f2a1c9e0b7d4f6a3e5c2b8d9f0a1c3e5b7d9f012")); + assert!(!is_commit_sha("8f2a1c9e0b7d4f6a3e5c2b8d9f0a1c3e5b7d9fzz")); + } + + #[test] + fn pinned_by_serialises_kebab_case() { + let r = RefRecord::new("main", "abc", PinnedBy::AutoFreeze); + let json = serde_json::to_string(&r).unwrap(); + assert!(json.contains("\"auto-freeze\""), "{json}"); + assert!(json.contains("\"ref\":\"main\""), "{json}"); + } +} diff --git a/crates/rustingface-entities/src/repo.rs b/crates/rustingface-entities/src/repo.rs new file mode 100644 index 0000000..a52e443 --- /dev/null +++ b/crates/rustingface-entities/src/repo.rs @@ -0,0 +1,232 @@ +//! Repository identity: type, id, and the `repos/…/repo.json` record. + +use std::fmt; +use std::str::FromStr; + +use serde::{Deserialize, Serialize}; +use time::OffsetDateTime; + +use crate::error::Error; + +/// Schema version of the `repos/…/repo.json` record. +pub const REPO_SCHEMA: u32 = 1; + +/// The two repository namespaces rustingface serves. +/// +/// Deliberately not an open enum: spaces are a non-goal (see spec §1). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum RepoType { + Models, + Datasets, +} + +impl RepoType { + /// The bucket prefix segment and the `/api/{…}` path segment. These are + /// the same string upstream, and we keep them the same here. + pub const fn as_str(self) -> &'static str { + match self { + Self::Models => "models", + Self::Datasets => "datasets", + } + } + + /// Every repo type rustingface knows about, for iteration in the CLI. + pub const ALL: [Self; 2] = [Self::Models, Self::Datasets]; +} + +impl fmt::Display for RepoType { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.as_str()) + } +} + +impl FromStr for RepoType { + type Err = Error; + + fn from_str(s: &str) -> Result { + match s { + "models" | "model" => Ok(Self::Models), + "datasets" | "dataset" => Ok(Self::Datasets), + other => Err(Error::BadRequest(format!("unknown repo type {other:?}"))), + } + } +} + +/// A Hub repository id: either `namespace/name` or a bare `name`. +/// +/// Bare names are legitimate on the Hub (`gpt2`, `bert-base-uncased`) and must +/// round-trip through the bucket layout, so [`RepoId::namespace_segment`] +/// substitutes a sentinel that cannot collide with a real namespace. +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct RepoId { + namespace: Option, + name: String, +} + +/// Key segment standing in for the namespace of a bare-named repo. +/// +/// Hub namespaces must begin with an alphanumeric character, so a segment of +/// two underscores is unreachable by any real namespace and the mapping is +/// injective in both directions. +pub const BARE_NAMESPACE: &str = "__"; + +impl RepoId { + /// Construct from parts. Neither part may be empty or contain a `/`. + pub fn new(namespace: Option, name: String) -> Result { + if name.is_empty() || name.contains('/') { + return Err(Error::BadRequest(format!("invalid repo name {name:?}"))); + } + if let Some(ns) = &namespace + && (ns.is_empty() || ns.contains('/') || ns == BARE_NAMESPACE) + { + return Err(Error::BadRequest(format!("invalid namespace {ns:?}"))); + } + Ok(Self { namespace, name }) + } + + /// The namespace, if the repo has one. + pub fn namespace(&self) -> Option<&str> { + self.namespace.as_deref() + } + + /// The repo name, without namespace. + pub fn name(&self) -> &str { + &self.name + } + + /// The namespace as it appears in a bucket key, substituting + /// [`BARE_NAMESPACE`] for a bare-named repo. + pub fn namespace_segment(&self) -> &str { + self.namespace.as_deref().unwrap_or(BARE_NAMESPACE) + } + + /// Reconstruct from the two key segments produced by + /// [`RepoId::namespace_segment`] and [`RepoId::name`]. + pub fn from_segments(namespace: &str, name: &str) -> Result { + let namespace = if namespace == BARE_NAMESPACE { + None + } else { + Some(namespace.to_owned()) + }; + Self::new(namespace, name.to_owned()) + } + + /// How many path segments this id occupies in a URL (1 or 2). + pub fn segment_count(&self) -> usize { + if self.namespace.is_some() { 2 } else { 1 } + } +} + +impl fmt::Display for RepoId { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match &self.namespace { + Some(ns) => write!(f, "{ns}/{}", self.name), + None => f.write_str(&self.name), + } + } +} + +impl FromStr for RepoId { + type Err = Error; + + fn from_str(s: &str) -> Result { + let s = s.trim_matches('/'); + let mut parts = s.split('/'); + let first = parts.next().unwrap_or_default(); + match parts.next() { + None => Self::new(None, first.to_owned()), + Some(second) => { + if parts.next().is_some() { + return Err(Error::BadRequest(format!( + "repo id {s:?} has too many segments" + ))); + } + Self::new(Some(first.to_owned()), second.to_owned()) + } + } + } +} + +impl Serialize for RepoId { + fn serialize(&self, s: S) -> Result { + s.serialize_str(&self.to_string()) + } +} + +impl<'de> Deserialize<'de> for RepoId { + fn deserialize>(d: D) -> Result { + let s = String::deserialize(d)?; + s.parse().map_err(serde::de::Error::custom) + } +} + +/// The `repos////repo.json` record. +/// +/// Its only job is to make "which repos does this bucket know about" a prefix +/// listing rather than a full scan of every manifest. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RepoRecord { + pub schema: u32, + pub repo_type: RepoType, + pub repo_id: RepoId, + #[serde(with = "time::serde::rfc3339")] + pub first_seen: OffsetDateTime, + /// Upstream this repo was first recorded from, for provenance. + pub upstream: Option, +} + +impl RepoRecord { + /// A record for a repo being seen for the first time, now. + pub fn new(repo_type: RepoType, repo_id: RepoId, upstream: Option) -> Self { + Self { + schema: REPO_SCHEMA, + repo_type, + repo_id, + first_seen: OffsetDateTime::now_utc(), + upstream, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn namespaced_id_round_trips() { + let id: RepoId = "Qwen/Qwen3-32B".parse().unwrap(); + assert_eq!(id.namespace(), Some("Qwen")); + assert_eq!(id.name(), "Qwen3-32B"); + assert_eq!(id.to_string(), "Qwen/Qwen3-32B"); + assert_eq!(RepoId::from_segments("Qwen", "Qwen3-32B").unwrap(), id); + } + + #[test] + fn bare_id_round_trips_through_key_segments() { + let id: RepoId = "gpt2".parse().unwrap(); + assert_eq!(id.namespace(), None); + assert_eq!(id.namespace_segment(), BARE_NAMESPACE); + assert_eq!(id.to_string(), "gpt2"); + let back = RepoId::from_segments(id.namespace_segment(), id.name()).unwrap(); + assert_eq!(back, id); + assert_eq!(back.to_string(), "gpt2"); + } + + #[test] + fn three_segment_id_is_rejected() { + assert!("a/b/c".parse::().is_err()); + } + + #[test] + fn sentinel_namespace_is_not_accepted_as_a_real_one() { + assert!(RepoId::new(Some(BARE_NAMESPACE.into()), "x".into()).is_err()); + } + + #[test] + fn repo_type_parses_singular_and_plural() { + assert_eq!("models".parse::().unwrap(), RepoType::Models); + assert_eq!("dataset".parse::().unwrap(), RepoType::Datasets); + assert!("spaces".parse::().is_err()); + } +} diff --git a/crates/rustingface/Cargo.toml b/crates/rustingface/Cargo.toml new file mode 100644 index 0000000..3c2001c --- /dev/null +++ b/crates/rustingface/Cargo.toml @@ -0,0 +1,38 @@ +[package] +name = "rustingface" +description = "A sovereign, HuggingFace-compatible model registry: server and admin CLI in one binary." +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +authors.workspace = true +repository.workspace = true + +[[bin]] +name = "rustingface" +path = "src/main.rs" + +[dependencies] +rustingface-api.workspace = true +rustingface-core.workspace = true +rustingface-data.workspace = true +rustingface-entities.workspace = true + +anyhow.workspace = true +axum.workspace = true +clap.workspace = true +figment.workspace = true +futures.workspace = true +serde_json.workspace = true +time.workspace = true +tokio.workspace = true +toml.workspace = true +tracing.workspace = true +tracing-subscriber.workspace = true + +[dev-dependencies] +bytes.workspace = true +futures.workspace = true +http-body-util.workspace = true +reqwest.workspace = true +tower.workspace = true diff --git a/crates/rustingface/src/cmd/doctor.rs b/crates/rustingface/src/cmd/doctor.rs new file mode 100644 index 0000000..b6c7d72 --- /dev/null +++ b/crates/rustingface/src/cmd/doctor.rs @@ -0,0 +1,145 @@ +//! `rustingface doctor` — prove the deployment works before a client does. +//! +//! Every check reports what it actually attempted. A doctor that only proves a +//! TCP connection opened is a doctor that passes on a bucket the service +//! cannot write to. + +use anyhow::{Result, bail}; +use rustingface_entities::config::{AuthMode, BlobDelivery, RefResolution}; +use rustingface_entities::repo::RepoType; + +use crate::cmd::human_bytes; +use crate::wire; + +/// Run the checks, reporting each and failing on the first that matters. +pub async fn run(config: rustingface_entities::config::Config) -> Result<()> { + let mut problems = Vec::new(); + + println!("configuration"); + println!(" listen {}", config.server.listen); + println!( + " external url {}", + config.server.external_url.as_deref().unwrap_or("(unset)") + ); + println!( + " storage {} bucket {:?}{}", + config + .storage + .endpoint + .as_deref() + .unwrap_or("(aws default endpoint)"), + config.storage.bucket, + if config.storage.path_style { + " path-style" + } else { + "" + } + ); + println!(" blob delivery {:?}", config.storage.blob_delivery); + println!( + " part size {}", + human_bytes(config.storage.multipart_part_size) + ); + println!( + " upstream {}", + if config.upstream.enabled { + config.upstream.endpoint.clone() + } else { + "sealed (upstream.enabled = false)".into() + } + ); + println!(" ref resolution {:?}", config.policy.ref_resolution); + println!(" client auth {:?}", config.auth.mode); + println!( + " allowlist {}", + if config.policy.allowlist.is_empty() { + "(everything)".into() + } else { + config.policy.allowlist.join(", ") + } + ); + + if config.policy.ref_resolution == RefResolution::Follow { + problems.push( + "policy.ref_resolution = \"follow\" breaks reference stability; use \"freeze\" \ + outside development" + .to_owned(), + ); + } + if config.storage.blob_delivery == BlobDelivery::Redirect && config.storage.endpoint.is_none() { + problems.push( + "blob_delivery = \"redirect\" mints presigned URLs against storage.endpoint, which \ + is unset; clients would be redirected to AWS" + .to_owned(), + ); + } + if config.auth.mode == AuthMode::None && config.server.listen.starts_with("0.0.0.0") { + problems.push( + "auth.mode = \"none\" with a wildcard bind publishes the registry on every address \ + this host carries; bind the mesh address or enable bearer auth" + .to_owned(), + ); + } + + println!("\nstorage"); + let registry = wire::registry(&config)?; + match registry.probe_store().await { + Ok(()) => println!(" write/read/delete ok"), + Err(err) => { + println!(" write/read/delete FAILED: {err}"); + problems.push(format!("object storage is not usable: {err}")); + } + } + + let mut repos = 0; + for repo_type in RepoType::ALL { + match registry.list_repos(repo_type).await { + Ok(list) => { + println!(" {:<19} {} stored", repo_type.as_str(), list.len()); + repos += list.len(); + } + Err(err) => { + println!(" {:<19} FAILED: {err}", repo_type.as_str()); + problems.push(format!("listing {} failed: {err}", repo_type.as_str())); + } + } + } + if repos == 0 && config.sealed() { + problems.push( + "this instance is sealed and the bucket holds no repositories, so every request \ + will 404" + .to_owned(), + ); + } + + println!("\nupstream"); + if config.upstream.enabled { + let probe: rustingface_entities::repo::RepoId = "gpt2".parse()?; + match registry + .upstream() + .resolve_ref(RepoType::Models, &probe, "main") + .await + { + Ok(commit) => println!(" reachable yes (gpt2@main = {commit})"), + Err(err) => { + println!(" reachable no: {err}"); + problems.push(format!( + "upstream is enabled but unreachable: {err}. Cold fetches will fail; \ + anything already stored still serves." + )); + } + } + } else { + println!(" reachable n/a, sealed"); + } + + if problems.is_empty() { + println!("\nno problems found"); + return Ok(()); + } + println!("\n{} problem(s):", problems.len()); + for problem in &problems { + println!(" - {problem}"); + } + bail!("doctor found {} problem(s)", problems.len()); +} diff --git a/crates/rustingface/src/cmd/fetch.rs b/crates/rustingface/src/cmd/fetch.rs new file mode 100644 index 0000000..e5f2be9 --- /dev/null +++ b/crates/rustingface/src/cmd/fetch.rs @@ -0,0 +1,123 @@ +//! `rustingface fetch` — warm the bucket ahead of need. +//! +//! Drives the same resolve path the server does, so what lands in the bucket +//! is byte-identical to what a client pull would have left there. + +use anyhow::{Context, Result, bail}; +use futures::StreamExt; +use rustingface_entities::config::Config; +use rustingface_entities::repo::RepoType; +use serde_json::Value; + +use crate::cmd::{human_bytes, split_repo}; +use crate::wire; + +/// Fetch a revision, or a named subset of its files. +pub async fn run(config: Config, repo_type: RepoType, repo: &str, files: &[String]) -> Result<()> { + if config.sealed() { + bail!("fetch needs upstream; this instance is sealed (upstream.enabled = false)"); + } + let (repo_id, revision) = split_repo(repo)?; + let revision = revision.unwrap_or_else(|| "main".into()); + let registry = wire::registry(&config)?; + + let commit = registry + .resolve_revision(repo_type, &repo_id, &revision) + .await + .with_context(|| format!("resolving {repo_id}@{revision}"))?; + println!("{repo_id}@{revision} -> {commit}"); + + let wanted: Vec = if files.is_empty() { + let tree = registry + .repo_tree(repo_type, &repo_id, &commit, "", true) + .await + .with_context(|| format!("listing {repo_id}@{commit}"))?; + tree_paths(&tree) + } else { + files.to_vec() + }; + + if wanted.is_empty() { + println!("nothing to fetch: the revision lists no files"); + return Ok(()); + } + + let mut fetched = 0u64; + let mut bytes = 0u64; + let mut failures = Vec::new(); + + for path in &wanted { + // Draining the body is what commits the bytes: the same tee the server + // runs, with this process standing in for the client. + match registry + .get_file(repo_type, &repo_id, &commit, path, None) + .await + { + Ok((file, _)) => { + let mut size = 0u64; + if let rustingface_core::Body::Stream { mut stream, .. } = file.body { + while let Some(chunk) = stream.next().await { + size += chunk.with_context(|| format!("fetching {path}"))?.len() as u64; + } + } else { + size = file.size; + } + fetched += 1; + bytes += size; + println!(" {path} {}", human_bytes(size)); + } + Err(err) => { + eprintln!(" {path} FAILED: {err}"); + failures.push(path.clone()); + } + } + } + + println!( + "{fetched}/{} files, {} stored", + wanted.len(), + human_bytes(bytes) + ); + if !failures.is_empty() { + bail!( + "{} file(s) could not be fetched: {}", + failures.len(), + failures.join(", ") + ); + } + Ok(()) +} + +/// Every file path in a tree document. +fn tree_paths(tree: &Value) -> Vec { + tree.as_array() + .map(|items| { + items + .iter() + .filter(|item| item.get("type").and_then(Value::as_str) != Some("directory")) + .filter_map(|item| item.get("path").and_then(Value::as_str)) + .map(str::to_owned) + .collect() + }) + .unwrap_or_default() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn directories_are_not_fetched_as_files() { + let tree = serde_json::json!([ + {"type": "file", "path": "config.json"}, + {"type": "directory", "path": "onnx"}, + {"type": "file", "path": "onnx/model.onnx"}, + ]); + assert_eq!(tree_paths(&tree), vec!["config.json", "onnx/model.onnx"]); + } + + #[test] + fn a_non_array_tree_yields_nothing_rather_than_panicking() { + assert!(tree_paths(&serde_json::json!({"error": "nope"})).is_empty()); + } +} diff --git a/crates/rustingface/src/cmd/gc.rs b/crates/rustingface/src/cmd/gc.rs new file mode 100644 index 0000000..53bfec0 --- /dev/null +++ b/crates/rustingface/src/cmd/gc.rs @@ -0,0 +1,65 @@ +//! `rustingface gc` and `rustingface verify`. + +use anyhow::{Result, bail}; +use rustingface_entities::config::Config; +use rustingface_entities::repo::RepoType; + +use crate::cmd::{human_bytes, split_repo}; +use crate::wire; + +/// Delete blobs no manifest references. +pub async fn run(config: Config, dry_run: bool) -> Result<()> { + let registry = wire::registry(&config)?; + let result = registry.gc(dry_run).await?; + + for key in &result.deleted { + println!("{} {key}", if dry_run { "would delete" } else { "deleted" }); + } + println!( + "{} blobs scanned, {} referenced, {} {} ({})", + result.scanned, + result.referenced, + result.deleted.len(), + if dry_run { "reclaimable" } else { "reclaimed" }, + human_bytes(result.bytes) + ); + if dry_run && !result.deleted.is_empty() { + println!("re-run without --dry-run to reclaim them"); + } + Ok(()) +} + +/// Re-read blobs and check them against their content-addressed keys. +pub async fn verify(config: Config, repo_type: RepoType, repo: Option<&str>) -> Result<()> { + let registry = wire::registry(&config)?; + + let parsed = repo.map(split_repo).transpose()?; + let target = parsed.as_ref().map(|(id, _)| (repo_type, id)); + let result = registry.verify(target).await?; + + for key in &result.missing { + println!("MISSING {key}"); + } + for corruption in &result.corrupt { + println!( + "CORRUPT {}\n expected {}\n found {}", + corruption.key, corruption.expected, corruption.found + ); + } + println!( + "{} blobs read ({}), {} missing, {} corrupt", + result.blobs, + human_bytes(result.bytes), + result.missing.len(), + result.corrupt.len() + ); + + if !result.corrupt.is_empty() || !result.missing.is_empty() { + bail!( + "verify found {} corrupt and {} missing blob(s)", + result.corrupt.len(), + result.missing.len() + ); + } + Ok(()) +} diff --git a/crates/rustingface/src/cmd/list.rs b/crates/rustingface/src/cmd/list.rs new file mode 100644 index 0000000..aaec3ea --- /dev/null +++ b/crates/rustingface/src/cmd/list.rs @@ -0,0 +1,96 @@ +//! `rustingface list` and `rustingface show`. + +use anyhow::{Context, Result}; +use rustingface_entities::config::Config; +use rustingface_entities::repo::RepoType; + +use crate::cmd::{human_bytes, split_repo}; +use crate::wire; + +/// List stored repos with their pins and sizes. +pub async fn run(config: Config, repo_type: Option) -> Result<()> { + let registry = wire::registry(&config)?; + let types: Vec = repo_type + .map(|t| vec![t]) + .unwrap_or_else(|| RepoType::ALL.to_vec()); + + let mut total = 0u64; + let mut count = 0usize; + for repo_type in types { + for repo_id in registry.list_repos(repo_type).await? { + let summary = registry.summarise(repo_type, &repo_id).await?; + let pins = if summary.refs.is_empty() { + "-".to_owned() + } else { + summary + .refs + .iter() + .map(|(name, commit)| format!("{name}={}", short(commit))) + .collect::>() + .join(" ") + }; + println!( + "{:<9} {:<44} {:>3} rev {:>5} files {:>10} {pins}", + repo_type.as_str(), + repo_id.to_string(), + summary.commits.len(), + summary.files, + human_bytes(summary.bytes), + ); + total += summary.bytes; + count += 1; + } + } + + if count == 0 { + println!("no repositories stored in {}", config.storage.bucket); + } else { + println!("\n{count} repositories, {} referenced", human_bytes(total)); + } + Ok(()) +} + +/// Print a manifest. +pub async fn show(config: Config, repo_type: RepoType, repo: &str) -> Result<()> { + let (repo_id, revision) = split_repo(repo)?; + let registry = wire::registry(&config)?; + + // Resolve against the bucket only: `show` must never have the side effect + // of pulling a fresh pin from upstream. + let commit = registry + .resolve_stored_revision( + repo_type, + &repo_id, + &revision.unwrap_or_else(|| "main".into()), + ) + .await + .with_context(|| format!("resolving {repo_id} in the bucket"))?; + + let manifest = registry + .load_manifest(repo_type, &repo_id, &commit) + .await? + .with_context(|| format!("no manifest stored for {repo_id}@{commit}"))?; + + println!("{}", serde_json::to_string_pretty(&manifest)?); + Ok(()) +} + +/// Abbreviate a commit for a table. +fn short(commit: &str) -> &str { + &commit[..commit.len().min(12)] +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn commits_are_abbreviated_without_panicking_on_short_input() { + assert_eq!( + short("8f2a1c9e0b7d4f6a3e5c2b8d9f0a1c3e5b7d9f01"), + "8f2a1c9e0b7d" + ); + assert_eq!(short("abc"), "abc"); + assert_eq!(short(""), ""); + } +} diff --git a/crates/rustingface/src/cmd/mod.rs b/crates/rustingface/src/cmd/mod.rs new file mode 100644 index 0000000..5cd53fd --- /dev/null +++ b/crates/rustingface/src/cmd/mod.rs @@ -0,0 +1,69 @@ +//! Subcommand implementations. +//! +//! Each one builds the same registry the server would and drives the same core +//! operations, so the CLI cannot drift from the service's behaviour. + +pub mod doctor; +pub mod fetch; +pub mod gc; +pub mod list; +pub mod pin; +pub mod rm; +pub mod serve; + +use anyhow::{Result, bail}; +use rustingface_entities::repo::RepoId; + +/// Split a `namespace/name[@revision]` argument. +pub fn split_repo(argument: &str) -> Result<(RepoId, Option)> { + let (repo, revision) = match argument.split_once('@') { + Some((repo, revision)) if !revision.is_empty() => (repo, Some(revision.to_owned())), + Some((_, _)) => bail!("{argument:?} ends with `@` but names no revision"), + None => (argument, None), + }; + Ok((repo.parse()?, revision)) +} + +/// Render a byte count the way an operator reads it. +pub fn human_bytes(bytes: u64) -> String { + const UNITS: [&str; 5] = ["B", "KiB", "MiB", "GiB", "TiB"]; + let mut value = bytes as f64; + let mut unit = 0; + while value >= 1024.0 && unit < UNITS.len() - 1 { + value /= 1024.0; + unit += 1; + } + if unit == 0 { + format!("{bytes} B") + } else { + format!("{value:.1} {}", UNITS[unit]) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_repo_argument_splits_on_the_revision_marker() { + let (repo, revision) = split_repo("Qwen/Qwen3-32B@main").unwrap(); + assert_eq!(repo.to_string(), "Qwen/Qwen3-32B"); + assert_eq!(revision.as_deref(), Some("main")); + + let (repo, revision) = split_repo("gpt2").unwrap(); + assert_eq!(repo.to_string(), "gpt2"); + assert_eq!(revision, None); + } + + #[test] + fn a_trailing_marker_with_no_revision_is_an_error() { + assert!(split_repo("gpt2@").is_err()); + } + + #[test] + fn byte_counts_read_the_way_an_operator_expects() { + assert_eq!(human_bytes(512), "512 B"); + assert_eq!(human_bytes(1536), "1.5 KiB"); + assert_eq!(human_bytes(4 * 1024 * 1024 * 1024), "4.0 GiB"); + } +} diff --git a/crates/rustingface/src/cmd/pin.rs b/crates/rustingface/src/cmd/pin.rs new file mode 100644 index 0000000..61706d4 --- /dev/null +++ b/crates/rustingface/src/cmd/pin.rs @@ -0,0 +1,88 @@ +//! `rustingface pin` and `rustingface refresh`. +//! +//! Together these are the only ways a ref moves. That is the point of freeze +//! mode: upstream cannot change what a fleet resolves to, an operator can. + +use anyhow::{Context, Result, bail}; +use rustingface_entities::config::Config; +use rustingface_entities::refs::{PinnedBy, is_commit_sha}; +use rustingface_entities::repo::RepoType; + +use crate::cmd::split_repo; +use crate::wire; + +/// Set a ref to a commit explicitly. +pub async fn run( + config: Config, + repo_type: RepoType, + repo: &str, + ref_name: &str, + commit: &str, +) -> Result<()> { + if !is_commit_sha(commit) { + bail!("{commit:?} is not a 40-character commit SHA"); + } + let (repo_id, _) = split_repo(repo)?; + let registry = wire::registry(&config)?; + + let before = registry.load_ref(repo_type, &repo_id, ref_name).await?; + registry + .write_ref(repo_type, &repo_id, ref_name, commit, PinnedBy::Operator) + .await + .with_context(|| format!("pinning {repo_id}@{ref_name}"))?; + registry.ensure_repo_record(repo_type, &repo_id).await?; + + match before { + Some(before) if before.commit == commit => { + println!("{repo_id}@{ref_name} already at {commit}") + } + Some(before) => println!("{repo_id}@{ref_name}: {} -> {commit}", before.commit), + None => println!("{repo_id}@{ref_name} -> {commit}"), + } + + if registry + .load_manifest(repo_type, &repo_id, commit) + .await? + .is_none() + { + println!( + "note: no manifest is stored for {commit}. \ + Run `rustingface fetch {repo_id}@{commit}` to populate it before sealing." + ); + } + Ok(()) +} + +/// Re-resolve a ref upstream and repin it. +pub async fn refresh( + config: Config, + repo_type: RepoType, + repo: &str, + ref_name: Option<&str>, +) -> Result<()> { + let (repo_id, revision) = split_repo(repo)?; + let ref_name = ref_name + .map(str::to_owned) + .or(revision) + .unwrap_or_else(|| "main".into()); + let registry = wire::registry(&config)?; + + let (before, after) = registry + .refresh(repo_type, &repo_id, &ref_name) + .await + .with_context(|| format!("refreshing {repo_id}@{ref_name}"))?; + + match before { + Some(before) if before == after => println!("{repo_id}@{ref_name} unchanged at {after}"), + Some(before) => { + println!("{repo_id}@{ref_name}: {before} -> {after}"); + println!( + "note: the previous manifest for {before} is untouched, so anything pinned to \ + that commit still resolves. Run `rustingface fetch {repo_id}@{after}` to store \ + the new revision." + ); + } + None => println!("{repo_id}@{ref_name} -> {after}"), + } + Ok(()) +} diff --git a/crates/rustingface/src/cmd/rm.rs b/crates/rustingface/src/cmd/rm.rs new file mode 100644 index 0000000..59259b6 --- /dev/null +++ b/crates/rustingface/src/cmd/rm.rs @@ -0,0 +1,51 @@ +//! `rustingface rm` — detach metadata. +//! +//! Blobs survive. Reclaiming them is `gc`, a second explicit step that can be +//! dry-run first, so nothing deletes bytes without an operator typing `gc`. + +use anyhow::{Result, bail}; +use rustingface_entities::config::Config; +use rustingface_entities::repo::RepoType; + +use crate::cmd::split_repo; +use crate::wire; + +/// Remove a repo, or one revision of it. +pub async fn run(config: Config, repo_type: RepoType, repo: &str, confirmed: bool) -> Result<()> { + let (repo_id, revision) = split_repo(repo)?; + let registry = wire::registry(&config)?; + + if !confirmed { + let summary = registry.summarise(repo_type, &repo_id).await?; + match &revision { + Some(revision) => println!("would remove {repo_id}@{revision}"), + None => println!( + "would remove {repo_id}: {} revisions, {} files, {} refs", + summary.commits.len(), + summary.files, + summary.refs.len() + ), + } + bail!("refusing to remove anything without --yes"); + } + + let removed = registry + .remove(repo_type, &repo_id, revision.as_deref()) + .await?; + + for key in &removed.manifests { + println!("removed manifest {key}"); + } + for key in &removed.refs { + println!("removed ref {key}"); + } + if removed.repo_record { + println!("removed repo record for {repo_id}"); + } + if removed.manifests.is_empty() && removed.refs.is_empty() && !removed.repo_record { + println!("nothing to remove for {repo_id}"); + return Ok(()); + } + println!("blobs are untouched; run `rustingface gc --dry-run` to see what is now reclaimable"); + Ok(()) +} diff --git a/crates/rustingface/src/cmd/serve.rs b/crates/rustingface/src/cmd/serve.rs new file mode 100644 index 0000000..d9a2edc --- /dev/null +++ b/crates/rustingface/src/cmd/serve.rs @@ -0,0 +1,70 @@ +//! `rustingface serve`. + +use anyhow::{Context, Result}; +use rustingface_api::auth::Tokens; +use rustingface_api::{AppState, router, telemetry}; +use rustingface_entities::config::Config; +use tokio::net::TcpListener; +use tokio::signal::unix::{SignalKind, signal}; +use tracing::info; + +use crate::{config::read_optional_secret, wire}; + +/// Run the HTTP service until `SIGTERM` or `SIGINT`. +pub async fn run(config: Config) -> Result<()> { + let registry = wire::registry(&config)?; + + let token_file = + read_optional_secret(config.auth.token_file.as_ref()).context("auth.token_file")?; + let tokens = Tokens::new(config.auth.mode, token_file.as_deref())?; + + let metrics = if config.observability.metrics { + Some(telemetry::install()?) + } else { + None + }; + + let listener = TcpListener::bind(&config.server.listen) + .await + .with_context(|| format!("binding {}", config.server.listen))?; + let local = listener.local_addr()?; + + info!( + listen = %local, + bucket = %config.storage.bucket, + upstream = if config.upstream.enabled { config.upstream.endpoint.as_str() } else { "sealed" }, + delivery = ?config.storage.blob_delivery, + auth = ?config.auth.mode, + "rustingface listening" + ); + + let app = router(AppState::new(registry, tokens.into(), metrics)); + + axum::serve(listener, app) + .with_graceful_shutdown(shutdown()) + .await + .context("serving") +} + +/// Stop accepting on `SIGTERM`/`SIGINT` and let in-flight requests drain. +/// +/// Detached transfers are not requests and are not waited on here: the process +/// exiting mid-upload leaves an orphan blob, which is safe and reclaimable, +/// while a manifest is only ever written after an upload completes. +async fn shutdown() { + let mut term = match signal(SignalKind::terminate()) { + Ok(signal) => signal, + Err(err) => { + tracing::error!(%err, "could not install the SIGTERM handler"); + return; + } + }; + tokio::select! { + _ = term.recv() => info!("SIGTERM received; draining"), + result = tokio::signal::ctrl_c() => { + if result.is_ok() { + info!("interrupt received; draining"); + } + } + } +} diff --git a/crates/rustingface/src/config.rs b/crates/rustingface/src/config.rs new file mode 100644 index 0000000..ce4a348 --- /dev/null +++ b/crates/rustingface/src/config.rs @@ -0,0 +1,139 @@ +//! Loading configuration and the secrets it points at. +//! +//! Layering is file → environment → CLI, and every credential arrives from a +//! file path rather than inline, so `pass` or systemd credentials can own it +//! and it never appears in the config, the environment, or a log line. + +use std::path::{Path, PathBuf}; + +use anyhow::{Context, Result, bail}; +use figment::Figment; +use figment::providers::{Env, Format, Toml}; +use rustingface_entities::config::{Config, ENV_PREFIX}; + +/// Load and validate configuration. +/// +/// A missing file is only acceptable when the caller did not name one: an +/// explicit `--config` that does not exist is an error, not a silent fallback +/// to defaults that would point at the wrong bucket. +pub fn load(path: Option<&Path>, explicit: bool) -> Result { + let mut figment = Figment::new(); + if let Some(path) = path { + if explicit && !path.exists() { + bail!("{} does not exist", path.display()); + } + if path.exists() { + figment = figment.merge(Toml::file(path)); + } + } + let config: Config = figment + .merge(Env::prefixed(ENV_PREFIX).split("__")) + .extract() + .context("loading configuration")?; + config.validate()?; + Ok(config) +} + +/// Read a secret from a file, trimming the trailing newline an editor adds. +/// +/// systemd's `LoadCredential=` places credentials under +/// `$CREDENTIALS_DIRECTORY` with names rather than paths, so a configured path +/// that is absent is retried there by basename. That lets one config file work +/// both under systemd and when run by hand. +pub fn read_secret(path: &Path) -> Result { + if path.exists() { + return read_trimmed(path); + } + if let Some(dir) = std::env::var_os("CREDENTIALS_DIRECTORY") + && let Some(name) = path.file_name() + { + let candidate = PathBuf::from(dir).join(name); + if candidate.exists() { + return read_trimmed(&candidate); + } + } + bail!( + "{} does not exist, and no systemd credential of that name was provided", + path.display() + ) +} + +/// Read a secret if its path is configured at all. +pub fn read_optional_secret(path: Option<&PathBuf>) -> Result> { + match path { + None => Ok(None), + Some(path) => read_secret(path).map(Some).map(|s| { + // An empty credential file is almost always a deployment mistake + // — a rendered template that got no value — and it fails far more + // usefully here than as a 401 from upstream. + match s { + Some(v) if v.is_empty() => None, + other => other, + } + }), + } +} + +fn read_trimmed(path: &Path) -> Result { + let raw = + std::fs::read_to_string(path).with_context(|| format!("reading {}", path.display()))?; + Ok(raw.trim().to_owned()) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Write; + + fn tempdir() -> PathBuf { + let dir = std::env::temp_dir().join(format!( + "rustingface-config-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + std::fs::create_dir_all(&dir).unwrap(); + dir + } + + #[test] + fn a_missing_explicit_config_is_an_error_not_a_silent_default() { + let missing = tempdir().join("nope.toml"); + assert!(load(Some(&missing), true).is_err()); + assert!( + load(Some(&missing), false).is_ok(), + "the default path may be absent; defaults then apply" + ); + } + + #[test] + fn a_secret_file_loses_its_trailing_newline() { + let dir = tempdir(); + let path = dir.join("token"); + let mut f = std::fs::File::create(&path).unwrap(); + writeln!(f, "hf_secret_value").unwrap(); + assert_eq!(read_secret(&path).unwrap(), "hf_secret_value"); + } + + #[test] + fn an_empty_secret_file_reads_as_no_secret_at_all() { + let dir = tempdir(); + let path = dir.join("empty"); + std::fs::write(&path, "\n").unwrap(); + assert_eq!(read_optional_secret(Some(&path)).unwrap(), None); + } + + #[test] + fn a_systemd_credential_stands_in_for_a_missing_path() { + let dir = tempdir(); + std::fs::write(dir.join("hf-token"), "from-systemd\n").unwrap(); + // SAFETY: single-threaded test process, and the variable is read back + // only by this test. + unsafe { std::env::set_var("CREDENTIALS_DIRECTORY", &dir) }; + let configured = PathBuf::from("/etc/rustingface/hf-token"); + assert_eq!(read_secret(&configured).unwrap(), "from-systemd"); + unsafe { std::env::remove_var("CREDENTIALS_DIRECTORY") }; + } +} diff --git a/crates/rustingface/src/logging.rs b/crates/rustingface/src/logging.rs new file mode 100644 index 0000000..16001ac --- /dev/null +++ b/crates/rustingface/src/logging.rs @@ -0,0 +1,45 @@ +//! Log setup. +//! +//! Structured JSON when systemd is listening, pretty text when a person is. +//! Detection is by configuration first and `JOURNAL_STREAM` second, so a +//! deliberate `log_format = "text"` is not overridden by the environment. + +use rustingface_entities::config::{Config, LogFormat}; +use tracing_subscriber::EnvFilter; + +/// A minimal subscriber for the window before configuration is loaded, so a +/// failure to parse the config is still reported through the same channel. +pub fn init_bootstrap(interactive: bool) { + let _ = tracing_subscriber::fmt() + .with_env_filter(filter(interactive)) + .with_writer(std::io::stderr) + .try_init(); +} + +/// Install the configured subscriber. +/// +/// A no-op when the bootstrap subscriber is already installed, which is the +/// normal case; the bootstrap format is close enough that swapping it out +/// mid-process would cost more than it buys. +pub fn init(config: &Config, interactive: bool) { + let json = !interactive + && (config.observability.log_format == LogFormat::Json + || std::env::var_os("JOURNAL_STREAM").is_some()); + if json { + let _ = tracing_subscriber::fmt() + .json() + .with_env_filter(filter(interactive)) + .with_writer(std::io::stderr) + .try_init(); + } +} + +/// The default filter, quieter for the CLI than for the daemon. +fn filter(interactive: bool) -> EnvFilter { + let default = if interactive { + "rustingface=warn,warn" + } else { + "rustingface=info,rustingface_core=info,rustingface_data=info,rustingface_api=info,warn" + }; + EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(default)) +} diff --git a/crates/rustingface/src/main.rs b/crates/rustingface/src/main.rs new file mode 100644 index 0000000..0afffa7 --- /dev/null +++ b/crates/rustingface/src/main.rs @@ -0,0 +1,185 @@ +//! rustingface: a sovereign, HuggingFace-compatible model registry. +//! +//! One binary, two jobs. `serve` runs the HTTP service; the other subcommands +//! are the operator interface to the bucket. They share the same configuration +//! and the same code paths, so what the CLI does to the bucket is exactly what +//! the server would have done. + +mod cmd; +mod config; +mod logging; +mod wire; + +use std::path::PathBuf; + +use anyhow::Result; +use clap::{Parser, Subcommand}; +use rustingface_entities::config::DEFAULT_CONFIG_PATH; +use rustingface_entities::repo::RepoType; + +/// Command-line interface. +#[derive(Debug, Parser)] +#[command(name = "rustingface", version, about, long_about = None)] +struct Cli { + /// Configuration file. + #[arg(short, long, global = true, env = "RUSTINGFACE_CONFIG")] + config: Option, + + #[command(subcommand)] + command: Option, +} + +/// The subcommands. +#[derive(Debug, Subcommand)] +enum Command { + /// Run the HTTP service. The default when no subcommand is given. + Serve, + + /// Warm the bucket ahead of need. + Fetch { + /// `namespace/name[@revision]`. + repo: String, + /// Restrict to these paths. Without it, every file in the revision's + /// tree is fetched. + #[arg(long = "files", num_args = 1..)] + files: Vec, + #[arg(long, value_enum, default_value_t = RepoKind::Models)] + repo_type: RepoKind, + }, + + /// Set a ref to a commit explicitly. + Pin { + repo: String, + r#ref: String, + commit: String, + #[arg(long, value_enum, default_value_t = RepoKind::Models)] + repo_type: RepoKind, + }, + + /// Re-resolve a ref upstream and repin it. + Refresh { + repo: String, + /// Defaults to `main`. + r#ref: Option, + #[arg(long, value_enum, default_value_t = RepoKind::Models)] + repo_type: RepoKind, + }, + + /// List stored repos, revisions and sizes. + List { + #[arg(long, value_enum)] + repo_type: Option, + }, + + /// Print a manifest. + Show { + /// `namespace/name[@revision]`. + repo: String, + #[arg(long, value_enum, default_value_t = RepoKind::Models)] + repo_type: RepoKind, + }, + + /// Remove refs and manifests. Blobs are left for `gc`. + Rm { + /// `namespace/name[@revision]`. + repo: String, + #[arg(long, value_enum, default_value_t = RepoKind::Models)] + repo_type: RepoKind, + /// Required, because this is the step that makes data unreachable. + #[arg(long)] + yes: bool, + }, + + /// Delete blobs no manifest references. + Gc { + /// Report what would be deleted without deleting it. + #[arg(long)] + dry_run: bool, + }, + + /// Re-read stored blobs and check them against their digests. + Verify { + /// Restrict to one repository. + repo: Option, + #[arg(long, value_enum, default_value_t = RepoKind::Models)] + repo_type: RepoKind, + }, + + /// Check configuration, storage reachability and permissions. + Doctor, +} + +/// Repo type as a CLI value. +#[derive(Debug, Clone, Copy, clap::ValueEnum)] +enum RepoKind { + Models, + Datasets, +} + +impl From for RepoType { + fn from(kind: RepoKind) -> Self { + match kind { + RepoKind::Models => Self::Models, + RepoKind::Datasets => Self::Datasets, + } + } +} + +fn main() -> Result<()> { + let cli = Cli::parse(); + let explicit = cli.config.is_some(); + let path = cli + .config + .unwrap_or_else(|| PathBuf::from(DEFAULT_CONFIG_PATH)); + let command = cli.command.unwrap_or(Command::Serve); + + // The server logs structurally for journald; the CLI writes to a terminal + // an operator is reading, so it gets human output and its results go to + // stdout rather than into a log line. + let interactive = !matches!(command, Command::Serve); + let config = { + logging::init_bootstrap(interactive); + config::load(Some(&path), explicit)? + }; + logging::init(&config, interactive); + + let runtime = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build()?; + + runtime.block_on(async move { + match command { + Command::Serve => cmd::serve::run(config).await, + Command::Fetch { + repo, + files, + repo_type, + } => cmd::fetch::run(config, repo_type.into(), &repo, &files).await, + Command::Pin { + repo, + r#ref, + commit, + repo_type, + } => cmd::pin::run(config, repo_type.into(), &repo, &r#ref, &commit).await, + Command::Refresh { + repo, + r#ref, + repo_type, + } => cmd::pin::refresh(config, repo_type.into(), &repo, r#ref.as_deref()).await, + Command::List { repo_type } => cmd::list::run(config, repo_type.map(Into::into)).await, + Command::Show { repo, repo_type } => { + cmd::list::show(config, repo_type.into(), &repo).await + } + Command::Rm { + repo, + repo_type, + yes, + } => cmd::rm::run(config, repo_type.into(), &repo, yes).await, + Command::Gc { dry_run } => cmd::gc::run(config, dry_run).await, + Command::Verify { repo, repo_type } => { + cmd::gc::verify(config, repo_type.into(), repo.as_deref()).await + } + Command::Doctor => cmd::doctor::run(config).await, + } + }) +} diff --git a/crates/rustingface/src/wire.rs b/crates/rustingface/src/wire.rs new file mode 100644 index 0000000..17647db --- /dev/null +++ b/crates/rustingface/src/wire.rs @@ -0,0 +1,48 @@ +//! Wiring: turn a validated [`Config`] into a running [`Registry`]. +//! +//! Every binary subcommand needs the same stack, so it is assembled once here +//! rather than in each command. + +use std::sync::Arc; + +use anyhow::{Context, Result}; +use rustingface_core::Registry; +use rustingface_core::ports::{Store, Upstream}; +use rustingface_data::{HubClient, ObjectStoreAdapter, SealedUpstream}; +use rustingface_entities::config::Config; +use tracing::warn; + +use crate::config::read_optional_secret; + +/// Build the registry described by `config`. +pub fn registry(config: &Config) -> Result { + let access_key_id = read_optional_secret(config.storage.access_key_id_file.as_ref()) + .context("storage.access_key_id_file")?; + let secret_access_key = read_optional_secret(config.storage.secret_access_key_file.as_ref()) + .context("storage.secret_access_key_file")?; + + if config.storage.local_path.is_some() { + warn!( + "storage.local_path is set: state lives on this host's filesystem rather than in a \ + bucket, so losing the host loses the registry. This forfeits state portability and \ + is intended for tests and trials only." + ); + } + + let store: Arc = Arc::new( + ObjectStoreAdapter::new(&config.storage, access_key_id, secret_access_key) + .context("configuring object storage")?, + ); + + let upstream: Arc = if config.upstream.enabled { + let token = read_optional_secret(config.upstream.token_file.as_ref()) + .context("upstream.token_file")?; + Arc::new(HubClient::new(&config.upstream, token).context("configuring upstream client")?) + } else { + // Sealed: no client is constructed at all, so there is no code path + // that could reach the network even if a bug tried to. + Arc::new(SealedUpstream::new(config.upstream.endpoint.clone())) + }; + + Registry::new(store, upstream, config.clone()).context("assembling the registry") +} diff --git a/rust-toolchain.toml b/rust-toolchain.toml new file mode 100644 index 0000000..d80fdf0 --- /dev/null +++ b/rust-toolchain.toml @@ -0,0 +1,4 @@ +[toolchain] +channel = "stable" +components = ["rustfmt", "clippy"] +targets = ["x86_64-unknown-linux-musl"] diff --git a/script/infra-setup.sh b/script/infra-setup.sh new file mode 100755 index 0000000..51d7811 --- /dev/null +++ b/script/infra-setup.sh @@ -0,0 +1,295 @@ +#!/usr/bin/env bash +# +# One-time host provisioning for rustingface. +# +# Run by an operator from a workstation with full sudo (not by CI, and not as +# the scoped gitea_ci account). Idempotent: re-running with nothing to change +# is a no-op. Skips past unreachable hosts so one offline node does not block +# the rest. +# +# script/infra-setup.sh # provision everything +# script/infra-setup.sh --skip-minio # leave the bucket and credentials alone +# +# What it does: +# 1. bob — gitea_ci user, runner key, scoped sudoers, SELinux port label +# 2. caveman — the rustingface bucket and a scoped MinIO service account +# 3. hanzalova— rf.internal cert, nginx vhost, step@ renewal timer +# +# Application config is NOT shipped here. The deploy workflow renders it from +# Gitea secrets on every deploy. + +set -euo pipefail + +readonly APP=rustingface +readonly PORT=20482 +readonly SERVICE_HOST=bob.hanzalova.internal +readonly PROXY_HOST=hanzalova.internal +readonly MINIO_HOST=caveman.kosherinata.internal +readonly MINIO_URL=http://caveman.kosherinata.internal:9000 +readonly BUCKET=rustingface +readonly VHOST=rf +readonly RUNNER_PUBKEY="${RUNNER_PUBKEY:-$HOME/.ssh/id_gitea_ci.pub}" +readonly PROVISIONER_PASSWORD="${PROVISIONER_PASSWORD:-$HOME/.step/secrets/provisioner}" +readonly ROOT_CA=/etc/pki/ca-trust/source/anchors/root-internal.pem + +SKIP_MINIO=0 +for arg in "$@"; do + case "$arg" in + --skip-minio) SKIP_MINIO=1 ;; + -h|--help) sed -n '2,22p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;; + *) echo "unknown argument: $arg" >&2; exit 2 ;; + esac +done + +info() { printf '\033[1;34m==>\033[0m %s\n' "$*"; } +ok() { printf ' \033[32mok\033[0m %s\n' "$*"; } +warn() { printf ' \033[33m!!\033[0m %s\n' "$*" >&2; } +die() { printf '\033[1;31m!!\033[0m %s\n' "$*" >&2; exit 1; } + +# Returns non-zero (visibly) when a host cannot be reached, so the caller can +# skip it deliberately rather than by swallowing an error. +reachable() { + local host=$1 + if ssh -o BatchMode=yes -o ConnectTimeout=8 -o StrictHostKeyChecking=accept-new \ + "$host" true; then + return 0 + fi + warn "$host is unreachable; skipping it. Re-run this script when it is back." + return 1 +} + +# --------------------------------------------------------------------------- +# 1. The service host: bob +# --------------------------------------------------------------------------- + +provision_service_host() { + info "service host $SERVICE_HOST" + + [ -f "$RUNNER_PUBKEY" ] || die "runner public key not found at $RUNNER_PUBKEY. +The keypair is maintained at ~/.ssh/id_gitea_ci on the operator workstation and is +shared by every project's deploy. Copy it from there; do not generate a new one, or +every other project's deploy breaks." + + # The gitea_ci account, its key, and journal access. Creating the user is + # idempotent via `useradd` guarded on `id`. + ssh "$SERVICE_HOST" 'sudo bash -s' <<'REMOTE' +set -euo pipefail +if id gitea_ci >/dev/null 2>&1; then + echo " gitea_ci exists" +else + useradd --system --create-home --home-dir /var/lib/gitea_ci --shell /bin/bash gitea_ci + echo " created gitea_ci" +fi +install -d -o gitea_ci -g gitea_ci -m 0700 /var/lib/gitea_ci/.ssh +usermod -aG systemd-journal gitea_ci +REMOTE + ok "gitea_ci account" + + # shellcheck disable=SC2029 # the key must expand locally + ssh "$SERVICE_HOST" "sudo bash -c ' + umask 077 + touch /var/lib/gitea_ci/.ssh/authorized_keys + grep -qxF \"$(cat "$RUNNER_PUBKEY")\" /var/lib/gitea_ci/.ssh/authorized_keys \ + || echo \"$(cat "$RUNNER_PUBKEY")\" >> /var/lib/gitea_ci/.ssh/authorized_keys + chown -R gitea_ci:gitea_ci /var/lib/gitea_ci/.ssh + chmod 600 /var/lib/gitea_ci/.ssh/authorized_keys'" + ok "runner key installed" + + # Scoped sudoers: exactly the commands the deploy runs, pinned to literal + # destinations, and verified before it can lock anyone out. + ssh "$SERVICE_HOST" 'sudo bash -s' < /etc/sudoers.d/.${APP}_gitea_ci.new <<'SUDOERS' +gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /usr/local/bin/${APP} +gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/${APP}/config.toml +gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/${APP}/s3-access-key +gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/${APP}/s3-secret-key +gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/${APP}/hf-token +gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/systemd/system/${APP}.service +gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/sysusers.d/${APP}.conf +gitea_ci ALL=(root) NOPASSWD: /usr/bin/rsync * /etc/firewalld/services/${APP}.xml +gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemd-sysusers +gitea_ci ALL=(root) NOPASSWD: /usr/bin/install -d -o root -g ${APP} -m 0750 /etc/${APP} +gitea_ci ALL=(root) NOPASSWD: /usr/bin/chown -R root\:${APP} /etc/${APP} +gitea_ci ALL=(root) NOPASSWD: /usr/bin/chmod 0640 /etc/${APP}/config.toml /etc/${APP}/s3-access-key /etc/${APP}/s3-secret-key /etc/${APP}/hf-token +gitea_ci ALL=(root) NOPASSWD: /usr/bin/chmod 0755 /usr/local/bin/${APP} +gitea_ci ALL=(root) NOPASSWD: /usr/sbin/restorecon -R /usr/local/bin/${APP} /etc/${APP} +gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl daemon-reload +gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl enable ${APP}.service +gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl restart ${APP}.service +gitea_ci ALL=(root) NOPASSWD: /usr/bin/systemctl is-active ${APP}.service +gitea_ci ALL=(root) NOPASSWD: /usr/bin/firewall-cmd --reload +gitea_ci ALL=(root) NOPASSWD: /usr/bin/firewall-cmd --get-default-zone +gitea_ci ALL=(root) NOPASSWD: /usr/bin/firewall-cmd --zone=* --query-service=${APP} +gitea_ci ALL=(root) NOPASSWD: /usr/bin/firewall-cmd --permanent --zone=* --add-service=${APP} +gitea_ci ALL=(root) NOPASSWD: /usr/bin/firewall-cmd --zone=* --add-service=${APP} +SUDOERS +chmod 0440 /etc/sudoers.d/.${APP}_gitea_ci.new +# Verify before moving it into place: a typo in a sudoers file that is already +# installed can lock the host out of sudo entirely. +visudo -cf /etc/sudoers.d/.${APP}_gitea_ci.new +mv /etc/sudoers.d/.${APP}_gitea_ci.new /etc/sudoers.d/${APP}_gitea_ci +REMOTE + ok "scoped sudoers installed and visudo-verified" + + # SELinux: the daemon cannot bind an unlabelled port. This must happen + # before the first service start, which is why it lives here and not in + # the deploy workflow. + ssh "$SERVICE_HOST" "sudo bash -s" < + mc mb local/$BUCKET + mc admin user svcacct add local --name rustingface + +Put the resulting access key and secret in the Gitea repo secrets +S3_ACCESS_KEY_ID and S3_SECRET_ACCESS_KEY." + return 0 + fi + + warn "bucket and service-account creation needs MinIO root credentials, which +this script deliberately does not read. Run on $MINIO_HOST as an operator: + + mc alias set local $MINIO_URL \"\$MINIO_ROOT_USER\" \"\$MINIO_ROOT_PASSWORD\" + mc mb --ignore-existing local/$BUCKET + mc admin user svcacct add local \"\$MINIO_ROOT_USER\" --name rustingface + +then set the Gitea repo secrets S3_ACCESS_KEY_ID and S3_SECRET_ACCESS_KEY. +Re-run this script with --skip-minio once that is done." +} + +# --------------------------------------------------------------------------- +# 3. The edge proxy: hanzalova +# --------------------------------------------------------------------------- + +provision_proxy() { + info "edge proxy $PROXY_HOST" + + local cert=/etc/nginx/tls/cert/${VHOST}.internal.pem + local key=/etc/nginx/tls/key/${VHOST}.internal.pem + + # `step certificate verify` checks chain and expiry, not the name, which is + # enough to decide whether a fresh mint is needed. + local state + state=$(ssh "$PROXY_HOST" "if sudo test -f $cert && sudo step certificate verify $cert --roots $ROOT_CA; then echo valid; else echo missing; fi" \ + | tail -1) + + if [ "$state" != valid ]; then + [ -f "$PROVISIONER_PASSWORD" ] \ + || die "provisioner password not found at $PROVISIONER_PASSWORD; cannot mint ${VHOST}.internal" + rsync -az --rsync-path='sudo rsync' --chmod=0600 \ + "$PROVISIONER_PASSWORD" "$PROXY_HOST:/tmp/${VHOST}-provisioner" + # Remove the credential whether or not the mint succeeds, then + # propagate the failure. + ssh "$PROXY_HOST" " + set -euo pipefail + sudo mkdir -p /etc/nginx/tls/cert /etc/nginx/tls/key + rc=0 + sudo step ca certificate --force \ + --provisioner lair \ + --provisioner-password-file /tmp/${VHOST}-provisioner \ + --ca-url https://ca.internal \ + --root $ROOT_CA \ + --san ${VHOST}.internal \ + ${VHOST}.internal $cert $key || rc=\$? + sudo rm -f /tmp/${VHOST}-provisioner + [ \$rc -eq 0 ] || { echo 'minting ${VHOST}.internal failed' >&2; exit \$rc; } + sudo chown root:root $cert $key + sudo chmod 0644 $cert + sudo chmod 0640 $key + sudo setfacl -m u:nginx:r $key" + ok "minted ${VHOST}.internal" + else + ok "${VHOST}.internal cert is present and valid" + fi + + ssh "$PROXY_HOST" "sudo systemctl enable --now step@${VHOST}.timer" + ok "renewal timer enabled" + + # The vhost references the cert paths, so it is only installed after the + # cert exists: nginx -t fails on a missing ssl_certificate and that blocks + # every unrelated reload on the host. + rsync -az --rsync-path='sudo rsync' --mkpath \ + asset/nginx/${VHOST}.internal.conf \ + "$PROXY_HOST:/etc/nginx/sites-available/${VHOST}.internal.conf" + ssh "$PROXY_HOST" " + set -euo pipefail + sudo ln -sfn ../sites-available/${VHOST}.internal.conf \ + /etc/nginx/sites-enabled/${VHOST}.internal.conf + sudo nginx -t + sudo systemctl reload nginx" + ok "vhost installed and nginx reloaded" + + # A reload is a request, not a guarantee: nginx keeps its old cycle if it + # cannot rebind, and then serves the certificate it loaded then. Compare + # what is actually on the wire against what is on disk, by serial. + local served disk + served=$(echo | openssl s_client -servername ${VHOST}.internal -connect ${PROXY_HOST}:443 2>/dev/null \ + | openssl x509 -noout -serial || echo "serial=unavailable") + disk=$(ssh "$PROXY_HOST" "sudo openssl x509 -noout -serial -in $cert") + if [ "$served" = "$disk" ]; then + ok "the certificate on the wire matches the one on disk" + else + warn "nginx is serving a different certificate than the one on disk + wire: $served + disk: $disk +A reload was requested and reported success, but nginx kept its previous cycle. +With 24h certs this puts an expired certificate on the wire within a day. +Investigate the bind failure in the nginx error log, then restart nginx." + fi + + info "DNS" + echo " Add the split-horizon record if it is not already present:" + echo " opn-cli --config ~/.opn-cli/hanzalova.yml unbound host create \\" + echo " --hostname ${VHOST} --domain internal --server 10.6.0.46 \\" + echo " --description 'rustingface via hanzalova reverse proxy (bob backend)'" + echo " opn-cli --config ~/.opn-cli/kosherinata.yml unbound host create \\" + echo " --hostname ${VHOST} --domain internal --server 10.6.0.46 \\" + echo " --description 'rustingface via hanzalova reverse proxy (bob backend)'" + echo " Never give ${VHOST}.internal a public / Cloudflare record." +} + +# --------------------------------------------------------------------------- + +main() { + cd "$(dirname "$0")/.." + + if reachable "$SERVICE_HOST"; then + provision_service_host + fi + + if [ "$SKIP_MINIO" -eq 0 ] && reachable "$MINIO_HOST"; then + provision_bucket + fi + + if reachable "$PROXY_HOST"; then + provision_proxy + fi + + info "done" + echo " Next: set the Gitea repo secrets (RSYNC_SSH_KEY, S3_ACCESS_KEY_ID," + echo " S3_SECRET_ACCESS_KEY, HF_TOKEN) and push to main to deploy." +} + +main "$@" diff --git a/test/conformance/conformance.py b/test/conformance/conformance.py new file mode 100755 index 0000000..3866eb8 --- /dev/null +++ b/test/conformance/conformance.py @@ -0,0 +1,555 @@ +#!/usr/bin/env python3 +"""Conformance suite: test against real clients, not against the document. + +The spec is only as good as the client compatibility, so this drives an +unmodified ``huggingface_hub`` (and the ``hf`` CLI) through rustingface and +checks the behaviour a stack pinned to a model revision actually depends on. + +The test that matters is the offline one. Its steps here mirror spec §11: + +1. fresh bucket, ``HF_ENDPOINT`` set, pull a model, confirm bytes landed; +2. cut rustingface off from upstream; +3. wipe every client-side cache; +4. pull again -- it must succeed identically, with no upstream traffic; +5. kill rustingface, start a *new* process against the same bucket, repeat. + +Step 2 differs from the spec in one honest respect. The spec null-routes +``huggingface.co`` at the site router and verifies there, which CI cannot do. +This suite instead does something strictly stronger for the sealed run -- +``upstream.enabled = false`` means no upstream client is constructed at all, so +there is no code path that *could* reach the network -- and, for the +"upstream enabled but unreachable" run, points the endpoint at a closed port +and asserts that a miss fails loudly rather than hanging. Run the real +null-route test by hand before trusting a deployment. + +Usage:: + + python3 test/conformance/conformance.py --binary target/release/rustingface + +Set ``--s3-endpoint``/``--s3-bucket`` (plus ``AWS_ACCESS_KEY_ID`` and +``AWS_SECRET_ACCESS_KEY``) to run against a real bucket instead of a directory. +""" + +from __future__ import annotations + +import argparse +import json +import os +import shutil +import socket +import subprocess +import sys +import tempfile +import time +import urllib.error +import urllib.request +from dataclasses import dataclass, field +from pathlib import Path + +# A small, ungated, stable model. Small enough that CI can pull it repeatedly, +# real enough that it exercises the LFS bridge and a safetensors file. +REPO = os.environ.get("CONFORMANCE_REPO", "hf-internal-testing/tiny-random-gpt2") +REVISION = os.environ.get("CONFORMANCE_REVISION", "main") + +# The X-Error-Code values rustingface emits. These are client-internal +# constants upstream and are re-checked against the installed client below. +EXPECTED_ERROR_CODES = { + "RepoNotFound", + "RevisionNotFound", + "EntryNotFound", + "GatedRepo", +} + +# A disabled repo is the odd one out: the client dispatches DisabledRepoError +# on this exact X-Error-Message, not on an X-Error-Code, so rustingface emits +# the message verbatim. If this string moves, the mapping in +# rustingface-api/src/error.rs moves with it. +DISABLED_REPO_MESSAGE = "Access to this resource is disabled." + + +class Failure(Exception): + """A conformance check that did not hold.""" + + +@dataclass +class Results: + passed: list[str] = field(default_factory=list) + failed: list[tuple[str, str]] = field(default_factory=list) + + def check(self, name: str, fn) -> None: + print(f"\n=== {name}") + try: + fn() + except Exception as err: # noqa: BLE001 - a suite reports, it does not raise + print(f" FAILED: {err}") + self.failed.append((name, str(err))) + else: + print(" ok") + self.passed.append(name) + + def report(self) -> int: + print(f"\n{len(self.passed)} passed, {len(self.failed)} failed") + for name, err in self.failed: + print(f" FAILED {name}: {err}") + return 1 if self.failed else 0 + + +def free_port() -> int: + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + return sock.getsockname()[1] + + +def closed_port() -> int: + """A port nothing is listening on, standing in for an unreachable Hub.""" + return free_port() + + +class Instance: + """A running rustingface, configured for one phase of the suite.""" + + def __init__(self, binary: Path, workdir: Path, storage: dict, upstream: dict): + self.binary = binary + self.workdir = workdir + self.port = free_port() + self.config_path = workdir / f"config-{self.port}.toml" + self.process: subprocess.Popen | None = None + + storage_lines = "\n".join(f"{k} = {v}" for k, v in storage.items()) + upstream_lines = "\n".join(f"{k} = {v}" for k, v in upstream.items()) + self.config_path.write_text( + f""" +[server] +listen = "127.0.0.1:{self.port}" +request_timeout = "600s" + +[storage] +{storage_lines} + +[upstream] +{upstream_lines} + +[policy] +ref_resolution = "freeze" + +[observability] +metrics = true +log_format = "text" +""" + ) + + @property + def endpoint(self) -> str: + return f"http://127.0.0.1:{self.port}" + + def __enter__(self) -> "Instance": + self.process = subprocess.Popen( + [str(self.binary), "--config", str(self.config_path), "serve"], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + ) + for _ in range(100): + if self.process.poll() is not None: + raise Failure(f"rustingface exited early:\n{self.process.stdout.read()}") + try: + with urllib.request.urlopen(f"{self.endpoint}/healthz", timeout=1): + return self + except (urllib.error.URLError, ConnectionError, TimeoutError): + time.sleep(0.1) + raise Failure("rustingface did not become healthy") + + def __exit__(self, *_exc) -> None: + if self.process is not None: + self.process.terminate() + try: + self.process.wait(timeout=30) + except subprocess.TimeoutExpired: + self.process.kill() + + def metrics(self) -> str: + with urllib.request.urlopen(f"{self.endpoint}/metrics", timeout=5) as response: + return response.read().decode() + + def cli(self, *args: str) -> str: + result = subprocess.run( + [str(self.binary), "--config", str(self.config_path), *args], + capture_output=True, + text=True, + ) + if result.returncode != 0: + raise Failure(f"`rustingface {' '.join(args)}` failed:\n{result.stdout}\n{result.stderr}") + return result.stdout + + +def wipe_client_caches(home: Path) -> None: + """Remove every client-side cache, so a pull cannot be served locally.""" + for sub in ("hub", "assets", "xet"): + shutil.rmtree(home / sub, ignore_errors=True) + shutil.rmtree(home.parent / ".cache" / "huggingface", ignore_errors=True) + + +def client_env(endpoint: str, home: Path) -> dict: + env = dict(os.environ) + env.update( + { + "HF_ENDPOINT": endpoint, + "HF_HOME": str(home), + "HF_HUB_CACHE": str(home / "hub"), + # rustingface implements no Xet client and never advertises the + # capability; setting this makes the client's side explicit too. + "HF_HUB_DISABLE_XET": "1", + "HF_HUB_DISABLE_TELEMETRY": "1", + "HF_HUB_DISABLE_PROGRESS_BARS": "1", + } + ) + env.pop("HF_TOKEN", None) + return env + + +def run_client(script: str, endpoint: str, home: Path) -> str: + """Run a snippet in a fresh interpreter, so no client state leaks between phases.""" + result = subprocess.run( + [sys.executable, "-c", script], + capture_output=True, + text=True, + env=client_env(endpoint, home), + ) + if result.returncode != 0: + raise Failure(f"client failed:\n{result.stdout}\n{result.stderr}") + return result.stdout + + +DOWNLOAD_ONE = """ +import hashlib, json, sys +from huggingface_hub import hf_hub_download +path = hf_hub_download({repo!r}, "config.json", revision={revision!r}) +data = open(path, "rb").read() +print(json.dumps({{"sha256": hashlib.sha256(data).hexdigest(), "len": len(data)}})) +""" + +SNAPSHOT = """ +import hashlib, json, os +from huggingface_hub import snapshot_download +root = snapshot_download({repo!r}, revision={revision!r}) +files = {{}} +for base, _dirs, names in os.walk(root): + for name in names: + full = os.path.join(base, name) + if os.path.islink(full) and not os.path.exists(full): + continue + rel = os.path.relpath(full, root) + files[rel] = hashlib.sha256(open(full, "rb").read()).hexdigest() +print(json.dumps(files, sort_keys=True)) +""" + +MODEL_INFO = """ +import json +from huggingface_hub import HfApi +info = HfApi().model_info({repo!r}, revision={revision!r}) +print(json.dumps({{"sha": info.sha, "siblings": sorted(s.rfilename for s in info.siblings)}})) +""" + +FILE_METADATA = """ +import json +from huggingface_hub import get_hf_file_metadata, hf_hub_url +url = hf_hub_url({repo!r}, "config.json", revision={revision!r}) +meta = get_hf_file_metadata(url) +print(json.dumps({{"commit": meta.commit_hash, "etag": meta.etag, "size": meta.size}})) +""" + +MISSING_FILE = """ +import json +from huggingface_hub import hf_hub_download +from huggingface_hub.errors import EntryNotFoundError, RepositoryNotFoundError, RevisionNotFoundError +try: + hf_hub_download({repo!r}, "no-such-file.bin", revision={revision!r}) +except EntryNotFoundError: + print(json.dumps({{"raised": "EntryNotFoundError"}})) +except (RepositoryNotFoundError, RevisionNotFoundError) as err: + print(json.dumps({{"raised": type(err).__name__}})) +else: + print(json.dumps({{"raised": None}})) +""" + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--binary", type=Path, default=Path("target/release/rustingface")) + parser.add_argument("--s3-endpoint") + parser.add_argument("--s3-bucket") + parser.add_argument("--keep", action="store_true", help="keep the work directory") + args = parser.parse_args() + + if not args.binary.exists(): + print(f"{args.binary} does not exist; build it with `cargo build --release`", file=sys.stderr) + return 2 + + results = Results() + workdir = Path(tempfile.mkdtemp(prefix="rustingface-conformance-")) + print(f"work directory: {workdir}") + + bucket_dir = workdir / "bucket" + bucket_dir.mkdir() + hf_home = workdir / "hf" + + if args.s3_endpoint: + storage = { + "endpoint": json.dumps(args.s3_endpoint), + "bucket": json.dumps(args.s3_bucket or "rustingface-conformance"), + "region": '"us-east-1"', + "path_style": "true", + "multipart_part_size": '"5MiB"', + } + for var, key in (("AWS_ACCESS_KEY_ID", "access_key_id_file"), + ("AWS_SECRET_ACCESS_KEY", "secret_access_key_file")): + value = os.environ.get(var) + if not value: + print(f"{var} must be set to run against S3", file=sys.stderr) + return 2 + secret = workdir / key + secret.write_text(value) + storage[key] = json.dumps(str(secret)) + else: + storage = {"local_path": json.dumps(str(bucket_dir)), "multipart_part_size": '"5MiB"'} + + open_upstream = {"enabled": "true", "endpoint": '"https://huggingface.co"'} + unreachable_upstream = {"enabled": "true", "endpoint": f'"http://127.0.0.1:{closed_port()}"', + "connect_timeout": '"2s"'} + sealed_upstream = {"enabled": "false"} + + # ------------------------------------------------------------------ + # The client's own constants, re-checked rather than assumed. + # ------------------------------------------------------------------ + def check_error_codes() -> None: + import huggingface_hub + + print(f" huggingface_hub {huggingface_hub.__version__}") + source = Path(huggingface_hub.__file__).parent + found = set() + for path in source.rglob("*.py"): + text = path.read_text(errors="ignore") + for code in EXPECTED_ERROR_CODES: + if f'"{code}"' in text or f"'{code}'" in text: + found.add(code) + missing = EXPECTED_ERROR_CODES - found + if missing: + raise Failure( + f"this client does not mention {sorted(missing)}; the X-Error-Code strings " + "rustingface emits are version-coupled and need review" + ) + http = (source / "utils" / "_http.py").read_text() + if DISABLED_REPO_MESSAGE not in http: + raise Failure( + f"this client no longer dispatches DisabledRepoError on " + f"{DISABLED_REPO_MESSAGE!r}; rustingface-api/src/error.rs needs review" + ) + print(f" {len(found)} codes plus the disabled-repo message all match") + + results.check("X-Error-Code strings match the pinned client", check_error_codes) + + baseline: dict = {} + snapshot_baseline: dict = {} + + # ------------------------------------------------------------------ + # 1. Cold pull through rustingface. + # ------------------------------------------------------------------ + def cold_pull() -> None: + nonlocal baseline, snapshot_baseline + wipe_client_caches(hf_home) + with Instance(args.binary, workdir, storage, open_upstream) as rf: + baseline = json.loads( + run_client(DOWNLOAD_ONE.format(repo=REPO, revision=REVISION), rf.endpoint, hf_home) + ) + snapshot_baseline = json.loads( + run_client(SNAPSHOT.format(repo=REPO, revision=REVISION), rf.endpoint, hf_home) + ) + print(f" {len(snapshot_baseline)} files in the snapshot") + listing = rf.cli("list") + if REPO not in listing: + raise Failure(f"`rustingface list` does not mention {REPO}:\n{listing}") + if "rustingface_bytes_served_total" not in rf.metrics(): + raise Failure("/metrics did not expose the served-bytes counter") + + results.check("a cold pull succeeds and lands in the bucket", cold_pull) + + # ------------------------------------------------------------------ + # 2-4. Cut off upstream, wipe the client, pull again. + # ------------------------------------------------------------------ + def sealed_pull() -> None: + wipe_client_caches(hf_home) + with Instance(args.binary, workdir, storage, sealed_upstream) as rf: + again = json.loads( + run_client(DOWNLOAD_ONE.format(repo=REPO, revision=REVISION), rf.endpoint, hf_home) + ) + if again != baseline: + raise Failure(f"sealed pull differs from the first: {again} != {baseline}") + snapshot_again = json.loads( + run_client(SNAPSHOT.format(repo=REPO, revision=REVISION), rf.endpoint, hf_home) + ) + if snapshot_again != snapshot_baseline: + missing = set(snapshot_baseline) - set(snapshot_again) + raise Failure(f"sealed snapshot differs; missing {sorted(missing)}") + + results.check("a sealed instance serves the identical snapshot", sealed_pull) + + # ------------------------------------------------------------------ + # 5. A brand-new process against the same bucket. + # ------------------------------------------------------------------ + def portable_state() -> None: + wipe_client_caches(hf_home) + # Nothing carries over: a fresh process, a fresh client cache, only the + # bucket in common. This is the step that tests sovereignty rather + # than a cache. + with Instance(args.binary, workdir, storage, sealed_upstream) as rf: + again = json.loads( + run_client(DOWNLOAD_ONE.format(repo=REPO, revision=REVISION), rf.endpoint, hf_home) + ) + if again != baseline: + raise Failure(f"reinstalled instance differs: {again} != {baseline}") + + results.check("a reinstalled instance serves from the bucket alone", portable_state) + + # ------------------------------------------------------------------ + # Metadata replay. + # ------------------------------------------------------------------ + def metadata_replay() -> None: + with Instance(args.binary, workdir, storage, open_upstream) as rf: + live = json.loads( + run_client(FILE_METADATA.format(repo=REPO, revision=REVISION), rf.endpoint, hf_home) + ) + with Instance(args.binary, workdir, storage, sealed_upstream) as rf: + replayed = json.loads( + run_client(FILE_METADATA.format(repo=REPO, revision=REVISION), rf.endpoint, hf_home) + ) + if live != replayed: + raise Failure(f"file metadata changed once sealed: {live} != {replayed}") + print(f" commit {live['commit']} etag {live['etag']} size {live['size']}") + + results.check("file metadata replays byte-identically when sealed", metadata_replay) + + def model_info() -> None: + with Instance(args.binary, workdir, storage, open_upstream) as rf: + info = json.loads( + run_client(MODEL_INFO.format(repo=REPO, revision=REVISION), rf.endpoint, hf_home) + ) + if not info["sha"]: + raise Failure("model_info returned no commit sha") + print(f" {info['sha']} with {len(info['siblings'])} siblings") + + results.check("model_info answers through rustingface", model_info) + + # ------------------------------------------------------------------ + # Failures are loud. + # ------------------------------------------------------------------ + def missing_file_raises() -> None: + with Instance(args.binary, workdir, storage, sealed_upstream) as rf: + result = json.loads( + run_client(MISSING_FILE.format(repo=REPO, revision=REVISION), rf.endpoint, hf_home) + ) + if result["raised"] is None: + raise Failure("a missing file returned successfully instead of raising") + print(f" client raised {result['raised']}") + + results.check("a missing file raises a typed client exception", missing_file_raises) + + def unreachable_upstream_fails_loudly() -> None: + wipe_client_caches(hf_home) + with Instance(args.binary, workdir, storage, unreachable_upstream) as rf: + # Something already stored still serves. + served = json.loads( + run_client(DOWNLOAD_ONE.format(repo=REPO, revision=REVISION), rf.endpoint, hf_home) + ) + if served != baseline: + raise Failure("a stored file stopped serving when upstream became unreachable") + # Something not stored fails, rather than returning an empty file. + result = json.loads( + run_client(MISSING_FILE.format(repo=REPO, revision=REVISION), rf.endpoint, hf_home) + ) + if result["raised"] is None: + raise Failure("an unfetchable file returned successfully") + + results.check( + "an unreachable upstream serves what is stored and refuses what is not", + unreachable_upstream_fails_loudly, + ) + + # ------------------------------------------------------------------ + # The hf CLI. + # ------------------------------------------------------------------ + def hf_cli() -> None: + with Instance(args.binary, workdir, storage, sealed_upstream) as rf: + env = client_env(rf.endpoint, hf_home) + whoami = subprocess.run( + ["hf", "auth", "whoami"], capture_output=True, text=True, env=env + ) + # `hf auth whoami` without a token is allowed to report "not logged + # in"; what must not happen is a crash against /api/whoami-v2. + if "Traceback" in whoami.stderr: + raise Failure(f"hf auth whoami crashed:\n{whoami.stderr}") + download = subprocess.run( + ["hf", "download", REPO, "config.json", "--revision", REVISION], + capture_output=True, + text=True, + env=env, + ) + if download.returncode != 0: + raise Failure(f"hf download failed:\n{download.stdout}\n{download.stderr}") + + if shutil.which("hf"): + results.check("the hf CLI downloads through rustingface", hf_cli) + else: + print("\n=== the hf CLI downloads through rustingface\n skipped: hf not on PATH") + + # ------------------------------------------------------------------ + # safetensors, the path an inference stack actually takes. + # ------------------------------------------------------------------ + def safetensors_load() -> None: + script = """ +import glob, json, os +from huggingface_hub import snapshot_download +from safetensors import safe_open +root = snapshot_download({repo!r}, revision={revision!r}) +files = glob.glob(os.path.join(root, "**", "*.safetensors"), recursive=True) +if not files: + print(json.dumps({{"tensors": 0, "note": "repo has no safetensors"}})) +else: + total = 0 + for path in files: + with safe_open(path, framework="np") as handle: + total += len(handle.keys()) + print(json.dumps({{"tensors": total}})) +""".format(repo=REPO, revision=REVISION) + with Instance(args.binary, workdir, storage, sealed_upstream) as rf: + out = json.loads(run_client(script, rf.endpoint, hf_home)) + print(f" {out}") + + results.check("safetensors files load from a sealed snapshot", safetensors_load) + + # ------------------------------------------------------------------ + # Admin operations against the same bucket. + # ------------------------------------------------------------------ + def admin_operations() -> None: + with Instance(args.binary, workdir, storage, sealed_upstream) as rf: + show = rf.cli("show", REPO) + manifest = json.loads(show) + if not manifest["entries"]: + raise Failure("the manifest records no entries") + verify = rf.cli("verify", REPO) + if "0 corrupt" not in verify: + raise Failure(f"verify reported problems:\n{verify}") + dry = rf.cli("gc", "--dry-run") + if "0 reclaimable" not in dry: + raise Failure(f"gc found reclaimable blobs while everything is referenced:\n{dry}") + + results.check("show, verify and gc agree with what was stored", admin_operations) + + if not args.keep: + shutil.rmtree(workdir, ignore_errors=True) + else: + print(f"\nwork directory kept at {workdir}") + return results.report() + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/test/conformance/requirements.txt b/test/conformance/requirements.txt new file mode 100644 index 0000000..0a99967 --- /dev/null +++ b/test/conformance/requirements.txt @@ -0,0 +1,9 @@ +# Pinned deliberately. A client upgrade is a spec-review trigger (spec §11): +# the X-Error-Code strings rustingface emits are client-internal constants, not +# a stable API, and `conformance.py` re-checks them against whatever is +# installed here on every run. +huggingface_hub==0.36.0 +safetensors==0.6.2 +# safetensors needs a framework to open a file; numpy is the lightest. +numpy==2.3.4 +requests==2.32.5