diff --git a/doc/spec.md b/doc/spec.md index db7955e..9cbc0e1 100644 --- a/doc/spec.md +++ b/doc/spec.md @@ -461,7 +461,11 @@ file reconstruction information from a CAS rather than a URL. rustingface implements none of it. - **Downstream:** never advertise Xet capability. Clients then take the ordinary - resolve-and-redirect path. + resolve-and-redirect path. This is the half that has to keep working as the + Hub migrates, so the conformance suite runs a *Xet-capable* client — `hf_xet` + installs as a `huggingface_hub` dependency whether or not a user asks for it + — and asserts it still chooses HTTP. Testing it with Xet disabled client-side + would prove nothing about the configuration people actually run. - **Upstream:** set `HF_HUB_DISABLE_XET=1` semantics on outbound fetches so rustingface receives whole-file URLs via the Git LFS bridge, which reconstructs the file and returns a single resource URL for legacy clients. Storing whole diff --git a/test/conformance/conformance.py b/test/conformance/conformance.py index 7eb62df..3b52a34 100755 --- a/test/conformance/conformance.py +++ b/test/conformance/conformance.py @@ -208,9 +208,14 @@ def client_env(endpoint: str, home: Path) -> dict: "HF_ENDPOINT": endpoint, "HF_HOME": str(home), "HF_HUB_CACHE": str(home / "hub"), - # rustingface implements no Xet client and never advertises the - # capability; setting this makes the client's side explicit too. - "HF_HUB_DISABLE_XET": "1", + # Xet is deliberately *not* disabled here. `hf_xet` installs as a + # huggingface_hub dependency, so a real user's client is Xet-capable + # whether or not they asked for it, and the whole claim rustingface + # makes is that pointing HF_ENDPOINT at it works with the client + # people actually have. Setting HF_HUB_DISABLE_XET would test a + # configuration nobody runs, and would hide the one failure mode + # that matters here: a client choosing Xet against a service that + # implements none of it (spec §8). "HF_HUB_DISABLE_TELEMETRY": "1", "HF_HUB_DISABLE_PROGRESS_BARS": "1", } @@ -270,6 +275,32 @@ meta = get_hf_file_metadata(url) print(json.dumps({{"commit": meta.commit_hash, "etag": meta.etag, "size": meta.size}})) """ +# The Xet negotiation, from the client's own side. `hf_xet` ships as a +# huggingface_hub dependency, so a real user's client is Xet-capable whether or +# not they asked for it. What keeps rustingface compatible is that the *server* +# never advertises Xet: `parse_xet_file_data_from_response` returns None, and +# `hf_hub_download` dispatches to plain HTTP (file_download.py, "if +# xet_file_data is not None and is_xet_available()"). +# +# Asserted from inside the client rather than by inspecting headers ourselves, +# because the question is not "which headers did we send" but "what did the +# client conclude from them". +XET_FALLBACK = """ +import json +from huggingface_hub import get_session, hf_hub_url +from huggingface_hub.utils._runtime import is_xet_available +from huggingface_hub.utils._xet import parse_xet_file_data_from_response + +url = hf_hub_url({repo!r}, "config.json", revision={revision!r}) +response = get_session().get(url, headers={{"Accept-Encoding": "identity"}}, allow_redirects=False) +response.raise_for_status() +print(json.dumps({{ + "xet_capable": bool(is_xet_available()), + "xet_offered": parse_xet_file_data_from_response(response) is not None, + "xet_headers": sorted(h for h in response.headers if h.lower().startswith("x-xet-")), +}})) +""" + MISSING_FILE = """ import json from huggingface_hub import hf_hub_download @@ -451,6 +482,37 @@ def main() -> int: results.check("model_info answers through rustingface", model_info) + def xet_capable_client_falls_back_to_http() -> None: + """The central compatibility claim: point HF_ENDPOINT here and it works. + + Xet is the thing most likely to break that quietly. The Hub is moving to + it, `hf_xet` is installed alongside `huggingface_hub` whether or not the + user asked for it, and rustingface implements none of it (spec §8) -- + deliberately, because a Xet-backed fetch would leave the bucket + unreadable without a Xet client. That only holds while the client + *chooses* HTTP, so assert the choice rather than assume it. + """ + with Instance(args.binary, workdir, storage, open_upstream) as rf: + seen = json.loads( + run_client(XET_FALLBACK.format(repo=REPO, revision=REVISION), rf.endpoint, hf_home) + ) + if not seen["xet_capable"]: + raise Failure( + "hf_xet is not installed, so this check proves nothing: it would pass against a " + "client that could not use Xet even if rustingface offered it. Install the pinned " + "requirements." + ) + if seen["xet_offered"] or seen["xet_headers"]: + raise Failure( + "rustingface advertised Xet to the client " + f"(headers {seen['xet_headers']}). It implements none of it, and a client that " + "took that path would write a bucket nobody can read back without a Xet client, " + "which forfeits state portability (spec §8)." + ) + print(" client is Xet-capable, rustingface offered no Xet, download stays on HTTP") + + results.check("a Xet-capable client falls back to plain HTTP", xet_capable_client_falls_back_to_http) + # ------------------------------------------------------------------ # Failures are loud. # ------------------------------------------------------------------