From 059c781422a4ced36b6b343316038ee87cfb0535 Mon Sep 17 00:00:00 2001
From: rob thijssen <grenade@rob.tn>
Date: Mon, 31 Aug 2026 14:14:15 +0300
Subject: [PATCH] fix(conformance): the hf CLI check was silently skipping in
 CI

The suite's first run on a runner reported "10 passed, 0 failed" and
exited 0. Locally it is 11. The difference is the `hf` CLI check, which
did not run: the workflow invokes .venv/bin/python directly without
putting the venv's bin on PATH, so shutil.which("hf") found nothing and
the check skipped.

A skipped check reads exactly like a passing one, and this is the suite
that runs unattended on a weekly timer specifically to notice when a
client changes underneath us -- so a silent skip defeats its purpose.

Exports the venv's bin in the workflow and asserts `hf version` before
the suite, so a missing CLI fails the job outright rather than being
quietly stepped over. Also counts skips and lists them in the summary,
where they cannot be mistaken for passes.

Verified both ways: with the venv off PATH the suite now reports
"10 passed, 0 failed, 1 skipped" and names it; with it on PATH, 11
passed and 0 skipped.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01XZG2i4AmfSqE97EJGBVb64
---
 .gitea/workflows/conformance.yml |  9 ++++++++-
 test/conformance/conformance.py  | 20 ++++++++++++++++++--
 2 files changed, 26 insertions(+), 3 deletions(-)

diff --git a/.gitea/workflows/conformance.yml b/.gitea/workflows/conformance.yml
index d83e74b..e92bb51 100644
--- a/.gitea/workflows/conformance.yml
+++ b/.gitea/workflows/conformance.yml
@@ -26,5 +26,12 @@ jobs:
       # Runs against a directory rather than a bucket. The S3 path is covered
       # by the `doctor` step of the deploy workflow, which exercises the real
       # MinIO on caveman with the real credentials.
+      # The venv's bin goes on PATH so the `hf` CLI is found. Without it the
+      # suite reports "skipped: hf not on PATH" and still exits 0 -- a check
+      # that silently does not run, in the one suite whose job is to notice
+      # when a client changes underneath us.
       - name: conformance
-        run: .venv/bin/python test/conformance/conformance.py --binary target/release/rustingface
+        run: |
+          export PATH="$PWD/.venv/bin:$PATH"
+          hf version
+          .venv/bin/python test/conformance/conformance.py --binary target/release/rustingface
diff --git a/test/conformance/conformance.py b/test/conformance/conformance.py
index 3866eb8..0c53ba6 100755
--- a/test/conformance/conformance.py
+++ b/test/conformance/conformance.py
@@ -75,6 +75,11 @@ class Failure(Exception):
 class Results:
     passed: list[str] = field(default_factory=list)
     failed: list[tuple[str, str]] = field(default_factory=list)
+    skipped: list[tuple[str, str]] = field(default_factory=list)
+
+    def skip(self, name: str, why: str) -> None:
+        print(f"\n=== {name}\n    skipped: {why}")
+        self.skipped.append((name, why))
 
     def check(self, name: str, fn) -> None:
         print(f"\n=== {name}")
@@ -88,9 +93,17 @@ class Results:
             self.passed.append(name)
 
     def report(self) -> int:
-        print(f"\n{len(self.passed)} passed, {len(self.failed)} failed")
+        print(
+            f"\n{len(self.passed)} passed, {len(self.failed)} failed, "
+            f"{len(self.skipped)} skipped"
+        )
         for name, err in self.failed:
             print(f"  FAILED {name}: {err}")
+        # Surfaced in the summary, not just where it happened: a skipped check
+        # reads exactly like a passing one in a scrollback, and this suite runs
+        # unattended on a timer.
+        for name, why in self.skipped:
+            print(f"  SKIPPED {name}: {why}")
         return 1 if self.failed else 0
 
 
@@ -499,7 +512,10 @@ def main() -> int:
     if shutil.which("hf"):
         results.check("the hf CLI downloads through rustingface", hf_cli)
     else:
-        print("\n=== the hf CLI downloads through rustingface\n    skipped: hf not on PATH")
+        results.skip(
+            "the hf CLI downloads through rustingface",
+            "hf not on PATH (is the venv's bin directory exported?)",
+        )
 
     # ------------------------------------------------------------------
     # safetensors, the path an inference stack actually takes.